diff options
397 files changed, 15647 insertions, 6526 deletions
diff --git a/.github/actions/test_cpp_sdk_with_coverage/action.yaml b/.github/actions/test_cpp_sdk_with_coverage/action.yaml index 6a3822f64b8..73aaf7f67a9 100644 --- a/.github/actions/test_cpp_sdk_with_coverage/action.yaml +++ b/.github/actions/test_cpp_sdk_with_coverage/action.yaml @@ -41,6 +41,7 @@ runs: --clang-coverage --coverage-report -DCOVERAGE_TARGET_REGEXP=ydb/public/sdk/cpp --coverage-prefix-filter=ydb/public/sdk/cpp + --coverage-exclude-regexp='/(ut|tests)/|_(ut|it)\.(c|cc|cpp|cxx|h|hh|hpp|hxx)' --test-threads "${{ inputs.test_threads }}" --link-threads "${{ inputs.link_threads }}" --output "$out_dir" diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index df566eec68a..ae7070100b4 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -1,4 +1,3 @@ -ydb/core/blobstorage/ddisk/ut TDDiskActorPDiskTest.PDiskRestartWithReservedChunks_DDiskZombie_Uring ydb/core/blobstorage/dsproxy/ut TBlobStorageProxySequenceTest.TestBlock42PutWithChangingSlowDisk ydb/core/blobstorage/pdisk/ut RDMA.TestChunkReadWithRdmaAllocatorEncryptedChunks ydb/core/blobstorage/pdisk/ut RDMA.TestChunkReadWithRdmaAllocatorPlainChunks @@ -28,11 +27,8 @@ ydb/core/kqp/ut/federated_query/datastreams HasPathDatastreams.ExternalTableMatc ydb/core/kqp/ut/federated_query/datastreams HasPathDatastreams.FederatedEdsPathMatches ydb/core/kqp/ut/federated_query/datastreams HasPathDatastreams.FederatedTopicRemoteNameMatches ydb/core/kqp/ut/federated_query/datastreams unittest.[*/*] chunk -ydb/core/kqp/ut/olap/indexes KqpOlapIndexes.IndexesActualization-QueryService ydb/core/kqp/ut/olap/indexes KqpOlapIndexes.MinMaxIndexAppliedToDataAfterCompaction-QueryService-SchemeObjectDisabled ydb/core/kqp/ut/olap/indexes KqpOlapIndexes.MinMaxIndexAppliedToDataAfterCompaction-QueryService-SchemeObjectEnabled -ydb/core/kqp/ut/olap/storage KqpOlapDictionary.GroupBySomeDictionaryWithCompaction -ydb/core/kqp/ut/query KqpAnalyze.AnalyzeTable-ColumnStore ydb/core/kqp/ut/query KqpAnalyzeOperations.AnalyzeOperationsLifecycle ydb/core/kqp/ut/scheme KqpScheme.CreateDropTableMultipleTime ydb/core/load_test/ut GroupWriteTest.SimpleRdma @@ -48,17 +44,8 @@ ydb/public/sdk/cpp/tests/integration/topic gtest.[*/*] chunk ydb/public/sdk/cpp/tests/integration/topic/with_direct_read gtest.[*/*] chunk ydb/services/persqueue_v1/ut TPersQueueTest.TopicServiceCommitOffset ydb/services/persqueue_v1/ut unittest.[*/*] chunk -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_prestable-26-4-COLUMN] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-2-1] @@ -68,7 +55,6 @@ ydb/tests/compatibility/result_set_format py3test.[test_result_set_arrow.py */*] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-False] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-True] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRollingUpgradeAndDowngrade.test_rolling_upgrade[rolling_stable-26-3-1_to_prestable-26-4-False] -ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-3_and_stable-26-2-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-4_and_stable-26-3-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_stable-26-3-1_and_stable-26-2-1] ydb/tests/fq/http_api test_http_api.py.TestHttpApi.test_restart_idempotency @@ -80,6 +66,7 @@ ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_disk_creation ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_disk_creation_name_with_symbols ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_multiple_disks_creation ydb/tests/functional/query_cache/warmup test_warmup.py.TestCompileCacheViewPeerWarnings.test_peer_scan_warnings_increment_on_dead_peer +ydb/tests/functional/security/mon/canonical test_mon_endpoints_auth.py.test[require_healthcheck_authentication-with_schema_grants] ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud[fifo] ydb/tests/functional/sqs/migration/compatibility/common ydb.tests.functional.sqs.common.test_queue_counters.py.TestSqsGettingCounters.test_sqs_action_counters ydb/tests/functional/sqs/migration/finished/common ydb.tests.functional.sqs.common.test_queue_counters.py.TestSqsGettingCounters.test_sqs_action_counters @@ -87,6 +74,7 @@ ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional.sqs.multinode.test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[kick_tablets-std] ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional.sqs.multinode.test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[stop_node-fifo] ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional.sqs.multinode.test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[stop_node-std] +ydb/tests/functional/tenants test_remove_storage_groups.py.test_remove_storage_group ydb/tests/stress/kafka_serverless/tests test_kafka_streams.py.TestYdbTopicWorkload.test ydb/tests/stress/oltp_workload/tests py3test.sole chunk ydb/tests/stress/oltp_workload/tests test_workload.py.TestYdbWorkload.test @@ -97,7 +85,6 @@ ydb/tests/stress/topic_sqs/tests test_workload.py.TestYdbTopicSqsWorkload.test ydb/tools/mnc/viewer/ut py3test.sole chunk ydb/tools/mnc/viewer/ut test_cluster_config_validation.py.ClusterConfigSelectionStateTest.test_operations_tab_shows_running_steps_and_final_output ydb/tools/mnc/viewer/ut test_navigation.py.ViewerTabNavigationTest.test_operation_tab_navigation_matches_state_machine -ydb/tools/mnc/viewer/ut test_navigation.py.ViewerTabNavigationTest.test_tab_navigation_matches_state_machine_for_core_command_sequences ydb/tools/stress_tool/ut TDeviceTestTool.PDiskTestLogWrite ydb/tools/stress_tool/ut TDeviceTestTool.PersistentBufferTestWrite ydb/tools/stress_tool/ut unittest.sole chunk diff --git a/.github/config/muted_ya_asan.txt b/.github/config/muted_ya_asan.txt index a8ed49ec521..d23a7816d30 100644 --- a/.github/config/muted_ya_asan.txt +++ b/.github/config/muted_ya_asan.txt @@ -49,6 +49,7 @@ ydb/core/blobstorage/ut_blobstorage/ut_replication unittest.[*/*] chunk ydb/core/blobstorage/ut_blobstorage/ut_restart_pdisk BSCRestartPDisk.RestartOneByOneWithReconnects ydb/core/blobstorage/ut_blobstorage/ut_restart_pdisk unittest.[*/*] chunk ydb/core/blobstorage/ut_vdisk unittest.[*/*] chunk +ydb/core/formats/arrow/accessor/sub_columns/ut SubColumnsSparsedBuilder.BinaryJsonMatchesPlain ydb/core/graph/shard/ut unittest.sole chunk ydb/core/http_proxy/ut TestMalformedRequest.CompressedDeflateContentLengthCorrect ydb/core/http_proxy/ut TestMalformedRequest.CompressedDeflateContentLengthHigher @@ -474,7 +475,6 @@ ydb/core/kqp/ut/olap KqpOlapOptimizer.TilingPlusPlusTtlDeletion ydb/core/kqp/ut/olap/types KqpOlapJson.BloomMixIndexesOldSyntaxVariants[false,true,1024,0,0,0.5,BLOOM_FILTER] ydb/core/kqp/ut/olap/types KqpOlapJson.QuotedFilterVariants[10,false,1,0,100,0.5] ydb/core/kqp/ut/olap/types KqpOlapJson.SimpleExistsVariants[10,false,0,10,100,0.5] -ydb/core/kqp/ut/query KqpLimits.ComputeNodeMemoryLimit ydb/core/kqp/ut/query KqpLimits.OutOfSpaceYQLUpsertFail ydb/core/kqp/ut/query unittest.[*/*] chunk ydb/core/kqp/ut/rbo KqpRboOlap.BlockChannelAuto @@ -502,7 +502,7 @@ ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs+IsOlap-UseAdmin ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs-IsOlap+UseAdmin ydb/core/kqp/ut/scheme KqpAcl.AclInterruptInheritance ydb/core/kqp/ut/scheme KqpAcl.AclTemporaryInterruptInheritance-IsOlap -ydb/core/kqp/ut/scheme KqpConstraints.DropDefaultNotNullColumn_InsertOrReplace-IsInsert +ydb/core/kqp/ut/scheme KqpConstraints.CompileTimeDefaults_WithGlobalAsyncIndex-EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.AlterTableCompactAccessDenied+UseQueryService ydb/core/kqp/ut/scheme KqpScheme.CreateDropTableViaApiMultipleTime ydb/core/kqp/ut/scheme KqpScheme.ParallelDropStreamingQuery @@ -543,6 +543,7 @@ ydb/core/kqp/ut/view TCreateAndDropViewTest.DropViewIfExists ydb/core/kqp/ut/view TCreateAndDropViewTest.InvalidColumnRef ydb/core/kqp/ut/view TCreateAndDropViewTest.InvalidIndexRef ydb/core/kqp/ut/view TCreateAndDropViewTest.InvalidTableRef +ydb/core/kqp/ut/view TCreateAndDropViewTest.ParsingSecurityInvoker ydb/core/kqp/ut/view TSelectFromViewTest.LeftJoinViewsWithNoMatch ydb/core/load_test/ut GroupWriteTest.SimpleRdma ydb/core/load_test/ut GroupWriteTest.WriteHardRateDispatcher @@ -589,7 +590,6 @@ ydb/core/tx/columnshard/ut_rw TColumnShardTestReadWrite.CompactionSplitGranule_P ydb/core/tx/columnshard/ut_rw unittest.[*/*] chunk ydb/core/tx/datashard/build_index/ut TTxDataShardLocalKMeansScan.BuildToBuild_Ranges ydb/core/tx/datashard/build_index/ut TTxDataShardLocalKMeansScan.EmptyEmbeddingSkipped -ydb/core/tx/datashard/build_index/ut TTxDataShardLocalKMeansScan.MainToPostingWithKeyRange ydb/core/tx/datashard/build_index/ut TTxDataShardLocalKMeansScan.NullEmbedding ydb/core/tx/datashard/build_index/ut TTxDataShardPrefixKMeansScan.BadRequest ydb/core/tx/datashard/build_index/ut TTxDataShardPrefixKMeansScan.BuildToBuild @@ -614,6 +614,7 @@ ydb/core/tx/datashard/ut_stats DataShardStats.BackupTableStatsReportInterval ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleFollower-0-0 ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleFollower-0-1 ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleFollower-1-0 +ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleFollower-1-1 ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleLeader-0-0 ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleLeader-0-1 ydb/core/tx/datashard/ut_stats DataShardStats.CollectKeySampleLeader-1-0 @@ -660,7 +661,6 @@ ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut LocalPartition.Direct ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut LocalPartition.WithoutPartitionWithRestart ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Oltp_WriteToTopic_3_Query ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Oltp_WriteToTopic_3_Table -ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Oltp_WriteToTopics_2_Query ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Oltp_WriteToTopics_2_Table ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.WriteToTopic_Demo_27_Query ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut unittest.[*/*] chunk @@ -686,26 +686,15 @@ ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_UseUserPasswordWi ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_WithConnectionStringFakeTopicPath ydb/services/ydb/backup_ut BackupRestore.ReplicasAreNotBackedUp+UseSchemeSecret ydb/services/ydb/backup_ut BackupRestore.ReplicasAreNotBackedUp-UseSchemeSecret -ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_stable-26-2-1_to_prestable-26-3-row] ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_stable-26-3-1_to_stable-26-3-1-row] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_prestable-26-3] ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/common test_user_management.py.TestUserManagementRollingUpgrade.test_user_management[rolling_stable-26-2-1_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-2-1] @@ -713,42 +702,18 @@ ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex. ydb/tests/compatibility/olap test_encoding.py.TestEncoding.test_encoding[rolling_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/olap test_encoding.py.TestEncoding.test_encoding[rolling_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/result_set_format py3test.[test_result_set_arrow.py */*] chunk -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] @@ -761,30 +726,6 @@ ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] @@ -809,54 +750,24 @@ ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] @@ -869,30 +780,6 @@ ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] @@ -917,29 +804,14 @@ ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingMixedCluster.test_mixed_cluster[mixed_prestable-26-3_and_stable-26-2-1-False] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-False] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-True] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-False] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-True] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRollingUpgradeAndDowngrade.test_rolling_upgrade[rolling_stable-26-2-1_to_prestable-26-3-False] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_prestable-26-3] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_prestable-26-3_and_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_prestable-26-4] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_prestable-26-4_and_stable-26-3-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_stable-26-2-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_stable-26-2-1_and_stable-26-1-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicMixedClusterFixture.test_workload[mixed_stable-26-3-1_and_stable-26-2-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-1-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-3-1] -ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-3_and_stable-26-2-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-4_and_stable-26-3-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_stable-26-3-1_and_stable-26-2-1] ydb/tests/datashard/truncate/concurrency py3test.[test_truncate_table_concurrency.py */*] chunk @@ -959,6 +831,7 @@ ydb/tests/fq/s3 test_streaming_join.py.TestStreamingJoin.test_grace_join[v1-clie ydb/tests/fq/streaming test_scalar_topic_write.py.TestScalarTopicWriteInYdb.test_mixed_valid_invalid_statements[False] ydb/tests/fq/streaming test_scalar_topic_write.py.TestScalarTopicWriteInYdb.test_mixed_valid_invalid_statements[True] ydb/tests/fq/streaming test_udfs.py.TestUdfsUsage.test_precompute_recovery[False] +ydb/tests/fq/streaming/streaming_large test_cluster_restarts.py.TestStreamingLarge.test_rolling_restart[kikimr0] ydb/tests/fq/streaming_optimize py3test.[test_sql_negative.py */*] chunk ydb/tests/fq/streaming_optimize py3test.[test_sql_streaming.py */*] chunk ydb/tests/fq/streaming_optimize py3test.sole chunk @@ -991,12 +864,15 @@ ydb/tests/fq/yt/kqp_yt_file/part0 test.py.test[aggregate-group_by_gs_and_having- ydb/tests/fq/yt/kqp_yt_file/part1 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part1 test.py.test[aggregate-group_by_expr_mul_col-] ydb/tests/fq/yt/kqp_yt_file/part1 test.py.test[aggregate-group_by_expr_with_join-] +ydb/tests/fq/yt/kqp_yt_file/part1 test.py.test[aggregate-group_by_rollup_grouping_hum-] ydb/tests/fq/yt/kqp_yt_file/part10 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part10 test.py.test[action-eval_skip_take-] ydb/tests/fq/yt/kqp_yt_file/part10 test.py.test[aggregate-avg_and_sum_float-] +ydb/tests/fq/yt/kqp_yt_file/part10 test.py.test[aggregate-avg_with_having-default.txt] ydb/tests/fq/yt/kqp_yt_file/part11 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part11 test.py.test[aggregate-group_by_expr_alias_on_subexp-] ydb/tests/fq/yt/kqp_yt_file/part11 test.py.test[aggregate-group_by_expr_order_by_expr-] +ydb/tests/fq/yt/kqp_yt_file/part11 test.py.test[aggregate-group_by_gs_grouping-] ydb/tests/fq/yt/kqp_yt_file/part12 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part12 test.py.test[action-eval_for_over_subquery-default.txt] ydb/tests/fq/yt/kqp_yt_file/part13 py3test.sole chunk @@ -1019,12 +895,14 @@ ydb/tests/fq/yt/kqp_yt_file/part19 test.py.test[aggregate-aggregate_with_const_y ydb/tests/fq/yt/kqp_yt_file/part2 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part2 test.py.test[aggregate-compare_tuple-] ydb/tests/fq/yt/kqp_yt_file/part2 test.py.test[aggregate-group_by_expr_semi_join-] +ydb/tests/fq/yt/kqp_yt_file/part2 test.py.test[aggregate-group_by_gs_alt_duo-] ydb/tests/fq/yt/kqp_yt_file/part3 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part3 test.py.test[action-eval_column-] ydb/tests/fq/yt/kqp_yt_file/part3 test.py.test[aggregate-aggregate_distinct_expr_with_groupby_expr-default.txt] ydb/tests/fq/yt/kqp_yt_file/part4 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part4 test.py.test[aggregate-aggregate_with_deep_aggregated_column-] ydb/tests/fq/yt/kqp_yt_file/part4 test.py.test[aggregate-aggregate_with_lambda_inside_avg-] +ydb/tests/fq/yt/kqp_yt_file/part4 test.py.test[aggregate-aggrs_no_grouping_via_map-default.txt] ydb/tests/fq/yt/kqp_yt_file/part5 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part5 test.py.test[action-action_nested_query-default.txt] ydb/tests/fq/yt/kqp_yt_file/part6 py3test.sole chunk @@ -1034,13 +912,14 @@ ydb/tests/fq/yt/kqp_yt_file/part7 test.py.test[action-eval_if-default.txt] ydb/tests/fq/yt/kqp_yt_file/part8 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part8 test.py.test[aggregate-agg_filter_pushdown-] ydb/tests/fq/yt/kqp_yt_file/part8 test.py.test[aggregate-aggrs_no_grouping-] +ydb/tests/fq/yt/kqp_yt_file/part8 test.py.test[aggregate-group_by_cube_expr_trio-] ydb/tests/fq/yt/kqp_yt_file/part9 py3test.sole chunk ydb/tests/fq/yt/kqp_yt_file/part9 test.py.test[aggregate-aggregate_distinct_expr_with_udf-] ydb/tests/fq/yt/kqp_yt_file/part9 test.py.test[aggregate-group_by_expr-] +ydb/tests/fq/yt/kqp_yt_file/part9 test.py.test[aggregate-group_by_rollup_aggr_expr-] ydb/tests/functional/audit test_canonical_records.py.test_dstool_evict_vdisk_grpc ydb/tests/functional/blobstorage test_node_warden_cache.py.TestNodeWardenCacheFile.test_cache_file_lifecycle_and_restart ydb/tests/functional/config test_distconf_sentinel_node_status.py.TestKiKiMRDistConfSelfHealOverrides.test_state_storage -ydb/tests/functional/dstool test_canonical_requests.py.Test.test_group_add ydb/tests/functional/kqp/kqp_indexes ConsistentIndexRead.InteractiveTx ydb/tests/functional/kqp/kqp_indexes unittest.sole chunk ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach @@ -1069,7 +948,6 @@ ydb/tests/functional/statistics test_restarts.py.test_basic ydb/tests/functional/tpc/large py3test.[test_tpcds.py] chunk ydb/tests/functional/tpc/large py3test.[test_tpch_spilling.py] chunk ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[14] -ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[47] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[50] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[64] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1DecimalNative.test_tpcds[14] @@ -1228,8 +1106,11 @@ ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_doc ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_docs_search_list_with_path_filter ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_docs_search_tool_schema_in_request ydb/tests/olap data_read_correctness.py.TestDataReadPerformanceNoIntersections.test +ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_from_compression[zstd_20_compression-algorithm=zstd,level=20] +ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_from_compression[zstd_8_compression-algorithm=zstd,level=8] ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_to_compression[zstd_20_compression-algorithm=zstd,level=20] ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_zstd_level_from_0_to_19_compression +ydb/tests/olap/column_compression/alter py3test.[*/*] chunk ydb/tests/olap/s3_import py3test.[*/*] chunk ydb/tests/olap/s3_import test_simple_table.py.TestSimpleTable.test_drop_table_during_export ydb/tests/olap/s3_import test_simple_table.py.TestSimpleTable.test_rename_table_during_export diff --git a/.github/config/muted_ya_msan.txt b/.github/config/muted_ya_msan.txt index 1bd95b8fc85..6f15d482871 100644 --- a/.github/config/muted_ya_msan.txt +++ b/.github/config/muted_ya_msan.txt @@ -1,11 +1,10 @@ -ydb/core/blobstorage/backpressure/ut_client Backpressure.MonteCarlo -ydb/core/blobstorage/backpressure/ut_client unittest.sole chunk ydb/core/blobstorage/ddisk/ut TDDiskActorTest.LateOutdatedSyncReadResultDoesNotLogUnknownSync ydb/core/blobstorage/ut_blobstorage/ut_balancing VDiskBalancing.TestRandom_Block42 ydb/core/blobstorage/ut_blobstorage/ut_balancing VDiskBalancing.TestRandom_Mirror3dc ydb/core/blobstorage/ut_blobstorage/ut_balancing unittest.[*/*] chunk ydb/core/blobstorage/ut_blobstorage/ut_bridge BridgeGet.PartRestorationAcrossBridgeOnRange ydb/core/blobstorage/ut_blobstorage/ut_bridge unittest.[*/*] chunk +ydb/core/blobstorage/ut_blobstorage/ut_ddisk DDisk.PersistentBufferFillAndRead ydb/core/blobstorage/ut_blobstorage/ut_ddisk unittest.[*/*] chunk ydb/core/blobstorage/ut_blobstorage/ut_phantom_blobs PhantomBlobs.TestBlobSizeLimitAboveLimit ydb/core/blobstorage/ut_blobstorage/ut_phantom_blobs PhantomBlobs.TestBlobSizeLimitBelowLimit @@ -41,66 +40,31 @@ ydb/library/actors/interconnect/ut Interconnect.RdmaXdcCatchReplayAfterPartialPa ydb/library/actors/interconnect/ut Interconnect.RdmaXdcCatchReplayAfterPartialRdmaRead ydb/library/actors/interconnect/ut Interconnect.RdmaXdcCatchReplayAfterPartialRdmaReadOnPeerSocketClose ydb/library/actors/interconnect/ut Interconnect.SetupRdmaSession -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_prestable-26-3] ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/common test_user_management.py.TestUserManagementRollingUpgrade.test_user_management[rolling_stable-26-2-1_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-3-1] -ydb/tests/compatibility/olap test_encoding.py.TestEncoding.test_encoding[rolling_stable-26-2-1_to_prestable-26-3] ydb/tests/compatibility/result_set_format py3test.[test_result_set_arrow.py */*] chunk -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] @@ -113,30 +77,6 @@ ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] @@ -161,54 +101,24 @@ ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] @@ -221,30 +131,6 @@ ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] @@ -269,12 +155,8 @@ ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingMixedCluster.test_mixed_cluster[mixed_prestable-26-3_and_stable-26-2-1-True] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-False] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-True] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-False] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-True] -ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-3_and_stable-26-2-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-4_and_stable-26-3-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_stable-26-3-1_and_stable-26-2-1] ydb/tests/functional/audit test_canonical_records.py.test_dstool_evict_vdisk_grpc diff --git a/.github/config/muted_ya_tsan.txt b/.github/config/muted_ya_tsan.txt index a6cc808299d..c9e7b923422 100644 --- a/.github/config/muted_ya_tsan.txt +++ b/.github/config/muted_ya_tsan.txt @@ -85,7 +85,6 @@ ydb/core/kqp/executer_actor/ut TKqpTasksGraphBuild.TpchQuery22 ydb/core/kqp/executer_actor/ut unittest.sole chunk ydb/core/kqp/rm_service/ut KqpRm.ConcurrentChannels ydb/core/kqp/rm_service/ut KqpRm.ConcurrentTasks -ydb/core/kqp/ut/arrow unittest.[*/*] chunk ydb/core/kqp/ut/indexes/fulltext unittest.[*/*] chunk ydb/core/kqp/ut/opt KqpNamedExpressions.NamedExpressionRandomUpsertRevert+UseDataQuery-EnableIndexStreamWrite ydb/core/kqp/ut/opt unittest.[*/*] chunk @@ -128,7 +127,6 @@ ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut TVChunkTest.Shoul ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut TVChunkTest.ShouldSwitchHostToOfflineAndBack ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut TVChunkTest.ShouldSwitchHostToTemporaryOfflineAndBack ydb/core/nbs/cloud/blockstore/libs/vhost/ut TServerTest.ShouldCancelRequestsInFlightWhenStopEndpointOrStopServer -ydb/core/persqueue/pqtablet/partition/mlp/ut TMLPConsumerFIFOWithSplitDedup.Dedup_DoubleSplit_3min ydb/core/persqueue/pqtablet/partition/mlp/ut TMLPConsumerFIFOWithSplitDedup.Dedup_WithGroup_SameIds_6min ydb/core/persqueue/public/describer/ut TDescriberTests.TopicNotCanonizedPath ydb/core/persqueue/public/describer/ut TDescriberTests.TopicWithoutDatabase @@ -172,81 +170,41 @@ ydb/core/transfer/ut/functional Transfer.WriteToNotExists ydb/core/transfer/ut/functional Transfer.WriteToNotTable ydb/core/transfer/ut/functional unittest.sole chunk ydb/core/transfer/ut/row_table Transfer_RowTable.ComplexKey -ydb/core/transfer/ut/row_table Transfer_RowTable.TableWithAsyncIndex ydb/core/transfer/ut/row_table unittest.sole chunk ydb/core/tx/columnshard/backup/iscan/ut IScan.MultiExport ydb/core/tx/columnshard/backup/iscan/ut IScan.ShouldRejectParquetExportWithEncryption ydb/core/tx/columnshard/backup/iscan/ut IScan.UploaderExport +ydb/core/tx/datashard/ut_truncate unittest.[*/*] chunk ydb/core/tx/schemeshard/ut_set_column_constraint SetNotNullTest.CancelAtLockingStage ydb/core/tx/schemeshard/ut_set_column_constraint unittest.[*/*] chunk ydb/core/tx/tx_proxy/ut_ext_tenant TExtSubDomainTest.CreateTableInsideAndAlterDomainAndTable-AlterDatabaseCreateHiveFirst-true ydb/core/tx/tx_proxy/ut_storage_tenant TStorageTenantTest.CreateDummyTabletsInDifferentDomains -ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_prestable-26-3_to_stable-26-2-1-row] -ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_stable-26-2-1_to_prestable-26-3-row] -ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_stable-26-2-1_to_stable-26-1-1-row] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-3_to_prestable-26-3-ROW] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_prestable-26-3] ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/indexes py3test.[test_vector_index.py */*] chunk -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-3-1] -ydb/tests/compatibility/indexes test_vector_index.py.TestVectorIndex.test_vector_index[rolling_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/indexes test_vector_index.py.TestVectorIndex.test_vector_index[rolling_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/olap test_encoding.py.TestEncoding.test_encoding[rolling_stable-26-2-1_to_prestable-26-3] ydb/tests/compatibility/olap test_encoding.py.TestEncoding.test_encoding[rolling_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/result_set_format py3test.[test_result_set_arrow.py */*] chunk -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] @@ -259,30 +217,6 @@ ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] @@ -307,54 +241,24 @@ ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_arrow.py.TestResultSetArrow.test_types_mapping[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_column_alias[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-3_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_prestable-26-4_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-1-1_to_stable-26-2-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_prestable-26-3-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-1-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-2-1_to_stable-26-3-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_prestable-26-4-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-2-1-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_limit_ordered_columns[restart_stable-26-3-1_to_stable-26-3-1-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-3_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_prestable-26-4-1-2048-COLUMN] @@ -367,30 +271,6 @@ ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_prestable-26-4_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-1-1_to_stable-26-2-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_prestable-26-3-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-1-1-2-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-0-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-1-8388608-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-2048-COLUMN] -ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-2-1-2-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-0-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-2-1_to_stable-26-3-1-1-2048-COLUMN] @@ -415,24 +295,14 @@ ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSet ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-1-8388608-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-2048-COLUMN] ydb/tests/compatibility/result_set_format test_result_set_value.py.TestResultSetValue.test_schema_inclusion_mode[restart_stable-26-3-1_to_stable-26-3-1-2-8388608-COLUMN] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-False] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_prestable-26-3_to_stable-26-2-1-True] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-False] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-True] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRollingUpgradeAndDowngrade.test_rolling_upgrade[rolling_stable-26-2-1_to_prestable-26-3-False] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-3_to_prestable-26-3] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-3_to_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_prestable-26-4_to_stable-26-3-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-1-1_to_stable-26-2-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_prestable-26-3] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-1-1] -ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-2-1_to_stable-26-3-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-3-1_to_stable-26-2-1] ydb/tests/compatibility/topics test_kafka_topic.py.TestKafkaTopicRestartToAnotherVersion.test_workload[restart_stable-26-3-1_to_stable-26-3-1] -ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-3_and_stable-26-2-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_prestable-26-4_and_stable-26-3-1] ydb/tests/compatibility/udf test_datetime2.py.TestDatetime2.test_all[mixed_stable-26-3-1_and_stable-26-2-1] ydb/tests/fq/http_api py3test.sole chunk @@ -915,7 +785,6 @@ ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpcdsDiffProcessing. ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpcdsDiffProcessing.test_tpcds[CheckCanonicalPolicy.NO] ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpcdsDiffProcessing.test_tpcds[CheckCanonicalPolicy.WARNING] ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpchDiffProcessing.test_tpch[CheckCanonicalPolicy.ERROR] -ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpchDiffProcessing.test_tpch[CheckCanonicalPolicy.WARNING] ydb/tests/functional/tpc/medium test_import_csv.py.TestExternalImportCsv.test ydb/tests/functional/tpc/medium test_tpcc.py.TestTpccW10T4SdkCompaction.test ydb/tests/functional/tpc/medium test_tpcc.py.TestTpccW5T4LegacyCompaction.test @@ -1097,7 +966,6 @@ ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnComp ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_zstd_level_from_0_to_19_compression ydb/tests/olap/scenario py3test.[*/*] chunk ydb/tests/olap/scenario test_insert.py.TestInsert.test_multi[read_data_during_bulk_upsert] -ydb/tests/stress/backup/tests test_workload.py.TestYdbBackupWorkload.test ydb/tests/stress/compare_index_performance/tests test_compare.py.TestCompareIndexPerformance.test_vector ydb/tests/stress/kafka/tests test_kafka_streams.py.TestYdbTopicWorkload.test ydb/tests/stress/kafka_serverless/tests test_kafka_streams.py.TestYdbTopicWorkload.test @@ -1105,11 +973,10 @@ ydb/tests/stress/kv_volume_tool/tests py3test.sole chunk ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upload_download[128MB] ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upload_download[1GB] ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upload_download[256MB] +ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upload_download[512MB] ydb/tests/stress/mixedpy/tests py3test.sole chunk ydb/tests/stress/mixedpy/tests test_workload.py.TestYdbMixedWorkload.test[column] ydb/tests/stress/mixedpy/tests test_workload.py.TestYdbMixedWorkload.test[row] -ydb/tests/stress/nfs_backups/tests py3test.sole chunk -ydb/tests/stress/nfs_backups/tests test_workload.py.TestYdbNfsWorkload.test ydb/tests/stress/node_broker/tests py3test.sole chunk ydb/tests/stress/node_broker/tests test_workload.py.TestDeltaProtocol.test ydb/tests/stress/olap_workload/tests py3test.sole chunk @@ -1120,7 +987,6 @@ ydb/tests/stress/reconfig_state_storage_workload/tests test_scheme_board_workloa ydb/tests/stress/result_set_format/tests py3test.sole chunk ydb/tests/stress/result_set_format/tests test_arrow_workload.py.TestResultSetArrowWorkload.test ydb/tests/stress/result_set_format/tests test_value_workload.py.TestResultSetValueWorkload.test -ydb/tests/stress/scheme_board/pile_promotion/tests test_scheme_board_workload.py.TestSchemeBoard.test_scheme_board ydb/tests/stress/show_create/table/tests test_workload.py.TestYdbWorkload.test_show_create_table_workload[30-None] ydb/tests/stress/simple_queue/tests py3test.sole chunk ydb/tests/stress/simple_queue/tests test_workload.py.TestYdbWorkload.test[heavy-column] diff --git a/.github/workflows/cpp_sdk_peerdirs_check.yml b/.github/workflows/cpp_sdk_peerdirs_check.yml index ad124c0c8fc..d4000fb789a 100644 --- a/.github/workflows/cpp_sdk_peerdirs_check.yml +++ b/.github/workflows/cpp_sdk_peerdirs_check.yml @@ -3,18 +3,27 @@ name: Check CPP SDK PEERDIRs on: pull_request_target: branches: [main, stable-*] + types: [opened, synchronize, reopened, labeled] paths: ['ydb/public/sdk/cpp/**', '.github/workflows/cpp_sdk_peerdirs_check.yml'] workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # Irrelevant label events must not cancel a running check for the same PR. + group: >- + ${{ github.workflow }}-${{ + github.event_name == 'pull_request_target' + && format('pr-{0}{1}', github.event.pull_request.number, + github.event.action == 'labeled' && github.event.label.name != 'ok-to-test' && format('-ignore-{0}', github.run_id) || '') + || github.ref + }} cancel-in-progress: true jobs: check: if: >- github.event_name == 'workflow_dispatch' || - (vars.CHECKS_SWITCH != '' && fromJSON(vars.CHECKS_SWITCH).cpp_sdk_peerdirs_check == true) + (github.event.pull_request.state == 'open' && + (github.event.action != 'labeled' || github.event.label.name == 'ok-to-test')) runs-on: [self-hosted, tiny-worker] steps: - uses: actions/github-script@v8 diff --git a/.github/workflows/docs_build.yaml b/.github/workflows/docs_build.yaml index cdf6adbc47d..f478f7c836b 100644 --- a/.github/workflows/docs_build.yaml +++ b/.github/workflows/docs_build.yaml @@ -59,6 +59,7 @@ jobs: 'stable-26-1-1', 'stable-26-2', 'stable-26-2-1', + 'stable-26-1-1-enterprise', ]; let baseRef; if (context.eventName === 'workflow_dispatch') { diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index a2d4fc87259..0235b0a2b3a 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -248,6 +248,9 @@ jobs: with: ref: ${{ needs.check-running-allowed.outputs.commit_sha }} fetch-depth: 2 + # Fork code is checked out only after the check-running-allowed gate + # (collaborator/owner or a maintainer-applied `ok-to-test` label) + allow-unsafe-pr-checkout: true - name: Setup ydb access uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials with: diff --git a/build/conf/ts/ts_check.conf b/build/conf/ts/ts_check.conf index 366665f9bf3..430c9b57572 100644 --- a/build/conf/ts/ts_check.conf +++ b/build/conf/ts/ts_check.conf @@ -5,8 +5,9 @@ macro _TS_CHECK(SCRIPT_NAME, IS_MEDIUM, CHECK_TYPE) { SET_APPEND(_TS_CHECK_LIST ${_TS_CHECK_SEPARATOR} ${SCRIPT_NAME} ${IS_MEDIUM} ${CHECK_TYPE}) } +# todo: FBP-3291: когда-нибудь прибраться и убрать тернарник, оставить медиум для всех проверок macro TS_LINT(SCRIPT_NAME, TIMEOUT_MEDIUM?"yes":"no") { - _TS_CHECK($SCRIPT_NAME $TIMEOUT_MEDIUM lint) + _TS_CHECK($SCRIPT_NAME yes lint) } macro TS_TEST(SCRIPT_NAME) { diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json index 308f8343545..75dcc48d3b9 100644 --- a/build/external_resources/ymake/public.resources.json +++ b/build/external_resources/ymake/public.resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:12759441213" + "uri": "sbr:12836461311" }, "darwin-arm64": { - "uri": "sbr:12759440320" + "uri": "sbr:12836459950" }, "linux": { - "uri": "sbr:12759442670" + "uri": "sbr:12836463794" }, "linux-aarch64": { - "uri": "sbr:12759439073" + "uri": "sbr:12836458633" }, "win32": { - "uri": "sbr:12759441947" + "uri": "sbr:12836462531" } } } diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json index 3366243d6dc..cde2455e67d 100644 --- a/build/external_resources/ymake/resources.json +++ b/build/external_resources/ymake/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:12759447218" + "uri": "sbr:12836456989" }, "darwin-arm64": { - "uri": "sbr:12759446141" + "uri": "sbr:12836455885" }, "linux": { - "uri": "sbr:12759448668" + "uri": "sbr:12836459916" }, "linux-aarch64": { - "uri": "sbr:12759445467" + "uri": "sbr:12836454677" }, "win32": { - "uri": "sbr:12759447921" + "uri": "sbr:12836458452" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index aa1d8378f29..685cb5b6154 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -623,6 +623,7 @@ "12765030473": "{registry_endpoint}/12765030473", "12778660629": "{registry_endpoint}/12778660629", "12807488115": "{registry_endpoint}/12807488115", + "12845364131": "{registry_endpoint}/12845364131", "5486713852": "{registry_endpoint}/5486713852", "5514352253": "{registry_endpoint}/5514352253", "5523579199": "{registry_endpoint}/5523579199", @@ -1021,6 +1022,7 @@ "12592467517": "{registry_endpoint}/12592467517", "12675295547": "{registry_endpoint}/12675295547", "12759441213": "{registry_endpoint}/12759441213", + "12836461311": "{registry_endpoint}/12836461311", "5766172292": "{registry_endpoint}/5766172292", "5805431504": "{registry_endpoint}/5805431504", "5829027626": "{registry_endpoint}/5829027626", @@ -1165,6 +1167,7 @@ "12592466452": "{registry_endpoint}/12592466452", "12675293631": "{registry_endpoint}/12675293631", "12759440320": "{registry_endpoint}/12759440320", + "12836459950": "{registry_endpoint}/12836459950", "5766171800": "{registry_endpoint}/5766171800", "5805430761": "{registry_endpoint}/5805430761", "5829025456": "{registry_endpoint}/5829025456", @@ -1309,6 +1312,7 @@ "12592469929": "{registry_endpoint}/12592469929", "12675298739": "{registry_endpoint}/12675298739", "12759442670": "{registry_endpoint}/12759442670", + "12836463794": "{registry_endpoint}/12836463794", "5766173070": "{registry_endpoint}/5766173070", "5805432830": "{registry_endpoint}/5805432830", "5829031598": "{registry_endpoint}/5829031598", @@ -1453,6 +1457,7 @@ "12592464955": "{registry_endpoint}/12592464955", "12675292272": "{registry_endpoint}/12675292272", "12759439073": "{registry_endpoint}/12759439073", + "12836458633": "{registry_endpoint}/12836458633", "5766171341": "{registry_endpoint}/5766171341", "5805430188": "{registry_endpoint}/5805430188", "5829023352": "{registry_endpoint}/5829023352", @@ -1597,6 +1602,7 @@ "12592468857": "{registry_endpoint}/12592468857", "12675297027": "{registry_endpoint}/12675297027", "12759441947": "{registry_endpoint}/12759441947", + "12836462531": "{registry_endpoint}/12836462531", "8270821739": "{registry_endpoint}/8270821739", "8295446553": "{registry_endpoint}/8295446553", "8326170338": "{registry_endpoint}/8326170338", @@ -3051,6 +3057,7 @@ "12765030473": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "12778660629": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "12807488115": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "12845364131": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5486713852": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5514352253": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5523579199": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", @@ -3449,6 +3456,7 @@ "12592467517": "devtools/ymake/bin/ymake for darwin", "12675295547": "devtools/ymake/bin/ymake for darwin", "12759441213": "devtools/ymake/bin/ymake for darwin", + "12836461311": "devtools/ymake/bin/ymake for darwin", "5766172292": "devtools/ymake/bin/ymake for darwin", "5805431504": "devtools/ymake/bin/ymake for darwin", "5829027626": "devtools/ymake/bin/ymake for darwin", @@ -3593,6 +3601,7 @@ "12592466452": "devtools/ymake/bin/ymake for darwin-arm64", "12675293631": "devtools/ymake/bin/ymake for darwin-arm64", "12759440320": "devtools/ymake/bin/ymake for darwin-arm64", + "12836459950": "devtools/ymake/bin/ymake for darwin-arm64", "5766171800": "devtools/ymake/bin/ymake for darwin-arm64", "5805430761": "devtools/ymake/bin/ymake for darwin-arm64", "5829025456": "devtools/ymake/bin/ymake for darwin-arm64", @@ -3737,6 +3746,7 @@ "12592469929": "devtools/ymake/bin/ymake for linux", "12675298739": "devtools/ymake/bin/ymake for linux", "12759442670": "devtools/ymake/bin/ymake for linux", + "12836463794": "devtools/ymake/bin/ymake for linux", "5766173070": "devtools/ymake/bin/ymake for linux", "5805432830": "devtools/ymake/bin/ymake for linux", "5829031598": "devtools/ymake/bin/ymake for linux", @@ -3881,6 +3891,7 @@ "12592464955": "devtools/ymake/bin/ymake for linux-aarch64", "12675292272": "devtools/ymake/bin/ymake for linux-aarch64", "12759439073": "devtools/ymake/bin/ymake for linux-aarch64", + "12836458633": "devtools/ymake/bin/ymake for linux-aarch64", "5766171341": "devtools/ymake/bin/ymake for linux-aarch64", "5805430188": "devtools/ymake/bin/ymake for linux-aarch64", "5829023352": "devtools/ymake/bin/ymake for linux-aarch64", @@ -4025,6 +4036,7 @@ "12592468857": "devtools/ymake/bin/ymake for win32", "12675297027": "devtools/ymake/bin/ymake for win32", "12759441947": "devtools/ymake/bin/ymake for win32", + "12836462531": "devtools/ymake/bin/ymake for win32", "8270821739": "devtools/ymake/bin/ymake for win32", "8295446553": "devtools/ymake/bin/ymake for win32", "8326170338": "devtools/ymake/bin/ymake for win32", diff --git a/build/platform/test_tool/host.ya.make.inc b/build/platform/test_tool/host.ya.make.inc index 45904018c0f..691b5b613d5 100644 --- a/build/platform/test_tool/host.ya.make.inc +++ b/build/platform/test_tool/host.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807476180) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845357942) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807473502) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845356994) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807481043) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845359902) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807470623) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845355796) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807478544) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845358723) ENDIF() diff --git a/build/platform/test_tool/host_os.ya.make.inc b/build/platform/test_tool/host_os.ya.make.inc index f6061502b1c..5411b2969fb 100644 --- a/build/platform/test_tool/host_os.ya.make.inc +++ b/build/platform/test_tool/host_os.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807484254) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845362044) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807482262) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845360921) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807488115) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845364131) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807480382) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845360238) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12807486192) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:12845363322) ENDIF() diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 48d3d1f862e..1b0f8d6c02d 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -14,7 +14,7 @@ GO_FAKEID=2025-11-05 ANDROID_FAKEID=2023-05-17 CLANG_TIDY_FAKEID=2023-06-06 IWYU_FAKEID=2025-01-08 -CYTHON_FAKEID=2026-06-01 +CYTHON_FAKEID=2026-07-18 JAVA_FAKEID=13086370 PROTO_FAKEID=0 FBS_FAKEID=2024-03-13 diff --git a/contrib/libs/backtrace/dwarf.c b/contrib/libs/backtrace/dwarf.c index 58f6e4e1e2b..7a99eabb7c9 100644 --- a/contrib/libs/backtrace/dwarf.c +++ b/contrib/libs/backtrace/dwarf.c @@ -2026,6 +2026,113 @@ read_abbrevs (struct backtrace_state *state, uint64_t abbrev_offset, return 0; } +/* A cache of abbreviation tables, keyed by their offset in the + .debug_abbrev section. Compilation units may share abbrev tables: + tools that rewrite debug info (e.g. dwz, llvm-bolt + --update-debug-sections) deduplicate identical tables, which can + leave many thousands of units referring to one table. Without a + cache we would parse and allocate that table anew for every unit, + multiplying memory usage by orders of magnitude. The cache owns + the parsed abbrevs; units hold shallow copies. */ + +struct abbrevs_cache_entry +{ + /* The offset of the table in the .debug_abbrev section. */ + uint64_t abbrev_offset; + /* The parsed table. */ + struct abbrevs abbrevs; +}; + +/* A growable vector of abbrevs_cache_entry, sorted by + abbrev_offset. */ + +struct abbrevs_cache +{ + /* Memory for the entries. */ + struct backtrace_vector vec; + /* The number of entries. */ + size_t count; +}; + +/* Read the abbreviations at ABBREV_OFFSET, using CACHE. On a cache + miss read them with read_abbrevs and add them to the cache. The + abbrevs stored in *ABBREVS are owned by the cache and may be shared + with other units; the caller must not free them. They stay valid + until free_abbrevs_cache. Returns 1 on success, 0 on failure. */ + +static int +read_abbrevs_cached (struct backtrace_state *state, uint64_t abbrev_offset, + const unsigned char *dwarf_abbrev, + size_t dwarf_abbrev_size, int is_bigendian, + backtrace_error_callback error_callback, void *data, + struct abbrevs_cache *cache, struct abbrevs *abbrevs) +{ + struct abbrevs_cache_entry *entries; + size_t lo; + size_t hi; + struct abbrevs parsed; + + /* Binary search for ABBREV_OFFSET; if it is not present, LO is the + insertion point. */ + entries = (struct abbrevs_cache_entry *) cache->vec.base; + lo = 0; + hi = cache->count; + while (lo < hi) + { + size_t mid; + + mid = lo + (hi - lo) / 2; + if (entries[mid].abbrev_offset < abbrev_offset) + lo = mid + 1; + else if (entries[mid].abbrev_offset > abbrev_offset) + hi = mid; + else + { + *abbrevs = entries[mid].abbrevs; + return 1; + } + } + + if (!read_abbrevs (state, abbrev_offset, dwarf_abbrev, dwarf_abbrev_size, + is_bigendian, error_callback, data, &parsed)) + return 0; + + if (backtrace_vector_grow (state, sizeof (struct abbrevs_cache_entry), + error_callback, data, &cache->vec) == NULL) + { + free_abbrevs (state, &parsed, error_callback, data); + return 0; + } + + /* Growing the vector may have moved the entries. */ + entries = (struct abbrevs_cache_entry *) cache->vec.base; + memmove (entries + lo + 1, entries + lo, + (cache->count - lo) * sizeof (struct abbrevs_cache_entry)); + entries[lo].abbrev_offset = abbrev_offset; + entries[lo].abbrevs = parsed; + ++cache->count; + + *abbrevs = parsed; + return 1; +} + +/* Free the abbrevs held by CACHE. */ + +static void +free_abbrevs_cache (struct backtrace_state *state, + struct abbrevs_cache *cache, + backtrace_error_callback error_callback, void *data) +{ + struct abbrevs_cache_entry *entries; + size_t i; + + entries = (struct abbrevs_cache_entry *) cache->vec.base; + for (i = 0; i < cache->count; i++) + free_abbrevs (state, &entries[i].abbrevs, error_callback, data); + backtrace_vector_free (state, &cache->vec, error_callback, data); + cache->count = 0; +} + /* Return the abbrev information for an abbrev code. */ static const struct abbrev * @@ -2645,6 +2752,7 @@ build_address_map (struct backtrace_state *state, struct unit **pu; size_t unit_offset = 0; struct unit_addrs *pa; + struct abbrevs_cache abbrevs_cache; memset (&addrs->vec, 0, sizeof addrs->vec); memset (&unit_vec->vec, 0, sizeof unit_vec->vec); @@ -2666,6 +2774,7 @@ build_address_map (struct backtrace_state *state, memset (&units, 0, sizeof units); units_count = 0; + memset (&abbrevs_cache, 0, sizeof abbrevs_cache); while (info.left > 0) { @@ -2732,10 +2841,11 @@ build_address_map (struct backtrace_state *state, memset (&u->abbrevs, 0, sizeof u->abbrevs); abbrev_offset = read_offset (&unit_buf, is_dwarf64); - if (!read_abbrevs (state, abbrev_offset, - dwarf_sections->data[DEBUG_ABBREV], - dwarf_sections->size[DEBUG_ABBREV], - is_bigendian, error_callback, data, &u->abbrevs)) + if (!read_abbrevs_cached (state, abbrev_offset, + dwarf_sections->data[DEBUG_ABBREV], + dwarf_sections->size[DEBUG_ABBREV], + is_bigendian, error_callback, data, + &abbrevs_cache, &u->abbrevs)) goto fail; if (version < 5) @@ -2799,19 +2909,22 @@ build_address_map (struct backtrace_state *state, pa->high = pa->low; pa->u = NULL; + /* The units hold shallow copies of the cached abbrevs; the abbrevs + themselves are intentionally never freed after this point. + Release just the cache bookkeeping vector. */ + backtrace_vector_free (state, &abbrevs_cache.vec, error_callback, data); + unit_vec->vec = units; unit_vec->count = units_count; return 1; fail: + free_abbrevs_cache (state, &abbrevs_cache, error_callback, data); if (units_count > 0) { pu = (struct unit **) units.base; for (i = 0; i < units_count; i++) - { - free_abbrevs (state, &pu[i]->abbrevs, error_callback, data); - backtrace_free (state, pu[i], sizeof **pu, error_callback, data); - } + backtrace_free (state, pu[i], sizeof **pu, error_callback, data); backtrace_vector_free (state, &units, error_callback, data); } if (addrs->count > 0) diff --git a/contrib/libs/backtrace/patches/abbrev-cache.patch b/contrib/libs/backtrace/patches/abbrev-cache.patch new file mode 100644 index 00000000000..2978cbdfc16 --- /dev/null +++ b/contrib/libs/backtrace/patches/abbrev-cache.patch @@ -0,0 +1,175 @@ +--- a/dwarf.c ++++ b/dwarf.c +@@ -2026,6 +2026,113 @@ read_abbrevs (struct backtrace_state *state, uint64_t abbrev_offset, + return 0; + } + ++/* A cache of abbreviation tables, keyed by their offset in the ++ .debug_abbrev section. Compilation units may share abbrev tables: ++ tools that rewrite debug info (e.g. dwz, llvm-bolt ++ --update-debug-sections) deduplicate identical tables, which can ++ leave many thousands of units referring to one table. Without a ++ cache we would parse and allocate that table anew for every unit, ++ multiplying memory usage by orders of magnitude. The cache owns ++ the parsed abbrevs; units hold shallow copies. */ ++ ++struct abbrevs_cache_entry ++{ ++ /* The offset of the table in the .debug_abbrev section. */ ++ uint64_t abbrev_offset; ++ /* The parsed table. */ ++ struct abbrevs abbrevs; ++}; ++ ++/* A growable vector of abbrevs_cache_entry, sorted by ++ abbrev_offset. */ ++ ++struct abbrevs_cache ++{ ++ /* Memory for the entries. */ ++ struct backtrace_vector vec; ++ /* The number of entries. */ ++ size_t count; ++}; ++ ++/* Read the abbreviations at ABBREV_OFFSET, using CACHE. On a cache ++ miss read them with read_abbrevs and add them to the cache. The ++ abbrevs stored in *ABBREVS are owned by the cache and may be shared ++ with other units; the caller must not free them. They stay valid ++ until free_abbrevs_cache. Returns 1 on success, 0 on failure. */ ++ ++static int ++read_abbrevs_cached (struct backtrace_state *state, uint64_t abbrev_offset, ++ const unsigned char *dwarf_abbrev, ++ size_t dwarf_abbrev_size, int is_bigendian, ++ backtrace_error_callback error_callback, void *data, ++ struct abbrevs_cache *cache, struct abbrevs *abbrevs) ++{ ++ struct abbrevs_cache_entry *entries; ++ size_t lo; ++ size_t hi; ++ struct abbrevs parsed; ++ ++ /* Binary search for ABBREV_OFFSET; if it is not present, LO is the ++ insertion point. */ ++ entries = (struct abbrevs_cache_entry *) cache->vec.base; ++ lo = 0; ++ hi = cache->count; ++ while (lo < hi) ++ { ++ size_t mid; ++ ++ mid = lo + (hi - lo) / 2; ++ if (entries[mid].abbrev_offset < abbrev_offset) ++ lo = mid + 1; ++ else if (entries[mid].abbrev_offset > abbrev_offset) ++ hi = mid; ++ else ++ { ++ *abbrevs = entries[mid].abbrevs; ++ return 1; ++ } ++ } ++ ++ if (!read_abbrevs (state, abbrev_offset, dwarf_abbrev, dwarf_abbrev_size, ++ is_bigendian, error_callback, data, &parsed)) ++ return 0; ++ ++ if (backtrace_vector_grow (state, sizeof (struct abbrevs_cache_entry), ++ error_callback, data, &cache->vec) == NULL) ++ { ++ free_abbrevs (state, &parsed, error_callback, data); ++ return 0; ++ } ++ ++ /* Growing the vector may have moved the entries. */ ++ entries = (struct abbrevs_cache_entry *) cache->vec.base; ++ memmove (entries + lo + 1, entries + lo, ++ (cache->count - lo) * sizeof (struct abbrevs_cache_entry)); ++ entries[lo].abbrev_offset = abbrev_offset; ++ entries[lo].abbrevs = parsed; ++ ++cache->count; ++ ++ *abbrevs = parsed; ++ return 1; ++} ++ ++/* Free the abbrevs held by CACHE. */ ++ ++static void ++free_abbrevs_cache (struct backtrace_state *state, ++ struct abbrevs_cache *cache, ++ backtrace_error_callback error_callback, void *data) ++{ ++ struct abbrevs_cache_entry *entries; ++ size_t i; ++ ++ entries = (struct abbrevs_cache_entry *) cache->vec.base; ++ for (i = 0; i < cache->count; i++) ++ free_abbrevs (state, &entries[i].abbrevs, error_callback, data); ++ backtrace_vector_free (state, &cache->vec, error_callback, data); ++ cache->count = 0; ++} ++ + /* Return the abbrev information for an abbrev code. */ + + static const struct abbrev * +@@ -2645,6 +2752,7 @@ build_address_map (struct backtrace_state *state, + struct unit **pu; + size_t unit_offset = 0; + struct unit_addrs *pa; ++ struct abbrevs_cache abbrevs_cache; + + memset (&addrs->vec, 0, sizeof addrs->vec); + memset (&unit_vec->vec, 0, sizeof unit_vec->vec); +@@ -2666,6 +2774,7 @@ build_address_map (struct backtrace_state *state, + + memset (&units, 0, sizeof units); + units_count = 0; ++ memset (&abbrevs_cache, 0, sizeof abbrevs_cache); + + while (info.left > 0) + { +@@ -2732,10 +2841,11 @@ build_address_map (struct backtrace_state *state, + + memset (&u->abbrevs, 0, sizeof u->abbrevs); + abbrev_offset = read_offset (&unit_buf, is_dwarf64); +- if (!read_abbrevs (state, abbrev_offset, +- dwarf_sections->data[DEBUG_ABBREV], +- dwarf_sections->size[DEBUG_ABBREV], +- is_bigendian, error_callback, data, &u->abbrevs)) ++ if (!read_abbrevs_cached (state, abbrev_offset, ++ dwarf_sections->data[DEBUG_ABBREV], ++ dwarf_sections->size[DEBUG_ABBREV], ++ is_bigendian, error_callback, data, ++ &abbrevs_cache, &u->abbrevs)) + goto fail; + + if (version < 5) +@@ -2799,19 +2909,22 @@ build_address_map (struct backtrace_state *state, + pa->high = pa->low; + pa->u = NULL; + ++ /* The units hold shallow copies of the cached abbrevs; the abbrevs ++ themselves are intentionally never freed after this point. ++ Release just the cache bookkeeping vector. */ ++ backtrace_vector_free (state, &abbrevs_cache.vec, error_callback, data); ++ + unit_vec->vec = units; + unit_vec->count = units_count; + return 1; + + fail: ++ free_abbrevs_cache (state, &abbrevs_cache, error_callback, data); + if (units_count > 0) + { + pu = (struct unit **) units.base; + for (i = 0; i < units_count; i++) +- { +- free_abbrevs (state, &pu[i]->abbrevs, error_callback, data); +- backtrace_free (state, pu[i], sizeof **pu, error_callback, data); +- } ++ backtrace_free (state, pu[i], sizeof **pu, error_callback, data); + backtrace_vector_free (state, &units, error_callback, data); + } + if (addrs->count > 0) diff --git a/contrib/libs/expat/.yandex_meta/__init__.py b/contrib/libs/expat/.yandex_meta/__init__.py index 8344f573151..372eabf2f4f 100644 --- a/contrib/libs/expat/.yandex_meta/__init__.py +++ b/contrib/libs/expat/.yandex_meta/__init__.py @@ -42,7 +42,10 @@ def post_build(self): ), ), ) - expat.PEERDIR.add("contrib/libs/libc_compat") + expat.PEERDIR |= { + "contrib/libs/libc_compat", + "library/cpp/sanitizer/include", + } expat = GNUMakeNixProject( diff --git a/contrib/libs/expat/.yandex_meta/devtools.copyrights.report b/contrib/libs/expat/.yandex_meta/devtools.copyrights.report index 072776c75e9..b3280199fc1 100644 --- a/contrib/libs/expat/.yandex_meta/devtools.copyrights.report +++ b/contrib/libs/expat/.yandex_meta/devtools.copyrights.report @@ -48,7 +48,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL 12db9abedf3e0b98a87d6f4ec1377a19 BELONGS ya.make @@ -61,9 +61,9 @@ BELONGS ya.make expat.h [9:23] expat_external.h [9:19] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok.h [9:15] - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL 16cc365f87360f9e773cd8812039915f BELONGS ya.make @@ -73,8 +73,8 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] - lib/xmltok_impl.c [9:20] + lib/xmlparse.c [9:53] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL 17566be0ee85deadbd5b2fcedc8b66a9 BELONGS ya.make @@ -84,7 +84,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 1916cbefc2e0a780a3d503ba26f3780a BELONGS ya.make @@ -107,7 +107,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 2309e9d42c2779decb776323d1f5dabd BELONGS ya.make @@ -117,8 +117,8 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] - lib/xmltok.c [9:28] + lib/xmlparse.c [9:53] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 262c58e3a627f5cee77a882379e1364f BELONGS ya.make @@ -139,7 +139,19 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] + +KEEP COPYRIGHT_SERVICE_LABEL 2d8e554003b116aae13ce1d358a8a9b9 +BELONGS ya.make + License text: + Copyright (c) 2022 Sebastian Pipping <[email protected]> + Licensed under the MIT license: + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + lib/xcsinc.c [9:10] KEEP COPYRIGHT_SERVICE_LABEL 338b8ad8ee9b8449a90a88a0559aefd9 BELONGS ya.make @@ -149,7 +161,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 347b7c5a6d0cfaa5712b5fb2bfd1a69a BELONGS ya.make @@ -180,7 +192,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 4010f67351b9e656cc500aa367c0c393 BELONGS ya.make @@ -190,9 +202,9 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 4588cbaabde7eebe28cc29740402f156 BELONGS ya.make @@ -202,8 +214,8 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] - lib/xmltok_impl.c [9:20] + lib/xmltok.c [9:29] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL 477adc1652fe6ac004ea66fbbcc0bb19 BELONGS ya.make @@ -214,8 +226,8 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: lib/random_getrandom.c [9:13] - lib/xmlparse.c [9:50] - lib/xmltok.c [9:28] + lib/xmlparse.c [9:53] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 4ffe0aad4a07d95c6b3c7d06e1dfbead BELONGS ya.make @@ -225,7 +237,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 50da2a76d12ee3df6d928d81ca59a715 BELONGS ya.make @@ -245,7 +257,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 5619621de1e7b76391ee5fff9c1b1f91 BELONGS ya.make @@ -255,7 +267,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 576486d7fb5eeab148dcf39027acec46 BELONGS ya.make @@ -265,7 +277,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 581b53ae6f0fb8a0cc30c73b46bc3441 BELONGS ya.make @@ -308,7 +320,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: expat.h [9:23] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 5fe78d330bca84f8e39670e48c92ba39 BELONGS ya.make @@ -319,7 +331,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: lib/internal.h [28:36] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 60e5da68428d71447ddd4bd7750453bb BELONGS ya.make @@ -329,7 +341,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 610290a5946ae0c99de2de99646ba086 BELONGS ya.make @@ -349,7 +361,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 6451d5e490271b354ad3b567c7a03423 BELONGS ya.make @@ -360,7 +372,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: lib/random_rand_s.c [9:14] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 647bb74c2d45ce0423d4c9a6b28bddac BELONGS ya.make @@ -380,7 +392,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 66d3ecc8836b1f9801bfad98582d2a32 BELONGS ya.make @@ -390,7 +402,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 7588a9020c401c85f6b542a5629c0c92 BELONGS ya.make @@ -410,7 +422,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 7c09099ef5f35bf3be4611e6cbb14510 BELONGS ya.make @@ -420,7 +432,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL 7d65e46a306fed19dd12ba56b69d764c BELONGS ya.make @@ -440,9 +452,9 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok_ns.c [9:16] KEEP COPYRIGHT_SERVICE_LABEL 868bf74335c8c4c3a6c576a3120e708c @@ -484,12 +496,12 @@ BELONGS ya.make lib/nametab.h [9:11] lib/utf8tab.h [9:13] lib/winconfig.h [9:14] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] lib/xmlrole.h [9:14] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok.h [9:15] - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] lib/xmltok_impl.h [9:12] lib/xmltok_ns.c [9:16] @@ -503,7 +515,7 @@ BELONGS ya.make Files with this license: lib/winconfig.h [9:14] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok_ns.c [9:16] KEEP COPYRIGHT_SERVICE_LABEL 962bda2bd5ce73f8fa7de2620a3507f9 @@ -516,6 +528,16 @@ BELONGS ya.make Files with this license: lib/xmltok.h [9:15] +KEEP COPYRIGHT_SERVICE_LABEL 9842d67f5d734f4ca2f5f6c1c52393fe +BELONGS ya.make + Note: matched license text is too long. Read it in the source files. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + lib/xmlparse.c [9:53] + KEEP COPYRIGHT_SERVICE_LABEL 99dbe0c5cacb0e3d1bc5b15bef337ea4 BELONGS ya.make Note: matched license text is too long. Read it in the source files. @@ -527,10 +549,10 @@ BELONGS ya.make lib/internal.h [28:36] lib/random_rand_s.c [9:14] lib/random_rand_s.h [9:11] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] - lib/xmltok_impl.c [9:20] + lib/xmltok.c [9:29] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL 9b348497698951665d1e43c1d376a510 BELONGS ya.make @@ -540,7 +562,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL 9fdb85dcaaf74d518b27233b48fffa52 BELONGS ya.make @@ -552,7 +574,7 @@ BELONGS ya.make Files with this license: expat.h [9:23] expat_external.h [9:19] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL a0fdd1392c0b9b2558b9ccfe44592143 BELONGS ya.make @@ -574,7 +596,7 @@ BELONGS ya.make Files with this license: expat.h [9:23] lib/internal.h [28:36] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL a5c66b2d6998a1fbabcc4f2885ad5707 BELONGS ya.make @@ -584,7 +606,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL a6a07a1f1768704011ca547d57a193f2 BELONGS ya.make @@ -594,7 +616,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL aa19fc04b7b0134a0a4b71a6c182c1ae BELONGS ya.make @@ -605,7 +627,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: expat.h [9:23] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL aafe06df8255f48781ac9d4e96e1ea4e BELONGS ya.make @@ -616,9 +638,9 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: expat.h [9:23] - lib/xmlparse.c [9:50] - lib/xmltok.c [9:28] - lib/xmltok_impl.c [9:20] + lib/xmlparse.c [9:53] + lib/xmltok.c [9:29] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL abbaf6167a20f6ba0eee322937821c15 BELONGS ya.make @@ -628,7 +650,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL ac721fcd634b3e5674a847f5ed2f1c8e BELONGS ya.make @@ -638,7 +660,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL acbc3573ff7fdf431f94f6aea99de1e0 BELONGS ya.make @@ -648,7 +670,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL ad31071eb61955b9cdd9e0a463d6714f BELONGS ya.make @@ -658,7 +680,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL b667d591a50b1e424d66b58f6e0ff4a1 BELONGS ya.make @@ -668,7 +690,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL b927d5afef47e3ae01d339429012d6a7 BELONGS ya.make @@ -719,12 +741,12 @@ BELONGS ya.make lib/iasciitab.h [9:13] lib/latin1tab.h [9:13] lib/utf8tab.h [9:13] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] lib/xmlrole.h [9:14] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok.h [9:15] - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] lib/xmltok_impl.h [9:12] lib/xmltok_ns.c [9:16] @@ -751,7 +773,22 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] + +KEEP COPYRIGHT_SERVICE_LABEL d363fbc5cd02e7b4ee3c5c0d9fca8733 +BELONGS ya.make + License text: + Copyright (c) 2026 Nick Begg <[email protected]> + Licensed under the MIT license: + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + lib/fallthrough.h [9:10] + lib/xmlparse.c [9:53] + lib/xmltok.c [9:29] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL d548c6beaeae204247905b60d5feff91 BELONGS ya.make @@ -761,8 +798,8 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] - lib/xmltok_impl.c [9:20] + lib/xmlparse.c [9:53] + lib/xmltok_impl.c [9:21] KEEP COPYRIGHT_SERVICE_LABEL d9ac29affff64c14b3c82b78ed157a77 BELONGS ya.make @@ -772,9 +809,9 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] lib/xmlrole.c [9:20] - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] KEEP COPYRIGHT_SERVICE_LABEL dd3c5623e58aa85a367a6638299f50f3 BELONGS ya.make @@ -784,7 +821,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL dd3e49797101e1ae2e966f0ebdd58da4 BELONGS ya.make @@ -817,10 +854,10 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: lib/random_arc4random.h [9:10] - lib/random_arc4random_buf.c [9:10] + lib/random_arc4random_buf.c [9:11] lib/random_arc4random_buf.h [9:10] lib/random_dev_urandom.h [9:10] - lib/random_getentropy.c [9:10] + lib/random_getentropy.c [9:11] lib/random_getentropy.h [9:10] lib/random_getrandom.h [9:10] lib/random_rand_s.h [9:11] @@ -833,7 +870,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL e2f8f506923a26765706e1ad8cc8edc6 BELONGS ya.make @@ -843,7 +880,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL e3d6c1b6030b59aad9996cc0a9efeda5 BELONGS ya.make @@ -853,7 +890,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL e526eb605bd716769dd1ed15786af827 BELONGS ya.make @@ -874,8 +911,11 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: expat_external.h [9:19] + lib/memory_sanitizer.h [9:10] lib/random_arc4random.c [9:11] + lib/random_arc4random_buf.c [9:11] lib/random_dev_urandom.c [9:11] + lib/random_getentropy.c [9:11] lib/random_getrandom.c [9:13] lib/random_rand_s.c [9:14] @@ -888,7 +928,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: lib/random_getrandom.c [9:13] - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL ef0dda0153a00710149f327147a79b7f BELONGS ya.make @@ -939,7 +979,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL f792720cb266393d3d3fb2199660c4ea BELONGS ya.make @@ -949,9 +989,19 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmltok.c [9:28] + lib/xmltok.c [9:29] lib/xmltok.h [9:15] +KEEP COPYRIGHT_SERVICE_LABEL fddb9185e786dd5d20eac5402cf4f745 +BELONGS ya.make + Note: matched license text is too long. Read it in the source files. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + lib/xmlparse.c [9:53] + KEEP COPYRIGHT_SERVICE_LABEL fea018c6e4e19bc6bd4ac263c015567a BELONGS ya.make Note: matched license text is too long. Read it in the source files. @@ -967,7 +1017,7 @@ BELONGS ya.make lib/utf8tab.h [9:13] lib/xmlrole.h [9:14] lib/xmltok.h [9:15] - lib/xmltok_impl.c [9:20] + lib/xmltok_impl.c [9:21] lib/xmltok_ns.c [9:16] KEEP COPYRIGHT_SERVICE_LABEL ff4131ca4bcbb6079f40a140651275ac @@ -978,7 +1028,7 @@ BELONGS ya.make Score : 100.00 Match type : COPYRIGHT Files with this license: - lib/xmlparse.c [9:50] + lib/xmlparse.c [9:53] KEEP COPYRIGHT_SERVICE_LABEL ffabe442e64111946b696c0384a52a77 BELONGS ya.make diff --git a/contrib/libs/expat/.yandex_meta/devtools.licenses.report b/contrib/libs/expat/.yandex_meta/devtools.licenses.report index 0402a3133b7..7d0b620fe8f 100644 --- a/contrib/libs/expat/.yandex_meta/devtools.licenses.report +++ b/contrib/libs/expat/.yandex_meta/devtools.licenses.report @@ -42,7 +42,7 @@ BELONGS ya.make KEEP MIT 6bb6514a1d779748b76a73215a89ae66 BELONGS ya.make -FILE_INCLUDE AUTHORS found in files: expat.h at line 39, expat_external.h at line 35, lib/ascii.h at line 30, lib/asciitab.h at line 29, lib/iasciitab.h at line 29, lib/internal.h at line 52, lib/latin1tab.h at line 29, lib/random_arc4random.c at line 27, lib/random_arc4random.h at line 26, lib/random_arc4random_buf.c at line 26, lib/random_arc4random_buf.h at line 26, lib/random_dev_urandom.c at line 27, lib/random_dev_urandom.h at line 26, lib/random_getentropy.c at line 26, lib/random_getentropy.h at line 26, lib/random_getrandom.c at line 29, lib/random_getrandom.h at line 26, lib/random_rand_s.c at line 30, lib/random_rand_s.h at line 27, lib/utf8tab.h at line 29, lib/winconfig.h at line 30, lib/xmlparse.c at line 66, lib/xmlrole.c at line 36, lib/xmlrole.h at line 30, lib/xmltok.c at line 44, lib/xmltok.h at line 31, lib/xmltok_impl.c at line 36, lib/xmltok_impl.h at line 28, lib/xmltok_ns.c at line 32 +FILE_INCLUDE AUTHORS found in files: expat.h at line 39, expat_external.h at line 35, lib/ascii.h at line 30, lib/asciitab.h at line 29, lib/fallthrough.h at line 26, lib/iasciitab.h at line 29, lib/internal.h at line 52, lib/latin1tab.h at line 29, lib/memory_sanitizer.h at line 26, lib/random_arc4random.c at line 27, lib/random_arc4random.h at line 26, lib/random_arc4random_buf.c at line 27, lib/random_arc4random_buf.h at line 26, lib/random_dev_urandom.c at line 27, lib/random_dev_urandom.h at line 26, lib/random_getentropy.c at line 27, lib/random_getentropy.h at line 26, lib/random_getrandom.c at line 29, lib/random_getrandom.h at line 26, lib/random_rand_s.c at line 30, lib/random_rand_s.h at line 27, lib/utf8tab.h at line 29, lib/winconfig.h at line 30, lib/xcsinc.c at line 26, lib/xmlparse.c at line 69, lib/xmlrole.c at line 36, lib/xmlrole.h at line 30, lib/xmltok.c at line 45, lib/xmltok.h at line 31, lib/xmltok_impl.c at line 37, lib/xmltok_impl.h at line 28, lib/xmltok_ns.c at line 32 Note: matched license text is too long. Read it in the source files. Scancode info: Original SPDX id: MIT @@ -54,16 +54,18 @@ FILE_INCLUDE AUTHORS found in files: expat.h at line 39, expat_external.h at lin expat_external.h [21:38] lib/ascii.h [16:33] lib/asciitab.h [15:32] + lib/fallthrough.h [12:29] lib/iasciitab.h [15:32] lib/internal.h [38:55] lib/latin1tab.h [15:32] + lib/memory_sanitizer.h [12:29] lib/random_arc4random.c [13:30] lib/random_arc4random.h [12:29] - lib/random_arc4random_buf.c [12:29] + lib/random_arc4random_buf.c [13:30] lib/random_arc4random_buf.h [12:29] lib/random_dev_urandom.c [13:30] lib/random_dev_urandom.h [12:29] - lib/random_getentropy.c [12:29] + lib/random_getentropy.c [13:30] lib/random_getentropy.h [12:29] lib/random_getrandom.c [15:32] lib/random_getrandom.h [12:29] @@ -71,12 +73,13 @@ FILE_INCLUDE AUTHORS found in files: expat.h at line 39, expat_external.h at lin lib/random_rand_s.h [13:30] lib/utf8tab.h [15:32] lib/winconfig.h [16:33] - lib/xmlparse.c [52:69] + lib/xcsinc.c [12:29] + lib/xmlparse.c [55:72] lib/xmlrole.c [22:39] lib/xmlrole.h [16:33] - lib/xmltok.c [30:47] + lib/xmltok.c [31:48] lib/xmltok.h [17:34] - lib/xmltok_impl.c [22:39] + lib/xmltok_impl.c [23:40] lib/xmltok_impl.h [14:31] lib/xmltok_ns.c [18:35] @@ -106,17 +109,19 @@ BELONGS ya.make expat_external.h [19:19] lib/ascii.h [14:14] lib/asciitab.h [13:13] + lib/fallthrough.h [10:10] lib/iasciitab.h [13:13] lib/internal.h [36:36] lib/latin1tab.h [13:13] + lib/memory_sanitizer.h [10:10] lib/nametab.h [11:11] lib/random_arc4random.c [11:11] lib/random_arc4random.h [10:10] - lib/random_arc4random_buf.c [10:10] + lib/random_arc4random_buf.c [11:11] lib/random_arc4random_buf.h [10:10] lib/random_dev_urandom.c [11:11] lib/random_dev_urandom.h [10:10] - lib/random_getentropy.c [10:10] + lib/random_getentropy.c [11:11] lib/random_getentropy.h [10:10] lib/random_getrandom.c [13:13] lib/random_getrandom.h [10:10] @@ -124,12 +129,13 @@ BELONGS ya.make lib/random_rand_s.h [11:11] lib/utf8tab.h [13:13] lib/winconfig.h [14:14] - lib/xmlparse.c [50:50] + lib/xcsinc.c [10:10] + lib/xmlparse.c [53:53] lib/xmlrole.c [20:20] lib/xmlrole.h [14:14] - lib/xmltok.c [28:28] + lib/xmltok.c [29:29] lib/xmltok.h [15:15] - lib/xmltok_impl.c [20:20] + lib/xmltok_impl.c [21:21] lib/xmltok_impl.h [12:12] lib/xmltok_ns.c [16:16] diff --git a/contrib/libs/expat/.yandex_meta/licenses.list.txt b/contrib/libs/expat/.yandex_meta/licenses.list.txt index 8b7e30d3de9..687cd921a14 100644 --- a/contrib/libs/expat/.yandex_meta/licenses.list.txt +++ b/contrib/libs/expat/.yandex_meta/licenses.list.txt @@ -76,6 +76,9 @@ Copyright (c) 2026 Rosen Penev <[email protected]> Copyright (c) 2026 Francesco Bertolaccini Copyright (c) 2026 Christian Ng <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> + Copyright (c) 2026 Kartik Kenchi <[email protected]> + Copyright (c) 2026 Haris Hussain <[email protected]> Licensed under the MIT license: @@ -99,6 +102,7 @@ Copyright (c) 2022 Sean McBride <[email protected]> Copyright (c) 2023 Hanno Böck <[email protected]> Copyright (c) 2025 Alfonso Gregory <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> Licensed under the MIT license: @@ -124,6 +128,7 @@ Copyright (c) 2019 David Loffredo <[email protected]> Copyright (c) 2020 Boris Kolpackov <[email protected]> Copyright (c) 2022 Martin Ettl <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> Licensed under the MIT license: @@ -223,6 +228,16 @@ ====================COPYRIGHT==================== + Copyright (c) 2022 Sebastian Pipping <[email protected]> + Licensed under the MIT license: + + +====================COPYRIGHT==================== + Copyright (c) 2026 Nick Begg <[email protected]> + Licensed under the MIT license: + + +====================COPYRIGHT==================== Copyright (c) 2026 Sebastian Pipping <[email protected]> Licensed under the MIT license: diff --git a/contrib/libs/expat/.yandex_meta/override.nix b/contrib/libs/expat/.yandex_meta/override.nix index 19266e058c2..ae1a5b09b61 100644 --- a/contrib/libs/expat/.yandex_meta/override.nix +++ b/contrib/libs/expat/.yandex_meta/override.nix @@ -1,12 +1,12 @@ pkgs: attrs: with pkgs; with attrs; rec { - version = "2.8.1"; + version = "2.8.2"; versionTag = "R_${lib.replaceStrings ["."] ["_"] version}"; src = fetchFromGitHub { owner = "libexpat"; repo = "libexpat"; rev = "${versionTag}"; - hash = "sha256-XHQN6Wd29Furdu8VB1UokZMmIFKq4sCG+Aa8/KfSoi4="; + hash = "sha256-YNrsTYgY4gyRDpYPZPmkCTOF1eKA7rCcyfS0jR/SItY="; }; nativeBuildInputs = [ autoreconfHook ]; diff --git a/contrib/libs/expat/Changes b/contrib/libs/expat/Changes index 313b3dc60cd..6a1b89160fb 100644 --- a/contrib/libs/expat/Changes +++ b/contrib/libs/expat/Changes @@ -29,6 +29,94 @@ !! THANK YOU! Sebastian Pipping -- Berlin, 2026-03-17 !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +Release 2.8.2 Thu June 25 2026 + Security fixes: + #1246 CVE-2026-50219 -- Disallow calls to functions + `XML_GetBuffer`, `XML_Parse`, `XML_ParseBuffer`, + `XML_ParserFree`, `XML_ParserReset` to guard e.g. + Expat bindings from memory corruption; + this CPython issue is related: + https://github.com/python/cpython/issues/146169 + #1267 CVE-2026-56131 -- Protect XML_ResumeParser from being called + from a handler, plugging a hole in the fix + to CVE-2026-50219 + #1272 CVE-2026-56132 -- Fix out-of-bound scaffolding index store + in `doProlog` + #1229 #1232 CVE-2026-56403 -- Integer overflow in `storeAtts` + #1249 CVE-2026-56404 -- Integer overflow in `addBinding` + #1251 CVE-2026-56405 -- Integer overflow in `getAttributeId` + #1255 CVE-2026-56406 -- Integer overflow in `XML_ParseBuffer` + #1262 CVE-2026-56407 -- Integer overflow in `textLen` handling + #565 CVE-2026-56408 -- Integer overflow in `copyString` + (commit 16e2efd867ea8567ffa012210b52ef5918e20817) + #1259 CVE-2026-56409 -- xmlwf: Integer overflow in output path join + #1252 CVE-2026-56410 -- xmlwf: Integer overflow in + `resolveSystemId` + #1263 CVE-2026-56411 -- xmlwf: Integer overflow in notation list + allocation + #1278 CVE-2026-56412 -- Guard XML_TOK_DATA_CHARS handler calls in + `doCdataSection`, plugging a hole in the fix to + CVE-2026-50219 + + Bug fixes: + #1260 xmlwf: Escape names and base URI in meta output + #1266 xmlwf: Pick a safe quote for notation system and public IDs + + Other changes: + #1257 CMake|Autotools: Stop using /dev/urandom by default + #1244 #1254 CMake: Fix guard for Unix sources of entropy + #1183 #1270 CMake|Windows: Add missing export for symbol + `XML_SetHashSalt16Bytes` + #1236 CMake: Mark option EXPAT_OSSFUZZ_BUILD as advanced + #1283 Limit output indentation for EXPAT_ENTITY_DEBUG=1 and + allow unlimited indentation via EXPAT_ENTITY_DEBUG=2 + #565 Replace some loops by use of `memcpy`, `strlen`, `wcslen` + #1220 lib: Use a size_t for group sizes + #1221 lib: Fix too-conservative integer overflow check when + appending raw name + #1222 lib: Simplify attribute allocation/management logic + #1224 Update fallthrough annotations to satisfy Clang and GCC + #1226 lib: Remove unnecessary void * casts in random code + #1228 lib: Reduce scope of locals in storeAtts + #1230 lib: Count attributes with size_t variables + #1238 Minor get-buffer improvements + #1239 #1240 lib|tests: Include header expat_config.h first + #1241 lib: Shrink size of XML_GetBuffer + #1242 lib: Remove a legacy comment + #1243 lib: XML_ParserReset: Extract repeated linked-list move logic + #1243 lib: Unify entity free lists + #1247 lib: Fix use of '0' as boolean literal + #1248 lib: Make XML_Index overflow check more intuitive + #1256 lib: Use size_t for counting string/URI lengths + #1258 lib: XML_GetInputContext: Remove use of 0 for NULL + #1261 Comment typo fixes + #1275 Teach Memory Sanitizer semantics of randomization functions + #1276 #1281 Version info bumped from 13:1:12 (libexpat*.so.1.12.1) + to 13:2:12 (libexpat*.so.1.12.2); see https://verbump.de/ + for what these numbers do + + Infrastructure: + #1231 perl-integration.yml: Bump to XML::Parser 2.59 + #1237 emscripten.yml: Bump from Ubuntu 22.04 to 24.04 + #1183 #1271 windows-build.yml: Cover completeness of file + libexpat.def.cmake + #1274 linux.yml: Make llvm-symbolizer available in CI + + Special thanks to: + Alessandro Gario + Asher Darden + Christoph Reiter + Haris Hussain + Matthew Fernandez + Kartik Kenchi + Nick Begg + Sajin S + Yousef Shanableh + and + Anthropic + Astra Security + Trail of Bits + Release 2.8.1 Sun May 10 2026 Security fixes: #1216 CVE-2026-45186 -- Fix quadratic runtime from attribute name @@ -1820,7 +1908,7 @@ Release 2.1.0 Sat March 24 2012 - Bug Fixes: #1742315: Harmful XML_ParserCreateNS suggestion. #1785430: Expat build fails on linux-amd64 with gcc version>=4.1 -O3. - #1983953, 2517952, 2517962, 2649838: + #1983953, 2517952, 2517962, 2649838: Build modifications using autoreconf instead of buildconf.sh. #2815947, #2884086: OBJEXT and EXEEXT support while building. #2517938: xmlwf should return non-zero exit status if not well-formed. @@ -1846,7 +1934,7 @@ Release 2.1.0 Sat March 24 2012 See bug #2990652 and patch #3312568. Added run-benchmark target to Makefile.in - relies on testdata module present in the same relative location as in the repository. - + Release 2.0.1 Tue June 5 2007 - Fixed bugs #1515266, #1515600: The character data handler's calling of XML_StopParser() was not handled properly; if the parser was diff --git a/contrib/libs/expat/README.md b/contrib/libs/expat/README.md index 7a5ed463b6d..5ad5e1e85af 100644 --- a/contrib/libs/expat/README.md +++ b/contrib/libs/expat/README.md @@ -11,7 +11,7 @@ > at the top of the `Changes` file. -# Expat, Release 2.8.1 +# Expat, Release 2.8.2 This is Expat, a C99 library for parsing [XML 1.0 Fourth Edition](https://www.w3.org/TR/2006/REC-xml-20060816/), started by @@ -286,9 +286,6 @@ EXPAT_ENABLE_INSTALL:BOOL=ON // Use /MT flag (static CRT) when compiling in MSVC EXPAT_MSVC_STATIC_CRT:BOOL=OFF -// Build fuzzers via OSS-Fuzz for the expat library -EXPAT_OSSFUZZ_BUILD:BOOL=OFF - // Build a shared expat library EXPAT_SHARED_LIBS:BOOL=ON diff --git a/contrib/libs/expat/expat.h b/contrib/libs/expat/expat.h index ec3f58544cb..c493c70441c 100644 --- a/contrib/libs/expat/expat.h +++ b/contrib/libs/expat/expat.h @@ -1094,7 +1094,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 1 +# define XML_MICRO_VERSION 2 # ifdef __cplusplus } diff --git a/contrib/libs/expat/expat_config-linux.h b/contrib/libs/expat/expat_config-linux.h index c97ae4de7e6..2a91d46e67e 100644 --- a/contrib/libs/expat/expat_config-linux.h +++ b/contrib/libs/expat/expat_config-linux.h @@ -83,7 +83,7 @@ #define PACKAGE_NAME "expat" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "expat 2.8.1" +#define PACKAGE_STRING "expat 2.8.2" /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "expat" @@ -92,7 +92,7 @@ #define PACKAGE_URL "" /* Define to the version of this package. */ -#define PACKAGE_VERSION "2.8.1" +#define PACKAGE_VERSION "2.8.2" /* Define to 1 if all of the C89 standard headers exist (not just the ones required in a freestanding environment). This macro is provided for @@ -100,7 +100,7 @@ #define STDC_HEADERS 1 /* Version number of package */ -#define VERSION "2.8.1" +#define VERSION "2.8.2" /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel). */ @@ -123,7 +123,7 @@ #define XML_CONTEXT_BYTES 1024 /* Define to include code reading entropy from `/dev/urandom'. */ -#define XML_DEV_URANDOM 1 +/* #undef XML_DEV_URANDOM */ /* Define to make parameter entity parsing functionality available. */ #define XML_DTD 1 diff --git a/contrib/libs/expat/lib/fallthrough.h b/contrib/libs/expat/lib/fallthrough.h new file mode 100644 index 00000000000..707dbdd44bf --- /dev/null +++ b/contrib/libs/expat/lib/fallthrough.h @@ -0,0 +1,49 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Nick Begg <[email protected]> + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#ifndef FALLTHROUGH_H +# define FALLTHROUGH_H 1 + +// Explicit fallthrough in switch case to avoid warnings +// with compiler flag -Wimplicit-fallthrough. + +# define EXPAT_FALLTHROUGH \ + do { \ + } while (0) + +# if defined(__has_attribute) +# if __has_attribute(fallthrough) +# undef EXPAT_FALLTHROUGH +# define EXPAT_FALLTHROUGH __attribute__((fallthrough)) +# endif +# endif + +#endif // FALLTHROUGH_H diff --git a/contrib/libs/expat/lib/memory_sanitizer.h b/contrib/libs/expat/lib/memory_sanitizer.h new file mode 100644 index 00000000000..a8a8006ccde --- /dev/null +++ b/contrib/libs/expat/lib/memory_sanitizer.h @@ -0,0 +1,51 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Matthew Fernandez <[email protected]> + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#if ! defined(MEMORY_SANITIZER_H) +# define MEMORY_SANITIZER_H 1 + +# if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# include <sanitizer/msan_interface.h> + +// inform Memory Sanitizer that [base, base + extent) is now initialized +# define MSAN_UNPOISON(base, extent) __msan_unpoison((base), (extent)) + +# endif +# endif + +# if ! defined(MSAN_UNPOISON) +# define MSAN_UNPOISON(base, extent) \ + do { \ + } while (0) +# endif + +#endif // ! defined(MEMORY_SANITIZER_H) diff --git a/contrib/libs/expat/lib/random_arc4random_buf.c b/contrib/libs/expat/lib/random_arc4random_buf.c index 1476f07ba1a..5212b9f192c 100644 --- a/contrib/libs/expat/lib/random_arc4random_buf.c +++ b/contrib/libs/expat/lib/random_arc4random_buf.c @@ -7,6 +7,7 @@ |_| XML parser Copyright (c) 2026 Sebastian Pipping <[email protected]> + Copyright (c) 2026 Matthew Fernandez <[email protected]> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -35,9 +36,12 @@ # define _DEFAULT_SOURCE 1 /* for glibc */ #endif +#include "memory_sanitizer.h" #include <stdlib.h> // for arc4random_buf void writeRandomBytes_arc4random_buf(void *target, size_t count) { arc4random_buf(target, count); + // MSan does not understand `arc4random_buf`, so explain its effects + MSAN_UNPOISON(target, count); } diff --git a/contrib/libs/expat/lib/random_dev_urandom.c b/contrib/libs/expat/lib/random_dev_urandom.c index e508fe9f3e3..acae6ba4b04 100644 --- a/contrib/libs/expat/lib/random_dev_urandom.c +++ b/contrib/libs/expat/lib/random_dev_urandom.c @@ -49,11 +49,11 @@ writeRandomBytes_dev_urandom(void *target, size_t count) { const int fd = open("/dev/urandom", O_RDONLY | O_CLOEXEC); if (fd < 0) { - return 0; + return false; } do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); + void *const currentTarget = (char *)target + bytesWrittenTotal; const size_t bytesToWrite = count - bytesWrittenTotal; errno = 0; diff --git a/contrib/libs/expat/lib/random_getentropy.c b/contrib/libs/expat/lib/random_getentropy.c index b4be70ed73c..5cea2d8838b 100644 --- a/contrib/libs/expat/lib/random_getentropy.c +++ b/contrib/libs/expat/lib/random_getentropy.c @@ -7,6 +7,7 @@ |_| XML parser Copyright (c) 2026 Sebastian Pipping <[email protected]> + Copyright (c) 2026 Matthew Fernandez <[email protected]> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -45,10 +46,15 @@ # include <unistd.h> #endif // ! defined(__APPLE__) +#include "memory_sanitizer.h" #include <errno.h> bool writeRandomBytes_getentropy(void *target, size_t count) { errno = 0; - return getentropy(target, count) == 0; + const bool success = getentropy(target, count); + // MSan does not understand `getentropy`, so explain its effects + if (success) + MSAN_UNPOISON(target, count); + return success; } diff --git a/contrib/libs/expat/lib/random_getrandom.c b/contrib/libs/expat/lib/random_getrandom.c index e5724a211cc..72e2cc7093b 100644 --- a/contrib/libs/expat/lib/random_getrandom.c +++ b/contrib/libs/expat/lib/random_getrandom.c @@ -32,10 +32,10 @@ USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "random_getrandom.h" - #include "expat_config.h" // for HAVE_GETRANDOM, HAVE_SYSCALL_GETRANDOM +#include "random_getrandom.h" + #if defined(HAVE_GETRANDOM) # include <sys/random.h> /* getrandom */ #endif @@ -52,6 +52,7 @@ # define GRND_NONBLOCK 0x0001 #endif /* defined(GRND_NONBLOCK) */ +#include "memory_sanitizer.h" #include <assert.h> #include <errno.h> #include <limits.h> // for INT_MAX @@ -64,7 +65,7 @@ writeRandomBytes_getrandom_nonblock(void *target, size_t count) { const unsigned int getrandomFlags = GRND_NONBLOCK; do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); + void *const currentTarget = (char *)target + bytesWrittenTotal; const size_t bytesToWrite = count - bytesWrittenTotal; assert(bytesToWrite <= INT_MAX); @@ -74,9 +75,13 @@ writeRandomBytes_getrandom_nonblock(void *target, size_t count) { const int bytesWrittenMore = #if defined(HAVE_GETRANDOM) (int)getrandom(currentTarget, bytesToWrite, getrandomFlags); + // MSan understands `getrandom`, so does not need extra guidance #else (int)syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags); + // MSan does not understand `syscall`, so explain its effects + if (bytesWrittenMore > 0) + MSAN_UNPOISON(currentTarget, bytesWrittenMore); #endif if (bytesWrittenMore > 0) { diff --git a/contrib/libs/expat/lib/siphash.h b/contrib/libs/expat/lib/siphash.h index 04f6f74585b..be216b4006d 100644 --- a/contrib/libs/expat/lib/siphash.h +++ b/contrib/libs/expat/lib/siphash.h @@ -100,6 +100,7 @@ #include <stddef.h> /* size_t */ #include <stdint.h> /* uint64_t uint32_t uint8_t */ +#include "fallthrough.h" /* * Workaround to not require a C++11 compiler for using ULL suffix @@ -234,25 +235,25 @@ sip24_final(struct siphash *H) { switch (left) { case 7: b |= (uint64_t)H->buf[6] << 48; - /* fall through */ + EXPAT_FALLTHROUGH; case 6: b |= (uint64_t)H->buf[5] << 40; - /* fall through */ + EXPAT_FALLTHROUGH; case 5: b |= (uint64_t)H->buf[4] << 32; - /* fall through */ + EXPAT_FALLTHROUGH; case 4: b |= (uint64_t)H->buf[3] << 24; - /* fall through */ + EXPAT_FALLTHROUGH; case 3: b |= (uint64_t)H->buf[2] << 16; - /* fall through */ + EXPAT_FALLTHROUGH; case 2: b |= (uint64_t)H->buf[1] << 8; - /* fall through */ + EXPAT_FALLTHROUGH; case 1: b |= (uint64_t)H->buf[0] << 0; - /* fall through */ + EXPAT_FALLTHROUGH; case 0: break; } diff --git a/contrib/libs/expat/lib/xcsinc.c b/contrib/libs/expat/lib/xcsinc.c new file mode 100644 index 00000000000..3597c2480bc --- /dev/null +++ b/contrib/libs/expat/lib/xcsinc.c @@ -0,0 +1,48 @@ +/* This file is included from other .c files! + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2022 Sebastian Pipping <[email protected]> + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +static size_t +xcslen(const XML_Char *s) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return wcslen(s); +# else + // XML_Char is unsigned short + size_t len = 0; + while (s[len]) { + len++; + } + return len; +# endif +#else + return strlen(s); +#endif +} diff --git a/contrib/libs/expat/lib/xmlparse.c b/contrib/libs/expat/lib/xmlparse.c index 890a9389f15..60f6cc70054 100644 --- a/contrib/libs/expat/lib/xmlparse.c +++ b/contrib/libs/expat/lib/xmlparse.c @@ -1,4 +1,4 @@ -/* 75ef4224f81c052e9e5aeea2ac7de75357d2169ff9908e39edc08b9dc3052513 (2.8.1+) +/* 5de44e6750c6cc78818f06ed552f522a1241df0299395250e1792cb339389daf (2.8.2+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -47,6 +47,9 @@ Copyright (c) 2026 Rosen Penev <[email protected]> Copyright (c) 2026 Francesco Bertolaccini Copyright (c) 2026 Christian Ng <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> + Copyright (c) 2026 Kartik Kenchi <[email protected]> + Copyright (c) 2026 Haris Hussain <[email protected]> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -90,7 +93,7 @@ #include <stddef.h> #include <string.h> /* memset(), memcpy() */ #include <assert.h> -#include <limits.h> /* INT_MAX, UINT_MAX */ +#include <limits.h> /* INT_MAX, LLONG_MAX, LONG_MAX, UINT_MAX */ #include <stdio.h> /* fprintf */ #include <stdlib.h> /* getenv */ #include <stdint.h> /* SIZE_MAX, uintptr_t */ @@ -114,6 +117,7 @@ #include "ascii.h" #include "expat.h" #include "siphash.h" +#include "xcsinc.c" #if defined(HAVE_ARC4RANDOM) # include "random_arc4random.h" @@ -156,7 +160,7 @@ * Windows >=Vista (rand_s): _WIN32. \ \ If you insist on not using any of these, bypass this error by defining \ - XML_POOR_ENTROPY; you have been warned. \ + XML_POOR_ENTROPY and be vulnerable to hash flooding; you have been warned. \ \ If you have reasons to patch this detection code away or need changes \ to the build system, please open a bug. Thank you! @@ -207,6 +211,12 @@ typedef char ICHAR; #endif +#ifdef XML_LARGE_SIZE +# define XML_INDEX_MAX LLONG_MAX +#else +# define XML_INDEX_MAX LONG_MAX +#endif + /* Round up n to be a multiple of sz, where sz is a power of 2. */ #define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1)) @@ -270,8 +280,8 @@ typedef struct binding { struct binding *prevPrefixBinding; const struct attribute_id *attId; XML_Char *uri; - int uriLen; - int uriAlloc; + size_t uriLen; + size_t uriAlloc; } BINDING; typedef struct prefix { @@ -283,9 +293,9 @@ typedef struct { const XML_Char *str; const XML_Char *localPart; const XML_Char *prefix; - int strLen; - int uriLen; - int prefixLen; + size_t strLen; + size_t uriLen; + size_t prefixLen; } TAG_NAME; /* TAG represents an open element. @@ -384,8 +394,8 @@ typedef struct { const XML_Char *name; PREFIX *prefix; const ATTRIBUTE_ID *idAtt; - int nDefaultAtts; - int allocDefaultAtts; + size_t nDefaultAtts; + size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; HASH_TABLE defaultAttsNames; } ELEMENT_TYPE; @@ -417,6 +427,7 @@ typedef struct { unsigned scaffCount; int scaffLevel; int *scaffIndex; + size_t scaffIndexSize; } DTD; enum EntityType { @@ -588,6 +599,7 @@ static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); static XML_Bool FASTCALL poolGrow(STRING_POOL *pool); +static bool FASTCALL poolGrowUntil(STRING_POOL *pool, size_t needed); static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, const XML_Char *s); static const XML_Char *FASTCALL poolCopyStringNoFinish(STRING_POOL *pool, @@ -642,16 +654,40 @@ static XML_Parser getRootParserOf(XML_Parser parser, static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel); +static bool poolAppendChar(STRING_POOL *pool, XML_Char c); + +static bool poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len); + #define poolStart(pool) ((pool)->start) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) #define poolLastChar(pool) (((pool)->ptr)[-1]) #define poolDiscard(pool) ((pool)->ptr = (pool)->start) #define poolFinish(pool) ((pool)->start = (pool)->ptr) -#define poolAppendChar(pool, c) \ - (((pool)->ptr == (pool)->end && ! poolGrow(pool)) \ - ? 0 \ - : ((*((pool)->ptr)++ = c), 1)) + +bool +poolAppendChar(STRING_POOL *pool, XML_Char c) { + if (pool->ptr == pool->end && ! poolGrow(pool)) + return false; + + *(pool->ptr)++ = c; + return true; +} + +bool +poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len) { + // Detect and prevent integer overflow + if (len > SIZE_MAX / sizeof(XML_Char)) + return false; + + if (! poolGrowUntil(pool, len)) + return false; + + memcpy(pool->ptr, s, len * sizeof(XML_Char)); + pool->ptr += len; + + return true; +} #if ! defined(XML_TESTING) const @@ -730,11 +766,9 @@ struct XML_ParserStruct { const char *m_eventEndPtr; const char *m_positionPtr; OPEN_INTERNAL_ENTITY *m_openInternalEntities; - OPEN_INTERNAL_ENTITY *m_freeInternalEntities; OPEN_INTERNAL_ENTITY *m_openAttributeEntities; - OPEN_INTERNAL_ENTITY *m_freeAttributeEntities; OPEN_INTERNAL_ENTITY *m_openValueEntities; - OPEN_INTERNAL_ENTITY *m_freeValueEntities; + OPEN_INTERNAL_ENTITY *m_freeEntities; XML_Bool m_defaultExpandInternalEntities; int m_tagLevel; ENTITY *m_declEntity; @@ -754,7 +788,7 @@ struct XML_ParserStruct { TAG *m_freeTagList; BINDING *m_inheritedBindings; BINDING *m_freeBindingList; - int m_attsSize; + size_t m_attsSize; int m_nSpecifiedAtts; int m_idAttIndex; ATTRIBUTE *m_atts; @@ -768,7 +802,7 @@ struct XML_ParserStruct { STRING_POOL m_tempPool; STRING_POOL m_temp2Pool; char *m_groupConnector; - unsigned int m_groupSize; + size_t m_groupSize; XML_Char m_namespaceSeparator; XML_Parser m_parentParser; XML_ParsingStatus m_parsingStatus; @@ -785,6 +819,7 @@ struct XML_ParserStruct { ENTITY_STATS m_entity_stats; #endif XML_Bool m_reenter; + unsigned m_handlerCallDepth; }; #if XML_GE == 1 @@ -1128,6 +1163,23 @@ generate_hash_secret_salt(void) { #endif } +static void +beforeHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth < UINT_MAX); + parser->m_handlerCallDepth++; +} + +static void +afterHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth > 0); + parser->m_handlerCallDepth--; +} + +static bool +isCalledFromInsideHandler(XML_Parser parser) { + return parser->m_handlerCallDepth > 0; +} + static enum XML_Error callProcessor(XML_Parser parser, const char *start, const char *end, const char **endPtr) { @@ -1364,9 +1416,7 @@ parserCreate(const XML_Char *encodingName, parser->m_freeBindingList = NULL; parser->m_freeTagList = NULL; - parser->m_freeInternalEntities = NULL; - parser->m_freeAttributeEntities = NULL; - parser->m_freeValueEntities = NULL; + parser->m_freeEntities = NULL; parser->m_groupSize = 0; parser->m_groupConnector = NULL; @@ -1482,6 +1532,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_parsingStatus.parsing = XML_INITIALIZED; // Reentry can only be triggered inside m_processor calls parser->m_reenter = XML_FALSE; + parser->m_handlerCallDepth = 0; #ifdef XML_DTD parser->m_isParamEntity = XML_FALSE; parser->m_useForeignDTD = XML_FALSE; @@ -1515,12 +1566,22 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings) { } } +/* Moves a list of entities onto the start of another list. */ +static void +moveEntityList(OPEN_INTERNAL_ENTITY **dst, OPEN_INTERNAL_ENTITY **src) { + for (OPEN_INTERNAL_ENTITY *head = *src; head != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = head; + head = head->next; + openEntity->next = *dst; + *dst = openEntity; + } +} + XML_Bool XMLCALL XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { TAG *tStk; - OPEN_INTERNAL_ENTITY *openEntityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_FALSE; if (parser->m_parentParser) @@ -1535,32 +1596,14 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { tag->bindings = NULL; parser->m_freeTagList = tag; } - /* move m_openInternalEntities to m_freeInternalEntities */ - openEntityList = parser->m_openInternalEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; - } - /* move m_openAttributeEntities to m_freeAttributeEntities (i.e. same task but - * for attributes) */ - openEntityList = parser->m_openAttributeEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = openEntity; - } - /* move m_openValueEntities to m_freeValueEntities (i.e. same task but - * for value entities) */ - openEntityList = parser->m_openValueEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeValueEntities; - parser->m_freeValueEntities = openEntity; - } + /* move m_openInternalEntities to m_freeEntities */ + moveEntityList(&parser->m_freeEntities, &parser->m_openInternalEntities); + /* move m_openAttributeEntities to m_freeEntities (i.e. same task but for + * attributes) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openAttributeEntities); + /* move m_openValueEntities to m_freeEntities (i.e. same task but for value + * entities) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openValueEntities); moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) @@ -1712,11 +1755,6 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, newDtd = oldDtd; #endif /* XML_DTD */ - /* Note that the magical uses of the pre-processor to make field - access look more like C++ require that `parser' be overwritten - here. This makes this function more painful to follow than it - would be otherwise. - */ if (parser->m_ns) { XML_Char tmp[2] = {parser->m_namespaceSeparator, 0}; parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd, oldParser); @@ -1806,8 +1844,7 @@ destroyBindings(BINDING *bindings, XML_Parser parser) { void XMLCALL XML_ParserFree(XML_Parser parser) { TAG *tagList; - OPEN_INTERNAL_ENTITY *entityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return; /* free m_tagStack and m_freeTagList */ tagList = parser->m_tagStack; @@ -1825,48 +1862,35 @@ XML_ParserFree(XML_Parser parser) { destroyBindings(p->bindings, parser); FREE(parser, p); } - /* free m_openInternalEntities and m_freeInternalEntities */ - entityList = parser->m_openInternalEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeInternalEntities == NULL) - break; - entityList = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = NULL; - } - openEntity = entityList; + /* free m_openInternalEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openInternalEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } - /* free m_openAttributeEntities and m_freeAttributeEntities */ - entityList = parser->m_openAttributeEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeAttributeEntities == NULL) - break; - entityList = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = NULL; - } - openEntity = entityList; + /* free m_openAttributeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openAttributeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } - /* free m_openValueEntities and m_freeValueEntities */ - entityList = parser->m_openValueEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeValueEntities == NULL) - break; - entityList = parser->m_freeValueEntities; - parser->m_freeValueEntities = NULL; - } - openEntity = entityList; + /* free m_openValueEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openValueEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } + /* free m_freeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_freeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; + entityList = entityList->next; + FREE(parser, openEntity); + } + parser->m_freeEntities = NULL; destroyBindings(parser->m_freeBindingList, parser); destroyBindings(parser->m_inheritedBindings, parser); poolDestroy(&parser->m_tempPool); @@ -2263,6 +2287,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; return XML_STATUS_ERROR; } + if (isCalledFromInsideHandler(parser)) + return XML_STATUS_ERROR; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -2275,7 +2301,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } @@ -2286,7 +2312,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { int nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ - if ((XML_Size)len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { + if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -2317,7 +2343,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return XML_STATUS_OK; } - /* fall through */ + EXPAT_FALLTHROUGH; default: result = XML_STATUS_OK; } @@ -2372,7 +2398,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { const char *start; enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (len < 0) { @@ -2398,11 +2424,19 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } + // Detect and avoid integer overflow + if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; + } + start = parser->m_bufferPtr; parser->m_positionPtr = start; parser->m_bufferEnd += len; @@ -2428,6 +2462,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; /* should not happen */ } } @@ -2438,9 +2473,32 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { return result; } +/* Modifies `parser`’s buffer to be backed by `newBuf`. */ +static void +setParserBuffer(XML_Parser parser, char *newBuf, int newBufSize, int keep) { + parser->m_bufferLim = newBuf + newBufSize; + if (parser->m_bufferPtr) { + const int parsing + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); + memcpy(newBuf, parser->m_bufferPtr - keep, parsing + keep); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf + parsing + keep; + parser->m_bufferPtr = newBuf + keep; + } else { + /* This must be a brand new buffer with no data in it yet */ + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf; + parser->m_bufferPtr = newBuf; + } +} + void *XMLCALL XML_GetBuffer(XML_Parser parser, int len) { - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return NULL; if (len < 0) { parser->m_errorCode = XML_ERROR_NO_MEMORY; @@ -2461,9 +2519,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_lastBufferRequestSize = len; if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd) || parser->m_buffer == NULL) { -#if XML_CONTEXT_BYTES > 0 - int keep; -#endif /* XML_CONTEXT_BYTES > 0 */ /* Do not invoke signed arithmetic overflow: */ int neededSize = (int)((unsigned)len + (unsigned)EXPAT_SAFE_PTR_DIFF( @@ -2473,7 +2528,9 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } #if XML_CONTEXT_BYTES > 0 - keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + const int parsed + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + int keep = parsed; if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; /* Detect and prevent integer overflow */ @@ -2481,16 +2538,16 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - neededSize += keep; +#else + int keep = 0; #endif /* XML_CONTEXT_BYTES > 0 */ + neededSize += keep; if (parser->m_buffer && parser->m_bufferPtr && neededSize <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { #if XML_CONTEXT_BYTES > 0 - if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) { - int offset - = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) - - keep; + if (keep < parsed) { + int offset = parsed - keep; /* The buffer pointers cannot be NULL here; we have at least some bytes * in the buffer */ memmove(parser->m_buffer, &parser->m_buffer[offset], @@ -2507,7 +2564,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferPtr = parser->m_buffer; #endif /* XML_CONTEXT_BYTES > 0 */ } else { - char *newBuf; int bufferSize = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer); if (bufferSize == 0) @@ -2522,49 +2578,12 @@ XML_GetBuffer(XML_Parser parser, int len) { } // NOTE: We are avoiding MALLOC(..) here to leave limiting // the input size to the application using Expat. - newBuf = parser->m_mem.malloc_fcn(bufferSize); + char *const newBuf = parser->m_mem.malloc_fcn(bufferSize); if (newBuf == NULL) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - parser->m_bufferLim = newBuf + bufferSize; -#if XML_CONTEXT_BYTES > 0 - if (parser->m_bufferPtr) { - memcpy(newBuf, &parser->m_bufferPtr[-keep], - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep); - // NOTE: We are avoiding FREE(..) here because parser->m_buffer - // is not being allocated with MALLOC(..) but with plain - // .malloc_fcn(..). - parser->m_mem.free_fcn(parser->m_buffer); - parser->m_buffer = newBuf; - parser->m_bufferEnd - = parser->m_buffer - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep; - parser->m_bufferPtr = parser->m_buffer + keep; - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - parser->m_bufferPtr = parser->m_buffer = newBuf; - } -#else - if (parser->m_bufferPtr) { - memcpy(newBuf, parser->m_bufferPtr, - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); - // NOTE: We are avoiding FREE(..) here because parser->m_buffer - // is not being allocated with MALLOC(..) but with plain - // .malloc_fcn(..). - parser->m_mem.free_fcn(parser->m_buffer); - parser->m_bufferEnd - = newBuf - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - } - parser->m_bufferPtr = parser->m_buffer = newBuf; -#endif /* XML_CONTEXT_BYTES > 0 */ + setParserBuffer(parser, newBuf, bufferSize, keep); } parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_positionPtr = NULL; @@ -2617,7 +2636,7 @@ enum XML_Status XMLCALL XML_ResumeParser(XML_Parser parser) { enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (parser->m_parsingStatus.parsing != XML_SUSPENDED) { parser->m_errorCode = XML_ERROR_NOT_SUSPENDED; @@ -2643,6 +2662,7 @@ XML_ResumeParser(XML_Parser parser) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; } } @@ -2704,7 +2724,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)offset; (void)size; #endif /* XML_CONTEXT_BYTES > 0 */ - return (const char *)0; + return NULL; } XML_Size XMLCALL @@ -3063,7 +3083,7 @@ storeRawNames(XML_Parser parser) { */ rawNameLen = ROUND_UP(tag->rawNameLength, sizeof(XML_Char)); /* Detect and prevent integer overflow. */ - if (rawNameLen > (size_t)INT_MAX - nameLen) + if (rawNameLen > SIZE_MAX - nameLen) return XML_FALSE; bufSize = nameLen + rawNameLen; if (bufSize > (size_t)(tag->bufEnd - tag->buf.raw)) { @@ -3190,7 +3210,7 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default: start = next; } @@ -3269,7 +3289,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *eventEndPP = end; if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3322,9 +3344,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); #endif /* XML_GE == 1 */ - if (parser->m_characterDataHandler) + if (parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3344,9 +3368,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, else if (! entity->is_internal) return XML_ERROR_ENTITY_DECLARED_IN_PE; } else if (! entity) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3357,10 +3383,12 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (entity->textPtr) { enum XML_Error result; if (! parser->m_defaultExpandInternalEntities) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3374,9 +3402,12 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, entity->open = XML_FALSE; if (! context) return XML_ERROR_NO_MEMORY; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, context, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, context, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; poolDiscard(&parser->m_tempPool); } else if (parser->m_defaultHandler) @@ -3384,7 +3415,6 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, break; } case XML_TOK_START_TAG_NO_ATTS: - /* fall through */ case XML_TOK_START_TAG_WITH_ATTS: { TAG *tag; enum XML_Error result; @@ -3416,11 +3446,10 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const char *fromPtr = tag->rawName; toPtr = tag->buf.str; for (;;) { - int convLen; const enum XML_Convert_Result convert_res = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); - convLen = (int)(toPtr - tag->buf.str); + const size_t convLen = (size_t)(toPtr - tag->buf.str); if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; @@ -3445,16 +3474,17 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings), account); if (result) return result; - if (parser->m_startElementHandler) + if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, tag->name.str, (const XML_Char **)parser->m_atts); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); poolClear(&parser->m_tempPool); break; } case XML_TOK_EMPTY_ELEMENT_NO_ATTS: - /* fall through */ case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: { const char *rawName = s + enc->minBytesPerChar; enum XML_Error result; @@ -3474,14 +3504,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } poolFinish(&parser->m_tempPool); if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, name.str, (const XML_Char **)parser->m_atts); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (parser->m_endElementHandler) { if (parser->m_startElementHandler) *eventPP = *eventEndPP; + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, name.str); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (noElmHandlers && parser->m_defaultHandler) @@ -3539,14 +3573,19 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } *uri = XML_T('\0'); } + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, tag->name.str); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); while (tag->bindings) { BINDING *b = tag->bindings; - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } tag->bindings = tag->bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; parser->m_freeBindingList = b; @@ -3569,8 +3608,10 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return XML_ERROR_BAD_CHAR_REF; if (parser->m_characterDataHandler) { XML_Char buf[XML_ENCODE_MAX]; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, buf, XmlEncode(n, (ICHAR *)buf)); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3579,32 +3620,38 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; case XML_TOK_CDATA_SECT_OPEN: { enum XML_Error result; - if (parser->m_startCdataSectionHandler) + if (parser->m_startCdataSectionHandler) { + beforeHandler(parser); parser->m_startCdataSectionHandler(parser->m_handlerArg); - /* BEGIN disabled code */ - /* Suppose you doing a transformation on a document that involves - changing only the character data. You set up a defaultHandler - and a characterDataHandler. The defaultHandler simply copies - characters through. The characterDataHandler does the - transformation and writes the characters out escaping them as - necessary. This case will fail to work if we leave out the - following two lines (because & and < inside CDATA sections will - be incorrectly escaped). + afterHandler(parser); + /* BEGIN disabled code */ + /* Suppose you doing a transformation on a document that involves + changing only the character data. You set up a defaultHandler + and a characterDataHandler. The defaultHandler simply copies + characters through. The characterDataHandler does the + transformation and writes the characters out escaping them as + necessary. This case will fail to work if we leave out the + following two lines (because & and < inside CDATA sections will + be incorrectly escaped). - However, now we have a start/endCdataSectionHandler, so it seems - easier to let the user deal with this. - */ - else if ((0) && parser->m_characterDataHandler) + However, now we have a start/endCdataSectionHandler, so it seems + easier to let the user deal with this. + */ + } else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore, account); @@ -3624,13 +3671,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (MUST_CONVERT(enc, s)) { ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf; XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); - } else + afterHandler(parser); + } else { + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3655,16 +3707,21 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3702,7 +3759,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *nextPtr = next; return XML_ERROR_NONE; } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; *eventPP = s = next; } @@ -3722,8 +3779,11 @@ freeBindings(XML_Parser parser, BINDING *bindings) { /* m_startNamespaceDeclHandler will have been called for this * binding in addBindings(), so call the end handler now. */ - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } bindings = bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; @@ -3747,20 +3807,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, TAG_NAME *tagNamePtr, BINDING **bindingsPtr, enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ - ELEMENT_TYPE *elementType; - int nDefaultAtts; - const XML_Char **appAtts; /* the attribute list for the application */ int attIndex = 0; - int prefixLen; - int i; - int n; XML_Char *uri; int nPrefixes = 0; BINDING *binding; const XML_Char *localPart; /* lookup the element type name */ - elementType + ELEMENT_TYPE *elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0); if (! elementType) { const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str); @@ -3775,75 +3829,71 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } - nDefaultAtts = elementType->nDefaultAtts; + const size_t nDefaultAtts = elementType->nDefaultAtts; + + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; /* get the attributes from the tokenizer */ - n = XmlGetAttributes(enc, attStr, parser->m_attsSize, parser->m_atts); + size_t n = (size_t)XmlGetAttributes(enc, attStr, (int)parser->m_attsSize, + parser->m_atts); /* Detect and prevent integer overflow */ - if (n > INT_MAX - nDefaultAtts) { + if (n > SIZE_MAX - nDefaultAtts) { return XML_ERROR_NO_MEMORY; } if (n + nDefaultAtts > parser->m_attsSize) { - int oldAttsSize = parser->m_attsSize; - ATTRIBUTE *temp; -#ifdef XML_ATTR_INFO - XML_AttrInfo *temp2; -#endif + size_t oldAttsSize = parser->m_attsSize; /* Detect and prevent integer overflow */ - if ((nDefaultAtts > INT_MAX - INIT_ATTS_SIZE) - || (n > INT_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { + if ((nDefaultAtts > SIZE_MAX - INIT_ATTS_SIZE) + || (n > SIZE_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { return XML_ERROR_NO_MEMORY; } parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE; - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > SIZE_MAX / sizeof(ATTRIBUTE)) { + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > SIZE_MAX / sizeof(ATTRIBUTE)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } -#endif - temp = REALLOC(parser, parser->m_atts, - parser->m_attsSize * sizeof(ATTRIBUTE)); + ATTRIBUTE *const temp = REALLOC(parser, parser->m_atts, + parser->m_attsSize * sizeof(ATTRIBUTE)); if (temp == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } parser->m_atts = temp; #ifdef XML_ATTR_INFO - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -# if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > SIZE_MAX / sizeof(XML_AttrInfo)) { + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > SIZE_MAX / sizeof(XML_AttrInfo)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } -# endif - temp2 = REALLOC(parser, parser->m_attInfo, - parser->m_attsSize * sizeof(XML_AttrInfo)); + XML_AttrInfo *const temp2 = REALLOC( + parser, parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo)); if (temp2 == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } parser->m_attInfo = temp2; #endif - if (n > oldAttsSize) - XmlGetAttributes(enc, attStr, n, parser->m_atts); + if (n > oldAttsSize) { + /* Detect and prevent integer overflow. */ + if (n > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; + XmlGetAttributes(enc, attStr, (int)n, parser->m_atts); + } } - appAtts = (const XML_Char **)parser->m_atts; - for (i = 0; i < n; i++) { + /* the attribute list for the application */ + const XML_Char **const appAtts = (const XML_Char **)parser->m_atts; + for (size_t i = 0; i < n; i++) { ATTRIBUTE *currAtt = &parser->m_atts[i]; #ifdef XML_ATTR_INFO XML_AttrInfo *currAttInfo = &parser->m_attInfo[i]; @@ -3876,13 +3926,11 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, (attId->name)[-1] = 1; appAtts[attIndex++] = attId->name; if (! parser->m_atts[i].normalized) { - enum XML_Error result; XML_Bool isCdata = XML_TRUE; /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - int j; - for (j = 0; j < nDefaultAtts; j++) { + for (size_t j = 0; j < nDefaultAtts; j++) { if (attId == elementType->defaultAtts[j].id) { isCdata = elementType->defaultAtts[j].isCdata; break; @@ -3891,7 +3939,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* normalize the attribute value */ - result = storeAttributeValue( + const enum XML_Error result = storeAttributeValue( parser, enc, isCdata, parser->m_atts[i].valuePtr, parser->m_atts[i].valueEnd, &parser->m_tempPool, account); if (result) @@ -3929,7 +3977,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* set-up for XML_GetSpecifiedAttributeCount and XML_GetIdAttributeIndex */ parser->m_nSpecifiedAtts = attIndex; if (elementType->idAtt && (elementType->idAtt->name)[-1]) { - for (i = 0; i < attIndex; i += 2) + for (int i = 0; i < attIndex; i += 2) if (appAtts[i] == elementType->idAtt->name) { parser->m_idAttIndex = i; break; @@ -3938,7 +3986,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, parser->m_idAttIndex = -1; /* do attribute defaulting */ - for (i = 0; i < nDefaultAtts; i++) { + for (size_t i = 0; i < nDefaultAtts; i++) { const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i; if (! (da->id->name)[-1] && da->value) { if (da->id->prefix) { @@ -3964,7 +4012,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* expand prefixed attribute names, check for duplicates, and clear flags that say whether attributes were specified */ - i = 0; + int i = 0; if (nPrefixes) { unsigned int j; /* hash table index */ unsigned long version = parser->m_nsAttsVersion; @@ -3979,7 +4027,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* size of hash table must be at least 2 * (# of prefixed attributes) */ if ((nPrefixes << 1) >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ - NS_ATT *temp; /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -4007,7 +4054,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } #endif - temp = REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT)); + NS_ATT *const temp + = REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT)); if (! temp) { /* Restore actual size of memory in m_nsAtts */ parser->m_nsAttsPower = oldNsAttsPower; @@ -4028,9 +4076,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, for (; i < attIndex; i += 2) { const XML_Char *s = appAtts[i]; if (s[-1] == 2) { /* prefixed */ - ATTRIBUTE_ID *id; - const BINDING *b; - unsigned long uriHash; struct siphash sip_state; struct sipkey sip_key; @@ -4038,7 +4083,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_init(&sip_state, &sip_key); ((XML_Char *)s)[-1] = 0; /* clear flag */ - id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); + ATTRIBUTE_ID *const id + = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); if (! id || ! id->prefix) { /* This code is walking through the appAtts array, dealing * with (in this case) a prefixed attribute name. To be in @@ -4046,7 +4092,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * has to have passed through the hash table lookup once * already. That implies that an entry for it already * exists, so the lookup above will return a pointer to - * already allocated memory. There is no opportunaity for + * already allocated memory. There is no opportunity for * the allocator to fail, so the condition above cannot be * fulfilled. * @@ -4056,15 +4102,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, */ return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ } - b = id->prefix->binding; + const BINDING *const b = id->prefix->binding; if (! b) return XML_ERROR_UNBOUND_PREFIX; - for (j = 0; j < (unsigned int)b->uriLen; j++) { - const XML_Char c = b->uri[j]; - if (! poolAppendChar(&parser->m_tempPool, c)) - return XML_ERROR_NO_MEMORY; - } + if (! poolAppendChars(&parser->m_tempPool, b->uri, b->uriLen)) + return XML_ERROR_NO_MEMORY; sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char)); @@ -4073,12 +4116,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char)); - do { /* copies null terminator */ - if (! poolAppendChar(&parser->m_tempPool, *s)) + { + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) return XML_ERROR_NO_MEMORY; - } while (*s++); + } - uriHash = (unsigned long)sip24_final(&sip_state); + const unsigned long uriHash = (unsigned long)sip24_final(&sip_state); { /* Check hash table for duplicate of expanded name (uriName). Derived from code in lookup(parser, HASH_TABLE *table, ...). @@ -4106,10 +4150,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, if (parser->m_ns_triplets) { /* append namespace separator and prefix */ parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator; s = b->prefix->name; - do { - if (! poolAppendChar(&parser->m_tempPool, *s)) - return XML_ERROR_NO_MEMORY; - } while (*s++); + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) + return XML_ERROR_NO_MEMORY; } /* store expanded name in attribute list */ @@ -4152,48 +4195,36 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, localPart = tagNamePtr->str; } else return XML_ERROR_NONE; - prefixLen = 0; - if (parser->m_ns_triplets && binding->prefix->name) { - while (binding->prefix->name[prefixLen++]) - ; /* prefixLen includes null terminator */ - } + size_t prefixLen = 0; + if (parser->m_ns_triplets && binding->prefix->name) + prefixLen = xcslen(binding->prefix->name) + /*null terminator*/ 1; tagNamePtr->localPart = localPart; tagNamePtr->uriLen = binding->uriLen; tagNamePtr->prefix = binding->prefix->name; tagNamePtr->prefixLen = prefixLen; - for (i = 0; localPart[i++];) - ; /* i includes null terminator */ + + const size_t localPartLen = xcslen(localPart) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ - if (binding->uriLen > INT_MAX - prefixLen - || i > INT_MAX - (binding->uriLen + prefixLen)) { + if (binding->uriLen > SIZE_MAX - prefixLen + || localPartLen > SIZE_MAX - (binding->uriLen + prefixLen)) { return XML_ERROR_NO_MEMORY; } - n = i + binding->uriLen + prefixLen; - if (n > binding->uriAlloc) { - TAG *p; - + const size_t totalLen = localPartLen + binding->uriLen + prefixLen; + if (totalLen > binding->uriAlloc) { /* Detect and prevent integer overflow */ - if (n > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(n + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { + if (totalLen > SIZE_MAX - EXPAND_SPARE + || totalLen + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif - uri = MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char)); + uri = MALLOC(parser, (totalLen + EXPAND_SPARE) * sizeof(XML_Char)); if (! uri) return XML_ERROR_NO_MEMORY; - binding->uriAlloc = n + EXPAND_SPARE; + binding->uriAlloc = totalLen + EXPAND_SPARE; memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char)); - for (p = parser->m_tagStack; p; p = p->parent) + for (TAG *p = parser->m_tagStack; p; p = p->parent) if (p->name.str == binding->uri) p->name.str = uri; FREE(parser, binding->uri); @@ -4201,10 +4232,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* if m_namespaceSeparator != '\0' then uri includes it already */ uri = binding->uri + binding->uriLen; - memcpy(uri, localPart, i * sizeof(XML_Char)); + /* Detect and prevent integer overflow */ + if (localPartLen > SIZE_MAX / sizeof(XML_Char)) { + return XML_ERROR_NO_MEMORY; + } + memcpy(uri, localPart, localPartLen * sizeof(XML_Char)); /* we always have a namespace separator between localPart and prefix */ if (prefixLen) { - uri += i - 1; + uri += localPartLen - 1; *uri = parser->m_namespaceSeparator; /* replace null terminator */ memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char)); } @@ -4339,7 +4374,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'}; - static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlLen = sizeof(xmlNamespace) / sizeof(XML_Char) - 1; // "http://www.w3.org/2000/xmlns/" static const XML_Char xmlnsNamespace[] = {ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, @@ -4347,15 +4382,14 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_2, ASCII_0, ASCII_0, ASCII_0, ASCII_SLASH, ASCII_x, ASCII_m, ASCII_l, ASCII_n, ASCII_s, ASCII_SLASH, '\0'}; - static const int xmlnsLen - = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlnsLen = sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; XML_Bool mustBeXML = XML_FALSE; XML_Bool isXML = XML_TRUE; XML_Bool isXMLNS = XML_TRUE; BINDING *b; - int len; + size_t len; /* empty URI is only valid for default namespace per XML NS 1.0 (not 1.1) */ if (*uri == XML_T('\0') && prefix->name) @@ -4374,6 +4408,10 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, } for (len = 0; uri[len]; len++) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len])) isXML = XML_FALSE; @@ -4414,25 +4452,21 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, if (isXMLNS) return XML_ERROR_RESERVED_NAMESPACE_URI; - if (parser->m_namespaceSeparator) + if (parser->m_namespaceSeparator) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } len++; + } if (parser->m_freeBindingList) { b = parser->m_freeBindingList; if (len > b->uriAlloc) { /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif XML_Char *temp = REALLOC(parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE)); @@ -4448,18 +4482,10 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, return XML_ERROR_NO_MEMORY; /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { - return XML_ERROR_NO_MEMORY; - } -#endif b->uri = MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE)); if (! b->uri) { @@ -4483,9 +4509,12 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, b->nextTagBinding = *bindingsPtr; *bindingsPtr = b; /* if attId == NULL then we are not starting a namespace scope */ - if (attId && parser->m_startNamespaceDeclHandler) + if (attId && parser->m_startNamespaceDeclHandler) { + beforeHandler(parser); parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name, prefix->binding ? uri : 0); + afterHandler(parser); + } return XML_ERROR_NONE; } @@ -4547,15 +4576,20 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventEndPP = next; switch (tok) { case XML_TOK_CDATA_SECT_CLOSE: - if (parser->m_endCdataSectionHandler) + if (parser->m_endCdataSectionHandler) { + beforeHandler(parser); parser->m_endCdataSectionHandler(parser->m_handlerArg); + afterHandler(parser); + } /* BEGIN disabled code */ /* see comment under XML_TOK_CDATA_SECT_OPEN */ - else if ((0) && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); *startPtr = next; *nextPtr = next; @@ -4566,7 +4600,9 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; @@ -4579,16 +4615,21 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = next; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4633,7 +4674,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; *eventPP = s = next; } @@ -4827,8 +4868,10 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, if (! storedversion) return XML_ERROR_NO_MEMORY; } + beforeHandler(parser); parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName, standalone); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, parser->m_encoding, s, next); if (parser->m_protocolEncodingName == NULL) { @@ -4878,8 +4921,11 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { info.convert = NULL; info.data = NULL; info.release = NULL; - if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData, - encodingName, &info)) { + beforeHandler(parser); + const int status = parser->m_unknownEncodingHandler( + parser->m_unknownEncodingHandlerData, encodingName, &info); + afterHandler(parser); + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { @@ -5256,9 +5302,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; case XML_ROLE_DOCTYPE_INTERNAL_SUBSET: if (parser->m_startDoctypeDeclHandler) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 1); + afterHandler(parser); parser->m_doctypeName = NULL; poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; @@ -5297,7 +5345,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, handleDefault = XML_FALSE; goto alreadyChecked; } - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_PUBLIC_ID: if (! XmlIsPublicId(enc, s, next, eventPP)) return XML_ERROR_PUBLICID; @@ -5325,9 +5373,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_doctypeName) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 0); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -5354,14 +5404,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_useForeignDTD) entity->base = parser->m_curBase; dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -5374,7 +5432,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } #endif /* XML_DTD */ if (parser->m_endDoctypeDeclHandler) { + beforeHandler(parser); parser->m_endDoctypeDeclHandler(parser->m_handlerArg); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5394,14 +5454,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; entity->base = parser->m_curBase; dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -5494,10 +5562,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, 0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5532,10 +5602,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -5550,16 +5622,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar, XML_ACCOUNT_NONE); if (parser->m_declEntity) { + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(&dtd->entityValuePool) > (size_t)INT_MAX) { + return XML_ERROR_NO_MEMORY; + } parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool); parser->m_declEntity->textLen = (int)(poolLength(&dtd->entityValuePool)); poolFinish(&dtd->entityValuePool); if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } else @@ -5577,10 +5655,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5611,9 +5691,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #ifdef XML_DTD && ! parser->m_paramEntityParsing #endif /* XML_DTD */ - && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } #ifndef XML_DTD break; #else /* XML_DTD */ @@ -5625,7 +5709,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_declEntity->publicId = NULL; } #endif /* XML_DTD */ - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_SYSTEM_ID: if (dtd->keepProcessing && parser->m_declEntity) { parser->m_declEntity->systemId @@ -5656,10 +5740,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5672,17 +5758,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&dtd->pool); if (parser->m_unparsedEntityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_unparsedEntityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } else if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5789,9 +5879,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! systemId) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, systemId, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5799,9 +5891,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_NOTATION_NO_SYSTEM_ID: if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, 0, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5835,41 +5929,18 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_GROUP_OPEN: if (parser->m_prologState.level >= parser->m_groupSize) { if (parser->m_groupSize) { - { - /* Detect and prevent integer overflow */ - if (parser->m_groupSize > (unsigned int)(-1) / 2u) { - return XML_ERROR_NO_MEMORY; - } - - char *const new_connector = REALLOC( - parser, parser->m_groupConnector, parser->m_groupSize *= 2); - if (new_connector == NULL) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } - parser->m_groupConnector = new_connector; + /* Detect and prevent integer overflow */ + if (parser->m_groupSize > SIZE_MAX / 2) { + return XML_ERROR_NO_MEMORY; } - if (dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if (parser->m_groupSize > SIZE_MAX / sizeof(int)) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } -#endif - - int *const new_scaff_index = REALLOC( - parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int)); - if (new_scaff_index == NULL) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } - dtd->scaffIndex = new_scaff_index; + char *const new_connector = REALLOC(parser, parser->m_groupConnector, + parser->m_groupSize *= 2); + if (new_connector == NULL) { + parser->m_groupSize /= 2; + return XML_ERROR_NO_MEMORY; } + parser->m_groupConnector = new_connector; } else { parser->m_groupConnector = MALLOC(parser, parser->m_groupSize = 32); if (! parser->m_groupConnector) { @@ -5884,6 +5955,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (myindex < 0) return XML_ERROR_NO_MEMORY; assert(dtd->scaffIndex != NULL); + if ((size_t)dtd->scaffLevel >= dtd->scaffIndexSize) { + /* Detect and prevent integer overflow */ + if (dtd->scaffIndexSize > SIZE_MAX / 2 / sizeof(int)) { + return XML_ERROR_NO_MEMORY; + } + assert(dtd->scaffIndexSize > 0); + const size_t new_size = dtd->scaffIndexSize * 2; + int *const new_scaff_index + = REALLOC(parser, dtd->scaffIndex, new_size * sizeof(int)); + if (new_scaff_index == NULL) { + return XML_ERROR_NO_MEMORY; + } + dtd->scaffIndex = new_scaff_index; + dtd->scaffIndexSize = new_size; + } dtd->scaffIndex[dtd->scaffLevel] = myindex; dtd->scaffLevel++; dtd->scaffold[myindex].type = XML_CTYPE_SEQ; @@ -5964,7 +6050,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* cannot report skipped entities in declarations */ if ((role == XML_ROLE_PARAM_ENTITY_REF) && parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5985,9 +6073,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; @@ -6005,9 +6096,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } #endif /* XML_DTD */ - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } break; /* Element declaration stuff */ @@ -6042,8 +6137,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY : XML_CTYPE_EMPTY); *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, content); + afterHandler(parser); handleDefault = XML_FALSE; } dtd->in_eldecl = XML_FALSE; @@ -6087,9 +6184,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; name = el->name; dtd->scaffold[myindex].name = name; - nameLen = 0; - while (name[nameLen++]) - ; + nameLen = xcslen(name) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ if (nameLen > UINT_MAX - dtd->contentStringLen) { @@ -6125,8 +6220,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! model) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, model); + afterHandler(parser); } dtd->in_eldecl = XML_FALSE; dtd->contentStringLen = 0; @@ -6188,7 +6285,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, *nextPtr = next; return XML_ERROR_NONE; } - /* Fall through */ + EXPAT_FALLTHROUGH; default: s = next; tok = XmlPrologTok(enc, s, end, &next); @@ -6268,7 +6365,7 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; parser->m_eventPtr = s = next; } @@ -6278,20 +6375,18 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, static enum XML_Error processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, enum EntityType type) { - OPEN_INTERNAL_ENTITY *openEntity, **openEntityList, **freeEntityList; + OPEN_INTERNAL_ENTITY *openEntity, **openEntityList; + OPEN_INTERNAL_ENTITY **const freeEntityList = &parser->m_freeEntities; switch (type) { case ENTITY_INTERNAL: parser->m_processor = internalEntityProcessor; openEntityList = &parser->m_openInternalEntities; - freeEntityList = &parser->m_freeInternalEntities; break; case ENTITY_ATTRIBUTE: openEntityList = &parser->m_openAttributeEntities; - freeEntityList = &parser->m_freeAttributeEntities; break; case ENTITY_VALUE: openEntityList = &parser->m_openValueEntities; - freeEntityList = &parser->m_freeValueEntities; break; /* default case serves merely as a safety net in case of a * wrong entityType. Therefore we exclude the following lines @@ -6408,8 +6503,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, parser->m_openInternalEntities = parser->m_openInternalEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; if (parser->m_openInternalEntities == NULL) { parser->m_processor = entity->is_param ? prologProcessor : contentProcessor; @@ -6485,8 +6580,8 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; } // Break if an error occurred or there is nothing left to process @@ -6539,7 +6634,6 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, return XML_ERROR_INVALID_TOKEN; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, ptr); if (n < 0) { if (enc == parser->m_encoding) @@ -6559,10 +6653,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (! poolAppendChar(pool, buf[i])) - return XML_ERROR_NO_MEMORY; - } + + if (! poolAppendChars(pool, buf, n)) + return XML_ERROR_NO_MEMORY; } break; case XML_TOK_DATA_CHARS: if (! poolAppend(pool, enc, ptr, next)) @@ -6570,7 +6663,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, break; case XML_TOK_TRAILING_CR: next = ptr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_ATTRIBUTE_VALUE_S: case XML_TOK_DATA_NEWLINE: if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20)) @@ -6624,8 +6717,11 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } else if (! entity) { /* Cannot report skipped entity here - see comments on parser->m_skippedEntityHandler. - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ /* Cannot call the default handler because this would be out of sync with the call to the startElementHandler. @@ -6758,8 +6854,11 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, /* not a well-formedness error - see XML 1.0: WFC Entity Declared */ /* cannot report skipped entity here - see comments on parser->m_skippedEntityHandler - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ dtd->keepProcessing = dtd->standalone; goto endEntityValue; @@ -6775,9 +6874,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; @@ -6813,17 +6915,15 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, break; case XML_TOK_TRAILING_CR: next = entityTextPtr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_DATA_NEWLINE: - if (pool->end == pool->ptr && ! poolGrow(pool)) { + if (! poolAppendChar(pool, 0xA)) { result = XML_ERROR_NO_MEMORY; goto endEntityValue; } - *(pool->ptr)++ = 0xA; break; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, entityTextPtr); if (n < 0) { if (enc == parser->m_encoding) @@ -6841,12 +6941,9 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (pool->end == pool->ptr && ! poolGrow(pool)) { - result = XML_ERROR_NO_MEMORY; - goto endEntityValue; - } - *(pool->ptr)++ = buf[i]; + if (! poolAppendChars(pool, buf, n)) { + result = XML_ERROR_NO_MEMORY; + goto endEntityValue; } } break; case XML_TOK_PARTIAL: @@ -6943,8 +7040,8 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, parser->m_openValueEntities = parser->m_openValueEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeValueEntities; - parser->m_freeValueEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; } // Break if an error occurred or there is nothing left to process @@ -6974,6 +7071,11 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { return XML_ERROR_NO_MEMORY; } + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(pool) > (size_t)INT_MAX) { + poolDiscard(pool); + return XML_ERROR_NO_MEMORY; + } entity->textPtr = poolStart(pool); entity->textLen = (int)(poolLength(pool)); poolFinish(pool); @@ -7026,7 +7128,9 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_processingInstructionHandler(parser->m_handlerArg, target, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -7046,7 +7150,9 @@ reportComment(XML_Parser parser, const ENCODING *enc, const char *start, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_commentHandler(parser->m_handlerArg, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -7087,15 +7193,20 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); *eventPP = s; } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); - } else + } else { + beforeHandler(parser); parser->m_defaultHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } static int @@ -7106,48 +7217,34 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ NAMED *const nameFound - = (NAMED *)lookup(parser, &(type->defaultAttsNames), attId->name, 0); + = lookup(parser, &(type->defaultAttsNames), attId->name, 0); if (nameFound) return 1; if (isId && ! type->idAtt && ! attId->xmlns) type->idAtt = attId; } if (type->nDefaultAtts == type->allocDefaultAtts) { - if (type->allocDefaultAtts == 0) { - type->allocDefaultAtts = 8; - type->defaultAtts - = MALLOC(parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); - if (! type->defaultAtts) { - type->allocDefaultAtts = 0; - return 0; - } - } else { - DEFAULT_ATTRIBUTE *temp; - - /* Detect and prevent integer overflow */ - if (type->allocDefaultAtts > INT_MAX / 2) { - return 0; - } - - int count = type->allocDefaultAtts * 2; + /* Detect and prevent integer overflow */ + if (type->allocDefaultAtts > SIZE_MAX / 2) { + return 0; + } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)count > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { - return 0; - } -#endif + size_t count = type->allocDefaultAtts * 2; + if (count == 0) { + count = 8; + } - temp = REALLOC(parser, type->defaultAtts, - (count * sizeof(DEFAULT_ATTRIBUTE))); - if (temp == NULL) - return 0; - type->allocDefaultAtts = count; - type->defaultAtts = temp; + /* Detect and prevent integer overflow. */ + if (count > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { + return 0; } + + DEFAULT_ATTRIBUTE *const temp = REALLOC( + parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE))); + if (temp == NULL) + return 0; + type->allocDefaultAtts = count; + type->defaultAtts = temp; } att = type->defaultAtts + type->nDefaultAtts; att->id = attId; @@ -7156,8 +7253,8 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (! isCdata) attId->maybeTokenized = XML_TRUE; - NAMED *const nameAddedOrFound = (NAMED *)lookup( - parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); + NAMED *const nameAddedOrFound + = lookup(parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); if (! nameAddedOrFound) return 0; @@ -7230,13 +7327,14 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, } else { int i; for (i = 0; name[i]; i++) { + /* Detect and prevent signed integer overflow */ + if (i == INT_MAX) { + return NULL; + } /* attributes without prefix are *not* in the default namespace */ if (name[i] == XML_T(ASCII_COLON)) { - int j; - for (j = 0; j < i; j++) { - if (! poolAppendChar(&dtd->pool, name[j])) - return NULL; - } + if (! poolAppendChars(&dtd->pool, name, i)) + return NULL; if (! poolAppendChar(&dtd->pool, XML_T('\0'))) return NULL; id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, @@ -7264,46 +7362,39 @@ getContext(XML_Parser parser) { XML_Bool needSep = XML_FALSE; if (dtd->defaultPrefix.binding) { - int i; - int len; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = dtd->defaultPrefix.binding->uriLen; + size_t len = dtd->defaultPrefix.binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) { - if (! poolAppendChar(&parser->m_tempPool, - dtd->defaultPrefix.binding->uri[i])) { - /* Because of memory caching, I don't believe this line can be - * executed. - * - * This is part of a loop copying the default prefix binding - * URI into the parser's temporary string pool. Previously, - * that URI was copied into the same string pool, with a - * terminating NUL character, as part of setContext(). When - * the pool was cleared, that leaves a block definitely big - * enough to hold the URI on the free block list of the pool. - * The URI copy in getContext() therefore cannot run out of - * memory. - * - * If the pool is used between the setContext() and - * getContext() calls, the worst it can do is leave a bigger - * block on the front of the free list. Given that this is - * all somewhat inobvious and program logic can be changed, we - * don't delete the line but we do exclude it from the test - * coverage statistics. - */ - return NULL; /* LCOV_EXCL_LINE */ - } + if (! poolAppendChars(&parser->m_tempPool, dtd->defaultPrefix.binding->uri, + len)) { + /* Because of memory caching, I don't believe this line can be + * executed. + * + * This is part of a loop copying the default prefix binding + * URI into the parser's temporary string pool. Previously, + * that URI was copied into the same string pool, with a + * terminating NUL character, as part of setContext(). When + * the pool was cleared, that leaves a block definitely big + * enough to hold the URI on the free block list of the pool. + * The URI copy in getContext() therefore cannot run out of + * memory. + * + * If the pool is used between the setContext() and + * getContext() calls, the worst it can do is leave a bigger + * block on the front of the free list. Given that this is + * all somewhat inobvious and program logic can be changed, we + * don't delete the line but we do exclude it from the test + * coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ } needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->prefixes)); for (;;) { - int i; - int len; - const XML_Char *s; PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter); if (! prefix) break; @@ -7318,23 +7409,21 @@ getContext(XML_Parser parser) { } if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = prefix->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->name, + xcslen(prefix->name))) + return NULL; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = prefix->binding->uriLen; + size_t len = prefix->binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) - if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i])) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->binding->uri, len)) + return NULL; needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->generalEntities)); for (;;) { - const XML_Char *s; ENTITY *e = (ENTITY *)hashTableIterNext(&iter); if (! e) break; @@ -7342,9 +7431,8 @@ getContext(XML_Parser parser) { continue; if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = e->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return 0; + if (! poolAppendChars(&parser->m_tempPool, e->name, xcslen(e->name))) + return NULL; needSep = XML_TRUE; } @@ -7466,6 +7554,7 @@ dtdCreate(XML_Parser parser) { p->in_eldecl = XML_FALSE; p->scaffIndex = NULL; + p->scaffIndexSize = 0; p->scaffold = NULL; p->scaffLevel = 0; p->scaffSize = 0; @@ -7487,8 +7576,7 @@ dtdReset(DTD *p, XML_Parser parser) { if (! e) break; hashTableDestroy(&(e->defaultAttsNames)); - if (e->allocDefaultAtts != 0) - FREE(parser, e->defaultAtts); + FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); #ifdef XML_DTD @@ -7507,6 +7595,7 @@ dtdReset(DTD *p, XML_Parser parser) { FREE(parser, p->scaffIndex); p->scaffIndex = NULL; + p->scaffIndexSize = 0; FREE(parser, p->scaffold); p->scaffold = NULL; @@ -7529,8 +7618,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { if (! e) break; hashTableDestroy(&(e->defaultAttsNames)); - if (e->allocDefaultAtts != 0) - FREE(parser, e->defaultAtts); + FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); #ifdef XML_DTD @@ -7609,7 +7697,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, hashTableIterInit(&iter, &(oldDtd->elementTypes)); for (;;) { - int i; ELEMENT_TYPE *newE; const XML_Char *name; const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter); @@ -7627,15 +7714,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, hashTableInit(&(newE->defaultAttsNames), parser); if (oldE->nDefaultAtts) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((size_t)oldE->nDefaultAtts > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { + /* Detect and prevent integer overflow. */ + if (oldE->nDefaultAtts > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { return 0; } -#endif newE->defaultAtts = MALLOC(parser, oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (! newE->defaultAtts) { @@ -7649,7 +7731,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (oldE->prefix) newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes), oldE->prefix->name, 0); - for (i = 0; i < newE->nDefaultAtts; i++) { + for (size_t i = 0; i < newE->nDefaultAtts; i++) { const XML_Char *const attributeName = oldE->defaultAtts[i].id->name; newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup( oldParser, &(newDtd->attributeIds), attributeName, 0); @@ -7662,8 +7744,8 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, } else newE->defaultAtts[i].value = NULL; - NAMED *const nameAddedOrFound = (NAMED *)lookup( - parser, &(newE->defaultAttsNames), attributeName, sizeof(NAMED)); + NAMED *const nameAddedOrFound = lookup(parser, &(newE->defaultAttsNames), + attributeName, sizeof(NAMED)); if (! nameAddedOrFound) { return 0; } @@ -7693,6 +7775,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, newDtd->scaffSize = oldDtd->scaffSize; newDtd->scaffLevel = oldDtd->scaffLevel; newDtd->scaffIndex = oldDtd->scaffIndex; + newDtd->scaffIndexSize = oldDtd->scaffIndexSize; return 1; } /* End dtdCopy */ @@ -7764,18 +7847,23 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, static XML_Bool FASTCALL keyeq(KEY s1, KEY s2) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return (wcscmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; +# else for (; *s1 == *s2; s1++, s2++) if (*s1 == 0) return XML_TRUE; return XML_FALSE; +# endif +#else + return (strcmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; +#endif } static size_t keylen(KEY s) { - size_t len = 0; - for (; *s; s++, len++) - ; - return len; + return xcslen(s); } static void @@ -7993,10 +8081,8 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, const XML_Char *s) { - do { - if (! poolAppendChar(pool, *s)) - return NULL; - } while (*s++); + if (! poolAppendChars(pool, s, xcslen(s) + /*null terminator*/ 1)) + return NULL; s = pool->start; poolFinish(pool); return s; @@ -8035,10 +8121,8 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { */ return NULL; /* LCOV_EXCL_LINE */ } - for (; n > 0; --n, s++) { - if (! poolAppendChar(pool, *s)) - return NULL; - } + if (n > 0 && ! poolAppendChars(pool, s, n)) + return NULL; s = pool->start; poolFinish(pool); return s; @@ -8046,11 +8130,8 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool, const XML_Char *s) { - while (*s) { - if (! poolAppendChar(pool, *s)) - return NULL; - s++; - } + if (! poolAppendChars(pool, s, xcslen(s))) + return NULL; return pool->start; } @@ -8059,9 +8140,8 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end) { if (! poolAppend(pool, enc, ptr, end)) return NULL; - if (pool->ptr == pool->end && ! poolGrow(pool)) + if (! poolAppendChar(pool, 0)) return NULL; - *(pool->ptr)++ = 0; return pool->start; } @@ -8196,6 +8276,19 @@ poolGrow(STRING_POOL *pool) { return XML_TRUE; } +static bool FASTCALL +poolGrowUntil(STRING_POOL *pool, size_t needed) { + for (;;) { + const size_t available = pool->end - pool->ptr; + if (available >= needed) { + return true; + } + if (! poolGrow(pool)) { + return false; + } + } +} + static int FASTCALL nextScaffoldPart(XML_Parser parser) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ @@ -8203,18 +8296,14 @@ nextScaffoldPart(XML_Parser parser) { int next; if (! dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX + /* Detect and prevent integer overflow. */ if (parser->m_groupSize > SIZE_MAX / sizeof(int)) { return -1; } -#endif dtd->scaffIndex = MALLOC(parser, parser->m_groupSize * sizeof(int)); if (! dtd->scaffIndex) return -1; + dtd->scaffIndexSize = parser->m_groupSize; dtd->scaffIndex[0] = 0; } @@ -8375,12 +8464,21 @@ build_model(XML_Parser parser) { const XML_Char *src; dest->name = str; src = dtd->scaffold[src_node].name; - for (;;) { - *str++ = *src; - if (! *src) - break; - src++; + + const size_t nameLen = xcslen(src) + /* null terminator*/ 1; + + // Detect and prevent integer overflow + if (nameLen > SIZE_MAX / sizeof(XML_Char)) { + // NOTE: We are avoiding FREE(..) here because the model + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(ret); + return NULL; } + + memcpy(str, src, nameLen * sizeof(XML_Char)); + str += nameLen; + dest->numchildren = 0; dest->children = NULL; } else { @@ -8427,22 +8525,24 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, static XML_Char * copyString(const XML_Char *s, XML_Parser parser) { - size_t charsRequired = 0; - XML_Char *result; - /* First determine how long the string is */ - while (s[charsRequired] != 0) { - charsRequired++; - } - /* Include the terminator */ - charsRequired++; + const size_t charsRequired = xcslen(s) + /*null terminator*/ 1; + + /* Detect and prevent integer overflow */ + if (charsRequired > SIZE_MAX / sizeof(XML_Char)) + return NULL; + + const size_t bytesRequired = charsRequired * sizeof(XML_Char); /* Now allocate space for the copy */ - result = MALLOC(parser, charsRequired * sizeof(XML_Char)); + XML_Char *const result = MALLOC(parser, bytesRequired); + if (result == NULL) return NULL; + /* Copy the original into place */ - memcpy(result, s, charsRequired * sizeof(XML_Char)); + memcpy(result, s, bytesRequired); + return result; } @@ -8609,15 +8709,25 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, const char *const entityName = entity->name; # endif + const bool limitingWanted = rootParser->m_entity_stats.debugLevel < 2; + const int maxLimitedDepth = 10; // somewhat arbitrary + const int candidateIndentDepth + = (int)rootParser->m_entity_stats.currentDepth - 1; + const bool limitingNeeded + = limitingWanted && (candidateIndentDepth > maxLimitedDepth); + const char *const ellipisOrEmpty = limitingNeeded ? " [..] " : ""; + const int indentDepth + = limitingNeeded ? (maxLimitedDepth - /* make space for ellipis */ 2) + : candidateIndentDepth; + fprintf( stderr, - "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n", + "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s%s; %s length %d (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_entity_stats.countEverOpened, rootParser->m_entity_stats.currentDepth, - rootParser->m_entity_stats.maximumDepthSeen, - ((int)rootParser->m_entity_stats.currentDepth - 1) * 2, "", - entity->is_param ? "%" : "&", entityName, action, entity->textLen, - sourceLine); + rootParser->m_entity_stats.maximumDepthSeen, indentDepth * 2, "", + ellipisOrEmpty, entity->is_param ? "%" : "&", entityName, action, + entity->textLen, sourceLine); } static void diff --git a/contrib/libs/expat/lib/xmltok.c b/contrib/libs/expat/lib/xmltok.c index 32cd5f147e9..387c6e44a25 100644 --- a/contrib/libs/expat/lib/xmltok.c +++ b/contrib/libs/expat/lib/xmltok.c @@ -25,6 +25,7 @@ Copyright (c) 2022 Sean McBride <[email protected]> Copyright (c) 2023 Hanno Böck <[email protected]> Copyright (c) 2025 Alfonso Gregory <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -58,6 +59,7 @@ #endif #include "internal.h" +#include "fallthrough.h" #include "xmltok.h" #include "nametab.h" @@ -641,7 +643,7 @@ unicode_byte_type(char hi, char lo) { *(*toP)++ = lo; \ break; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case 0x1: \ case 0x2: \ case 0x3: \ @@ -1557,7 +1559,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc, case 0xEF: /* possibly first byte of UTF-8 BOM */ if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE) break; - /* fall through */ + EXPAT_FALLTHROUGH; case 0x00: case 0x3C: return XML_TOK_PARTIAL; diff --git a/contrib/libs/expat/lib/xmltok_impl.c b/contrib/libs/expat/lib/xmltok_impl.c index 239a2d06c45..eae11bdd968 100644 --- a/contrib/libs/expat/lib/xmltok_impl.c +++ b/contrib/libs/expat/lib/xmltok_impl.c @@ -17,6 +17,7 @@ Copyright (c) 2019 David Loffredo <[email protected]> Copyright (c) 2020 Boris Kolpackov <[email protected]> Copyright (c) 2022 Martin Ettl <[email protected]> + Copyright (c) 2026 Nick Begg <[email protected]> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -83,7 +84,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ case BT_DIGIT: \ @@ -112,7 +113,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ ptr += MINBPC(enc); \ @@ -209,7 +210,7 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_S: case BT_CR: case BT_LF: @@ -323,7 +324,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr + MINBPC(enc); return tok; } - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -615,7 +616,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_INVALID; } } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_EQUALS: { int open; # ifdef XML_NS @@ -898,7 +899,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_INVALID; } } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_AMP: case BT_LT: case BT_NONXML: @@ -1059,7 +1060,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* indicate that this might be part of a CR/LF pair */ return -XML_TOK_PROLOG_S; } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_S: case BT_LF: for (;;) { @@ -1074,7 +1075,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* don't split CR/LF pair */ if (ptr + MINBPC(enc) != end) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_PROLOG_S; @@ -1189,7 +1190,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, tok = XML_TOK_NMTOKEN; break; } - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -1484,7 +1485,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, case BT_NMSTRT: if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f)) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: switch (BYTE_TO_ASCII(enc, ptr)) { case 0x24: /* $ */ diff --git a/contrib/libs/expat/ya.make b/contrib/libs/expat/ya.make index cdd5cc2d3e5..0fd2f760564 100644 --- a/contrib/libs/expat/ya.make +++ b/contrib/libs/expat/ya.make @@ -10,12 +10,13 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(2.8.1) +VERSION(2.8.2) -ORIGINAL_SOURCE(https://github.com/libexpat/libexpat/archive/R_2_8_1.tar.gz) +ORIGINAL_SOURCE(https://github.com/libexpat/libexpat/archive/R_2_8_2.tar.gz) PEERDIR( contrib/libs/libc_compat + library/cpp/sanitizer/include ) ADDINCL( diff --git a/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report b/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report index 93fa9c4becc..f87d3c05433 100644 --- a/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report +++ b/contrib/libs/libtiff/.yandex_meta/devtools.copyrights.report @@ -392,7 +392,7 @@ BELONGS ya.make Match type : COPYRIGHT Files with this license: LICENSE.md [32:33] - tif_lzw.c [1445:1446] + tif_lzw.c [1463:1464] KEEP COPYRIGHT_SERVICE_LABEL ef9e24bc2ee738c92b1a0aae6c46127d BELONGS ya.make diff --git a/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report b/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report index 358bb3da041..a91c6140116 100644 --- a/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report +++ b/contrib/libs/libtiff/.yandex_meta/devtools.licenses.report @@ -61,7 +61,7 @@ BELONGS ya.make Match type : TEXT Links : https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/bsla.LICENSE Files with this license: - tif_lzw.c [1452:1462] + tif_lzw.c [1470:1480] KEEP BSLA 99516a4e1b8d6f6054da2439715a8bda BELONGS ya.make @@ -144,7 +144,6 @@ BELONGS ya.make tif_version.c [5:22] tif_warning.c [5:22] tif_webp.c [5:22] - tif_win32.c [5:22] tif_write.c [5:22] tif_zip.c [5:22] tif_zstd.c [5:22] @@ -164,3 +163,14 @@ BELONGS ya.make Files with this license: LICENSE.md [6:23] README.md [52:69] + +KEEP libtiff eb46110bca2ca592d93347fc2a74357a +BELONGS ya.make + Note: matched license text is too long. Read it in the source files. + Scancode info: + Original SPDX id: libtiff + Score : 93.33 + Match type : TEXT + Links : https://fedoraproject.org/wiki/Licensing/libtiff, https://spdx.org/licenses/libtiff + Files with this license: + tif_win32.c [5:23] diff --git a/contrib/libs/libtiff/.yandex_meta/licenses.list.txt b/contrib/libs/libtiff/.yandex_meta/licenses.list.txt index ca337b4f57f..054ff5c4672 100644 --- a/contrib/libs/libtiff/.yandex_meta/licenses.list.txt +++ b/contrib/libs/libtiff/.yandex_meta/licenses.list.txt @@ -215,6 +215,28 @@ Silicon Graphics has seen fit to allow us to give this work away. It ====================libtiff==================== + * Permission to use, copy, modify, distribute, and sell this software and + * its documentation for any purpose is hereby granted without fee, provided + * that (i) the above copyright notices and this permission notice appear in + * that (i) the above copyright notices and this permission notice appear in + * all copies of the software and related documentation, and (ii) the names of + * Sam Leffler and Silicon Graphics may not be used in any advertising or + * publicity relating to the software without the specific, prior written + * permission of Sam Leffler and Silicon Graphics. + * + * THE SOFTWARE IS PROVIDED "AS-IS" AND WITHOUT WARRANTY OF ANY KIND, + * EXPRESS, IMPLIED OR OTHERWISE, INCLUDING WITHOUT LIMITATION, ANY + * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + * + * IN NO EVENT SHALL SAM LEFFLER OR SILICON GRAPHICS BE LIABLE FOR + * ANY SPECIAL, INCIDENTAL, INDIRECT OR CONSEQUENTIAL DAMAGES OF ANY KIND, + * OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + * WHETHER OR NOT ADVISED OF THE POSSIBILITY OF DAMAGE, AND ON ANY THEORY OF + * LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE + * OF THIS SOFTWARE. + + +====================libtiff==================== Permission to use, copy, modify, distribute, and sell this software and its documentation for any purpose is hereby granted without fee, provided that (i) the above copyright notices and this permission notice appear in diff --git a/contrib/libs/libtiff/.yandex_meta/override.nix b/contrib/libs/libtiff/.yandex_meta/override.nix index 9ce9dd2f515..0a61f01b1fe 100644 --- a/contrib/libs/libtiff/.yandex_meta/override.nix +++ b/contrib/libs/libtiff/.yandex_meta/override.nix @@ -1,11 +1,11 @@ pkgs: attrs: with pkgs; rec { - version = "4.7.1"; + version = "4.7.2"; src = fetchFromGitLab { owner = "libtiff"; repo = "libtiff"; rev = "v${version}"; - hash = "sha256-UiC6s86i7UavW86EKm74oPVlEacvoKmwW7KETjpnNaI="; + hash = "sha256-60Lpg5WRfWMzlOoOUA+C6KLlYIZ+3BjXidOVqv4M2GA="; }; patches = []; diff --git a/contrib/libs/libtiff/README_for_libtiff_developpers.md b/contrib/libs/libtiff/README_for_libtiff_developpers.md new file mode 100644 index 00000000000..6f9b6e4a0bd --- /dev/null +++ b/contrib/libs/libtiff/README_for_libtiff_developpers.md @@ -0,0 +1,149 @@ +README for libtiff developpers +============================== + +## About TIFFGetMaxCompressionRatio() + +That function was introduced in libtiff 4.7.2 per https://gitlab.com/libtiff/libtiff/-/merge_requests/872 + +The estimation of the maximum compression ratio per codec has been performed by +generated a number of test files using [gdal raster create](https://gdal.org/en/stable/programs/gdal_raster_create.html), +with 0-byte content, for different image sizes, and guessing the compression ratio pattern. + +```console +gdal raster create --size 1024,1024 black_1024_packbits.tif --co COMPRESS=PACKBITS --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_rle.tif --co NBITS=1 --co COMPRESS=CCITTRLE --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_fax3.tif --co NBITS=1 --co COMPRESS=CCITTFAX3 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_fax4.tif --co NBITS=1 --co COMPRESS=CCITTFAX4 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_lzw.tif --co COMPRESS=LZW --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_deflate.tif --co COMPRESS=DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_zstd.tif --co COMPRESS=ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_lzma.tif --co COMPRESS=LZMA --co LZMA_PRESET=9 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_lerc.tif --co COMPRESS=LERC --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_lerc_zstd.tif --co COMPRESS=LERC_ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_lerc_deflate.tif --co COMPRESS=LERC_DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=1024 --overwrite +gdal raster create --band-count=3 --size 1024,1024 black_1024_rgb_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=1024 --overwrite +gdal raster create --band-count=3 --size 1024,1024 black_1024_ycbcr_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=1024 --co PHOTOMETRIC=YCBCR --overwrite +gdal raster create --band-count 3 --size 1024,1024 black_1024_rgb_webp.tif --co COMPRESS=WEBP --co WEBP_LEVEL=1 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --size 1024,1024 black_1024_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=1024 --overwrite +gdal raster create --band-count 3 --size 1024,1024 black_1024_rgb_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=1024 --overwrite + +gdal raster create --size 4096,4096 black_4096_packbits.tif --co COMPRESS=PACKBITS --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_rle.tif --co NBITS=1 --co COMPRESS=CCITTRLE --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_fax3.tif --co NBITS=1 --co COMPRESS=CCITTFAX3 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_fax4.tif --co NBITS=1 --co COMPRESS=CCITTFAX4 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_lzw.tif --co COMPRESS=LZW --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_deflate.tif --co COMPRESS=DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_zstd.tif --co COMPRESS=ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_lzma.tif --co COMPRESS=LZMA --co LZMA_PRESET=9 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_lerc.tif --co COMPRESS=LERC --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_lerc_zstd.tif --co COMPRESS=LERC_ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_lerc_deflate.tif --co COMPRESS=LERC_DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=4096 --overwrite +gdal raster create --band-count=3 --size 4096,4096 black_4096_rgb_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=4096 --overwrite +gdal raster create --band-count=3 --size 4096,4096 black_4096_ycbcr_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=4096 --co PHOTOMETRIC=YCBCR --overwrite +gdal raster create --band-count 3 --size 4096,4096 black_4096_rgb_webp.tif --co COMPRESS=WEBP --co WEBP_LEVEL=1 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --size 4096,4096 black_4096_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=4096 --overwrite +gdal raster create --band-count 3 --size 4096,4096 black_4096_rgb_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=4096 --overwrite + +gdal raster create --size 16383,16383 black_16383_packbits.tif --co COMPRESS=PACKBITS --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_rle.tif --co NBITS=1 --co COMPRESS=CCITTRLE --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_fax3.tif --co NBITS=1 --co COMPRESS=CCITTFAX3 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_fax4.tif --co NBITS=1 --co COMPRESS=CCITTFAX4 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_lzw.tif --co COMPRESS=LZW --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_deflate.tif --co COMPRESS=DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_zstd.tif --co COMPRESS=ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_lzma.tif --co COMPRESS=LZMA --co LZMA_PRESET=9 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_lerc.tif --co COMPRESS=LERC --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_lerc_zstd.tif --co COMPRESS=LERC_ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_lerc_deflate.tif --co COMPRESS=LERC_DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=16383 --overwrite +gdal raster create --band-count=3 --size 16383,16383 black_16383_rgb_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=16383 --overwrite +gdal raster create --band-count=3 --size 16383,16383 black_16383_ycbcr_jpg.tif --co JPEG_QUALITY=1 --co COMPRESS=JPEG --co BLOCKYSIZE=16383 --co PHOTOMETRIC=YCBCR --overwrite +gdal raster create --band-count 3 --size 16383,16383 black_16383_rgb_webp.tif --co COMPRESS=WEBP --co WEBP_LEVEL=1 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --band-count 4 --size 16383,16383 black_16383_rgba_webp.tif --co COMPRESS=WEBP --co WEBP_LEVEL=1 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --size 16383,16383 black_16383_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=16383 --overwrite +gdal raster create --band-count 3 --size 16383,16383 black_16383_rgb_jxl.tif --co COMPRESS=JXL --co JXL_LOSSLESS=NO --co JXL_DISTANCE=15 --co JXL_EFFORT=5 --co BLOCKYSIZE=16383 --overwrite + +gdal raster create --size 65536,65536 black_65536_rle.tif --co NBITS=1 --co COMPRESS=CCITTRLE --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_fax3.tif --co NBITS=1 --co COMPRESS=CCITTFAX3 --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_fax4.tif --co NBITS=1 --co COMPRESS=CCITTFAX4 --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_deflate.tif --co COMPRESS=DEFLATE --co ZLEVEL=12 --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_zstd.tif --co COMPRESS=ZSTD --co ZSTD_LEVEL=22 --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_lzma.tif --co COMPRESS=LZMA --co LZMA_PRESET=9 --co BLOCKYSIZE=65536 --overwrite +gdal raster create --size 65536,65536 black_65536_lzw.tif --co COMPRESS=LZW --co BLOCKYSIZE=65536 --overwrite +``` + +and then extracted the compression ratios with: + +```console +for i in black*.tif; do bytecounts=$(tiffdump $i | grep StripByteCounts | sed "s/StripByteCounts (279) LONG (4) 1<//" | sed "s/>//"); python3 -c "import math; uncompressed_size = 1024 * 1024 if '1024' in '$i' else 4096 * 4096 if '4096' in '$i' else 65536 * 65536 if '65536' in '$i' else 200000 * 200000 if '200000' in '$i' else 16363 * 16383; uncompressed_size = uncompressed_size * 4 if 'rgba' in '$i' else uncompressed_size * 3 if 'rgb' in '$i' or 'ycbcr' in '$i' else uncompressed_size; uncompressed_size = uncompressed_size / 8 if 'rle' in '$i' or 'fax' in '$i' else uncompressed_size; print('$i', math.ceil(uncompressed_size / ${bytecounts}))"; done +``` + +which outputs + +```console +black_1024_deflate.tif 975 +black_1024_fax3.tif 36 +black_1024_fax4.tif 1001 +black_1024_jpg.tif 252 +black_1024_jxl.tif 5323 +black_1024_lerc_deflate.tif 23832 +black_1024_lerc.tif 14980 +black_1024_lerc_zstd.tif 18397 +black_1024_lzma.tif 3800 +black_1024_lzw.tif 562 +black_1024_packbits.tif 64 +black_1024_rgb_jpg.tif 255 +black_1024_rgb_jxl.tif 16216 +black_1024_rgb_webp.tif 1634 +black_1024_rle.tif 43 +black_1024_ycbcr_jpg.tif 502 +black_1024_zstd.tif 21400 +black_16383_deflate.tif 989 +black_16383_fax3.tif 163 +black_16383_fax4.tif 16339 +black_16383_jpg.tif 256 +black_16383_jxl.tif 45360 +black_16383_lerc_deflate.tif 5703725 +black_16383_lerc.tif 3829644 +black_16383_lerc_zstd.tif 4394673 +black_16383_lzma.tif 6846 +black_16383_lzw.tif 1353 +black_16383_packbits.tif 64 +black_16383_rgba_webp_lossless.tif 104067 +black_16383_rgba_webp.tif 2199 +black_16383_rgb_jpg.tif 256 +black_16383_rgb_jxl.tif 136148 +black_16383_rgb_webp_lossless.tif 78050 +black_16383_rgb_webp.tif 1685 +black_16383_rle.tif 171 +black_16383_ycbcr_jpg.tif 512 +black_16383_zstd.tif 32669 +black_4096_deflate.tif 989 +black_4096_fax3.tif 100 +black_4096_fax4.tif 4073 +black_4096_jpg.tif 256 +black_4096_jxl_rgba.tif 90934 +black_4096_jxl.tif 25079 +black_4096_lerc_deflate.tif 381301 +black_4096_lerc.tif 239675 +black_4096_lerc_zstd.tif 294338 +black_4096_lzma.tif 6534 +black_4096_lzw.tif 1243 +black_4096_packbits.tif 64 +black_4096_rgb_jpg.tif 256 +black_4096_rgb_jxl.tif 75574 +black_4096_rgb_webp.tif 1684 +black_4096_rle.tif 128 +black_4096_ycbcr_jpg.tif 512 +black_4096_zstd.tif 31715 +black_65536_deflate.tif 990 +black_65536_fax3.tif 200 +black_65536_fax4.tif 65513 +black_65536_lzma.tif 6874 +black_65536_lzw.tif 1362 +black_65536_rle.tif 205 +black_65536_zstd.tif 32764 +``` + diff --git a/contrib/libs/libtiff/RELEASE-DATE b/contrib/libs/libtiff/RELEASE-DATE index cc3bc45669d..4fff66f6b5d 100644 --- a/contrib/libs/libtiff/RELEASE-DATE +++ b/contrib/libs/libtiff/RELEASE-DATE @@ -1 +1 @@ -20250911 +20260627 diff --git a/contrib/libs/libtiff/libtiff.map b/contrib/libs/libtiff/libtiff.map index 45641920874..b8a37705eb6 100644 --- a/contrib/libs/libtiff/libtiff.map +++ b/contrib/libs/libtiff/libtiff.map @@ -222,3 +222,15 @@ LIBTIFF_4.6.1 { LIBTIFF_4.7.1 { TIFFOpenOptionsSetWarnAboutUnknownTags; } LIBTIFF_4.6.1; + +LIBTIFF_4.7.2 { + _TIFFAdd64; + _TIFFAddSSize; + _TIFFCastUInt64ToSSize; + _TIFFCastUInt64ToUInt32; + _TIFFComputeBitOffset; + _TIFFComputeRowOffset; + _TIFFComputeRowSize64; + _TIFFMultiplySSize; + TIFFGetMaxCompressionRatio; +} LIBTIFF_4.7.1; diff --git a/contrib/libs/libtiff/tif_aux.c b/contrib/libs/libtiff/tif_aux.c index 31b240070c6..1b2db768670 100644 --- a/contrib/libs/libtiff/tif_aux.c +++ b/contrib/libs/libtiff/tif_aux.c @@ -56,6 +56,21 @@ uint64_t _TIFFMultiply64(TIFF *tif, uint64_t first, uint64_t second, return first * second; } +uint64_t _TIFFAdd64(TIFF *tif, uint64_t first, uint64_t second, + const char *where) +{ + if (first > UINT64_MAX - second) + { + if (tif != NULL && where != NULL) + { + TIFFErrorExtR(tif, where, "Integer overflow in %s", where); + } + return 0; + } + + return first + second; +} + tmsize_t _TIFFMultiplySSize(TIFF *tif, tmsize_t first, tmsize_t second, const char *where) { @@ -81,6 +96,30 @@ tmsize_t _TIFFMultiplySSize(TIFF *tif, tmsize_t first, tmsize_t second, return first * second; } +tmsize_t _TIFFAddSSize(TIFF *tif, tmsize_t first, tmsize_t second, + const char *where) +{ + if (first < 0 || second < 0) + { + if (tif != NULL && where != NULL) + { + TIFFErrorExtR(tif, where, + "Invalid argument to _TIFFAddSSize() in %s", where); + } + return 0; + } + + if (first > TIFF_TMSIZE_T_MAX - second) + { + if (tif != NULL && where != NULL) + { + TIFFErrorExtR(tif, where, "Integer overflow in %s", where); + } + return 0; + } + return first + second; +} + tmsize_t _TIFFCastUInt64ToSSize(TIFF *tif, uint64_t val, const char *module) { if (val > (uint64_t)TIFF_TMSIZE_T_MAX) @@ -94,6 +133,55 @@ tmsize_t _TIFFCastUInt64ToSSize(TIFF *tif, uint64_t val, const char *module) return (tmsize_t)val; } +uint32_t _TIFFCastUInt64ToUInt32(TIFF *tif, uint64_t val, const char *module) +{ + if (val > UINT32_MAX) + { + if (tif != NULL && module != NULL) + { + TIFFErrorExtR(tif, module, "Integer overflow"); + } + return 0; + } + return (uint32_t)val; +} + +tmsize_t _TIFFComputeRowOffset(TIFF *tif, tmsize_t rowstride, uint32_t row, + const char *where) +{ + if (row == 0) + return 0; + return _TIFFMultiplySSize(tif, rowstride, (tmsize_t)row, where); +} + +uint64_t _TIFFComputeBitOffset(TIFF *tif, uint32_t col, uint16_t spp, + uint16_t bps, const char *where) +{ + uint64_t samples = _TIFFMultiply64(tif, col, spp, where); + if (samples == 0 && col != 0) + return 0; + return _TIFFMultiply64(tif, samples, bps, where); +} + +/* + * Returns 0 on overflow or invalid zero-sized row inputs. Callers that + * intentionally allow empty rows should not use this helper directly. + */ +uint64_t _TIFFComputeRowSize64(TIFF *tif, uint32_t width, uint16_t spp, + uint16_t bps, const char *where) +{ + uint64_t samples = _TIFFMultiply64(tif, width, spp, where); + uint64_t bits; + if (samples == 0) + return 0; + + bits = _TIFFMultiply64(tif, samples, bps, where); + if (bits == 0) + return 0; + + return TIFFhowmany8_64(bits); +} + void *_TIFFCheckRealloc(TIFF *tif, void *buffer, tmsize_t nmemb, tmsize_t elem_size, const char *what) { @@ -137,8 +225,8 @@ static int TIFFDefaultTransferFunction(TIFF *tif, TIFFDirectory *td) if (td->td_bitspersample > 24) return 0; - n = ((tmsize_t)1) << td->td_bitspersample; - nbytes = n * sizeof(uint16_t); + n = (tmsize_t)(1ULL << td->td_bitspersample); + nbytes = (tmsize_t)((uint64_t)n * sizeof(uint16_t)); tf[0] = (uint16_t *)_TIFFmallocExt(tif, nbytes); if (tf[0] == NULL) return 0; @@ -186,10 +274,10 @@ static int TIFFDefaultRefBlackWhite(TIFF *tif, TIFFDirectory *td) * YCbCr (Class Y) images must have the ReferenceBlackWhite * tag set. Fix the broken images, which lacks that tag. */ - td->td_refblackwhite[0] = 0.0F; + td->td_refblackwhite[0] = 0.0f; td->td_refblackwhite[1] = td->td_refblackwhite[3] = - td->td_refblackwhite[5] = 255.0F; - td->td_refblackwhite[2] = td->td_refblackwhite[4] = 128.0F; + td->td_refblackwhite[5] = 255.0f; + td->td_refblackwhite[2] = td->td_refblackwhite[4] = 128.0f; } else { @@ -199,8 +287,11 @@ static int TIFFDefaultRefBlackWhite(TIFF *tif, TIFFDirectory *td) for (i = 0; i < 3; i++) { td->td_refblackwhite[2 * i + 0] = 0; - td->td_refblackwhite[2 * i + 1] = - (float)((1L << td->td_bitspersample) - 1L); + if (td->td_bitspersample < 64) + td->td_refblackwhite[2 * i + 1] = + (float)((1ULL << td->td_bitspersample) - 1ULL); + else + td->td_refblackwhite[2 * i + 1] = (float)UINT64_MAX; } } return 1; @@ -258,8 +349,8 @@ int TIFFVGetFieldDefaulted(TIFF *tif, uint32_t tag, va_list ap) * 65535 even if td_bitspersamle is > 16 */ if (td->td_bitspersample <= 16) { - maxsamplevalue = (1 << td->td_bitspersample) - - 1; /* 2**(BitsPerSample) - 1 */ + maxsamplevalue = (uint16_t)((1U << td->td_bitspersample) - + 1); /* 2**(BitsPerSample) - 1 */ } else { @@ -295,7 +386,11 @@ int TIFFVGetFieldDefaulted(TIFF *tif, uint32_t tag, va_list ap) } case TIFFTAG_DOTRANGE: *va_arg(ap, uint16_t *) = 0; - *va_arg(ap, uint16_t *) = (1 << td->td_bitspersample) - 1; + if (td->td_bitspersample <= 16) + *va_arg(ap, uint16_t *) = + (uint16_t)((1U << td->td_bitspersample) - 1); + else + *va_arg(ap, uint16_t *) = 65535; return (1); case TIFFTAG_INKSET: *va_arg(ap, uint16_t *) = INKSET_CMYK; @@ -309,14 +404,14 @@ int TIFFVGetFieldDefaulted(TIFF *tif, uint32_t tag, va_list ap) return (1); case TIFFTAG_MATTEING: *va_arg(ap, uint16_t *) = - (td->td_extrasamples == 1 && + (td->td_extrasamples == 1 && td->td_sampleinfo && td->td_sampleinfo[0] == EXTRASAMPLE_ASSOCALPHA); return (1); case TIFFTAG_TILEDEPTH: *va_arg(ap, uint32_t *) = td->td_tiledepth; return (1); case TIFFTAG_DATATYPE: - *va_arg(ap, uint16_t *) = td->td_sampleformat - 1; + *va_arg(ap, uint16_t *) = (uint16_t)(td->td_sampleformat - 1); return (1); case TIFFTAG_SAMPLEFORMAT: *va_arg(ap, uint16_t *) = td->td_sampleformat; @@ -369,6 +464,8 @@ int TIFFVGetFieldDefaulted(TIFF *tif, uint32_t tag, va_list ap) return (0); *va_arg(ap, const float **) = td->td_refblackwhite; return (1); + default: + break; } return 0; } @@ -390,9 +487,9 @@ int TIFFGetFieldDefaulted(TIFF *tif, uint32_t tag, ...) float _TIFFClampDoubleToFloat(double val) { - if (val > FLT_MAX) + if (val > (double)FLT_MAX) return FLT_MAX; - if (val < -FLT_MAX) + if (val < -(double)FLT_MAX) return -FLT_MAX; return (float)val; } @@ -401,7 +498,7 @@ uint32_t _TIFFClampDoubleToUInt32(double val) { if (val < 0) return 0; - if (val > 0xFFFFFFFFU || val != val) + if (val > 0xFFFFFFFFU || isnan(val)) return 0xFFFFFFFFU; return (uint32_t)val; } diff --git a/contrib/libs/libtiff/tif_close.c b/contrib/libs/libtiff/tif_close.c index d498010740c..e60aba7113d 100644 --- a/contrib/libs/libtiff/tif_close.c +++ b/contrib/libs/libtiff/tif_close.c @@ -88,7 +88,7 @@ void TIFFCleanup(TIFF *tif) */ TIFFFieldIsAnonymous(fld)) { - _TIFFfreeExt(tif, fld->field_name); + _TIFFfreeExt(tif, (void *)fld->field_name); _TIFFfreeExt(tif, fld); } } @@ -121,7 +121,7 @@ void TIFFCleanup(TIFF *tif) } /************************************************************************/ -/* _TIFFCleanupIFDOffsetAndNumberMaps() */ +/* _TIFFCleanupIFDOffsetAndNumberMaps() */ /************************************************************************/ void _TIFFCleanupIFDOffsetAndNumberMaps(TIFF *tif) @@ -139,7 +139,7 @@ void _TIFFCleanupIFDOffsetAndNumberMaps(TIFF *tif) } /************************************************************************/ -/* TIFFClose() */ +/* TIFFClose() */ /************************************************************************/ /** diff --git a/contrib/libs/libtiff/tif_color.c b/contrib/libs/libtiff/tif_color.c index a52fdacba59..4a17914a44c 100644 --- a/contrib/libs/libtiff/tif_color.c +++ b/contrib/libs/libtiff/tif_color.c @@ -55,34 +55,34 @@ void TIFFCIELabToXYZ(TIFFCIELabToRGB *cielab, uint32_t l, int32_t a, int32_t b, void TIFFCIELab16ToXYZ(TIFFCIELabToRGB *cielab, uint32_t l, int32_t a, int32_t b, float *X, float *Y, float *Z) { - float L = (float)l * 100.0F / 65535.0F; + float L = (float)l * 100.0f / 65535.0f; float cby, tmp; - if (L < 8.856F) + if (L < 8.856f) { - *Y = (L * cielab->Y0) / 903.292F; - cby = 7.787F * (*Y / cielab->Y0) + 16.0F / 116.0F; + *Y = (L * cielab->Y0) / 903.292f; + cby = 7.787f * (*Y / cielab->Y0) + 16.0f / 116.0f; } else { - cby = (L + 16.0F) / 116.0F; + cby = (L + 16.0f) / 116.0f; *Y = cielab->Y0 * cby * cby * cby; } - tmp = (float)a / 256.0F / 500.0F + cby; - if (tmp < 0.2069F) - *X = cielab->X0 * (tmp - 0.13793F) / 7.787F; + tmp = (float)a / 256.0f / 500.0f + cby; + if (tmp < 0.2069f) + *X = cielab->X0 * (tmp - 0.13793f) / 7.787f; else *X = cielab->X0 * tmp * tmp * tmp; - tmp = cby - (float)b / 256.0F / 200.0F; - if (tmp < 0.2069F) - *Z = cielab->Z0 * (tmp - 0.13793F) / 7.787F; + tmp = cby - (float)b / 256.0f / 200.0f; + if (tmp < 0.2069f) + *Z = cielab->Z0 * (tmp - 0.13793f) / 7.787f; else *Z = cielab->Z0 * tmp * tmp * tmp; } -#define RINT(R) ((uint32_t)((R) > 0 ? ((R) + 0.5) : ((R)-0.5))) +#define RINT(R) ((uint32_t)((R) > 0 ? ((R) + 0.5f) : ((R) - 0.5f))) /* * Convert color value from the XYZ space to RGB. */ @@ -143,32 +143,32 @@ int TIFFCIELabToRGBInit(TIFFCIELabToRGB *cielab, const TIFFDisplay *display, _TIFFmemcpy(&cielab->display, display, sizeof(TIFFDisplay)); /* Red */ - dfGamma = 1.0 / cielab->display.d_gammaR; + dfGamma = 1.0 / (double)cielab->display.d_gammaR; cielab->rstep = - (cielab->display.d_YCR - cielab->display.d_Y0R) / cielab->range; + (cielab->display.d_YCR - cielab->display.d_Y0R) / (float)cielab->range; for (i = 0; i <= (size_t)cielab->range; i++) { - cielab->Yr2r[i] = cielab->display.d_Vrwr * + cielab->Yr2r[i] = (float)cielab->display.d_Vrwr * ((float)pow((double)i / cielab->range, dfGamma)); } /* Green */ - dfGamma = 1.0 / cielab->display.d_gammaG; + dfGamma = 1.0 / (double)cielab->display.d_gammaG; cielab->gstep = - (cielab->display.d_YCR - cielab->display.d_Y0R) / cielab->range; + (cielab->display.d_YCR - cielab->display.d_Y0R) / (float)cielab->range; for (i = 0; i <= (size_t)cielab->range; i++) { - cielab->Yg2g[i] = cielab->display.d_Vrwg * + cielab->Yg2g[i] = (float)cielab->display.d_Vrwg * ((float)pow((double)i / cielab->range, dfGamma)); } /* Blue */ - dfGamma = 1.0 / cielab->display.d_gammaB; + dfGamma = 1.0 / (double)cielab->display.d_gammaB; cielab->bstep = - (cielab->display.d_YCR - cielab->display.d_Y0R) / cielab->range; + (cielab->display.d_YCR - cielab->display.d_Y0R) / (float)cielab->range; for (i = 0; i <= (size_t)cielab->range; i++) { - cielab->Yb2b[i] = cielab->display.d_Vrwb * + cielab->Yb2b[i] = (float)cielab->display.d_Vrwb * ((float)pow((double)i / cielab->range, dfGamma)); } @@ -186,11 +186,11 @@ int TIFFCIELabToRGBInit(TIFFCIELabToRGB *cielab, const TIFFDisplay *display, * see below for more information on how it works. */ #define SHIFT 16 -#define FIX(x) ((int32_t)((x) * (1L << SHIFT) + 0.5)) +#define FIX(x) ((int32_t)((double)(x) * (1L << SHIFT) + 0.5)) #define ONE_HALF ((int32_t)(1 << (SHIFT - 1))) #define Code2V(c, RB, RW, CR) \ - ((((c) - (int32_t)(RB)) * (float)(CR)) / \ - (float)(((RW) - (RB) != 0) ? ((RW) - (RB)) : 1)) + (((float)((c) - (int32_t)(RB)) * (float)(CR)) / \ + ((!TIFF_FLOAT_EQ((RW), (RB))) ? ((RW) - (RB)) : 1.0f)) /* !((f)>=(min)) written that way to deal with NaN */ #define CLAMP(f, min, max) \ ((!((f) >= (min))) ? (min) : (f) > (max) ? (max) : (f)) @@ -207,12 +207,12 @@ void TIFFYCbCrtoRGB(TIFFYCbCrToRGB *ycbcr, uint32_t Y, int32_t Cb, int32_t Cr, Cr = CLAMP(Cr, 0, 255); i = ycbcr->Y_tab[Y] + ycbcr->Cr_r_tab[Cr]; - *r = CLAMP(i, 0, 255); + *r = (uint32_t)CLAMP(i, 0, 255); i = ycbcr->Y_tab[Y] + - (int)((ycbcr->Cb_g_tab[Cb] + ycbcr->Cr_g_tab[Cr]) >> SHIFT); - *g = CLAMP(i, 0, 255); + ((ycbcr->Cb_g_tab[Cb] + ycbcr->Cr_g_tab[Cr]) >> SHIFT); + *g = (uint32_t)CLAMP(i, 0, 255); i = ycbcr->Y_tab[Y] + ycbcr->Cb_b_tab[Cb]; - *b = CLAMP(i, 0, 255); + *b = (uint32_t)CLAMP(i, 0, 255); } /* Clamp function for sanitization purposes. Normally clamping should not */ @@ -258,8 +258,7 @@ int TIFFYCbCrToRGBInit(TIFFYCbCrToRGB *ycbcr, float *luma, float *refBlackWhite) #define LumaBlue luma[2] clamptab = - (TIFFRGBValue *)((uint8_t *)ycbcr + - TIFFroundup_32(sizeof(TIFFYCbCrToRGB), sizeof(long))); + (uint8_t *)ycbcr + TIFFroundup_32(sizeof(TIFFYCbCrToRGB), sizeof(long)); _TIFFmemset(clamptab, 0, 256); /* v < 0 => 0 */ ycbcr->clamptab = (clamptab += 256); for (i = 0; i < 256; i++) @@ -273,13 +272,13 @@ int TIFFYCbCrToRGBInit(TIFFYCbCrToRGB *ycbcr, float *luma, float *refBlackWhite) { float f1 = 2 - 2 * LumaRed; - int32_t D1 = FIX(CLAMP(f1, 0.0F, 2.0F)); + int32_t D1 = FIX(CLAMP(f1, 0.0f, 2.0f)); float f2 = LumaRed * f1 / LumaGreen; - int32_t D2 = -FIX(CLAMP(f2, 0.0F, 2.0F)); + int32_t D2 = -FIX(CLAMP(f2, 0.0f, 2.0f)); float f3 = 2 - 2 * LumaBlue; - int32_t D3 = FIX(CLAMP(f3, 0.0F, 2.0F)); + int32_t D3 = FIX(CLAMP(f3, 0.0f, 2.0f)); float f4 = LumaBlue * f3 / LumaGreen; - int32_t D4 = -FIX(CLAMP(f4, 0.0F, 2.0F)); + int32_t D4 = -FIX(CLAMP(f4, 0.0f, 2.0f)); int x; #undef LumaBlue @@ -295,12 +294,12 @@ int TIFFYCbCrToRGBInit(TIFFYCbCrToRGB *ycbcr, float *luma, float *refBlackWhite) */ for (i = 0, x = -128; i < 256; i++, x++) { - int32_t Cr = (int32_t)CLAMPw(Code2V(x, refBlackWhite[4] - 128.0F, - refBlackWhite[5] - 128.0F, 127), - -128.0F * 32, 128.0F * 32); - int32_t Cb = (int32_t)CLAMPw(Code2V(x, refBlackWhite[2] - 128.0F, - refBlackWhite[3] - 128.0F, 127), - -128.0F * 32, 128.0F * 32); + int32_t Cr = (int32_t)CLAMPw(Code2V(x, refBlackWhite[4] - 128.0f, + refBlackWhite[5] - 128.0f, 127), + -128.0f * 32, 128.0f * 32); + int32_t Cb = (int32_t)CLAMPw(Code2V(x, refBlackWhite[2] - 128.0f, + refBlackWhite[3] - 128.0f, 127), + -128.0f * 32, 128.0f * 32); ycbcr->Cr_r_tab[i] = (int32_t)((D1 * Cr + ONE_HALF) >> SHIFT); ycbcr->Cb_b_tab[i] = (int32_t)((D3 * Cb + ONE_HALF) >> SHIFT); @@ -308,7 +307,7 @@ int TIFFYCbCrToRGBInit(TIFFYCbCrToRGB *ycbcr, float *luma, float *refBlackWhite) ycbcr->Cb_g_tab[i] = D4 * Cb + ONE_HALF; ycbcr->Y_tab[i] = (int32_t)CLAMPw( Code2V(x + 128, refBlackWhite[0], refBlackWhite[1], 255), - -128.0F * 32, 128.0F * 32); + -128.0f * 32, 128.0f * 32); } } diff --git a/contrib/libs/libtiff/tif_compress.c b/contrib/libs/libtiff/tif_compress.c index c6e17d3e114..3002083f12e 100644 --- a/contrib/libs/libtiff/tif_compress.c +++ b/contrib/libs/libtiff/tif_compress.c @@ -139,6 +139,18 @@ static int _TIFFtrue(TIFF *tif) } static void _TIFFvoid(TIFF *tif) { (void)tif; } +static uint64_t _TIFFDefaultGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + return 0; /* unknown */ +} + +static uint64_t _TIFFGetMaxCompressionRatioOne(TIFF *tif) +{ + (void)tif; + return 1; /* no compression */ +} + void _TIFFSetDefaultCompressionState(TIFF *tif) { tif->tif_fixuptags = _TIFFNoFixupTags; @@ -160,6 +172,7 @@ void _TIFFSetDefaultCompressionState(TIFF *tif) tif->tif_cleanup = _TIFFvoid; tif->tif_defstripsize = _TIFFDefaultStripSize; tif->tif_deftilesize = _TIFFDefaultTileSize; + tif->tif_getmaxcompressionratio = _TIFFDefaultGetMaxCompressionRatio; tif->tif_flags &= ~(TIFF_NOBITREV | TIFF_NOREADRAW); } @@ -168,6 +181,8 @@ int TIFFSetCompressionScheme(TIFF *tif, int scheme) const TIFFCodec *c = TIFFFindCODEC((uint16_t)scheme); _TIFFSetDefaultCompressionState(tif); + if (scheme == COMPRESSION_NONE) + tif->tif_getmaxcompressionratio = _TIFFGetMaxCompressionRatioOne; /* * Don't treat an unknown compression scheme as an error. * This permits applications to open files with data that @@ -177,6 +192,13 @@ int TIFFSetCompressionScheme(TIFF *tif, int scheme) return (c ? (*c->init)(tif, scheme) : 1); } +uint64_t TIFFGetMaxCompressionRatio(TIFF *tif) +{ + if (tif->tif_getmaxcompressionratio) + return tif->tif_getmaxcompressionratio(tif); + return 0; +} + /* * Other compression schemes may be registered. Registered * schemes can also override the builtin versions provided @@ -200,7 +222,7 @@ const TIFFCodec *TIFFFindCODEC(uint16_t scheme) for (c = _TIFFBuiltinCODECS; c->name; c++) if (c->scheme == scheme) return (c); - return ((const TIFFCodec *)0); + return NULL; } TIFFCodec *TIFFRegisterCODEC(uint16_t scheme, const char *name, @@ -212,9 +234,11 @@ TIFFCodec *TIFFRegisterCODEC(uint16_t scheme, const char *name, if (cd != NULL) { + char *codec_name; cd->info = (TIFFCodec *)((uint8_t *)cd + sizeof(codec_t)); - cd->info->name = (char *)((uint8_t *)cd->info + sizeof(TIFFCodec)); - strcpy(cd->info->name, name); + codec_name = (char *)((uint8_t *)cd->info + sizeof(TIFFCodec)); + strcpy(codec_name, name); + cd->info->name = codec_name; cd->info->scheme = scheme; cd->info->init = init; cd->next = registeredCODECS; @@ -247,7 +271,7 @@ void TIFFUnRegisterCODEC(TIFFCodec *c) } /************************************************************************/ -/* TIFFGetConfisuredCODECs() */ +/* TIFFGetConfiguredCODECs() */ /************************************************************************/ /** @@ -258,7 +282,7 @@ void TIFFUnRegisterCODEC(TIFFCodec *c) * or NULL if function failed. */ -TIFFCodec *TIFFGetConfiguredCODECs() +TIFFCodec *TIFFGetConfiguredCODECs(void) { int i = 1; codec_t *cd; @@ -268,8 +292,8 @@ TIFFCodec *TIFFGetConfiguredCODECs() for (cd = registeredCODECS; cd; cd = cd->next) { - new_codecs = - (TIFFCodec *)_TIFFreallocExt(NULL, codecs, i * sizeof(TIFFCodec)); + new_codecs = (TIFFCodec *)_TIFFreallocExt( + NULL, codecs, (tmsize_t)((size_t)i * sizeof(TIFFCodec))); if (!new_codecs) { _TIFFfreeExt(NULL, codecs); @@ -283,21 +307,21 @@ TIFFCodec *TIFFGetConfiguredCODECs() { if (TIFFIsCODECConfigured(c->scheme)) { - new_codecs = (TIFFCodec *)_TIFFreallocExt(NULL, codecs, - i * sizeof(TIFFCodec)); + new_codecs = (TIFFCodec *)_TIFFreallocExt( + NULL, codecs, (tmsize_t)((size_t)i * sizeof(TIFFCodec))); if (!new_codecs) { _TIFFfreeExt(NULL, codecs); return NULL; } codecs = new_codecs; - _TIFFmemcpy(codecs + i - 1, (const void *)c, sizeof(TIFFCodec)); + _TIFFmemcpy(codecs + i - 1, c, sizeof(TIFFCodec)); i++; } } - new_codecs = - (TIFFCodec *)_TIFFreallocExt(NULL, codecs, i * sizeof(TIFFCodec)); + new_codecs = (TIFFCodec *)_TIFFreallocExt( + NULL, codecs, (tmsize_t)((size_t)i * sizeof(TIFFCodec))); if (!new_codecs) { _TIFFfreeExt(NULL, codecs); diff --git a/contrib/libs/libtiff/tif_config.h b/contrib/libs/libtiff/tif_config.h index a4c90a65fbf..693d661d991 100644 --- a/contrib/libs/libtiff/tif_config.h +++ b/contrib/libs/libtiff/tif_config.h @@ -133,15 +133,7 @@ /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel). */ -#if defined AC_APPLE_UNIVERSAL_BUILD -# if defined __BIG_ENDIAN__ -# define WORDS_BIGENDIAN 1 -# endif -#else -# ifndef WORDS_BIGENDIAN -/* #undef WORDS_BIGENDIAN */ -# endif -#endif +#define WORDS_BIGENDIAN 0 #if !defined(__MINGW32__) # define TIFF_SIZE_FORMAT "zu" diff --git a/contrib/libs/libtiff/tif_dir.c b/contrib/libs/libtiff/tif_dir.c index 7421c738db2..999a2452f33 100644 --- a/contrib/libs/libtiff/tif_dir.c +++ b/contrib/libs/libtiff/tif_dir.c @@ -31,6 +31,7 @@ #include "tiffiop.h" #include <float.h> /*--: for Rational2Double */ #include <limits.h> +#include <math.h> /* * These are used in the backwards compatibility code... @@ -50,7 +51,8 @@ static void setByteArray(TIFF *tif, void **vpp, const void *vp, size_t nmemb, } if (vp) { - tmsize_t bytes = _TIFFMultiplySSize(NULL, nmemb, elem_size, NULL); + tmsize_t bytes = _TIFFMultiplySSize(NULL, (tmsize_t)nmemb, + (tmsize_t)elem_size, NULL); if (bytes) *vpp = (void *)_TIFFmallocExt(tif, bytes); if (*vpp) @@ -121,7 +123,8 @@ static void setDoubleArrayOneValue(TIFF *tif, double **vpp, double value, { if (*vpp) _TIFFfreeExt(tif, *vpp); - *vpp = _TIFFmallocExt(tif, nmemb * sizeof(double)); + *vpp = (double *)_TIFFmallocExt(tif, + (tmsize_t)nmemb * (tmsize_t)sizeof(double)); if (*vpp) { while (nmemb--) @@ -294,7 +297,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) /* * Setup new compression routine state. */ - if ((status = TIFFSetCompressionScheme(tif, v)) != 0) + if ((status = TIFFSetCompressionScheme(tif, (int)v)) != 0) td->td_compression = (uint16_t)v; else status = 0; @@ -402,13 +405,13 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) break; case TIFFTAG_XRESOLUTION: dblval = va_arg(ap, double); - if (dblval != dblval || dblval < 0) + if (isnan(dblval) || dblval < 0) goto badvaluedouble; td->td_xresolution = _TIFFClampDoubleToFloat(dblval); break; case TIFFTAG_YRESOLUTION: dblval = va_arg(ap, double); - if (dblval != dblval || dblval < 0) + if (isnan(dblval) || dblval < 0) goto badvaluedouble; td->td_yresolution = _TIFFClampDoubleToFloat(dblval); break; @@ -439,7 +442,12 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) td->td_halftonehints[1] = (uint16_t)va_arg(ap, uint16_vap); break; case TIFFTAG_COLORMAP: - v32 = (uint32_t)(1L << td->td_bitspersample); + if (td->td_bitspersample >= 32) + { + v = td->td_bitspersample; + goto badvalue; + } + v32 = 1U << td->td_bitspersample; _TIFFsetShortArrayExt(tif, &td->td_colormap[0], va_arg(ap, uint16_t *), v32); _TIFFsetShortArrayExt(tif, &td->td_colormap[1], @@ -557,11 +565,17 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) case TIFFTAG_TRANSFERFUNCTION: { uint32_t i; + uint32_t count; + if (td->td_bitspersample >= 32) + { + v = td->td_bitspersample; + goto badvalue; + } + count = 1U << td->td_bitspersample; v = (td->td_samplesperpixel - td->td_extrasamples) > 1 ? 3 : 1; for (i = 0; i < v; i++) _TIFFsetShortArrayExt(tif, &td->td_transferfunction[i], - va_arg(ap, uint16_t *), - 1U << td->td_bitspersample); + va_arg(ap, uint16_t *), count); break; } case TIFFTAG_REFERENCEBLACKWHITE: @@ -579,7 +593,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) if (ninksinstring > 0) { _TIFFsetNString(tif, &td->td_inknames, s, v); - td->td_inknameslen = v; + td->td_inknameslen = (int)v; /* Set NumberOfInks to the value ninksinstring */ if (TIFFFieldSet(tif, FIELD_NUMBEROFINKS)) { @@ -725,7 +739,8 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) new_customValues = (TIFFTagValue *)_TIFFreallocExt( tif, td->td_customValues, - sizeof(TIFFTagValue) * (td->td_customValueCount + 1)); + (tmsize_t)(sizeof(TIFFTagValue) * + (size_t)(td->td_customValueCount + 1))); if (!new_customValues) { TIFFErrorExtR(tif, module, @@ -754,7 +769,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) if (tv_size == 0) { status = 0; - TIFFErrorExtR(tif, module, "%s: Bad field type %d for \"%s\"", + TIFFErrorExtR(tif, module, "%s: Bad field type %u for \"%s\"", tif->tif_name, fip->field_type, fip->field_name); goto end; } @@ -784,7 +799,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) } ma = (uint32_t)len; } - tv->count = ma; + tv->count = (int)ma; setByteArray(tif, &tv->value, mb, ma, 1); } else @@ -792,9 +807,9 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) if (fip->field_passcount) { if (fip->field_writecount == TIFF_VARIABLE2) - tv->count = (uint32_t)va_arg(ap, uint32_t); + tv->count = (int)va_arg(ap, uint32_t); else - tv->count = (int)va_arg(ap, int); + tv->count = va_arg(ap, int); } else if (fip->field_writecount == TIFF_VARIABLE || fip->field_writecount == TIFF_VARIABLE2) @@ -808,7 +823,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) { TIFFWarningExtR(tif, module, "%s: Null count for \"%s\" (type " - "%d, writecount %d, passcount %d)", + "%u, writecount %d, passcount %d)", tif->tif_name, fip->field_name, fip->field_type, fip->field_writecount, fip->field_passcount); @@ -845,11 +860,12 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) * 4 or 8 according to fip->set_get_field_type! */ _TIFFmemcpy(tv->value, va_arg(ap, void *), tv->count * tv_size); - /* Test here for too big values for LONG8, SLONG8 in + /* Test here for too big values for LONG8, IFD8, SLONG8 in * ClassicTIFF and delete custom field from custom list */ if (!(tif->tif_flags & TIFF_BIGTIFF)) { - if (tv->info->field_type == TIFF_LONG8) + if (tv->info->field_type == TIFF_LONG8 || + tv->info->field_type == TIFF_IFD8) { uint64_t *pui64 = (uint64_t *)tv->value; for (int i = 0; i < tv->count; i++) @@ -858,12 +874,15 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) { TIFFErrorExtR( tif, module, - "%s: Bad LONG8 value %" PRIu64 + "%s: Bad %s value %" PRIu64 " at %d. array position for \"%s\" tag " - "%d in ClassicTIFF. Tag won't be " + "%u in ClassicTIFF. Tag won't be " "written to file", - tif->tif_name, pui64[i], i, - fip->field_name, tag); + tif->tif_name, + (tv->info->field_type == TIFF_LONG8 + ? "LONG8" + : "IFD8"), + pui64[i], i, fip->field_name, tag); goto badvalueifd8long8; } } @@ -880,7 +899,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) tif, module, "%s: Bad SLONG8 value %" PRIi64 " at %d. array position for \"%s\" tag " - "%d in ClassicTIFF. Tag won't be " + "%u in ClassicTIFF. Tag won't be " "written to file", tif->tif_name, pi64[i], i, fip->field_name, tag); @@ -948,7 +967,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) TIFFErrorExtR( tif, module, "%s: Bad LONG8 or IFD8 value %" PRIu64 - " for \"%s\" tag %d in ClassicTIFF. Tag " + " for \"%s\" tag %u in ClassicTIFF. Tag " "won't be written to file", tif->tif_name, v2, fip->field_name, tag); goto badvalueifd8long8; @@ -967,7 +986,7 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) TIFFErrorExtR( tif, module, "%s: Bad SLONG8 value %" PRIi64 - " for \"%s\" tag %d in ClassicTIFF. Tag " + " for \"%s\" tag %u in ClassicTIFF. Tag " "won't be written to file", tif->tif_name, v2, fip->field_name, tag); goto badvalueifd8long8; @@ -1018,6 +1037,8 @@ static int _TIFFVSetField(TIFF *tif, uint32_t tag, va_list ap) _TIFFmemcpy(val, &v2, tv_size); } break; + case TIFF_NOTYPE: + case TIFF_ASCII: default: _TIFFmemset(val, 0, tv_size); status = 0; @@ -1345,7 +1366,7 @@ static int _TIFFVGetField(TIFF *tif, uint32_t tag, va_list ap) break; case TIFFTAG_MATTEING: *va_arg(ap, uint16_t *) = - (td->td_extrasamples == 1 && + (td->td_extrasamples == 1 && td->td_sampleinfo && td->td_sampleinfo[0] == EXTRASAMPLE_ASSOCALPHA); break; case TIFFTAG_EXTRASAMPLES: @@ -1376,6 +1397,8 @@ static int _TIFFVGetField(TIFF *tif, uint32_t tag, va_list ap) case SAMPLEFORMAT_VOID: *va_arg(ap, uint16_t *) = DATATYPE_VOID; break; + default: + break; } break; case TIFFTAG_SAMPLEFORMAT: @@ -1562,6 +1585,8 @@ static int _TIFFVGetField(TIFF *tif, uint32_t tag, va_list ap) *va_arg(ap, double *) = *(double *)val; ret_val = 1; break; + case TIFF_NOTYPE: + case TIFF_ASCII: default: ret_val = 0; break; @@ -1614,6 +1639,21 @@ int TIFFVGetField(TIFF *tif, uint32_t tag, va_list ap) } /* + * Reset tif->tif_dir structure to zero and + * initialize some IFD strile counter and index parameters. + */ +void _TIFFResetTifDirAndInitStrileCounters(TIFFDirectory *td) +{ + _TIFFmemset(td, 0, sizeof(*td)); + td->td_curstrip = NOSTRIP; /* invalid strip = NOSTRIP */ + td->td_row = (uint32_t)-1; /* read/write pre-increment */ + td->td_col = (uint32_t)-1; /* read/write pre-increment */ + td->td_scanlinesize = 0; /* initialize to zero */ + td->td_curtile = NOTILE; /* invalid tile = NOTILE */ + td->td_tilesize = (tmsize_t)-1; /* invalidate tilezize */ +} + +/* * Release storage associated with a directory. */ void TIFFFreeDirectory(TIFF *tif) @@ -1664,6 +1704,7 @@ void TIFFFreeDirectory(TIFF *tif) tif->tif_dir.td_dirdatasize_Noffsets = 0; } tif->tif_dir.td_iswrittentofile = FALSE; + /* Note: tif->tif_dir structure is set to zero in TIFFDefaultDirectory() */ } #undef CleanupField @@ -1694,10 +1735,7 @@ int TIFFCreateDirectory(TIFF *tif) tif->tif_diroff = 0; tif->tif_nextdiroff = 0; tif->tif_curoff = 0; - tif->tif_row = (uint32_t)-1; - tif->tif_curstrip = (uint32_t)-1; tif->tif_dir.td_iswrittentofile = FALSE; - return 0; } @@ -1717,15 +1755,12 @@ int TIFFCreateCustomDirectory(TIFF *tif, const TIFFFieldArray *infoarray) tif->tif_diroff = 0; tif->tif_nextdiroff = 0; tif->tif_curoff = 0; - tif->tif_row = (uint32_t)-1; - tif->tif_curstrip = (uint32_t)-1; /* invalidate directory index */ tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; /* invalidate IFD loop lists */ _TIFFCleanupIFDOffsetAndNumberMaps(tif); /* To be able to return from SubIFD or custom-IFD to main-IFD */ tif->tif_setdirectory_force_absolute = TRUE; - return 0; } @@ -1751,13 +1786,14 @@ int TIFFCreateGPSDirectory(TIFF *tif) */ int TIFFDefaultDirectory(TIFF *tif) { - register TIFFDirectory *td = &tif->tif_dir; + TIFFDirectory *td = &tif->tif_dir; const TIFFFieldArray *tiffFieldArray; tiffFieldArray = _TIFFGetFields(); _TIFFSetupFields(tif, tiffFieldArray); - - _TIFFmemset(td, 0, sizeof(*td)); + /* Reset tif->tif_dir structure to zero and + * initialize some IFD strile counter and index parameters. */ + _TIFFResetTifDirAndInitStrileCounters(td); td->td_fillorder = FILLORDER_MSB2LSB; td->td_bitspersample = 1; td->td_threshholding = THRESHHOLD_BILEVEL; @@ -1860,10 +1896,8 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, tmsize_t poffa, poffb, poffc, poffd; uint16_t dircount; uint32_t nextdir32; - poffa = (tmsize_t)poff; - poffb = poffa + sizeof(uint16_t); - if (((uint64_t)poffa != poff) || (poffb < poffa) || - (poffb < (tmsize_t)sizeof(uint16_t)) || (poffb > tif->tif_size)) + if (poff > (uint64_t)TIFF_TMSIZE_T_MAX - sizeof(uint16_t) || + poff > (uint64_t)tif->tif_size - sizeof(uint16_t)) { TIFFErrorExtR(tif, module, "%s:%d: %s: Error fetching directory count", @@ -1871,13 +1905,20 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, *nextdiroff = 0; return (0); } + poffa = (tmsize_t)poff; + poffb = poffa + (tmsize_t)sizeof(uint16_t); _TIFFmemcpy(&dircount, tif->tif_base + poffa, sizeof(uint16_t)); if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort(&dircount); + if (poffb > + TIFF_TMSIZE_T_MAX - dircount * 12 - (tmsize_t)sizeof(uint32_t)) + { + TIFFErrorExtR(tif, module, "Error fetching directory link"); + return (0); + } poffc = poffb + dircount * 12; - poffd = poffc + sizeof(uint32_t); - if ((poffc < poffb) || (poffc < dircount * 12) || (poffd < poffc) || - (poffd < (tmsize_t)sizeof(uint32_t)) || (poffd > tif->tif_size)) + poffd = poffc + (tmsize_t)sizeof(uint32_t); + if (poffd > tif->tif_size) { TIFFErrorExtR(tif, module, "Error fetching directory link"); return (0); @@ -1893,7 +1934,6 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, { tmsize_t poffa, poffb, poffc, poffd; uint64_t dircount64; - uint16_t dircount16; if (poff > (uint64_t)TIFF_TMSIZE_T_MAX - sizeof(uint64_t)) { TIFFErrorExtR(tif, module, @@ -1902,7 +1942,7 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, return (0); } poffa = (tmsize_t)poff; - poffb = poffa + sizeof(uint64_t); + poffb = poffa + (tmsize_t)sizeof(uint64_t); if (poffb > tif->tif_size) { TIFFErrorExtR(tif, module, @@ -1919,15 +1959,14 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, "Sanity check on directory count failed"); return (0); } - dircount16 = (uint16_t)dircount64; - if (poffb > TIFF_TMSIZE_T_MAX - (tmsize_t)(dircount16 * 20) - + if (poffb > TIFF_TMSIZE_T_MAX - (tmsize_t)(dircount64 * 20) - (tmsize_t)sizeof(uint64_t)) { TIFFErrorExtR(tif, module, "Error fetching directory link"); return (0); } - poffc = poffb + dircount16 * 20; - poffd = poffc + sizeof(uint64_t); + poffc = poffb + (tmsize_t)(dircount64 * 20); + poffd = poffc + (tmsize_t)sizeof(uint64_t); if (poffd > tif->tif_size) { TIFFErrorExtR(tif, module, "Error fetching directory link"); @@ -1957,9 +1996,9 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort(&dircount); if (off != NULL) - *off = TIFFSeekFile(tif, dircount * 12, SEEK_CUR); + *off = TIFFSeekFile(tif, dircount * 12U, SEEK_CUR); else - (void)TIFFSeekFile(tif, dircount * 12, SEEK_CUR); + (void)TIFFSeekFile(tif, dircount * 12U, SEEK_CUR); if (!ReadOK(tif, &nextdir32, sizeof(uint32_t))) { TIFFErrorExtR(tif, module, "%s: Error fetching directory link", @@ -1973,7 +2012,6 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, else { uint64_t dircount64; - uint16_t dircount16; if (!SeekOK(tif, *nextdiroff) || !ReadOK(tif, &dircount64, sizeof(uint64_t))) { @@ -1991,11 +2029,10 @@ static int TIFFAdvanceDirectory(TIFF *tif, uint64_t *nextdiroff, uint64_t *off, __FILE__, __LINE__, tif->tif_name); return (0); } - dircount16 = (uint16_t)dircount64; if (off != NULL) - *off = TIFFSeekFile(tif, dircount16 * 20, SEEK_CUR); + *off = TIFFSeekFile(tif, dircount64 * 20, SEEK_CUR); else - (void)TIFFSeekFile(tif, dircount16 * 20, SEEK_CUR); + (void)TIFFSeekFile(tif, dircount64 * 20, SEEK_CUR); if (!ReadOK(tif, nextdiroff, sizeof(uint64_t))) { TIFFErrorExtR(tif, module, "%s: Error fetching directory link", @@ -2354,8 +2391,6 @@ int TIFFUnlinkDirectory(TIFF *tif, tdir_t dirn) tif->tif_nextdiroff = 0; /* next write must be at end */ tif->tif_lastdiroff = 0; /* will be updated on next link */ tif->tif_curoff = 0; - tif->tif_row = (uint32_t)-1; - tif->tif_curstrip = (uint32_t)-1; tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; if (tif->tif_curdircount > 0) tif->tif_curdircount--; diff --git a/contrib/libs/libtiff/tif_dir.h b/contrib/libs/libtiff/tif_dir.h index 44270f70970..aed97222288 100644 --- a/contrib/libs/libtiff/tif_dir.h +++ b/contrib/libs/libtiff/tif_dir.h @@ -104,6 +104,16 @@ typedef struct uint16_t td_halftonehints[2]; uint16_t td_extrasamples; uint16_t *td_sampleinfo; + /* strip support */ + uint32_t td_row; /* current scanline */ + uint32_t td_curstrip; /* current strip for read/write */ + tmsize_t td_scanlinesize; /* # of bytes in a scanline */ +#define NOSTRIP ((uint32_t)(-1)) /* undefined state */ + /* tiling support */ + uint32_t td_col; /* current column (offset by row too) */ + uint32_t td_curtile; /* current tile for read/write */ + tmsize_t td_tilesize; /* # of bytes in a tile */ +#define NOTILE ((uint32_t)(-1)) /* undefined state */ /* even though the name is misleading, td_stripsperimage is the number * of striles (=strips or tiles) per plane, and td_nstrips the total * number of striles */ @@ -164,6 +174,8 @@ typedef struct entries. */ } TIFFDirectory; +extern void _TIFFResetTifDirAndInitStrileCounters(TIFFDirectory *td); + /* * Field flags used to indicate fields that have been set in a directory, and * to reference fields when manipulating a directory. @@ -231,7 +243,7 @@ typedef struct #define FIELD_LAST (32 * FIELDSET_ITEMS - 1) -#define BITn(n) (((uint32_t)1L) << ((n)&0x1f)) +#define BITn(n) (1U << ((n) & 0x1f)) #define BITFIELDn(tif, n) ((tif)->tif_dir.td_fieldsset[(n) / 32]) #define TIFFFieldSet(tif, field) (BITFIELDn(tif, field) & BITn(field)) #define TIFFSetFieldBit(tif, field) (BITFIELDn(tif, field) |= BITn(field)) @@ -340,7 +352,7 @@ extern "C" unsigned short field_bit; /* bit in fieldsset bit vector */ unsigned char field_oktochange; /* if true, can change while writing */ unsigned char field_passcount; /* if true, pass dir count on set */ - char *field_name; /* ASCII name */ + const char *field_name; /* ASCII name */ TIFFFieldArray *field_subfields; /* if field points to child ifds, child ifd field definition array */ }; diff --git a/contrib/libs/libtiff/tif_dirinfo.c b/contrib/libs/libtiff/tif_dirinfo.c index c038497cf49..9218661b723 100644 --- a/contrib/libs/libtiff/tif_dirinfo.c +++ b/contrib/libs/libtiff/tif_dirinfo.c @@ -40,17 +40,47 @@ * values accordingly. */ -/* const object should be initialized */ -#ifdef _MSC_VER +/* Forward declarations - definition follows after field arrays. + * Note: In C, we can forward declare static const objects and define them + * later. In C++, we need extern for the declaration, then define without + * extern. Since these are only used within this file via pointers, we use a + * workaround that works in both C and C++: declare as extern here, define as + * static later, but actually we need a different approach for C++ + * compatibility. + * + * For C/C++ compatibility, we define a simple struct that holds the pointer + * and initialize it after the arrays are defined. + */ +#ifdef __cplusplus +/* C++ doesn't allow forward declaration of const objects, so we use extern */ +extern const TIFFFieldArray tiffFieldArray; +extern const TIFFFieldArray exifFieldArray; +extern const TIFFFieldArray gpsFieldArray; +#else +/* C allows forward declaration of const objects, but C++ doesn't. + * We disable the C++-compat warning for this section since these circular + * dependencies are unavoidable with static initialization. */ +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wc++-compat" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc++-compat" +#elif defined(_MSC_VER) #pragma warning(push) #pragma warning(disable : 4132) #endif static const TIFFFieldArray tiffFieldArray; static const TIFFFieldArray exifFieldArray; static const TIFFFieldArray gpsFieldArray; -#ifdef _MSC_VER +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#elif defined(_MSC_VER) #pragma warning(pop) #endif +#endif /*--: Rational2Double: -- * The Rational2Double upgraded libtiff functionality allows the definition and * achievement of true double-precision accuracy for TIFF tags of RATIONAL type @@ -152,9 +182,9 @@ static const TIFFField tiffFields[] = { /*--: EXIFIFD and GPSIFD specified as TIFF_LONG by Aware-Systems and not TIFF_IFD8 as in original LibTiff. However, for IFD-like tags, * libtiff uses the data type TIFF_IFD8 in tiffFields[]-tag definition combined with a special handling procedure in order to write either * a 32-bit value and the TIFF_IFD type-id into ClassicTIFF files or a 64-bit value and the TIFF_IFD8 type-id into BigTIFF files. */ - {TIFFTAG_EXIFIFD, 1, 1, TIFF_IFD8, 0, TIFF_SETGET_IFD8, FIELD_CUSTOM, 1, 0, "EXIFIFDOffset", (TIFFFieldArray *)&exifFieldArray}, + {TIFFTAG_EXIFIFD, 1, 1, TIFF_LONG8, 0, TIFF_SETGET_UINT64, FIELD_CUSTOM, 1, 0, "EXIFIFDOffset", (TIFFFieldArray *)&exifFieldArray}, {TIFFTAG_ICCPROFILE, -3, -3, TIFF_UNDEFINED, 0, TIFF_SETGET_C32_UINT8, FIELD_CUSTOM, 1, 1, "ICC Profile", NULL}, - {TIFFTAG_GPSIFD, 1, 1, TIFF_IFD8, 0, TIFF_SETGET_IFD8, FIELD_CUSTOM, 1, 0, "GPSIFDOffset", (TIFFFieldArray *)&gpsFieldArray}, + {TIFFTAG_GPSIFD, 1, 1, TIFF_LONG8, 0, TIFF_SETGET_UINT64, FIELD_CUSTOM, 1, 0, "GPSIFDOffset", (TIFFFieldArray *)&gpsFieldArray}, {TIFFTAG_FAXRECVPARAMS, 1, 1, TIFF_LONG, 0, TIFF_SETGET_UINT32, FIELD_CUSTOM, TRUE, FALSE, "FaxRecvParams", NULL}, {TIFFTAG_FAXSUBADDRESS, -1, -1, TIFF_ASCII, 0, TIFF_SETGET_ASCII, FIELD_CUSTOM, TRUE, FALSE, "FaxSubAddress", NULL}, {TIFFTAG_FAXRECVTIME, 1, 1, TIFF_LONG, 0, TIFF_SETGET_UINT32, FIELD_CUSTOM, TRUE, FALSE, "FaxRecvTime", NULL}, @@ -479,12 +509,36 @@ static const TIFFField gpsFields[] = { {GPSTAG_GPSHPOSITIONINGERROR, 1, 1, TIFF_RATIONAL, 0, TIFF_SETGET_DOUBLE, FIELD_CUSTOM, 1, 0, "HorizontalPositioningError", NULL}}; /* clang-format on */ /* was off for better readability of tag comments */ +#ifdef __cplusplus +/* In C++, the forward declaration used extern, so definitions must not be + * static */ +const TIFFFieldArray tiffFieldArray = { + tfiatImage, 0, TIFFArrayCount(tiffFields), (TIFFField *)tiffFields}; +const TIFFFieldArray exifFieldArray = {tfiatExif, 0, TIFFArrayCount(exifFields), + (TIFFField *)exifFields}; +const TIFFFieldArray gpsFieldArray = {tfiatGps, 0, TIFFArrayCount(gpsFields), + (TIFFField *)gpsFields}; +#else +/* Suppress C++-compat warning for the definitions as well */ +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wc++-compat" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc++-compat" +#endif static const TIFFFieldArray tiffFieldArray = { tfiatImage, 0, TIFFArrayCount(tiffFields), (TIFFField *)tiffFields}; static const TIFFFieldArray exifFieldArray = { tfiatExif, 0, TIFFArrayCount(exifFields), (TIFFField *)exifFields}; static const TIFFFieldArray gpsFieldArray = { tfiatGps, 0, TIFFArrayCount(gpsFields), (TIFFField *)gpsFields}; +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif +#endif /* * We have our own local lfind() equivalent to avoid subtle differences @@ -523,7 +577,7 @@ void _TIFFSetupFields(TIFF *tif, const TIFFFieldArray *fieldarray) { if (fld->field_bit == FIELD_CUSTOM && TIFFFieldIsAnonymous(fld)) { - _TIFFfreeExt(tif, fld->field_name); + _TIFFfreeExt(tif, (void *)fld->field_name); /* caution: tif_fields[i] must not be the beginning of a * fields-array. Otherwise the following tags are also freed * with the first free(). @@ -579,22 +633,26 @@ int _TIFFMergeFields(TIFF *tif, const TIFFField info[], uint32_t n) tif->tif_foundfield = NULL; + TIFFField **tif_newfields = NULL; + if (tif->tif_fields && tif->tif_nfields > 0) { - tif->tif_fields = (TIFFField **)_TIFFCheckRealloc( - tif, tif->tif_fields, (tif->tif_nfields + n), sizeof(TIFFField *), - reason); + tif_newfields = (TIFFField **)_TIFFCheckRealloc( + tif, tif->tif_fields, (tmsize_t)tif->tif_nfields + n, + (tmsize_t)sizeof(TIFFField *), reason); } else { - tif->tif_fields = + tif_newfields = (TIFFField **)_TIFFCheckMalloc(tif, n, sizeof(TIFFField *), reason); } - if (!tif->tif_fields) + if (!tif_newfields) { + tif->tif_nfields = 0; TIFFErrorExtR(tif, module, "Failed to allocate fields array"); return 0; } + tif->tif_fields = tif_newfields; /* tp = tif->tif_fields + tif->tif_nfields; */ for (i = 0; i < n; i++) @@ -612,7 +670,7 @@ int _TIFFMergeFields(TIFF *tif, const TIFFField info[], uint32_t n) /* Sort the field info by tag number */ qsort(tif->tif_fields, tif->tif_nfields, sizeof(TIFFField *), tagCompare); - return n; + return (int)n; } void _TIFFPrintFieldInfo(TIFF *tif, FILE *fd) @@ -623,7 +681,7 @@ void _TIFFPrintFieldInfo(TIFF *tif, FILE *fd) for (i = 0; i < tif->tif_nfields; i++) { const TIFFField *fip = tif->tif_fields[i]; - fprintf(fd, "field[%2d] %5lu, %2d, %2d, %d, %2d, %5s, %5s, %s\n", + fprintf(fd, "field[%2d] %5lu, %2d, %2d, %u, %2d, %5s, %5s, %s\n", (int)i, (unsigned long)fip->field_tag, fip->field_readcount, fip->field_writecount, fip->field_type, fip->field_bit, fip->field_oktochange ? "TRUE" : "FALSE", @@ -787,6 +845,34 @@ int TIFFFieldSetGetCountSize(const TIFFField *fip) case TIFF_SETGET_C32_DOUBLE: case TIFF_SETGET_C32_IFD8: return 4; + case TIFF_SETGET_UNDEFINED: + case TIFF_SETGET_ASCII: + case TIFF_SETGET_UINT8: + case TIFF_SETGET_SINT8: + case TIFF_SETGET_UINT16: + case TIFF_SETGET_SINT16: + case TIFF_SETGET_UINT32: + case TIFF_SETGET_SINT32: + case TIFF_SETGET_UINT64: + case TIFF_SETGET_SINT64: + case TIFF_SETGET_FLOAT: + case TIFF_SETGET_DOUBLE: + case TIFF_SETGET_IFD8: + case TIFF_SETGET_INT: + case TIFF_SETGET_UINT16_PAIR: + case TIFF_SETGET_C0_ASCII: + case TIFF_SETGET_C0_UINT8: + case TIFF_SETGET_C0_SINT8: + case TIFF_SETGET_C0_UINT16: + case TIFF_SETGET_C0_SINT16: + case TIFF_SETGET_C0_UINT32: + case TIFF_SETGET_C0_SINT32: + case TIFF_SETGET_C0_UINT64: + case TIFF_SETGET_C0_SINT64: + case TIFF_SETGET_C0_FLOAT: + case TIFF_SETGET_C0_DOUBLE: + case TIFF_SETGET_C0_IFD8: + case TIFF_SETGET_OTHER: default: return 0; } @@ -794,7 +880,8 @@ int TIFFFieldSetGetCountSize(const TIFFField *fip) const TIFFField *TIFFFindField(TIFF *tif, uint32_t tag, TIFFDataType dt) { - TIFFField key = {0, 0, 0, TIFF_NOTYPE, 0, 0, 0, 0, 0, NULL, NULL}; + TIFFField key = {0, 0, 0, TIFF_NOTYPE, 0, TIFF_SETGET_UNDEFINED, + 0, 0, 0, NULL, NULL}; TIFFField *pkey = &key; const TIFFField **ret; if (tif->tif_foundfield && tif->tif_foundfield->field_tag == tag && @@ -818,7 +905,8 @@ const TIFFField *TIFFFindField(TIFF *tif, uint32_t tag, TIFFDataType dt) static const TIFFField *_TIFFFindFieldByName(TIFF *tif, const char *field_name, TIFFDataType dt) { - TIFFField key = {0, 0, 0, TIFF_NOTYPE, 0, 0, 0, 0, 0, NULL, NULL}; + TIFFField key = {0, 0, 0, TIFF_NOTYPE, 0, TIFF_SETGET_UNDEFINED, + 0, 0, 0, NULL, NULL}; TIFFField *pkey = &key; const TIFFField **ret; if (tif->tif_foundfield && @@ -876,7 +964,10 @@ int TIFFFieldReadCount(const TIFFField *fip) { return fip->field_readcount; } int TIFFFieldWriteCount(const TIFFField *fip) { return fip->field_writecount; } -int TIFFFieldIsAnonymous(const TIFFField *fip) { return fip->field_anonymous; } +int TIFFFieldIsAnonymous(const TIFFField *fip) +{ + return (int)fip->field_anonymous; +} const TIFFField *_TIFFFindOrRegisterField(TIFF *tif, uint32_t tag, TIFFDataType dt) @@ -954,6 +1045,7 @@ TIFFField *_TIFFCreateAnonField(TIFF *tif, uint32_t tag, case TIFF_SLONG8: fld->set_get_field_type = TIFF_SETGET_C32_SINT64; break; + case TIFF_NOTYPE: default: fld->set_get_field_type = TIFF_SETGET_UNDEFINED; break; @@ -961,8 +1053,8 @@ TIFFField *_TIFFCreateAnonField(TIFF *tif, uint32_t tag, fld->field_bit = FIELD_CUSTOM; fld->field_oktochange = TRUE; fld->field_passcount = TRUE; - fld->field_name = (char *)_TIFFmallocExt(tif, 32); - if (fld->field_name == NULL) + char *field_name_buf = (char *)_TIFFmallocExt(tif, 32); + if (field_name_buf == NULL) { _TIFFfreeExt(tif, fld); return NULL; @@ -975,7 +1067,8 @@ TIFFField *_TIFFCreateAnonField(TIFF *tif, uint32_t tag, * Update: * This special sign is replaced by fld->field_anonymous flag. */ - (void)snprintf(fld->field_name, 32, "Tag %d", (int)tag); + (void)snprintf(field_name_buf, 32, "Tag %d", (int)tag); + fld->field_name = field_name_buf; return fld; } @@ -1025,6 +1118,7 @@ static TIFFSetGetFieldType _TIFFSetGetType(TIFFDataType type, short count, return TIFF_SETGET_UINT64; case TIFF_SLONG8: return TIFF_SETGET_SINT64; + case TIFF_NOTYPE: default: return TIFF_SETGET_UNDEFINED; } @@ -1062,6 +1156,7 @@ static TIFFSetGetFieldType _TIFFSetGetType(TIFFDataType type, short count, return TIFF_SETGET_C0_UINT64; case TIFF_SLONG8: return TIFF_SETGET_C0_SINT64; + case TIFF_NOTYPE: default: return TIFF_SETGET_UNDEFINED; } @@ -1099,6 +1194,7 @@ static TIFFSetGetFieldType _TIFFSetGetType(TIFFDataType type, short count, return TIFF_SETGET_C16_UINT64; case TIFF_SLONG8: return TIFF_SETGET_C16_SINT64; + case TIFF_NOTYPE: default: return TIFF_SETGET_UNDEFINED; } @@ -1136,6 +1232,7 @@ static TIFFSetGetFieldType _TIFFSetGetType(TIFFDataType type, short count, return TIFF_SETGET_C32_UINT64; case TIFF_SLONG8: return TIFF_SETGET_C32_SINT64; + case TIFF_NOTYPE: default: return TIFF_SETGET_UNDEFINED; } @@ -1152,22 +1249,25 @@ int TIFFMergeFieldInfo(TIFF *tif, const TIFFFieldInfo info[], uint32_t n) size_t nfields; uint32_t i; + TIFFFieldArray *tif_newfieldscompat = NULL; + if (tif->tif_nfieldscompat > 0) { - tif->tif_fieldscompat = (TIFFFieldArray *)_TIFFCheckRealloc( - tif, tif->tif_fieldscompat, tif->tif_nfieldscompat + 1, - sizeof(TIFFFieldArray), reason); + tif_newfieldscompat = (TIFFFieldArray *)_TIFFCheckRealloc( + tif, tif->tif_fieldscompat, (tmsize_t)tif->tif_nfieldscompat + 1, + (tmsize_t)sizeof(TIFFFieldArray), reason); } else { - tif->tif_fieldscompat = (TIFFFieldArray *)_TIFFCheckMalloc( + tif_newfieldscompat = (TIFFFieldArray *)_TIFFCheckMalloc( tif, 1, sizeof(TIFFFieldArray), reason); } - if (!tif->tif_fieldscompat) + if (!tif_newfieldscompat) { TIFFErrorExtR(tif, module, "Failed to allocate fields array"); return -1; } + tif->tif_fieldscompat = tif_newfieldscompat; nfields = tif->tif_nfieldscompat++; tif->tif_fieldscompat[nfields].type = tfiatOther; @@ -1309,6 +1409,8 @@ int _TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag) case TIFFTAG_JPEGPROC: case TIFFTAG_JPEGRESTARTINTERVAL: return 1; + default: + break; } break; case COMPRESSION_CCITTRLE: @@ -1329,6 +1431,8 @@ int _TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag) if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX4) return 1; break; + default: + break; } break; case COMPRESSION_JBIG: @@ -1359,6 +1463,8 @@ int _TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag) if (tag == TIFFTAG_LERC_PARAMETERS) return 1; break; + default: + break; } return 0; } diff --git a/contrib/libs/libtiff/tif_dirread.c b/contrib/libs/libtiff/tif_dirread.c index 0182f9b4194..50803caf4fd 100644 --- a/contrib/libs/libtiff/tif_dirread.c +++ b/contrib/libs/libtiff/tif_dirread.c @@ -597,7 +597,7 @@ TIFFReadDirEntrySshort(TIFF *tif, TIFFDirEntry *direntry, int16_t *value) err = TIFFReadDirEntryCheckRangeSshortShort(m); if (err != TIFFReadDirEntryErrOk) return (err); - *value = (uint16_t)m; + *value = (int16_t)m; return (TIFFReadDirEntryErrOk); } case TIFF_SSHORT: @@ -695,6 +695,7 @@ TIFFReadDirEntryLong(TIFF *tif, TIFFDirEntry *direntry, uint32_t *value) return (TIFFReadDirEntryErrOk); } case TIFF_LONG: + case TIFF_IFD: TIFFReadDirEntryCheckedLong(tif, direntry, value); return (TIFFReadDirEntryErrOk); case TIFF_SLONG: @@ -708,6 +709,7 @@ TIFFReadDirEntryLong(TIFF *tif, TIFFDirEntry *direntry, uint32_t *value) return (TIFFReadDirEntryErrOk); } case TIFF_LONG8: + case TIFF_IFD8: { uint64_t m; err = TIFFReadDirEntryCheckedLong8(tif, direntry, &m); @@ -857,6 +859,7 @@ TIFFReadDirEntryLong8(TIFF *tif, TIFFDirEntry *direntry, uint64_t *value) return (TIFFReadDirEntryErrOk); } case TIFF_LONG: + case TIFF_IFD: { uint32_t m; TIFFReadDirEntryCheckedLong(tif, direntry, &m); @@ -874,6 +877,7 @@ TIFFReadDirEntryLong8(TIFF *tif, TIFFDirEntry *direntry, uint64_t *value) return (TIFFReadDirEntryErrOk); } case TIFF_LONG8: + case TIFF_IFD8: err = TIFFReadDirEntryCheckedLong8(tif, direntry, value); return (err); case TIFF_SLONG8: @@ -1058,7 +1062,7 @@ TIFFReadDirEntryFloat(TIFF *tif, TIFFDirEntry *direntry, float *value) err = TIFFReadDirEntryCheckedDouble(tif, direntry, &m); if (err != TIFFReadDirEntryErrOk) return (err); - if ((m > FLT_MAX) || (m < -FLT_MAX)) + if ((m > (double)FLT_MAX) || (m < -(double)FLT_MAX)) return (TIFFReadDirEntryErrRange); *value = (float)m; return (TIFFReadDirEntryErrOk); @@ -1237,6 +1241,8 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryDataAndRealloc(TIFF *tif, *pdest = new_dest; bytes_read = TIFFReadFile(tif, (char *)*pdest + already_read, to_read); + if (bytes_read < 0) + return TIFFReadDirEntryErrIo; already_read += bytes_read; if (bytes_read != to_read) { @@ -1260,7 +1266,7 @@ TIFFReadDirEntryArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, void *data; uint64_t target_count64; int original_datasize_clamped; - typesize = TIFFDataWidth(direntry->tdir_type); + typesize = TIFFDataWidth((TIFFDataType)direntry->tdir_type); target_count64 = (direntry->tdir_count > maxcount) ? maxcount : direntry->tdir_count; @@ -1284,13 +1290,13 @@ TIFFReadDirEntryArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, * in either the current data type or the dest data type. This also * avoids problems with overflow of tmsize_t on 32bit systems. */ - if ((uint64_t)(MAX_SIZE_TAG_DATA / typesize) < target_count64) + if ((uint64_t)(MAX_SIZE_TAG_DATA / (unsigned int)typesize) < target_count64) return (TIFFReadDirEntryErrSizesan); if ((uint64_t)(MAX_SIZE_TAG_DATA / desttypesize) < target_count64) return (TIFFReadDirEntryErrSizesan); *count = (uint32_t)target_count64; - datasize = (*count) * typesize; + datasize = (uint32_t)(*count) * (unsigned int)typesize; assert((tmsize_t)datasize > 0); if (datasize > 100 * 1024 * 1024) @@ -1440,6 +1446,8 @@ TIFFReadDirEntryByteArray(TIFF *tif, TIFFDirEntry *direntry, uint8_t **value) *value = (uint8_t *)origdata; return (TIFFReadDirEntryErrOk); } + default: + break; } data = (uint8_t *)_TIFFmallocExt(tif, count); if (data == 0) @@ -1557,6 +1565,8 @@ TIFFReadDirEntryByteArray(TIFF *tif, TIFFDirEntry *direntry, uint8_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -1620,6 +1630,8 @@ TIFFReadDirEntrySbyteArray(TIFF *tif, TIFFDirEntry *direntry, int8_t **value) case TIFF_SBYTE: *value = (int8_t *)origdata; return (TIFFReadDirEntryErrOk); + default: + break; } data = (int8_t *)_TIFFmallocExt(tif, count); if (data == 0) @@ -1737,6 +1749,8 @@ TIFFReadDirEntrySbyteArray(TIFF *tif, TIFFDirEntry *direntry, int8_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -1802,6 +1816,8 @@ TIFFReadDirEntryShortArray(TIFF *tif, TIFFDirEntry *direntry, uint16_t **value) *value = (uint16_t *)origdata; return (TIFFReadDirEntryErrOk); } + default: + break; } data = (uint16_t *)_TIFFmallocExt(tif, count * 2); if (data == 0) @@ -1910,6 +1926,8 @@ TIFFReadDirEntryShortArray(TIFF *tif, TIFFDirEntry *direntry, uint16_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -1975,6 +1993,8 @@ TIFFReadDirEntrySshortArray(TIFF *tif, TIFFDirEntry *direntry, int16_t **value) if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfShort((uint16_t *)(*value), count); return (TIFFReadDirEntryErrOk); + default: + break; } data = (int16_t *)_TIFFmallocExt(tif, count * 2); if (data == 0) @@ -2078,6 +2098,8 @@ TIFFReadDirEntrySshortArray(TIFF *tif, TIFFDirEntry *direntry, int16_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -2106,6 +2128,8 @@ TIFFReadDirEntryLongArray(TIFF *tif, TIFFDirEntry *direntry, uint32_t **value) case TIFF_SLONG: case TIFF_LONG8: case TIFF_SLONG8: + case TIFF_IFD: + case TIFF_IFD8: break; default: return (TIFFReadDirEntryErrType); @@ -2119,6 +2143,7 @@ TIFFReadDirEntryLongArray(TIFF *tif, TIFFDirEntry *direntry, uint32_t **value) switch (direntry->tdir_type) { case TIFF_LONG: + case TIFF_IFD: *value = (uint32_t *)origdata; if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong(*value, count); @@ -2143,6 +2168,8 @@ TIFFReadDirEntryLongArray(TIFF *tif, TIFFDirEntry *direntry, uint32_t **value) *value = (uint32_t *)origdata; return (TIFFReadDirEntryErrOk); } + default: + break; } data = (uint32_t *)_TIFFmallocExt(tif, count * 4); if (data == 0) @@ -2213,6 +2240,7 @@ TIFFReadDirEntryLongArray(TIFF *tif, TIFFDirEntry *direntry, uint32_t **value) } break; case TIFF_LONG8: + case TIFF_IFD8: { uint64_t *ma; uint32_t *mb; @@ -2248,6 +2276,8 @@ TIFFReadDirEntryLongArray(TIFF *tif, TIFFDirEntry *direntry, uint32_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -2313,6 +2343,8 @@ TIFFReadDirEntrySlongArray(TIFF *tif, TIFFDirEntry *direntry, int32_t **value) if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong((uint32_t *)(*value), count); return (TIFFReadDirEntryErrOk); + default: + break; } data = (int32_t *)_TIFFmallocExt(tif, count * 4); if (data == 0) @@ -2410,6 +2442,8 @@ TIFFReadDirEntrySlongArray(TIFF *tif, TIFFDirEntry *direntry, int32_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -2439,6 +2473,8 @@ TIFFReadDirEntryLong8ArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, case TIFF_SLONG: case TIFF_LONG8: case TIFF_SLONG8: + case TIFF_IFD: + case TIFF_IFD8: break; default: return (TIFFReadDirEntryErrType); @@ -2453,6 +2489,7 @@ TIFFReadDirEntryLong8ArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, switch (direntry->tdir_type) { case TIFF_LONG8: + case TIFF_IFD8: *value = (uint64_t *)origdata; if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong8(*value, count); @@ -2477,6 +2514,8 @@ TIFFReadDirEntryLong8ArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, *value = (uint64_t *)origdata; return (TIFFReadDirEntryErrOk); } + default: + break; } data = (uint64_t *)_TIFFmallocExt(tif, count * 8); if (data == 0) @@ -2547,6 +2586,7 @@ TIFFReadDirEntryLong8ArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, } break; case TIFF_LONG: + case TIFF_IFD: { uint32_t *ma; uint64_t *mb; @@ -2579,6 +2619,8 @@ TIFFReadDirEntryLong8ArrayWithLimit(TIFF *tif, TIFFDirEntry *direntry, } } break; + default: + break; } _TIFFfreeExt(tif, origdata); if (err != TIFFReadDirEntryErrOk) @@ -2651,6 +2693,8 @@ TIFFReadDirEntrySlong8Array(TIFF *tif, TIFFDirEntry *direntry, int64_t **value) if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong8((uint64_t *)(*value), count); return (TIFFReadDirEntryErrOk); + default: + break; } data = (int64_t *)_TIFFmallocExt(tif, count * 8); if (data == 0) @@ -2742,6 +2786,8 @@ TIFFReadDirEntrySlong8Array(TIFF *tif, TIFFDirEntry *direntry, int64_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); *value = data; @@ -2787,8 +2833,11 @@ TIFFReadDirEntryFloatArray(TIFF *tif, TIFFDirEntry *direntry, float **value) TIFFCvtIEEEFloatToNative(tif, count, (float *)origdata); *value = (float *)origdata; return (TIFFReadDirEntryErrOk); + default: + break; } - data = (float *)_TIFFmallocExt(tif, count * sizeof(float)); + data = + (float *)_TIFFmallocExt(tif, (tmsize_t)((size_t)count * sizeof(float))); if (data == 0) { _TIFFfreeExt(tif, origdata); @@ -2970,14 +3019,16 @@ TIFFReadDirEntryFloatArray(TIFF *tif, TIFFDirEntry *direntry, float **value) for (n = 0; n < count; n++) { double val = *ma++; - if (val > FLT_MAX) - val = FLT_MAX; - else if (val < -FLT_MAX) - val = -FLT_MAX; + if (val > (double)FLT_MAX) + val = (double)FLT_MAX; + else if (val < -(double)FLT_MAX) + val = -(double)FLT_MAX; *mb++ = (float)val; } } break; + default: + break; } _TIFFfreeExt(tif, origdata); *value = data; @@ -3023,8 +3074,11 @@ TIFFReadDirEntryDoubleArray(TIFF *tif, TIFFDirEntry *direntry, double **value) TIFFCvtIEEEDoubleToNative(tif, count, (double *)origdata); *value = (double *)origdata; return (TIFFReadDirEntryErrOk); + default: + break; } - data = (double *)_TIFFmallocExt(tif, count * sizeof(double)); + data = (double *)_TIFFmallocExt(tif, + (tmsize_t)((size_t)count * sizeof(double))); if (data == 0) { _TIFFfreeExt(tif, origdata); @@ -3207,6 +3261,8 @@ TIFFReadDirEntryDoubleArray(TIFF *tif, TIFFDirEntry *direntry, double **value) *mb++ = (double)(*ma++); } break; + default: + break; } _TIFFfreeExt(tif, origdata); *value = data; @@ -3244,6 +3300,8 @@ TIFFReadDirEntryIfd8Array(TIFF *tif, TIFFDirEntry *direntry, uint64_t **value) if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong8(*value, count); return (TIFFReadDirEntryErrOk); + default: + break; } data = (uint64_t *)_TIFFmallocExt(tif, count * 8); if (data == 0) @@ -3269,6 +3327,8 @@ TIFFReadDirEntryIfd8Array(TIFF *tif, TIFFDirEntry *direntry, uint64_t **value) } } break; + default: + break; } _TIFFfreeExt(tif, origdata); *value = data; @@ -3283,13 +3343,42 @@ TIFFReadDirEntryPersampleShort(TIFF *tif, TIFFDirEntry *direntry, uint16_t *m; uint16_t *na; uint16_t nb; - if (direntry->tdir_count < (uint64_t)tif->tif_dir.td_samplesperpixel) - return (TIFFReadDirEntryErrCount); + if (direntry->tdir_count != (uint64_t)tif->tif_dir.td_samplesperpixel) + { + const TIFFField *fip = TIFFFieldWithTag(tif, direntry->tdir_tag); + if (direntry->tdir_count == 0) + { + return TIFFReadDirEntryErrCount; + } + else if (direntry->tdir_count < + (uint64_t)tif->tif_dir.td_samplesperpixel) + { + TIFFWarningExtR( + tif, "TIFFReadDirEntryPersampleShort", + "Tag %s entry count is %" PRIu64 + " , whereas it should be SamplesPerPixel=%d. Assuming that " + "missing entries are all at the value of the first one", + fip ? fip->field_name : "unknown tagname", direntry->tdir_count, + tif->tif_dir.td_samplesperpixel); + } + else + { + TIFFWarningExtR(tif, "TIFFReadDirEntryPersampleShort", + "Tag %s entry count is %" PRIu64 + " , whereas it should be SamplesPerPixel=%d. " + "Ignoring extra entries", + fip ? fip->field_name : "unknown tagname", + direntry->tdir_count, + tif->tif_dir.td_samplesperpixel); + } + } err = TIFFReadDirEntryShortArray(tif, direntry, &m); if (err != TIFFReadDirEntryErrOk || m == NULL) return (err); na = m; nb = tif->tif_dir.td_samplesperpixel; + if (direntry->tdir_count < nb) + nb = (uint16_t)direntry->tdir_count; *value = *na++; nb--; while (nb > 0) @@ -3908,7 +3997,7 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryData(TIFF *tif, uint64_t offset, { return TIFFReadDirEntryErrIo; } - mb = ma + size; + mb = (uint64_t)ma + (uint64_t)size; if (mb > (uint64_t)tif->tif_size) return (TIFFReadDirEntryErrIo); _TIFFmemcpy(dest, tif->tif_base + ma, size); @@ -3955,6 +4044,7 @@ static void TIFFReadDirEntryOutputErr(TIFF *tif, enum TIFFReadDirEntryErr err, TIFFErrorExtR(tif, module, "Out of memory reading of \"%s\"", tagname); break; + case TIFFReadDirEntryErrOk: default: assert(0); /* we should never get here */ break; @@ -4001,6 +4091,7 @@ static void TIFFReadDirEntryOutputErr(TIFF *tif, enum TIFFReadDirEntryErr err, "Out of memory reading of \"%s\"; tag ignored", tagname); break; + case TIFFReadDirEntryErrOk: default: assert(0); /* we should never get here */ break; @@ -4086,7 +4177,8 @@ static int ByteCountLooksBad(TIFF *tif) */ static bool EvaluateIFDdatasizeReading(TIFF *tif, TIFFDirEntry *dp) { - const uint64_t data_width = TIFFDataWidth(dp->tdir_type); + const uint64_t data_width = + (uint64_t)TIFFDataWidth((TIFFDataType)dp->tdir_type); if (data_width != 0 && dp->tdir_count > UINT64_MAX / data_width) { TIFFErrorExtR(tif, "EvaluateIFDdatasizeReading", @@ -4173,9 +4265,9 @@ static void CalcFinalIFDdatasizeReading(TIFF *tif, uint16_t dircount) /* Get offset of end of IFD entry space. */ uint64_t IFDendoffset; if (!(tif->tif_flags & TIFF_BIGTIFF)) - IFDendoffset = tif->tif_diroff + 2 + dircount * 12 + 4; + IFDendoffset = tif->tif_diroff + 2 + (uint64_t)dircount * 12 + 4; else - IFDendoffset = tif->tif_diroff + 8 + dircount * 20 + 8; + IFDendoffset = tif->tif_diroff + 8 + (uint64_t)dircount * 20 + 8; /* Check which offsets are right behind IFD entries. However, LibTIFF * increments the writing address for every external data to an even offset. @@ -4224,9 +4316,9 @@ static void CalcFinalIFDdatasizeReading(TIFF *tif, uint16_t dircount) /* Finally, add the size of the IFD tag entries themselves. */ if (!(tif->tif_flags & TIFF_BIGTIFF)) - tif->tif_dir.td_dirdatasize_read = 2 + dircount * 12 + 4 + size; + tif->tif_dir.td_dirdatasize_read = 2U + dircount * 12U + 4U + size; else - tif->tif_dir.td_dirdatasize_read = 8 + dircount * 20 + 8 + size; + tif->tif_dir.td_dirdatasize_read = 8U + dircount * 20U + 8U + size; } /*-- CalcFinalIFDdatasizeReading() --*/ /* @@ -4298,7 +4390,8 @@ int TIFFReadDirectory(TIFF *tif) { TIFFDirEntry *na; uint16_t nb; - for (na = ma + 1, nb = mb + 1; nb < dircount; na++, nb++) + for (na = ma + 1, nb = (uint16_t)(mb + 1); nb < dircount; + na++, nb++) { if (ma->tdir_tag == na->tdir_tag) { @@ -4312,7 +4405,15 @@ int TIFFReadDirectory(TIFF *tif) tif->tif_flags &= ~TIFF_BUF4WRITE; /* reset before new dir */ tif->tif_flags &= ~TIFF_CHOPPEDUPARRAYS; - /* free any old stuff and reinit */ + /* When changing directory, in deferred strile loading mode, we must also + * unset the TIFF_LAZYSTRILELOAD_DONE bit if it was initially set, + * to make sure the strile offset/bytecount are read again (when they fit + * in the tag data area). + */ + tif->tif_flags &= ~TIFF_LAZYSTRILELOAD_DONE; + + /* Free any old stuff and reinit i/o and other parameters within + * TIFFDefaultDirectory() since we are starting on a new directory. */ TIFFFreeDirectory(tif); TIFFDefaultDirectory(tif); @@ -4325,7 +4426,8 @@ int TIFFReadDirectory(TIFF *tif) * checking. Note: Counter are reset within TIFFFreeDirectory(). */ tif->tif_dir.td_dirdatasize_offsets = (TIFFEntryOffsetAndLength *)_TIFFmallocExt( - tif, dircount * sizeof(TIFFEntryOffsetAndLength)); + tif, + (tmsize_t)((size_t)dircount * sizeof(TIFFEntryOffsetAndLength))); if (tif->tif_dir.td_dirdatasize_offsets == NULL) { TIFFErrorExtR( @@ -4787,8 +4889,8 @@ int TIFFReadDirectory(TIFF *tif) (void)TIFFFetchNormalTag(tif, dp, TRUE); break; } /* -- switch (dp->tdir_tag) -- */ - } /* -- if (!dp->tdir_ignore) */ - } /* -- for-loop -- */ + } /* -- if (!dp->tdir_ignore) */ + } /* -- for-loop -- */ /* Evaluate final IFD data size. */ CalcFinalIFDdatasizeReading(tif, dircount); @@ -5097,7 +5199,7 @@ int TIFFReadDirectory(TIFF *tif) tif->tif_dir.td_maxsamplevalue = 0xFFFF; else tif->tif_dir.td_maxsamplevalue = - (uint16_t)((1L << tif->tif_dir.td_bitspersample) - 1); + (uint16_t)((1 << tif->tif_dir.td_bitspersample) - 1); } #ifdef STRIPBYTECOUNTSORTED_UNUSED @@ -5163,24 +5265,18 @@ int TIFFReadDirectory(TIFF *tif) tif->tif_flags &= ~TIFF_DIRTYSTRIP; /* - * Reinitialize i/o since we are starting on a new directory. + * Reinitialize some further i/o since we are starting on a new directory. */ - tif->tif_row = (uint32_t)-1; - tif->tif_curstrip = (uint32_t)-1; - tif->tif_col = (uint32_t)-1; - tif->tif_curtile = (uint32_t)-1; - tif->tif_tilesize = (tmsize_t)-1; - - tif->tif_scanlinesize = TIFFScanlineSize(tif); - if (!tif->tif_scanlinesize) + tif->tif_dir.td_scanlinesize = TIFFScanlineSize(tif); + if (!tif->tif_dir.td_scanlinesize) { TIFFWarningExtR(tif, module, "Cannot handle zero scanline size"); } if (isTiled(tif)) { - tif->tif_tilesize = TIFFTileSize(tif); - if (!tif->tif_tilesize) + tif->tif_dir.td_tilesize = TIFFTileSize(tif); + if (!tif->tif_dir.td_tilesize) { TIFFWarningExtR(tif, module, "Cannot handle zero tile size"); } @@ -5216,7 +5312,7 @@ static void TIFFReadDirectoryCheckOrder(TIFF *tif, TIFFDirEntry *dir, "ascending order"); break; } - m = o->tdir_tag + 1; + m = o->tdir_tag + 1U; } } @@ -5264,7 +5360,7 @@ static void TIFFReadDirectoryFindFieldInfo(TIFF *tif, uint16_t tagid, break; mb--; } - *fii = mb; + *fii = (uint32_t)mb; } /* @@ -5304,7 +5400,8 @@ int TIFFReadCustomDirectory(TIFF *tif, toff_t diroff, { TIFFDirEntry *na; uint16_t nb; - for (na = ma + 1, nb = mb + 1; nb < dircount; na++, nb++) + for (na = ma + 1, nb = (uint16_t)(mb + 1); nb < dircount; + na++, nb++) { if (ma->tdir_tag == na->tdir_tag) { @@ -5329,7 +5426,8 @@ int TIFFReadCustomDirectory(TIFF *tif, toff_t diroff, * checking. Note: Counter are reset within TIFFFreeDirectory(). */ tif->tif_dir.td_dirdatasize_offsets = (TIFFEntryOffsetAndLength *)_TIFFmallocExt( - tif, dircount * sizeof(TIFFEntryOffsetAndLength)); + tif, + (tmsize_t)((size_t)dircount * sizeof(TIFFEntryOffsetAndLength))); if (tif->tif_dir.td_dirdatasize_offsets == NULL) { TIFFErrorExtR( @@ -5511,15 +5609,17 @@ static int EstimateStripByteCounts(TIFF *tif, TIFFDirEntry *dir, uint64_t space; uint16_t n; if (!(tif->tif_flags & TIFF_BIGTIFF)) - space = sizeof(TIFFHeaderClassic) + 2 + dircount * 12 + 4; + space = sizeof(TIFFHeaderClassic) + 2 + + (unsigned long)dircount * 12 + 4; else - space = sizeof(TIFFHeaderBig) + 8 + dircount * 20 + 8; + space = + sizeof(TIFFHeaderBig) + 8 + (unsigned long)dircount * 20 + 8; /* calculate amount of space used by indirect values */ for (dp = dir, n = dircount; n > 0; n--, dp++) { uint32_t typewidth; uint64_t datasize; - typewidth = TIFFDataWidth((TIFFDataType)dp->tdir_type); + typewidth = (uint32_t)TIFFDataWidth((TIFFDataType)dp->tdir_type); if (typewidth == 0) { TIFFErrorExtR( @@ -5782,7 +5882,7 @@ int _TIFFCheckDirNumberAndOffset(TIFF *tif, tdir_t dirn, uint64_t diroff) { TIFFErrorExtR(tif, "_TIFFCheckDirNumberAndOffset", "Cannot handle more than %u TIFF directories", - TIFF_MAX_DIR_COUNT); + (unsigned)TIFF_MAX_DIR_COUNT); return 0; } @@ -6044,7 +6144,7 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, "to read TIFF directory"); if (origdir == NULL) return 0; - if (!ReadOK(tif, origdir, (tmsize_t)(dircount16 * dirsize))) + if (!ReadOK(tif, origdir, (tmsize_t)dircount16 * dirsize)) { TIFFErrorExtR(tif, module, "%.100s: Can not read TIFF directory", tif->tif_name); @@ -6097,9 +6197,9 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, */ if (!(tif->tif_flags & TIFF_BIGTIFF)) { - m = off + sizeof(uint16_t); - if ((m < off) || (m < (tmsize_t)sizeof(uint16_t)) || - (m > tif->tif_size)) + m = (tmsize_t)((uint64_t)off + sizeof(uint16_t)); + if ((m < off) || ((uint64_t)m < sizeof(uint16_t)) || + ((uint64_t)m > (uint64_t)tif->tif_size)) { TIFFErrorExtR(tif, module, "Can not read TIFF directory count"); return 0; @@ -6108,7 +6208,7 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, { _TIFFmemcpy(&dircount16, tif->tif_base + off, sizeof(uint16_t)); } - off += sizeof(uint16_t); + off = (tmsize_t)((uint64_t)off + sizeof(uint16_t)); if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort(&dircount16); if (dircount16 > 4096) @@ -6123,9 +6223,9 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, else { uint64_t dircount64; - m = off + sizeof(uint64_t); - if ((m < off) || (m < (tmsize_t)sizeof(uint64_t)) || - (m > tif->tif_size)) + m = (tmsize_t)((uint64_t)off + sizeof(uint64_t)); + if ((m < off) || ((uint64_t)m < sizeof(uint64_t)) || + ((uint64_t)m > (uint64_t)tif->tif_size)) { TIFFErrorExtR(tif, module, "Can not read TIFF directory count"); return 0; @@ -6134,7 +6234,7 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, { _TIFFmemcpy(&dircount64, tif->tif_base + off, sizeof(uint64_t)); } - off += sizeof(uint64_t); + off = (tmsize_t)((uint64_t)off + sizeof(uint64_t)); if (tif->tif_flags & TIFF_SWAB) TIFFSwabLong8(&dircount64); if (dircount64 > 4096) @@ -6173,7 +6273,7 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, if (origdir == NULL) return 0; m = off + dircount16 * dirsize; - if ((m < off) || (m < (tmsize_t)(dircount16 * dirsize)) || + if ((m < off) || (m < (tmsize_t)dircount16 * (tmsize_t)dirsize) || (m > tif->tif_size)) { TIFFErrorExtR(tif, module, "Can not read TIFF directory"); @@ -6182,7 +6282,8 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, } else { - _TIFFmemcpy(origdir, tif->tif_base + off, dircount16 * dirsize); + _TIFFmemcpy(origdir, tif->tif_base + off, + (tmsize_t)dircount16 * dirsize); } if (nextdiroff) { @@ -6190,9 +6291,9 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, if (!(tif->tif_flags & TIFF_BIGTIFF)) { uint32_t nextdiroff32; - m = off + sizeof(uint32_t); - if ((m < off) || (m < (tmsize_t)sizeof(uint32_t)) || - (m > tif->tif_size)) + m = (tmsize_t)((uint64_t)off + sizeof(uint32_t)); + if ((m < off) || ((uint64_t)m < sizeof(uint32_t)) || + ((uint64_t)m > (uint64_t)tif->tif_size)) nextdiroff32 = 0; else _TIFFmemcpy(&nextdiroff32, tif->tif_base + off, @@ -6203,9 +6304,9 @@ static uint16_t TIFFFetchDirectory(TIFF *tif, uint64_t diroff, } else { - m = off + sizeof(uint64_t); - if ((m < off) || (m < (tmsize_t)sizeof(uint64_t)) || - (m > tif->tif_size)) + m = (tmsize_t)((uint64_t)off + sizeof(uint64_t)); + if ((m < off) || ((uint64_t)m < sizeof(uint64_t)) || + ((uint64_t)m > (uint64_t)tif->tif_size)) *nextdiroff = 0; else _TIFFmemcpy(nextdiroff, tif->tif_base + off, @@ -6351,8 +6452,8 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) /* TIFFReadDirEntryArrayWithLimit() ensures this can't be * larger than MAX_SIZE_TAG_DATA */ assert((uint32_t)dp->tdir_count + 1 == dp->tdir_count + 1); - uint8_t *o = - _TIFFmallocExt(tif, (uint32_t)dp->tdir_count + 1); + uint8_t *o = (uint8_t *)_TIFFmallocExt( + tif, (uint32_t)dp->tdir_count + 1); if (o == NULL) { if (data != NULL) @@ -6494,7 +6595,7 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) { if (!EvaluateIFDdatasizeReading(tif, dp)) return 0; - if (!TIFFSetField(tif, dp->tdir_tag, data)) + if (!TIFFSetField(tif, dp->tdir_tag, (double)data)) return (0); } } @@ -6906,6 +7007,41 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) } } break; + case TIFF_SETGET_C0_IFD8: + { + uint64_t *data; + assert(fip->field_readcount >= 1); + assert(fip->field_passcount == 0); + if (dp->tdir_count != (uint64_t)fip->field_readcount) + { + TIFFWarningExtR(tif, module, + "incorrect count for field \"%s\", expected " + "%d, got %" PRIu64, + fip->field_name, (int)fip->field_readcount, + dp->tdir_count); + return (0); + } + else + { + err = TIFFReadDirEntryIfd8Array(tif, dp, &data); + if (err == TIFFReadDirEntryErrOk) + { + if (!EvaluateIFDdatasizeReading(tif, dp)) + { + if (data != 0) + _TIFFfreeExt(tif, data); + return 0; + } + int m; + m = TIFFSetField(tif, dp->tdir_tag, data); + if (data != 0) + _TIFFfreeExt(tif, data); + if (!m) + return (0); + } + } + } + break; case TIFF_SETGET_C16_ASCII: { uint8_t *data; @@ -6934,8 +7070,8 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) "byte. Forcing it to be null", fip->field_name); /* Enlarge buffer and add terminating null. */ - uint8_t *o = - _TIFFmallocExt(tif, (uint32_t)dp->tdir_count + 1); + uint8_t *o = (uint8_t *)_TIFFmallocExt( + tif, (uint32_t)dp->tdir_count + 1); if (o == NULL) { if (data != NULL) @@ -7305,8 +7441,8 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) "in null byte. Forcing it to be null", fip->field_name); /* Enlarge buffer and add terminating null. */ - uint8_t *o = - _TIFFmallocExt(tif, (uint32_t)dp->tdir_count + 1); + uint8_t *o = (uint8_t *)_TIFFmallocExt( + tif, (uint32_t)dp->tdir_count + 1); if (o == NULL) { if (data != NULL) @@ -7620,6 +7756,11 @@ static int TIFFFetchNormalTag(TIFF *tif, TIFFDirEntry *dp, int recover) } } break; + case TIFF_SETGET_INT: + case TIFF_SETGET_C0_ASCII: + case TIFF_SETGET_OTHER: + assert(0); /* these should not arrive here */ + break; default: assert(0); /* we should never get here */ break; @@ -7696,9 +7837,10 @@ static int TIFFFetchStripThing(TIFF *tif, TIFFDirEntry *dir, uint32_t nstrips, } if (dir->tdir_count) _TIFFmemcpy(resizeddata, data, - (uint32_t)dir->tdir_count * sizeof(uint64_t)); + (tmsize_t)((size_t)dir->tdir_count * sizeof(uint64_t))); _TIFFmemset(resizeddata + (uint32_t)dir->tdir_count, 0, - (nstrips - (uint32_t)dir->tdir_count) * sizeof(uint64_t)); + (tmsize_t)((size_t)(nstrips - (uint32_t)dir->tdir_count) * + sizeof(uint64_t))); _TIFFfreeExt(tif, data); data = resizeddata; } @@ -7862,7 +8004,7 @@ static void allocChoppedUpStripArrays(TIFF *tif, uint32_t nstrips, */ static void ChopUpSingleUncompressedStrip(TIFF *tif) { - register TIFFDirectory *td = &tif->tif_dir; + TIFFDirectory *td = &tif->tif_dir; uint64_t bytecount; uint64_t offset; uint32_t rowblock; @@ -7877,7 +8019,7 @@ static void ChopUpSingleUncompressedStrip(TIFF *tif) /* later ( StripOffsets and StripByteCounts improperly filled) . */ if (bytecount == 0 && tif->tif_mode != O_RDONLY) return; - offset = TIFFGetStrileByteCount(tif, 0); + offset = TIFFGetStrileOffset(tif, 0); assert(td->td_planarconfig == PLANARCONFIG_CONTIG); if ((td->td_photometric == PHOTOMETRIC_YCBCR) && (!isUpSampled(tif))) rowblock = td->td_ycbcrsubsampling[1]; @@ -8028,7 +8170,7 @@ static void TryChopUpUncompressedBigTiff(TIFF *tif) TIFF_NOSANITIZE_UNSIGNED_INT_OVERFLOW static uint64_t _TIFFUnsanitizedAddUInt64AndInt(uint64_t a, int b) { - return a + b; + return a + (uint64_t)b; } /* Read the value of [Strip|Tile]Offset or [Strip|Tile]ByteCount around @@ -8107,7 +8249,7 @@ static int _TIFFPartialReadStripArray(TIFF *tif, TIFFDirEntry *dirent, panVals[strile] = 0; return 0; } - nOffset = nBaseOffset + sizeofval * strile; + nOffset = nBaseOffset + (uint64_t)sizeofval * (uint64_t)strile; nOffsetStartPage = (nOffset / IO_CACHE_PAGE_SIZE) * IO_CACHE_PAGE_SIZE; nOffsetEndPage = nOffsetStartPage + IO_CACHE_PAGE_SIZE; @@ -8115,7 +8257,7 @@ static int _TIFFPartialReadStripArray(TIFF *tif, TIFFDirEntry *dirent, nOffsetEndPage += IO_CACHE_PAGE_SIZE; #undef IO_CACHE_PAGE_SIZE - nLastStripOffset = nBaseOffset + arraySize * sizeofval; + nLastStripOffset = nBaseOffset + (uint64_t)arraySize * sizeofval; if (nLastStripOffset < nOffsetEndPage) nOffsetEndPage = nLastStripOffset; if (nOffsetStartPage >= nOffsetEndPage) @@ -8250,9 +8392,9 @@ static int _TIFFFetchStrileValue(TIFF *tif, uint32_t strile, } #endif offsetArray = (uint64_t *)(_TIFFreallocExt(tif, td->td_stripoffset_p, - nArraySize)); + (tmsize_t)nArraySize)); bytecountArray = (uint64_t *)(_TIFFreallocExt( - tif, td->td_stripbytecount_p, nArraySize)); + tif, td->td_stripbytecount_p, (tmsize_t)nArraySize)); if (offsetArray) td->td_stripoffset_p = offsetArray; if (bytecountArray) @@ -8286,7 +8428,7 @@ static int _TIFFFetchStrileValue(TIFF *tif, uint32_t strile, if (~((*parray)[strile]) == 0) { - if (!_TIFFPartialReadStripArray(tif, dirent, strile, *parray)) + if (!_TIFFPartialReadStripArray(tif, dirent, (int)strile, *parray)) { (*parray)[strile] = 0; return 0; @@ -8304,10 +8446,26 @@ static uint64_t _TIFFGetStrileOffsetOrByteCountValue(TIFF *tif, uint32_t strile, TIFFDirectory *td = &tif->tif_dir; if (pbErr) *pbErr = 0; + + /* Avoid the "dirent->tdir_count <= 4" code path for one of + * StripOffsets/StripByteCounts, and the other code path for the other one, + * which will lead to inconsistencies and potential out-of-bounds reads. + */ + if ((td->td_stripoffset_entry.tdir_count <= 4) != + (td->td_stripbytecount_entry.tdir_count <= 4)) + { + TIFFErrorExtR(tif, "_TIFFGetStrileOffsetOrByteCountValue", + "Inconsistent directory count between StripOffsets and " + "StripByteCounts"); + if (pbErr) + *pbErr = 1; + return 0; + } + if ((tif->tif_flags & TIFF_DEFERSTRILELOAD) && !(tif->tif_flags & TIFF_CHOPPEDUPARRAYS)) { - if (!(tif->tif_flags & TIFF_LAZYSTRILELOAD) || + if (!(tif->tif_flags & TIFF_LAZYSTRILELOAD_ASKED) || /* If the values may fit in the toff_long/toff_long8 member */ /* then use _TIFFFillStriles to simplify _TIFFFetchStrileValue */ dirent->tdir_count <= 4) @@ -8378,7 +8536,7 @@ int _TIFFFillStriles(TIFF *tif) { return _TIFFFillStrilesInternal(tif, 1); } static int _TIFFFillStrilesInternal(TIFF *tif, int loadStripByteCount) { - register TIFFDirectory *td = &tif->tif_dir; + TIFFDirectory *td = &tif->tif_dir; int return_value = 1; /* Do not do anything if TIFF_DEFERSTRILELOAD is not set */ @@ -8386,7 +8544,8 @@ static int _TIFFFillStrilesInternal(TIFF *tif, int loadStripByteCount) (tif->tif_flags & TIFF_CHOPPEDUPARRAYS) != 0) return 1; - if (tif->tif_flags & TIFF_LAZYSTRILELOAD) + if ((tif->tif_flags & TIFF_LAZYSTRILELOAD_ASKED) && + !(tif->tif_flags & TIFF_LAZYSTRILELOAD_DONE)) { /* In case of lazy loading, reload completely the arrays */ _TIFFfreeExt(tif, td->td_stripoffset_p); @@ -8394,7 +8553,7 @@ static int _TIFFFillStrilesInternal(TIFF *tif, int loadStripByteCount) td->td_stripoffset_p = NULL; td->td_stripbytecount_p = NULL; td->td_stripoffsetbyteallocsize = 0; - tif->tif_flags &= ~TIFF_LAZYSTRILELOAD; + tif->tif_flags |= TIFF_LAZYSTRILELOAD_DONE; } /* If stripoffset array is already loaded, exit with success */ diff --git a/contrib/libs/libtiff/tif_dirwrite.c b/contrib/libs/libtiff/tif_dirwrite.c index bc5ada0f2b5..5da087e2954 100644 --- a/contrib/libs/libtiff/tif_dirwrite.c +++ b/contrib/libs/libtiff/tif_dirwrite.c @@ -365,7 +365,7 @@ static int TIFFRewriteDirectorySec(TIFF *tif, int isimage, int imagedone, } if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort(&dircount); - (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12U, SEEK_SET); if (!ReadOK(tif, &nextnextdir, 4)) { TIFFErrorExtR(tif, module, "Error fetching directory link"); @@ -377,7 +377,7 @@ static int TIFFRewriteDirectorySec(TIFF *tif, int isimage, int imagedone, { uint32_t m; m = 0; - (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12, + (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12U, SEEK_SET); if (!WriteOK(tif, &m, 4)) { @@ -417,7 +417,6 @@ static int TIFFRewriteDirectorySec(TIFF *tif, int isimage, int imagedone, while (1) { uint64_t dircount64; - uint16_t dircount; uint64_t nextnextdir; if (!SeekOK(tif, nextdir) || !ReadOK(tif, &dircount64, 8)) @@ -435,8 +434,8 @@ static int TIFFRewriteDirectorySec(TIFF *tif, int isimage, int imagedone, "corrupt TIFF"); return (0); } - dircount = (uint16_t)dircount64; - (void)TIFFSeekFile(tif, nextdir + 8 + dircount * 20, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 8 + dircount64 * 20, + SEEK_SET); if (!ReadOK(tif, &nextnextdir, 8)) { TIFFErrorExtR(tif, module, "Error fetching directory link"); @@ -448,7 +447,7 @@ static int TIFFRewriteDirectorySec(TIFF *tif, int isimage, int imagedone, { uint64_t m; m = 0; - (void)TIFFSeekFile(tif, nextdir + 8 + dircount * 20, + (void)TIFFSeekFile(tif, nextdir + 8 + dircount64 * 20, SEEK_SET); if (!WriteOK(tif, &m, 8)) { @@ -592,24 +591,24 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, } if (TIFFFieldSet(tif, FIELD_RESOLUTION)) { - if (!TIFFWriteDirectoryTagRational(tif, &ndir, dir, - TIFFTAG_XRESOLUTION, - tif->tif_dir.td_xresolution)) + if (!TIFFWriteDirectoryTagRational( + tif, &ndir, dir, TIFFTAG_XRESOLUTION, + (double)tif->tif_dir.td_xresolution)) goto bad; - if (!TIFFWriteDirectoryTagRational(tif, &ndir, dir, - TIFFTAG_YRESOLUTION, - tif->tif_dir.td_yresolution)) + if (!TIFFWriteDirectoryTagRational( + tif, &ndir, dir, TIFFTAG_YRESOLUTION, + (double)tif->tif_dir.td_yresolution)) goto bad; } if (TIFFFieldSet(tif, FIELD_POSITION)) { - if (!TIFFWriteDirectoryTagRational(tif, &ndir, dir, - TIFFTAG_XPOSITION, - tif->tif_dir.td_xposition)) + if (!TIFFWriteDirectoryTagRational( + tif, &ndir, dir, TIFFTAG_XPOSITION, + (double)tif->tif_dir.td_xposition)) goto bad; - if (!TIFFWriteDirectoryTagRational(tif, &ndir, dir, - TIFFTAG_YPOSITION, - tif->tif_dir.td_yposition)) + if (!TIFFWriteDirectoryTagRational( + tif, &ndir, dir, TIFFTAG_YPOSITION, + (double)tif->tif_dir.td_yposition)) goto bad; } if (TIFFFieldSet(tif, FIELD_SUBFILETYPE)) @@ -712,6 +711,14 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, } if (TIFFFieldSet(tif, FIELD_STRIPBYTECOUNTS)) { + /* Check td_stripbytecount_p for NULL pointer (bug#749) */ + if (tif->tif_dir.td_stripbytecount_p == NULL) + { + TIFFErrorExtR( + tif, module, + "StripByteCount array is not set, pointer is NULL"); + goto bad; + } if (!isTiled(tif)) { if (!TIFFWriteDirectoryTagLongLong8Array( @@ -731,6 +738,14 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, } if (TIFFFieldSet(tif, FIELD_STRIPOFFSETS)) { + /* Check td_stripoffset_p for NULL pointer (bug#749) */ + if (tif->tif_dir.td_stripoffset_p == NULL) + { + TIFFErrorExtR( + tif, module, + "StripByteOffset array is not set, pointer is NULL"); + goto bad; + } if (!isTiled(tif)) { /* td_stripoffset_p might be NULL in an odd OJPEG case. See @@ -851,7 +866,8 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, { if (!TIFFWriteDirectoryTagAscii( tif, &ndir, dir, TIFFTAG_INKNAMES, - tif->tif_dir.td_inknameslen, tif->tif_dir.td_inknames)) + (uint32_t)tif->tif_dir.td_inknameslen, + tif->tif_dir.td_inknames)) goto bad; } if (TIFFFieldSet(tif, FIELD_NUMBEROFINKS)) @@ -932,10 +948,58 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, TIFFGetField(tif, o->field_tag, &pa, &pb); if (!TIFFWriteDirectoryTagUndefinedArray( tif, &ndir, dir, (uint16_t)o->field_tag, - pa, pb)) + pa, (uint8_t *)pb)) goto bad; } break; + case TIFF_SETGET_UNDEFINED: + case TIFF_SETGET_SINT8: + case TIFF_SETGET_SINT16: + case TIFF_SETGET_SINT32: + case TIFF_SETGET_UINT64: + case TIFF_SETGET_SINT64: + case TIFF_SETGET_FLOAT: + case TIFF_SETGET_DOUBLE: + case TIFF_SETGET_IFD8: + case TIFF_SETGET_INT: + case TIFF_SETGET_UINT16_PAIR: + case TIFF_SETGET_C0_ASCII: + case TIFF_SETGET_C0_UINT8: + case TIFF_SETGET_C0_SINT8: + case TIFF_SETGET_C0_UINT16: + case TIFF_SETGET_C0_SINT16: + case TIFF_SETGET_C0_UINT32: + case TIFF_SETGET_C0_SINT32: + case TIFF_SETGET_C0_UINT64: + case TIFF_SETGET_C0_SINT64: + case TIFF_SETGET_C0_FLOAT: + case TIFF_SETGET_C0_DOUBLE: + case TIFF_SETGET_C0_IFD8: + case TIFF_SETGET_C16_ASCII: + case TIFF_SETGET_C16_UINT8: + case TIFF_SETGET_C16_SINT8: + case TIFF_SETGET_C16_UINT16: + case TIFF_SETGET_C16_SINT16: + case TIFF_SETGET_C16_UINT32: + case TIFF_SETGET_C16_SINT32: + case TIFF_SETGET_C16_UINT64: + case TIFF_SETGET_C16_SINT64: + case TIFF_SETGET_C16_FLOAT: + case TIFF_SETGET_C16_DOUBLE: + case TIFF_SETGET_C16_IFD8: + case TIFF_SETGET_C32_ASCII: + case TIFF_SETGET_C32_SINT8: + case TIFF_SETGET_C32_UINT16: + case TIFF_SETGET_C32_SINT16: + case TIFF_SETGET_C32_UINT32: + case TIFF_SETGET_C32_SINT32: + case TIFF_SETGET_C32_UINT64: + case TIFF_SETGET_C32_SINT64: + case TIFF_SETGET_C32_FLOAT: + case TIFF_SETGET_C32_DOUBLE: + case TIFF_SETGET_C32_IFD8: + case TIFF_SETGET_UINT8: + case TIFF_SETGET_OTHER: default: TIFFErrorExtR( tif, module, @@ -954,67 +1018,67 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, { uint16_t tag = (uint16_t)tif->tif_dir.td_customValues[m].info->field_tag; - uint32_t count = tif->tif_dir.td_customValues[m].count; + uint32_t count = (uint32_t)tif->tif_dir.td_customValues[m].count; switch (tif->tif_dir.td_customValues[m].info->field_type) { case TIFF_ASCII: if (!TIFFWriteDirectoryTagAscii( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (char *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_UNDEFINED: if (!TIFFWriteDirectoryTagUndefinedArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint8_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_BYTE: if (!TIFFWriteDirectoryTagByteArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint8_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_SBYTE: if (!TIFFWriteDirectoryTagSbyteArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (int8_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_SHORT: if (!TIFFWriteDirectoryTagShortArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint16_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_SSHORT: if (!TIFFWriteDirectoryTagSshortArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (int16_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_LONG: if (!TIFFWriteDirectoryTagLongArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint32_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_SLONG: if (!TIFFWriteDirectoryTagSlongArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (int32_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_LONG8: if (!TIFFWriteDirectoryTagLong8Array( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint64_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_SLONG8: if (!TIFFWriteDirectoryTagSlong8Array( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (int64_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_RATIONAL: @@ -1029,7 +1093,8 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, { if (!TIFFWriteDirectoryTagRationalDoubleArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (double *)tif->tif_dir.td_customValues[m] + .value)) goto bad; } else @@ -1037,7 +1102,7 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, /*-- default should be tv_size == 4 */ if (!TIFFWriteDirectoryTagRationalArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (float *)tif->tif_dir.td_customValues[m].value)) goto bad; /*-- ToDo: After Testing, this should be removed and * tv_size==4 should be set as default. */ @@ -1064,7 +1129,8 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, { if (!TIFFWriteDirectoryTagSrationalDoubleArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (double *)tif->tif_dir.td_customValues[m] + .value)) goto bad; } else @@ -1072,7 +1138,7 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, /*-- default should be tv_size == 4 */ if (!TIFFWriteDirectoryTagSrationalArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (float *)tif->tif_dir.td_customValues[m].value)) goto bad; /*-- ToDo: After Testing, this should be removed and * tv_size==4 should be set as default. */ @@ -1090,27 +1156,28 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, case TIFF_FLOAT: if (!TIFFWriteDirectoryTagFloatArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (float *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_DOUBLE: if (!TIFFWriteDirectoryTagDoubleArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (double *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_IFD: if (!TIFFWriteDirectoryTagIfdArray( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint32_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; case TIFF_IFD8: if (!TIFFWriteDirectoryTagIfdIfd8Array( tif, &ndir, dir, tag, count, - tif->tif_dir.td_customValues[m].value)) + (uint64_t *)tif->tif_dir.td_customValues[m].value)) goto bad; break; + case TIFF_NOTYPE: default: assert(0); /* we should never get here */ break; @@ -1128,7 +1195,8 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, tif->tif_dir.td_dirdatasize_write += 8 + ndir * 20 + 8; /* Setup a new directory within first pass. */ - dir = _TIFFmallocExt(tif, ndir * sizeof(TIFFDirEntry)); + dir = (TIFFDirEntry *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)ndir * sizeof(TIFFDirEntry))); if (dir == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1238,7 +1306,7 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, uint8_t *n; uint32_t nTmp; TIFFDirEntry *o; - n = dirmem; + n = (uint8_t *)dirmem; *(uint16_t *)n = (uint16_t)ndir; if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort((uint16_t *)n); @@ -1274,7 +1342,7 @@ static int TIFFWriteDirectorySec(TIFF *tif, int isimage, int imagedone, { uint8_t *n; TIFFDirEntry *o; - n = dirmem; + n = (uint8_t *)dirmem; *(uint64_t *)n = ndir; if (tif->tif_flags & TIFF_SWAB) TIFFSwabLong8((uint64_t *)n); @@ -1405,7 +1473,7 @@ static int8_t TIFFClampDoubleToInt8(double val) { if (val > 127) return 127; - if (val < -128 || val != val) + if (val < -128 || isnan(val)) return -128; return (int8_t)val; } @@ -1414,7 +1482,7 @@ static int16_t TIFFClampDoubleToInt16(double val) { if (val > 32767) return 32767; - if (val < -32768 || val != val) + if (val < -32768 || isnan(val)) return -32768; return (int16_t)val; } @@ -1423,7 +1491,7 @@ static int32_t TIFFClampDoubleToInt32(double val) { if (val > 0x7FFFFFFF) return 0x7FFFFFFF; - if (val < -0x7FFFFFFF - 1 || val != val) + if (val < -0x7FFFFFFF - 1 || isnan(val)) return -0x7FFFFFFF - 1; return (int32_t)val; } @@ -1432,7 +1500,7 @@ static uint8_t TIFFClampDoubleToUInt8(double val) { if (val < 0) return 0; - if (val > 255 || val != val) + if (val > 255 || isnan(val)) return 255; return (uint8_t)val; } @@ -1441,7 +1509,7 @@ static uint16_t TIFFClampDoubleToUInt16(double val) { if (val < 0) return 0; - if (val > 65535 || val != val) + if (val > 65535 || isnan(val)) return 65535; return (uint16_t)val; } @@ -1450,7 +1518,7 @@ static uint32_t TIFFClampDoubleToUInt32(double val) { if (val < 0) return 0; - if (val > 0xFFFFFFFFU || val != val) + if (val > 0xFFFFFFFFU || isnan(val)) return 0xFFFFFFFFU; return (uint32_t)val; } @@ -1464,7 +1532,7 @@ static int TIFFWriteDirectoryTagSampleformatArray(TIFF *tif, uint32_t *ndir, void *conv; uint32_t i; int ok; - conv = _TIFFmallocExt(tif, count * sizeof(double)); + conv = _TIFFmallocExt(tif, (tmsize_t)((size_t)count * sizeof(double))); if (conv == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1603,7 +1671,9 @@ static int TIFFWriteDirectoryTagShortPerSample(TIFF *tif, uint32_t *ndir, return (TIFFWriteDirectoryTagCheckedShortArray( tif, ndir, dir, tag, tif->tif_dir.td_samplesperpixel, NULL)); } - m = _TIFFmallocExt(tif, tif->tif_dir.td_samplesperpixel * sizeof(uint16_t)); + m = (uint16_t *)_TIFFmallocExt( + tif, + (tmsize_t)((size_t)tif->tif_dir.td_samplesperpixel * sizeof(uint16_t))); if (m == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1664,14 +1734,6 @@ static int TIFFWriteDirectoryTagLong8Array(TIFF *tif, uint32_t *ndir, uint32_t *q; int o; - /* is this just a counting pass? */ - if (dir == NULL) - { - /* only evaluate IFD data size and inc. ndir */ - return (TIFFWriteDirectoryTagCheckedLong8Array(tif, ndir, dir, tag, - count, value)); - } - /* We always write Long8 for BigTIFF, no checking needed. */ if (tif->tif_flags & TIFF_BIGTIFF) return (TIFFWriteDirectoryTagCheckedLong8Array(tif, ndir, dir, tag, @@ -1681,7 +1743,8 @@ static int TIFFWriteDirectoryTagLong8Array(TIFF *tif, uint32_t *ndir, ** For classic tiff we want to verify everything is in range for long ** and convert to long format. */ - p = _TIFFmallocExt(tif, count * sizeof(uint32_t)); + p = (uint32_t *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)count * sizeof(uint32_t))); if (p == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1725,13 +1788,6 @@ static int TIFFWriteDirectoryTagSlong8Array(TIFF *tif, uint32_t *ndir, int32_t *q; int o; - /* is this just a counting pass? */ - if (dir == NULL) - { - /* only evaluate IFD data size and inc. ndir */ - return (TIFFWriteDirectoryTagCheckedSlong8Array(tif, ndir, dir, tag, - count, value)); - } /* We always write SLong8 for BigTIFF, no checking needed. */ if (tif->tif_flags & TIFF_BIGTIFF) return (TIFFWriteDirectoryTagCheckedSlong8Array(tif, ndir, dir, tag, @@ -1741,7 +1797,8 @@ static int TIFFWriteDirectoryTagSlong8Array(TIFF *tif, uint32_t *ndir, ** For classic tiff we want to verify everything is in range for signed-long ** and convert to signed-long format. */ - p = _TIFFmallocExt(tif, count * sizeof(uint32_t)); + p = (int32_t *)_TIFFmallocExt(tif, + (tmsize_t)((size_t)count * sizeof(uint32_t))); if (p == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1960,7 +2017,8 @@ static int TIFFWriteDirectoryTagLongLong8Array(TIFF *tif, uint32_t *ndir, ** and convert to long format. */ - uint32_t *p = _TIFFmallocExt(tif, count * sizeof(uint32_t)); + uint32_t *p = (uint32_t *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)count * sizeof(uint32_t))); uint32_t *q; uint64_t *ma; uint32_t mb; @@ -1990,7 +2048,8 @@ static int TIFFWriteDirectoryTagLongLong8Array(TIFF *tif, uint32_t *ndir, } else { - uint16_t *p = _TIFFmallocExt(tif, count * sizeof(uint16_t)); + uint16_t *p = (uint16_t *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)count * sizeof(uint16_t))); uint16_t *q; uint64_t *ma; uint32_t mb; @@ -2050,7 +2109,8 @@ static int TIFFWriteDirectoryTagIfdIfd8Array(TIFF *tif, uint32_t *ndir, ** and convert to long format. */ - p = _TIFFmallocExt(tif, count * sizeof(uint32_t)); + p = (uint32_t *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)count * sizeof(uint32_t))); if (p == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2101,26 +2161,45 @@ static int TIFFWriteDirectoryTagColormap(TIFF *tif, uint32_t *ndir, { static const char module[] = "TIFFWriteDirectoryTagColormap"; uint32_t m; + uint32_t count; + uint64_t count64; + tmsize_t total_values; + tmsize_t plane_bytes; uint16_t *n; int o; - m = (1 << tif->tif_dir.td_bitspersample); + if (tif->tif_dir.td_bitspersample >= 32) + { + TIFFErrorExtR(tif, module, "BitsPerSample too large for Colormap"); + return (0); + } + m = 1U << tif->tif_dir.td_bitspersample; + count64 = _TIFFMultiply64(tif, 3U, m, module); + if (count64 == 0) + return (0); + count = _TIFFCastUInt64ToUInt32(tif, count64, module); + total_values = _TIFFCastUInt64ToSSize(tif, count64, module); + plane_bytes = _TIFFCastUInt64ToSSize( + tif, _TIFFMultiply64(tif, m, sizeof(uint16_t), module), module); + if (count == 0 || total_values == 0 || plane_bytes == 0) + return (0); if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ { - EvaluateIFDdatasizeWrite(tif, 3 * m, sizeof(uint16_t), ndir); + EvaluateIFDdatasizeWrite(tif, count, sizeof(uint16_t), ndir); return 1; } - n = _TIFFmallocExt(tif, 3 * m * sizeof(uint16_t)); + n = (uint16_t *)_TIFFCheckMalloc(tif, total_values, sizeof(uint16_t), + module); if (n == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); return (0); } - _TIFFmemcpy(&n[0], tif->tif_dir.td_colormap[0], m * sizeof(uint16_t)); - _TIFFmemcpy(&n[m], tif->tif_dir.td_colormap[1], m * sizeof(uint16_t)); - _TIFFmemcpy(&n[2 * m], tif->tif_dir.td_colormap[2], m * sizeof(uint16_t)); + _TIFFmemcpy(&n[0], tif->tif_dir.td_colormap[0], plane_bytes); + _TIFFmemcpy(&n[m], tif->tif_dir.td_colormap[1], plane_bytes); + _TIFFmemcpy(&n[2 * m], tif->tif_dir.td_colormap[2], plane_bytes); o = TIFFWriteDirectoryTagCheckedShortArray(tif, ndir, dir, TIFFTAG_COLORMAP, - 3 * m, n); + count, n); _TIFFfreeExt(tif, n); return (o); } @@ -2130,13 +2209,27 @@ static int TIFFWriteDirectoryTagTransferfunction(TIFF *tif, uint32_t *ndir, { static const char module[] = "TIFFWriteDirectoryTagTransferfunction"; uint32_t m; + uint32_t count; + uint64_t count64; + tmsize_t total_values; + tmsize_t plane_bytes; uint16_t n; uint16_t *o; int p; /* TIFFTAG_TRANSFERFUNCTION expects (1 or 3) pointer to arrays with - * (1 << BitsPerSample) * uint16_t values. + * 2**BitsPerSample uint16_t values. */ - m = (1 << tif->tif_dir.td_bitspersample); + if (tif->tif_dir.td_bitspersample >= 32) + { + TIFFErrorExtR(tif, module, + "BitsPerSample too large for TransferFunction"); + return (0); + } + m = 1U << tif->tif_dir.td_bitspersample; + plane_bytes = _TIFFCastUInt64ToSSize( + tif, _TIFFMultiply64(tif, m, sizeof(uint16_t), module), module); + if (plane_bytes == 0) + return (0); /* clang-format off */ n = (tif->tif_dir.td_samplesperpixel - tif->tif_dir.td_extrasamples) > 1 ? 3 : 1; /* clang-format on */ @@ -2161,35 +2254,39 @@ static int TIFFWriteDirectoryTagTransferfunction(TIFF *tif, uint32_t *ndir, if (n == 3) { if (!_TIFFmemcmp(tif->tif_dir.td_transferfunction[0], - tif->tif_dir.td_transferfunction[2], - m * sizeof(uint16_t)) && + tif->tif_dir.td_transferfunction[2], plane_bytes) && !_TIFFmemcmp(tif->tif_dir.td_transferfunction[0], - tif->tif_dir.td_transferfunction[1], - m * sizeof(uint16_t))) + tif->tif_dir.td_transferfunction[1], plane_bytes)) n = 1; } + count64 = _TIFFMultiply64(tif, n, m, module); + if (count64 == 0) + return (0); + count = _TIFFCastUInt64ToUInt32(tif, count64, module); + total_values = _TIFFCastUInt64ToSSize(tif, count64, module); + if (count == 0 || total_values == 0) + return (0); if (dir == NULL) /* Just evaluate IFD data size and increment ndir. */ { - EvaluateIFDdatasizeWrite(tif, n * m, 2, ndir); + EvaluateIFDdatasizeWrite(tif, count, 2, ndir); return 1; } - o = _TIFFmallocExt(tif, n * m * sizeof(uint16_t)); + o = (uint16_t *)_TIFFCheckMalloc(tif, total_values, sizeof(uint16_t), + module); if (o == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); return (0); } - _TIFFmemcpy(&o[0], tif->tif_dir.td_transferfunction[0], - m * sizeof(uint16_t)); + _TIFFmemcpy(&o[0], tif->tif_dir.td_transferfunction[0], plane_bytes); if (n > 1) - _TIFFmemcpy(&o[m], tif->tif_dir.td_transferfunction[1], - m * sizeof(uint16_t)); + _TIFFmemcpy(&o[m], tif->tif_dir.td_transferfunction[1], plane_bytes); if (n > 2) _TIFFmemcpy(&o[2 * m], tif->tif_dir.td_transferfunction[2], - m * sizeof(uint16_t)); + plane_bytes); p = TIFFWriteDirectoryTagCheckedShortArray( - tif, ndir, dir, TIFFTAG_TRANSFERFUNCTION, n * m, o); + tif, ndir, dir, TIFFTAG_TRANSFERFUNCTION, count, o); _TIFFfreeExt(tif, o); return (p); } @@ -2209,7 +2306,9 @@ static int TIFFWriteDirectoryTagSubifd(TIFF *tif, uint32_t *ndir, uint64_t *pa; uint32_t *pb; uint16_t p; - o = _TIFFmallocExt(tif, tif->tif_dir.td_nsubifd * sizeof(uint32_t)); + o = (uint32_t *)_TIFFmallocExt( + tif, + (tmsize_t)((size_t)tif->tif_dir.td_nsubifd * sizeof(uint32_t))); if (o == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2494,7 +2593,7 @@ static int TIFFWriteDirectoryTagCheckedRational(TIFF *tif, uint32_t *ndir, TIFFErrorExtR(tif, module, "Negative value is illegal"); return 0; } - else if (value != value) + else if (isnan(value)) { TIFFErrorExtR(tif, module, "Not-a-number value is illegal"); return 0; @@ -2537,7 +2636,8 @@ static int TIFFWriteDirectoryTagCheckedRationalArray(TIFF *tif, uint32_t *ndir, EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(uint32_t), ndir); return 1; } - m = _TIFFmallocExt(tif, count * 2 * sizeof(uint32_t)); + m = (uint32_t *)_TIFFCheckMalloc(tif, count, 2 * sizeof(uint32_t), + "for rational array"); if (m == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2545,7 +2645,7 @@ static int TIFFWriteDirectoryTagCheckedRationalArray(TIFF *tif, uint32_t *ndir, } for (na = value, nb = m, nc = 0; nc < count; na++, nb += 2, nc++) { - DoubleToRational(*na, &nb[0], &nb[1]); + DoubleToRational((double)*na, &nb[0], &nb[1]); } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong(m, count * 2); @@ -2573,7 +2673,8 @@ static int TIFFWriteDirectoryTagCheckedSrationalArray(TIFF *tif, uint32_t *ndir, EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(int32_t), ndir); return 1; } - m = _TIFFmallocExt(tif, count * 2 * sizeof(int32_t)); + m = (int32_t *)_TIFFCheckMalloc(tif, count, 2 * sizeof(int32_t), + "for srational array"); if (m == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2581,7 +2682,7 @@ static int TIFFWriteDirectoryTagCheckedSrationalArray(TIFF *tif, uint32_t *ndir, } for (na = value, nb = m, nc = 0; nc < count; na++, nb += 2, nc++) { - DoubleToSrational(*na, &nb[0], &nb[1]); + DoubleToSrational((double)*na, &nb[0], &nb[1]); } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong((uint32_t *)m, count * 2); @@ -2610,7 +2711,8 @@ TIFFWriteDirectoryTagCheckedRationalDoubleArray(TIFF *tif, uint32_t *ndir, EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(uint32_t), ndir); return 1; } - m = _TIFFmallocExt(tif, count * 2 * sizeof(uint32_t)); + m = (uint32_t *)_TIFFCheckMalloc(tif, count, 2 * sizeof(uint32_t), + "for rational double array"); if (m == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2618,7 +2720,7 @@ TIFFWriteDirectoryTagCheckedRationalDoubleArray(TIFF *tif, uint32_t *ndir, } for (na = value, nb = m, nc = 0; nc < count; na++, nb += 2, nc++) { - DoubleToRational(*na, &nb[0], &nb[1]); + DoubleToRational((double)*na, &nb[0], &nb[1]); } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong(m, count * 2); @@ -2645,7 +2747,8 @@ static int TIFFWriteDirectoryTagCheckedSrationalDoubleArray( EvaluateIFDdatasizeWrite(tif, count * 2, sizeof(int32_t), ndir); return 1; } - m = _TIFFmallocExt(tif, count * 2 * sizeof(int32_t)); + m = (int32_t *)_TIFFCheckMalloc(tif, count, 2 * sizeof(int32_t), + "for srational double array"); if (m == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2653,7 +2756,7 @@ static int TIFFWriteDirectoryTagCheckedSrationalDoubleArray( } for (na = value, nb = m, nc = 0; nc < count; na++, nb += 2, nc++) { - DoubleToSrational(*na, &nb[0], &nb[1]); + DoubleToSrational((double)*na, &nb[0], &nb[1]); } if (tif->tif_flags & TIFF_SWAB) TIFFSwabArrayOfLong((uint32_t *)m, count * 2); @@ -2744,7 +2847,8 @@ static void ToRationalEuclideanGCD(double value, int blnUseSignedRange, *the double-value of it reaches an integer number without fractional part. */ bigDenom = 1; - while ((value != floor(value)) && (value < fMax) && (bigDenom < nMax)) + while ((!TIFF_DOUBLE_EQ(value, floor(value))) && (value < fMax) && + (bigDenom < nMax)) { bigDenom <<= 1; value *= 2; @@ -2836,7 +2940,7 @@ static void DoubleToRational(double value, uint32_t *num, uint32_t *denom) return; } /*-- Check for easy integer numbers -- */ - if (value == (uint32_t)(value)) + if (TIFF_DOUBLE_EQ(value, (double)(uint32_t)value)) { *num = (uint32_t)value; *denom = 1; @@ -2911,7 +3015,7 @@ static void DoubleToSrational(double value, int32_t *num, int32_t *denom) return; } /*-- Check for easy numbers -- */ - if (value == (int32_t)(value)) + if (TIFF_DOUBLE_EQ(value, (double)(int32_t)value)) { *num = (int32_t)(neg * value); *denom = 1; @@ -3231,6 +3335,14 @@ static int TIFFLinkDirectory(TIFF *tif) uint16_t dircount; uint32_t nextnextdir; + /* Update IDF loop list and check for IFD loop. + * ndir is IFD ID plus one. */ + if (!_TIFFCheckDirNumberAndOffset(tif, ndir - 1, nextdir)) + { + TIFFErrorExtR(tif, module, "Error IFD loop detected"); + return 0; /* bad offset (IFD looping or more than + TIFF_MAX_DIR_COUNT IFDs) */ + } if (!SeekOK(tif, nextdir) || !ReadOK(tif, &dircount, 2)) { TIFFErrorExtR(tif, module, "Error fetching directory count"); @@ -3238,7 +3350,7 @@ static int TIFFLinkDirectory(TIFF *tif) } if (tif->tif_flags & TIFF_SWAB) TIFFSwabShort(&dircount); - (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12U, SEEK_SET); if (!ReadOK(tif, &nextnextdir, 4)) { TIFFErrorExtR(tif, module, "Error fetching directory link"); @@ -3248,7 +3360,7 @@ static int TIFFLinkDirectory(TIFF *tif) TIFFSwabLong(&nextnextdir); if (nextnextdir == 0) { - (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 2 + dircount * 12U, SEEK_SET); if (!WriteOK(tif, &m, 4)) { TIFFErrorExtR(tif, module, "Error writing directory link"); @@ -3306,9 +3418,15 @@ static int TIFFLinkDirectory(TIFF *tif) while (1) { uint64_t dircount64; - uint16_t dircount; uint64_t nextnextdir; + /* Update IDF loop list and check for IFD loop. */ + if (!_TIFFCheckDirNumberAndOffset(tif, ndir - 1, nextdir)) + { + TIFFErrorExtR(tif, module, "Error IFD loop detected"); + return 0; /* bad offset (IFD looping or more than + TIFF_MAX_DIR_COUNT IFDs) */ + } if (!SeekOK(tif, nextdir) || !ReadOK(tif, &dircount64, 8)) { TIFFErrorExtR(tif, module, "Error fetching directory count"); @@ -3323,8 +3441,7 @@ static int TIFFLinkDirectory(TIFF *tif) "likely corrupt TIFF"); return (0); } - dircount = (uint16_t)dircount64; - (void)TIFFSeekFile(tif, nextdir + 8 + dircount * 20, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 8 + dircount64 * 20, SEEK_SET); if (!ReadOK(tif, &nextnextdir, 8)) { TIFFErrorExtR(tif, module, "Error fetching directory link"); @@ -3334,7 +3451,8 @@ static int TIFFLinkDirectory(TIFF *tif) TIFFSwabLong8(&nextnextdir); if (nextnextdir == 0) { - (void)TIFFSeekFile(tif, nextdir + 8 + dircount * 20, SEEK_SET); + (void)TIFFSeekFile(tif, nextdir + 8 + dircount64 * 20, + SEEK_SET); if (!WriteOK(tif, &m, 8)) { TIFFErrorExtR(tif, module, "Error writing directory link"); @@ -3475,7 +3593,7 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, if (entry_tag == tag) break; - read_offset += dirsize; + read_offset += (uint64_t)dirsize; } if (entry_tag != tag) @@ -3584,13 +3702,13 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, if (in_datatype == TIFF_LONG8 && (entry_type == TIFF_SHORT || entry_type == TIFF_LONG || entry_type == TIFF_LONG8)) - datatype = entry_type; + datatype = (TIFFDataType)entry_type; else if (in_datatype == TIFF_SLONG8 && (entry_type == TIFF_SLONG || entry_type == TIFF_SLONG8)) - datatype = entry_type; + datatype = (TIFFDataType)entry_type; else if (in_datatype == TIFF_IFD8 && (entry_type == TIFF_IFD || entry_type == TIFF_IFD8)) - datatype = entry_type; + datatype = (TIFFDataType)entry_type; else datatype = in_datatype; } @@ -3605,7 +3723,8 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, return 0; if (datatype == in_datatype) - memcpy(buf_to_write, data, count * TIFFDataWidth(datatype)); + memcpy(buf_to_write, data, + (size_t)count * (size_t)TIFFDataWidth(datatype)); else if (datatype == TIFF_SLONG && in_datatype == TIFF_SLONG8) { tmsize_t i; @@ -3698,8 +3817,8 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, tif->tif_dir.td_stripoffset_entry.tdir_type == 0 && tif->tif_dir.td_stripoffset_entry.tdir_offset.toff_long8 == 0) { - tif->tif_dir.td_stripoffset_entry.tdir_type = datatype; - tif->tif_dir.td_stripoffset_entry.tdir_count = count; + tif->tif_dir.td_stripoffset_entry.tdir_type = (uint16_t)datatype; + tif->tif_dir.td_stripoffset_entry.tdir_count = (uint64_t)count; } else if ((tag == TIFFTAG_TILEBYTECOUNTS || tag == TIFFTAG_STRIPBYTECOUNTS) && @@ -3707,8 +3826,8 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, tif->tif_dir.td_stripbytecount_entry.tdir_type == 0 && tif->tif_dir.td_stripbytecount_entry.tdir_offset.toff_long8 == 0) { - tif->tif_dir.td_stripbytecount_entry.tdir_type = datatype; - tif->tif_dir.td_stripbytecount_entry.tdir_count = count; + tif->tif_dir.td_stripbytecount_entry.tdir_type = (uint16_t)datatype; + tif->tif_dir.td_stripbytecount_entry.tdir_count = (uint64_t)count; } /* -------------------------------------------------------------------- */ @@ -3756,13 +3875,14 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, if (count * TIFFDataWidth(datatype) == 4) { uint32_t value; - memcpy(&value, buf_to_write, count * TIFFDataWidth(datatype)); + memcpy(&value, buf_to_write, + (size_t)count * (size_t)TIFFDataWidth(datatype)); entry_offset = value; } else { memcpy(&entry_offset, buf_to_write, - count * TIFFDataWidth(datatype)); + (size_t)count * (size_t)TIFFDataWidth(datatype)); } } @@ -3772,7 +3892,7 @@ int _TIFFRewriteField(TIFF *tif, uint16_t tag, TIFFDataType in_datatype, /* -------------------------------------------------------------------- */ /* Adjust the directory entry. */ /* -------------------------------------------------------------------- */ - entry_type = datatype; + entry_type = (uint16_t)datatype; entry_count = (uint64_t)count; memcpy(direntry_raw + 2, &entry_type, sizeof(uint16_t)); if (tif->tif_flags & TIFF_SWAB) diff --git a/contrib/libs/libtiff/tif_dumpmode.c b/contrib/libs/libtiff/tif_dumpmode.c index 267d5d2d7a5..4e3133f4192 100644 --- a/contrib/libs/libtiff/tif_dumpmode.c +++ b/contrib/libs/libtiff/tif_dumpmode.c @@ -80,7 +80,7 @@ static int DumpModeDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) "Not enough data for scanline %" PRIu32 ", expected a request for at most %" TIFF_SSIZE_FORMAT " bytes, got a request for %" TIFF_SSIZE_FORMAT " bytes", - tif->tif_row, tif->tif_rawcc, cc); + tif->tif_dir.td_row, tif->tif_rawcc, cc); return (0); } /* @@ -99,8 +99,23 @@ static int DumpModeDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) */ static int DumpModeSeek(TIFF *tif, uint32_t nrows) { - tif->tif_rawcp += nrows * tif->tif_scanlinesize; - tif->tif_rawcc -= nrows * tif->tif_scanlinesize; + tmsize_t seek_size; + if (nrows > 0 && + tif->tif_dir.td_scanlinesize > (tmsize_t)(TIFF_TMSIZE_T_MAX / nrows)) + { + TIFFErrorExtR(tif, "DumpModeSeek", + "Integer overflow computing seek size"); + return (0); + } + seek_size = (tmsize_t)nrows * tif->tif_dir.td_scanlinesize; + if (seek_size > tif->tif_rawcc) + { + TIFFErrorExtR(tif, "DumpModeSeek", + "Seek beyond end of raw data buffer"); + return (0); + } + tif->tif_rawcp += seek_size; + tif->tif_rawcc -= seek_size; return (1); } diff --git a/contrib/libs/libtiff/tif_fax3.c b/contrib/libs/libtiff/tif_fax3.c index 34e1c7882f7..06b9508df58 100644 --- a/contrib/libs/libtiff/tif_fax3.c +++ b/contrib/libs/libtiff/tif_fax3.c @@ -123,7 +123,7 @@ typedef struct static const char module[] = mod; \ Fax3CodecState *sp = DecoderState(tif); \ int a0; /* reference element */ \ - int lastx = sp->b.rowpixels; /* last element in row */ \ + int lastx = (int)sp->b.rowpixels; /* last element in row */ \ uint32_t BitAcc; /* bit accumulator */ \ int BitsAvail; /* # valid bits in BitAcc */ \ int RunLength; /* length of current run */ \ @@ -209,61 +209,68 @@ static int Fax3PreDecode(TIFF *tif, uint16_t s) static void Fax3Unexpected(const char *module, TIFF *tif, uint32_t line, uint32_t a0) { - TIFFErrorExtR(tif, module, - "Bad code word at line %" PRIu32 " of %s %" PRIu32 - " (x %" PRIu32 ")", - line, isTiled(tif) ? "tile" : "strip", - (isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip), a0); + TIFFErrorExtR( + tif, module, + "Bad code word at line %" PRIu32 " of %s %" PRIu32 " (x %" PRIu32 ")", + line, isTiled(tif) ? "tile" : "strip", + (isTiled(tif) ? tif->tif_dir.td_curtile : tif->tif_dir.td_curstrip), + a0); } #define unexpected(table, a0) \ do \ { \ - Fax3Unexpected(module, tif, sp->line, a0); \ + Fax3Unexpected(module, tif, (uint32_t)sp->line, (uint32_t)(a0)); \ ++sp->unexpectedReachedCount; \ } while (0) static void Fax3Extension(const char *module, TIFF *tif, uint32_t line, uint32_t a0) { - TIFFErrorExtR(tif, module, - "Uncompressed data (not supported) at line %" PRIu32 - " of %s %" PRIu32 " (x %" PRIu32 ")", - line, isTiled(tif) ? "tile" : "strip", - (isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip), a0); + TIFFErrorExtR( + tif, module, + "Uncompressed data (not supported) at line %" PRIu32 " of %s %" PRIu32 + " (x %" PRIu32 ")", + line, isTiled(tif) ? "tile" : "strip", + (isTiled(tif) ? tif->tif_dir.td_curtile : tif->tif_dir.td_curstrip), + a0); } -#define extension(a0) Fax3Extension(module, tif, sp->line, a0) +#define extension(a0) \ + Fax3Extension(module, tif, (uint32_t)sp->line, (uint32_t)(a0)) static void Fax3BadLength(const char *module, TIFF *tif, uint32_t line, uint32_t a0, uint32_t lastx) { - TIFFWarningExtR(tif, module, - "%s at line %" PRIu32 " of %s %" PRIu32 " (got %" PRIu32 - ", expected %" PRIu32 ")", - a0 < lastx ? "Premature EOL" : "Line length mismatch", line, - isTiled(tif) ? "tile" : "strip", - (isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip), a0, - lastx); + TIFFWarningExtR( + tif, module, + "%s at line %" PRIu32 " of %s %" PRIu32 " (got %" PRIu32 + ", expected %" PRIu32 ")", + a0 < lastx ? "Premature EOL" : "Line length mismatch", line, + isTiled(tif) ? "tile" : "strip", + (isTiled(tif) ? tif->tif_dir.td_curtile : tif->tif_dir.td_curstrip), a0, + lastx); } #define badlength(a0, lastx) \ do \ { \ - Fax3BadLength(module, tif, sp->line, a0, lastx); \ + Fax3BadLength(module, tif, (uint32_t)sp->line, (uint32_t)(a0), \ + (uint32_t)(lastx)); \ ++sp->eolReachedCount; \ } while (0) static void Fax3PrematureEOF(const char *module, TIFF *tif, uint32_t line, uint32_t a0) { - TIFFWarningExtR(tif, module, - "Premature EOF at line %" PRIu32 " of %s %" PRIu32 - " (x %" PRIu32 ")", - line, isTiled(tif) ? "tile" : "strip", - (isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip), a0); + TIFFWarningExtR( + tif, module, + "Premature EOF at line %" PRIu32 " of %s %" PRIu32 " (x %" PRIu32 ")", + line, isTiled(tif) ? "tile" : "strip", + (isTiled(tif) ? tif->tif_dir.td_curtile : tif->tif_dir.td_curstrip), + a0); } #define prematureEOF(a0) \ do \ { \ - Fax3PrematureEOF(module, tif, sp->line, a0); \ + Fax3PrematureEOF(module, tif, (uint32_t)sp->line, (uint32_t)(a0)); \ ++sp->eofReachedCount; \ } while (0) @@ -275,16 +282,15 @@ static void Fax3TryG3WithoutEOL(const char *module, TIFF *tif, uint32_t line, "Try to decode (read) fax Group 3 data without EOL at line %" PRIu32 " of %s %" PRIu32 " (x %" PRIu32 "). Please check result", line, isTiled(tif) ? "tile" : "strip", - (isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip), a0); + (isTiled(tif) ? tif->tif_dir.td_curtile : tif->tif_dir.td_curstrip), + a0); } #define tryG3WithoutEOL(a0) \ do \ { \ - Fax3TryG3WithoutEOL(module, tif, sp->line, a0); \ + Fax3TryG3WithoutEOL(module, tif, (uint32_t)sp->line, (uint32_t)(a0)); \ } while (0) -#define Nop - static int CheckReachedCounters(TIFF *tif, const char *module, Fax3CodecState *sp) { @@ -344,12 +350,12 @@ RETRY_WITHOUT_EOL_1D: pa = thisrun; #ifdef FAX3_DEBUG printf("\nBitAcc=%08" PRIX32 ", BitsAvail = %d\n", BitAcc, BitsAvail); - printf("-------------------- %" PRIu32 "\n", tif->tif_row); + printf("-------------------- %" PRIu32 "\n", tif->tif_dir.td_row); fflush(stdout); #endif SYNC_EOL(EOF1D, RETRY_WITHOUT_EOL_1D); EXPAND1D(EOF1Da); - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); buf += sp->b.rowbytes; occ -= sp->b.rowbytes; sp->line++; @@ -357,7 +363,7 @@ RETRY_WITHOUT_EOL_1D: EOF1D: /* premature EOF */ CLEANUP_RUNS(); EOF1Da: /* premature EOF */ - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); UNCACHE_STATE(tif, sp); return (-1); } @@ -404,16 +410,16 @@ RETRY_WITHOUT_EOL_2D: ClrBits(1); #ifdef FAX3_DEBUG printf(" %s\n-------------------- %" PRIu32 "\n", is1D ? "1D" : "2D", - tif->tif_row); + tif->tif_dir.td_row); fflush(stdout); #endif pb = sp->refruns; - b1 = *pb++; + b1 = (int)*pb++; if (is1D) EXPAND1D(EOF2Da); else EXPAND2D(EOF2Da); - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); if (pa < thisrun + sp->nruns) { SETVALUE(0); /* imaginary change for reference */ @@ -426,7 +432,7 @@ RETRY_WITHOUT_EOL_2D: EOF2D: /* premature EOF */ CLEANUP_RUNS(); EOF2Da: /* premature EOF */ - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); UNCACHE_STATE(tif, sp); return (-1); } @@ -479,12 +485,12 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, { if (bx) { /* align to byte boundary */ - *cp++ &= 0xff << (8 - bx); + *cp++ &= (unsigned char)(0xff << (8 - bx)); run -= 8 - bx; } - if ((n = run >> 3) != 0) + if ((n = (int32_t)(run >> 3)) != 0) { /* multiple bytes to fill */ - if ((n / sizeof(int64_t)) > 1) + if (((size_t)n / sizeof(int64_t)) > 1) { /* * Align to int64_tword boundary and fill. @@ -492,8 +498,8 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, for (; n && !isAligned(cp, int64_t); n--) *cp++ = 0x00; lp = (int64_t *)cp; - nw = (int32_t)(n / sizeof(int64_t)); - n -= nw * sizeof(int64_t); + nw = (int32_t)((size_t)n / sizeof(int64_t)); + n -= (int32_t)((size_t)nw * sizeof(int64_t)); do { *lp++ = 0L; @@ -504,10 +510,10 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, run &= 7; } if (run) - cp[0] &= 0xff >> run; + cp[0] &= (unsigned char)(0xff >> run); } else - cp[0] &= ~(_fillmasks[run] >> bx); + cp[0] &= (unsigned char)~(_fillmasks[run] >> bx); x += runs[0]; } run = runs[1]; @@ -521,12 +527,12 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, { if (bx) { /* align to byte boundary */ - *cp++ |= 0xff >> bx; + *cp++ |= (unsigned char)(0xff >> bx); run -= 8 - bx; } - if ((n = run >> 3) != 0) + if ((n = (int32_t)(run >> 3)) != 0) { /* multiple bytes to fill */ - if ((n / sizeof(int64_t)) > 1) + if (((size_t)n / sizeof(int64_t)) > 1) { /* * Align to int64_t boundary and fill. @@ -534,8 +540,8 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, for (; n && !isAligned(cp, int64_t); n--) *cp++ = 0xff; lp = (int64_t *)cp; - nw = (int32_t)(n / sizeof(int64_t)); - n -= nw * sizeof(int64_t); + nw = (int32_t)((size_t)n / sizeof(int64_t)); + n -= (int32_t)((size_t)nw * sizeof(int64_t)); do { *lp++ = -1L; @@ -550,7 +556,7 @@ void _TIFFFax3fillruns(unsigned char *buf, uint32_t *runs, uint32_t *erun, cp[0] = (unsigned char)((cp[0] | (0xff00 >> run)) & 0xff); } else - cp[0] |= _fillmasks[run] >> bx; + cp[0] |= (unsigned char)(_fillmasks[run] >> bx); x += runs[1]; } } @@ -726,12 +732,13 @@ static const int _msbmask[9] = {0x00, 0x01, 0x03, 0x07, 0x0f, { \ while (length > bit) \ { \ - data |= bits >> (length - bit); \ + data |= (int)((unsigned int)bits >> (length - bit)); \ length -= bit; \ _FlushBits(tif); \ } \ assert(length < 9); \ - data |= (bits & _msbmask[length]) << (bit - length); \ + data |= (int)(((unsigned int)bits & (unsigned int)_msbmask[length]) \ + << (unsigned int)(bit - length)); \ bit -= length; \ if (bit == 0) \ _FlushBits(tif); \ @@ -745,13 +752,13 @@ static const int _msbmask[9] = {0x00, 0x01, 0x03, 0x07, 0x0f, static int Fax3PutBits(TIFF *tif, unsigned int bits, unsigned int length) { Fax3CodecState *sp = EncoderState(tif); - unsigned int bit = sp->bit; - int data = sp->data; + unsigned int bit = (unsigned int)sp->bit; + int data = (int)sp->data; _PutBits(tif, bits, length); - sp->data = data; - sp->bit = bit; + sp->data = (uint32_t)data; + sp->bit = (int)bit; return 1; } @@ -782,8 +789,8 @@ static int Fax3PutBits(TIFF *tif, unsigned int bits, unsigned int length) static int putspan(TIFF *tif, int32_t span, const tableentry *tab) { Fax3CodecState *sp = EncoderState(tif); - unsigned int bit = sp->bit; - int data = sp->data; + unsigned int bit = (unsigned int)sp->bit; + int data = (int)sp->data; unsigned int code, length; while (span >= 2624) @@ -816,8 +823,8 @@ static int putspan(TIFF *tif, int32_t span, const tableentry *tab) #endif _PutBits(tif, code, length); - sp->data = data; - sp->bit = bit; + sp->data = (uint32_t)data; + sp->bit = (int)bit; return 1; } @@ -831,8 +838,8 @@ static int putspan(TIFF *tif, int32_t span, const tableentry *tab) static int Fax3PutEOL(TIFF *tif) { Fax3CodecState *sp = EncoderState(tif); - unsigned int bit = sp->bit; - int data = sp->data; + unsigned int bit = (unsigned int)sp->bit; + int data = (int)sp->data; unsigned int code, length, tparm; if (sp->b.groupoptions & GROUP3OPT_FILLBITS) @@ -849,7 +856,7 @@ static int Fax3PutEOL(TIFF *tif) align = sp->bit + (8 - align); else align = sp->bit - align; - tparm = align; + tparm = (unsigned int)align; _PutBits(tif, 0, tparm); } } @@ -862,8 +869,8 @@ static int Fax3PutEOL(TIFF *tif) } _PutBits(tif, code, length); - sp->data = data; - sp->bit = bit; + sp->data = (uint32_t)data; + sp->bit = (int)bit; return 1; } @@ -991,10 +998,10 @@ static inline int32_t find0span(unsigned char *bp, int32_t bs, int32_t be) bp++; } lp = (int64_t *)bp; - while ((bits >= (int32_t)(8 * sizeof(int64_t))) && (0 == *lp)) + while ((bits >= (int32_t)(size_t)(8 * sizeof(int64_t))) && (0 == *lp)) { - span += 8 * sizeof(int64_t); - bits -= 8 * sizeof(int64_t); + span += (int32_t)(size_t)(8 * sizeof(int64_t)); + bits -= (int32_t)(size_t)(8 * sizeof(int64_t)); lp++; } bp = (unsigned char *)lp; @@ -1059,11 +1066,11 @@ static inline int32_t find1span(unsigned char *bp, int32_t bs, int32_t be) bp++; } lp = (int64_t *)bp; - while ((bits >= (int32_t)(8 * sizeof(int64_t))) && + while ((bits >= (int32_t)(size_t)(8 * sizeof(int64_t))) && (~((uint64_t)0) == (uint64_t)*lp)) { - span += 8 * sizeof(int64_t); - bits -= 8 * sizeof(int64_t); + span += (int32_t)(size_t)(8 * sizeof(int64_t)); + bits -= (int32_t)(size_t)(8 * sizeof(int64_t)); lp++; } bp = (unsigned char *)lp; @@ -1118,16 +1125,16 @@ static int Fax3Encode1DRow(TIFF *tif, unsigned char *bp, uint32_t bits) for (;;) { - span = find0span(bp, bs, bits); /* white span */ + span = find0span(bp, (int32_t)bs, (int32_t)bits); /* white span */ if (!putspan(tif, span, TIFFFaxWhiteCodes)) return 0; - bs += span; + bs += (uint32_t)span; if (bs >= bits) break; - span = find1span(bp, bs, bits); /* black span */ + span = find1span(bp, (int32_t)bs, (int32_t)bits); /* black span */ if (!putspan(tif, span, TIFFFaxBlackCodes)) return 0; - bs += span; + bs += (uint32_t)span; if (bs >= bits) break; } @@ -1161,15 +1168,20 @@ static const tableentry vcodes[7] = { static int Fax3Encode2DRow(TIFF *tif, unsigned char *bp, unsigned char *rp, uint32_t bits) { -#define PIXEL(buf, ix) ((((buf)[(ix) >> 3]) >> (7 - ((ix)&7))) & 1) +#define PIXEL(buf, ix) ((((buf)[(ix) >> 3]) >> (7 - ((ix) & 7))) & 1) uint32_t a0 = 0; - uint32_t a1 = (PIXEL(bp, 0) != 0 ? 0 : finddiff(bp, 0, bits, 0)); - uint32_t b1 = (PIXEL(rp, 0) != 0 ? 0 : finddiff(rp, 0, bits, 0)); + uint32_t a1 = (PIXEL(bp, 0) != 0 + ? 0 + : (uint32_t)finddiff(bp, (int32_t)0, (int32_t)bits, 0)); + uint32_t b1 = (PIXEL(rp, 0) != 0 + ? 0 + : (uint32_t)finddiff(rp, (int32_t)0, (int32_t)bits, 0)); uint32_t a2, b2; for (;;) { - b2 = finddiff2(rp, b1, bits, PIXEL(rp, b1)); + b2 = (uint32_t)finddiff2(rp, (int32_t)b1, (int32_t)bits, + (int32_t)PIXEL(rp, b1)); if (b2 >= a1) { /* Naive computation triggers @@ -1182,21 +1194,22 @@ static int Fax3Encode2DRow(TIFF *tif, unsigned char *bp, unsigned char *rp, : 0x7FFFFFFF; if (!(-3 <= d && d <= 3)) { /* horizontal mode */ - a2 = finddiff2(bp, a1, bits, PIXEL(bp, a1)); + a2 = (uint32_t)finddiff2(bp, (int32_t)a1, (int32_t)bits, + (int32_t)PIXEL(bp, a1)); if (!putcode(tif, &horizcode)) return 0; if (a0 + a1 == 0 || PIXEL(bp, a0) == 0) { - if (!putspan(tif, a1 - a0, TIFFFaxWhiteCodes)) + if (!putspan(tif, (int32_t)(a1 - a0), TIFFFaxWhiteCodes)) return 0; - if (!putspan(tif, a2 - a1, TIFFFaxBlackCodes)) + if (!putspan(tif, (int32_t)(a2 - a1), TIFFFaxBlackCodes)) return 0; } else { - if (!putspan(tif, a1 - a0, TIFFFaxBlackCodes)) + if (!putspan(tif, (int32_t)(a1 - a0), TIFFFaxBlackCodes)) return 0; - if (!putspan(tif, a2 - a1, TIFFFaxWhiteCodes)) + if (!putspan(tif, (int32_t)(a2 - a1), TIFFFaxWhiteCodes)) return 0; } a0 = a2; @@ -1216,9 +1229,12 @@ static int Fax3Encode2DRow(TIFF *tif, unsigned char *bp, unsigned char *rp, } if (a0 >= bits) break; - a1 = finddiff(bp, a0, bits, PIXEL(bp, a0)); - b1 = finddiff(rp, a0, bits, !PIXEL(bp, a0)); - b1 = finddiff(rp, b1, bits, PIXEL(bp, a0)); + a1 = (uint32_t)finddiff(bp, (int32_t)a0, (int32_t)bits, + (int32_t)PIXEL(bp, a0)); + b1 = (uint32_t)finddiff(rp, (int32_t)a0, (int32_t)bits, + (int32_t)!PIXEL(bp, a0)); + b1 = (uint32_t)finddiff(rp, (int32_t)b1, (int32_t)bits, + (int32_t)PIXEL(bp, a0)); } return (1); #undef PIXEL @@ -1487,6 +1503,8 @@ static void Fax3PrintDir(TIFF *tif, FILE *fd, long flags) case CLEANFAXDATA_UNCLEAN: fprintf(fd, " uncorrected errors"); break; + default: + break; } fprintf(fd, " (%" PRIu16 " = 0x%" PRIx16 ")\n", sp->cleanfaxdata, sp->cleanfaxdata); @@ -1500,6 +1518,22 @@ static void Fax3PrintDir(TIFF *tif, FILE *fd, long flags) (*sp->printdir)(tif, fd, flags); } +static uint64_t Fax3GetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + /* 1024x1024: 36 */ + /* 4096x4096: 100 */ + /* 16383x16383: 163 */ + /* 65536x65536: 200 */ + /* 200000x200000: 208 */ + + return 250; +} + static int InitCCITTFax3(TIFF *tif) { static const char module[] = "InitCCITTFax3"; @@ -1564,6 +1598,7 @@ static int InitCCITTFax3(TIFF *tif) tif->tif_encodetile = Fax3Encode; tif->tif_close = Fax3Close; tif->tif_cleanup = Fax3Cleanup; + tif->tif_getmaxcompressionratio = Fax3GetMaxCompressionRatio; return (1); } @@ -1626,10 +1661,10 @@ static int Fax4Decode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) RunLength = 0; pa = thisrun = sp->curruns; pb = sp->refruns; - b1 = *pb++; + b1 = (int)*pb++; #ifdef FAX3_DEBUG printf("\nBitAcc=%08" PRIX32 ", BitsAvail = %d\n", BitAcc, BitsAvail); - printf("-------------------- %d\n", tif->tif_row); + printf("-------------------- %d\n", tif->tif_dir.td_row); fflush(stdout); #endif EXPAND2D(EOFG4); @@ -1643,7 +1678,7 @@ static int Fax4Decode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) occ, lastx); return -1; } - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); SETVALUE(0); /* imaginary change for reference */ SWAP(uint32_t *, sp->curruns, sp->refruns); buf += sp->b.rowbytes; @@ -1666,7 +1701,7 @@ static int Fax4Decode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) occ, lastx); return -1; } - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); UNCACHE_STATE(tif, sp); return (sp->line != start ? 1 @@ -1713,6 +1748,16 @@ static int Fax4PostEncode(TIFF *tif) return (1); } +static uint64_t Fax4GetMaxCompressionRatio(TIFF *tif) +{ + /* FAX4 can compress up to almost one byte per line, so the compression + * ratio can be up to the tile/strip width. + * See README_for_libtiff_developpers.md for raw data + */ + return isTiled(tif) ? tif->tif_dir.td_tilewidth + : tif->tif_dir.td_imagewidth; +} + int TIFFInitCCITTFax4(TIFF *tif, int scheme) { (void)scheme; @@ -1735,6 +1780,7 @@ int TIFFInitCCITTFax4(TIFF *tif, int scheme) tif->tif_encodestrip = Fax4Encode; tif->tif_encodetile = Fax4Encode; tif->tif_postencode = Fax4PostEncode; + tif->tif_getmaxcompressionratio = Fax4GetMaxCompressionRatio; /* * Suppress RTC at the end of each strip. */ @@ -1773,11 +1819,11 @@ static int Fax3DecodeRLE(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) pa = thisrun; #ifdef FAX3_DEBUG printf("\nBitAcc=%08" PRIX32 ", BitsAvail = %d\n", BitAcc, BitsAvail); - printf("-------------------- %" PRIu32 "\n", tif->tif_row); + printf("-------------------- %" PRIu32 "\n", tif->tif_dir.td_row); fflush(stdout); #endif EXPAND1D(EOFRLE); - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); /* * Cleanup at the end of the row. */ @@ -1798,7 +1844,7 @@ static int Fax3DecodeRLE(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) sp->line++; continue; EOFRLE: /* premature EOF */ - (*sp->fill)(buf, thisrun, pa, lastx); + (*sp->fill)(buf, thisrun, pa, (uint32_t)lastx); UNCACHE_STATE(tif, sp); return (-1); } @@ -1806,6 +1852,21 @@ static int Fax3DecodeRLE(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) return (1); } +static uint64_t Fax3RLEGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + /* 1024x1024: 43 */ + /* 4096x4096: 128 */ + /* 16383x16383: 171 */ + /* 65536x65536: 205 */ + /* 200000x200000: 211 */ + + return 250; +} + int TIFFInitCCITTRLE(TIFF *tif, int scheme) { (void)scheme; @@ -1814,6 +1875,7 @@ int TIFFInitCCITTRLE(TIFF *tif, int scheme) tif->tif_decoderow = Fax3DecodeRLE; tif->tif_decodestrip = Fax3DecodeRLE; tif->tif_decodetile = Fax3DecodeRLE; + tif->tif_getmaxcompressionratio = Fax3RLEGetMaxCompressionRatio; /* * Suppress RTC+EOLs when encoding and byte-align data. */ @@ -1832,6 +1894,7 @@ int TIFFInitCCITTRLEW(TIFF *tif, int scheme) tif->tif_decoderow = Fax3DecodeRLE; tif->tif_decodestrip = Fax3DecodeRLE; tif->tif_decodetile = Fax3DecodeRLE; + tif->tif_getmaxcompressionratio = Fax3RLEGetMaxCompressionRatio; /* * Suppress RTC+EOLs when encoding and word-align data. */ diff --git a/contrib/libs/libtiff/tif_fax3.h b/contrib/libs/libtiff/tif_fax3.h index c3100ce2c6c..60d709c7e5c 100644 --- a/contrib/libs/libtiff/tif_fax3.h +++ b/contrib/libs/libtiff/tif_fax3.h @@ -267,14 +267,14 @@ static const char *StateNames[] = { { \ if (pa >= thisrun + sp->nruns) \ { \ - TIFFErrorExtR(tif, module, "Buffer overflow at line %u of %s %u", \ + TIFFErrorExtR(tif, module, "Buffer overflow at line %d of %s %u", \ sp->line, isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile \ - : tif->tif_curstrip); \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ - *pa++ = RunLength + (x); \ - a0 += (x); \ + *pa++ = (uint32_t)((uint32_t)RunLength + (uint32_t)(x)); \ + a0 += (int)(uint32_t)(x); \ RunLength = 0; \ } while (0) #endif @@ -334,20 +334,20 @@ static const char *StateNames[] = { { \ if (RunLength) \ SETVALUE(0); \ - if (a0 != lastx) \ + if (a0 != (int)lastx) \ { \ badlength(a0, lastx); \ - while (a0 > lastx && pa > thisrun) \ - a0 -= *--pa; \ - if (a0 < lastx) \ + while (a0 > (int)lastx && pa > thisrun) \ + a0 -= (int)*--pa; \ + if (a0 < (int)lastx) \ { \ if (a0 < 0) \ a0 = 0; \ if ((pa - thisrun) & 1) \ SETVALUE(0); \ - SETVALUE(lastx - a0); \ + SETVALUE((uint32_t)((int)lastx - a0)); \ } \ - else if (a0 > lastx) \ + else if (a0 > (int)lastx) \ { \ SETVALUE(lastx); \ SETVALUE(0); \ @@ -385,8 +385,9 @@ static const char *StateNames[] = { goto doneWhite1d; \ case S_MakeUpW: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 = (int)((uint32_t)a0 + TabEnt->Param); \ + RunLength = \ + (int)((uint32_t)RunLength + TabEnt->Param); \ break; \ default: \ unexpected("WhiteTable", a0); \ @@ -394,7 +395,7 @@ static const char *StateNames[] = { } \ } \ doneWhite1d: \ - if (a0 >= lastx) \ + if (a0 >= (int)lastx) \ goto done1d; \ for (;;) \ { \ @@ -409,8 +410,8 @@ static const char *StateNames[] = { goto doneBlack1d; \ case S_MakeUpB: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 += (int)TabEnt->Param; \ + RunLength += (int)TabEnt->Param; \ break; \ default: \ unexpected("BlackTable", a0); \ @@ -418,7 +419,7 @@ static const char *StateNames[] = { } \ } \ doneBlack1d: \ - if (a0 >= lastx) \ + if (a0 >= (int)lastx) \ goto done1d; \ if (*(pa - 1) == 0 && *(pa - 2) == 0) \ pa -= 2; \ @@ -439,17 +440,18 @@ static const char *StateNames[] = { do \ { \ if (pa != thisrun) \ - while (b1 <= a0 && b1 < lastx) \ + while (b1 <= a0 && b1 < (int)lastx) \ { \ if (pb + 1 >= sp->refruns + sp->nruns) \ { \ - TIFFErrorExtR( \ - tif, module, "Buffer overflow at line %u of %s %u", \ - sp->line, isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip); \ + TIFFErrorExtR(tif, module, \ + "Buffer overflow at line %d of %s %u", \ + sp->line, isTiled(tif) ? "tile" : "strip", \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ - b1 += pb[0] + pb[1]; \ + b1 += (int)(pb[0] + pb[1]); \ pb += 2; \ } \ } while (0) @@ -460,14 +462,15 @@ static const char *StateNames[] = { #define EXPAND2D(eoflab) \ do \ { \ - while (a0 < lastx) \ + while (a0 < (int)lastx) \ { \ if (pa >= thisrun + sp->nruns) \ { \ - TIFFErrorExtR( \ - tif, module, "Buffer overflow at line %u of %s %u", \ - sp->line, isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip); \ + TIFFErrorExtR(tif, module, \ + "Buffer overflow at line %d of %s %u", sp->line, \ + isTiled(tif) ? "tile" : "strip", \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ LOOKUP8(7, TIFFFaxMainTable, eof2d); \ @@ -477,18 +480,19 @@ static const char *StateNames[] = { CHECK_b1; \ if (pb + 1 >= sp->refruns + sp->nruns) \ { \ - TIFFErrorExtR(tif, module, \ - "Buffer overflow at line %u of %s %u", \ - sp->line, \ - isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile \ - : tif->tif_curstrip); \ + TIFFErrorExtR( \ + tif, module, \ + "Buffer overflow at line %d of %s %u", sp->line, \ + isTiled(tif) ? "tile" : "strip", \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ - b1 += *pb++; \ - RunLength += b1 - a0; \ + b1 = b1 + (int)*pb++; \ + RunLength = \ + (int)((uint32_t)RunLength + (uint32_t)(b1 - a0)); \ a0 = b1; \ - b1 += *pb++; \ + b1 = b1 + (int)*pb++; \ break; \ case S_Horiz: \ if ((pa - thisrun) & 1) \ @@ -503,8 +507,9 @@ static const char *StateNames[] = { goto doneWhite2da; \ case S_MakeUpB: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 = (int)((uint32_t)a0 + TabEnt->Param); \ + RunLength = (int)((uint32_t)RunLength + \ + TabEnt->Param); \ break; \ default: \ goto badBlack2d; \ @@ -521,8 +526,9 @@ static const char *StateNames[] = { goto doneBlack2da; \ case S_MakeUpW: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 = (int)((uint32_t)a0 + TabEnt->Param); \ + RunLength = (int)((uint32_t)RunLength + \ + TabEnt->Param); \ break; \ default: \ goto badWhite2d; \ @@ -542,8 +548,9 @@ static const char *StateNames[] = { goto doneWhite2db; \ case S_MakeUpW: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 = (int)((uint32_t)a0 + TabEnt->Param); \ + RunLength = (int)((uint32_t)RunLength + \ + TabEnt->Param); \ break; \ default: \ goto badWhite2d; \ @@ -560,8 +567,9 @@ static const char *StateNames[] = { goto doneBlack2db; \ case S_MakeUpB: \ case S_MakeUp: \ - a0 += TabEnt->Param; \ - RunLength += TabEnt->Param; \ + a0 = (int)((uint32_t)a0 + TabEnt->Param); \ + RunLength = (int)((uint32_t)RunLength + \ + TabEnt->Param); \ break; \ default: \ goto badBlack2d; \ @@ -576,47 +584,47 @@ static const char *StateNames[] = { SETVALUE(b1 - a0); \ if (pb >= sp->refruns + sp->nruns) \ { \ - TIFFErrorExtR(tif, module, \ - "Buffer overflow at line %u of %s %u", \ - sp->line, \ - isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile \ - : tif->tif_curstrip); \ + TIFFErrorExtR( \ + tif, module, \ + "Buffer overflow at line %d of %s %u", sp->line, \ + isTiled(tif) ? "tile" : "strip", \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ - b1 += *pb++; \ + b1 = b1 + (int)*pb++; \ break; \ case S_VR: \ CHECK_b1; \ - SETVALUE(b1 - a0 + TabEnt->Param); \ + SETVALUE((int)((uint32_t)(b1 - a0) + TabEnt->Param)); \ if (pb >= sp->refruns + sp->nruns) \ { \ - TIFFErrorExtR(tif, module, \ - "Buffer overflow at line %u of %s %u", \ - sp->line, \ - isTiled(tif) ? "tile" : "strip", \ - isTiled(tif) ? tif->tif_curtile \ - : tif->tif_curstrip); \ + TIFFErrorExtR( \ + tif, module, \ + "Buffer overflow at line %d of %s %u", sp->line, \ + isTiled(tif) ? "tile" : "strip", \ + isTiled(tif) ? tif->tif_dir.td_curtile \ + : tif->tif_dir.td_curstrip); \ return (-1); \ } \ - b1 += *pb++; \ + b1 = b1 + (int)*pb++; \ break; \ case S_VL: \ CHECK_b1; \ - if (b1 < (int)(a0 + TabEnt->Param)) \ + if (b1 < (int)((uint32_t)a0 + TabEnt->Param)) \ { \ unexpected("VL", a0); \ goto eol2d; \ } \ - SETVALUE(b1 - a0 - TabEnt->Param); \ - b1 -= *--pb; \ + SETVALUE((int)((uint32_t)(b1 - a0) - TabEnt->Param)); \ + b1 = b1 - (int)*--pb; \ break; \ case S_Ext: \ - *pa++ = lastx - a0; \ + *pa++ = (uint32_t)((int)lastx - a0); \ extension(a0); \ goto eol2d; \ case S_EOL: \ - *pa++ = lastx - a0; \ + *pa++ = (uint32_t)((int)lastx - a0); \ NeedBits8(4, eof2d); \ if (GetBits(4)) \ unexpected("EOL", a0); \ diff --git a/contrib/libs/libtiff/tif_getimage.c b/contrib/libs/libtiff/tif_getimage.c index 9d4259ef831..4a759a65482 100644 --- a/contrib/libs/libtiff/tif_getimage.c +++ b/contrib/libs/libtiff/tif_getimage.c @@ -29,6 +29,7 @@ */ #include "tiffiop.h" #include <limits.h> +#include <math.h> #include <stdio.h> static int gtTileContig(TIFFRGBAImage *, uint32_t *, uint32_t, uint32_t); @@ -57,21 +58,21 @@ static const char photoTag[] = "PhotometricInterpretation"; static const TIFFDisplay display_sRGB = { {/* XYZ -> luminance matrix */ - {3.2410F, -1.5374F, -0.4986F}, - {-0.9692F, 1.8760F, 0.0416F}, - {0.0556F, -0.2040F, 1.0570F}}, - 100.0F, - 100.0F, - 100.0F, /* Light o/p for reference white */ + {3.2410f, -1.5374f, -0.4986f}, + {-0.9692f, 1.8760f, 0.0416f}, + {0.0556f, -0.2040f, 1.0570f}}, + 100.0f, + 100.0f, + 100.0f, /* Light o/p for reference white */ 255, 255, 255, /* Pixel values for ref. white */ - 1.0F, - 1.0F, - 1.0F, /* Residual light o/p for black pixel */ - 2.4F, - 2.4F, - 2.4F, /* Gamma values for the three guns */ + 1.0f, + 1.0f, + 1.0f, /* Residual light o/p for black pixel */ + 2.4f, + 2.4f, + 2.4f, /* Gamma values for the three guns */ }; /* @@ -316,7 +317,7 @@ int TIFFRGBAImageBegin(TIFFRGBAImage *img, TIFF *tif, int stop, uint16_t compress; int colorchannels; uint16_t *red_orig, *green_orig, *blue_orig; - int n_color; + uint32_t n_color; if (!TIFFRGBAImageOK(tif, emsg)) return 0; @@ -372,6 +373,8 @@ int TIFFRGBAImageBegin(TIFFRGBAImage *img, TIFF *tif, int stop, case EXTRASAMPLE_UNASSALPHA: /* data is not pre-multiplied */ img->alpha = sampleinfo[0]; break; + default: + break; } } @@ -421,13 +424,13 @@ int TIFFRGBAImageBegin(TIFFRGBAImage *img, TIFF *tif, int stop, } /* copy the colormaps so we can modify them */ - n_color = (1U << img->bitspersample); - img->redcmap = - (uint16_t *)_TIFFmallocExt(tif, sizeof(uint16_t) * n_color); - img->greencmap = - (uint16_t *)_TIFFmallocExt(tif, sizeof(uint16_t) * n_color); - img->bluecmap = - (uint16_t *)_TIFFmallocExt(tif, sizeof(uint16_t) * n_color); + n_color = (uint32_t)(1U << img->bitspersample); + img->redcmap = (uint16_t *)_TIFFmallocExt( + tif, (tmsize_t)(sizeof(uint16_t) * (size_t)n_color)); + img->greencmap = (uint16_t *)_TIFFmallocExt( + tif, (tmsize_t)(sizeof(uint16_t) * (size_t)n_color)); + img->bluecmap = (uint16_t *)_TIFFmallocExt( + tif, (tmsize_t)(sizeof(uint16_t) * (size_t)n_color)); if (!img->redcmap || !img->greencmap || !img->bluecmap) { snprintf(emsg, EMSG_BUF_SIZE, @@ -604,7 +607,7 @@ int TIFFRGBAImageGet(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, * Width is checked in img->get() function individually. */ if (0 <= img->row_offset && (uint32_t)img->row_offset < img->height) { - uint32_t hx = img->height - img->row_offset; + uint32_t hx = img->height - (uint32_t)img->row_offset; if (h > hx) { /* Adapt parameters to read only available lines and put image @@ -617,7 +620,7 @@ int TIFFRGBAImageGet(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, { TIFFErrorExtR(img->tif, TIFFFileName(img->tif), "Error in TIFFRGBAImageGet: row offset %d exceeds " - "image height %d", + "image height %u", img->row_offset, img->height); return 0; } @@ -751,13 +754,13 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, if (0 <= img->col_offset && (uint32_t)img->col_offset < img->width) { - wmin = TIFFmin(w, img->width - img->col_offset); + wmin = TIFFmin(w, img->width - (uint32_t)img->col_offset); } else { TIFFErrorExtR(tif, TIFFFileName(tif), "Error in gtTileContig: column offset %d exceeds " - "image width %d", + "image width %u", img->col_offset, img->width); return 0; } @@ -785,7 +788,7 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, } else { - if (tw > ((int64_t)INT_MAX + w)) + if (tw > ((int64_t)INT_MAX + w) || w > ((int64_t)INT_MAX + tw)) { TIFFErrorExtR(tif, TIFFFileName(tif), "%s", "unsupported tile size (too wide)"); @@ -804,9 +807,9 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, /* * Leftmost tile is clipped on left side if col_offset > 0. */ - leftmost_fromskew = img->col_offset % tw; - leftmost_tw = tw - leftmost_fromskew; - int64_t skew_i64 = (int64_t)toskew + leftmost_fromskew; + leftmost_fromskew = (int32_t)((uint32_t)img->col_offset % tw); + leftmost_tw = (uint32_t)((int32_t)tw - leftmost_fromskew); + int64_t skew_i64 = (int64_t)toskew + (int64_t)(int32_t)leftmost_fromskew; if (skew_i64 > INT_MAX || skew_i64 < INT_MIN) { TIFFErrorExtR(tif, TIFFFileName(tif), "%s %" PRId64, "Invalid skew", @@ -816,34 +819,36 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, leftmost_toskew = (int32_t)skew_i64; for (row = 0; ret != 0 && row < h; row += nrow) { - rowstoread = th - (row + img->row_offset) % th; + rowstoread = th - ((uint32_t)row + (uint32_t)img->row_offset) % th; nrow = (row + rowstoread > h ? h - row : rowstoread); fromskew = leftmost_fromskew; this_tw = leftmost_tw; this_toskew = leftmost_toskew; tocol = 0; - col = img->col_offset; + col = (uint32_t)img->col_offset; /* wmin: only write imagewidth if raster is bigger. */ while (tocol < wmin) { if (_TIFFReadTileAndAllocBuffer(tif, (void **)&buf, bufsize, col, - row + img->row_offset, 0, - 0) == (tmsize_t)(-1) && + (uint32_t)row + + (uint32_t)img->row_offset, + 0, 0) == (tmsize_t)(-1) && (buf == NULL || img->stoponerr)) { ret = 0; break; } - pos = ((row + img->row_offset) % th) * TIFFTileRowSize(tif) + + pos = (((uint32_t)row + (uint32_t)img->row_offset) % th) * + TIFFTileRowSize(tif) + ((tmsize_t)fromskew * img->samplesperpixel); if (tocol + this_tw > wmin) { /* * Rightmost tile is clipped on right side. */ - fromskew = tw - (wmin - tocol); - this_tw = tw - fromskew; - this_toskew = toskew + fromskew; + fromskew = (int32_t)(tw - (wmin - tocol)); + this_tw = (uint32_t)((int32_t)tw - fromskew); + this_toskew = (int32_t)(toskew + (int32_t)(uint32_t)fromskew); } tmsize_t roffset = (tmsize_t)y * w + tocol; (*put)(img, raster + roffset, tocol, y, this_tw, nrow, fromskew, @@ -859,7 +864,8 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, this_toskew = toskew; } - y += ((flip & FLIP_VERTICALLY) ? -(int32_t)nrow : (int32_t)nrow); + y += (uint32_t)((flip & FLIP_VERTICALLY) ? -(int32_t)nrow + : (int32_t)nrow); } _TIFFfreeExt(img->tif, buf); @@ -869,7 +875,7 @@ static int gtTileContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, for (line = 0; line < h; line++) { - uint32_t *left = raster + (line * w); + uint32_t *left = raster + (size_t)line * w; /* Use wmin to only flip horizontally data in place and not complete * raster-row. */ uint32_t *right = left + wmin - 1; @@ -924,13 +930,13 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, uint32_t wmin; if (0 <= img->col_offset && (uint32_t)img->col_offset < img->width) { - wmin = TIFFmin(w, img->width - img->col_offset); + wmin = TIFFmin(w, img->width - (uint32_t)img->col_offset); } else { TIFFErrorExtR(tif, TIFFFileName(tif), "Error in gtTileSeparate: column offset %d exceeds " - "image width %d", + "image width %u", img->col_offset, img->width); return 0; } @@ -960,7 +966,7 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, } else { - if (tw > ((int64_t)INT_MAX + w)) + if (tw > ((int64_t)INT_MAX + w) || w > ((int64_t)INT_MAX + tw)) { TIFFErrorExtR(tif, TIFFFileName(tif), "%s", "unsupported tile size (too wide)"); @@ -992,9 +998,9 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, /* * Leftmost tile is clipped on left side if col_offset > 0. */ - leftmost_fromskew = img->col_offset % tw; - leftmost_tw = tw - leftmost_fromskew; - int64_t skew_i64 = (int64_t)toskew + leftmost_fromskew; + leftmost_fromskew = (int32_t)((uint32_t)img->col_offset % tw); + leftmost_tw = (uint32_t)((int32_t)tw - leftmost_fromskew); + int64_t skew_i64 = (int64_t)toskew + (int64_t)(int32_t)leftmost_fromskew; if (skew_i64 > INT_MAX || skew_i64 < INT_MIN) { TIFFErrorExtR(tif, TIFFFileName(tif), "%s %" PRId64, "Invalid skew", @@ -1004,21 +1010,22 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, leftmost_toskew = (int32_t)skew_i64; for (row = 0; ret != 0 && row < h; row += nrow) { - rowstoread = th - (row + img->row_offset) % th; + rowstoread = th - ((uint32_t)row + (uint32_t)img->row_offset) % th; nrow = (row + rowstoread > h ? h - row : rowstoread); fromskew = leftmost_fromskew; this_tw = leftmost_tw; this_toskew = leftmost_toskew; tocol = 0; - col = img->col_offset; + col = (uint32_t)img->col_offset; /* wmin: only write imagewidth if raster is bigger. */ while (tocol < wmin) { if (buf == NULL) { - if (_TIFFReadTileAndAllocBuffer(tif, (void **)&buf, bufsize, - col, row + img->row_offset, 0, - 0) == (tmsize_t)(-1) && + if (_TIFFReadTileAndAllocBuffer( + tif, (void **)&buf, bufsize, col, + (uint32_t)((int32_t)row + img->row_offset), 0, + 0) == (tmsize_t)(-1) && (buf == NULL || img->stoponerr)) { ret = 0; @@ -1037,32 +1044,36 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, pa = (alpha ? (p2 + tilesize) : NULL); } } - else if (TIFFReadTile(tif, p0, col, row + img->row_offset, 0, 0) == - (tmsize_t)(-1) && + else if (TIFFReadTile(tif, p0, col, + (uint32_t)((int32_t)row + img->row_offset), 0, + 0) == (tmsize_t)(-1) && img->stoponerr) { ret = 0; break; } if (colorchannels > 1 && - TIFFReadTile(tif, p1, col, row + img->row_offset, 0, 1) == - (tmsize_t)(-1) && + TIFFReadTile(tif, p1, col, + (uint32_t)((int32_t)row + img->row_offset), 0, + (uint16_t)1) == (tmsize_t)(-1) && img->stoponerr) { ret = 0; break; } if (colorchannels > 1 && - TIFFReadTile(tif, p2, col, row + img->row_offset, 0, 2) == - (tmsize_t)(-1) && + TIFFReadTile(tif, p2, col, + (uint32_t)((int32_t)row + img->row_offset), 0, + (uint16_t)2) == (tmsize_t)(-1) && img->stoponerr) { ret = 0; break; } if (alpha && - TIFFReadTile(tif, pa, col, row + img->row_offset, 0, - colorchannels) == (tmsize_t)(-1) && + TIFFReadTile(tif, pa, col, + (uint32_t)((int32_t)row + img->row_offset), 0, + (uint16_t)colorchannels) == (tmsize_t)(-1) && img->stoponerr) { ret = 0; @@ -1071,16 +1082,17 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, /* For SEPARATE the pos-offset is per sample and should not be * multiplied by img->samplesperpixel. */ - pos = ((row + img->row_offset) % th) * TIFFTileRowSize(tif) + + pos = (tmsize_t)((uint32_t)((int32_t)row + img->row_offset) % th) * + TIFFTileRowSize(tif) + (tmsize_t)fromskew; if (tocol + this_tw > wmin) { /* * Rightmost tile is clipped on right side. */ - fromskew = tw - (wmin - tocol); - this_tw = tw - fromskew; - this_toskew = toskew + fromskew; + fromskew = (int32_t)(tw - (wmin - tocol)); + this_tw = (uint32_t)((int32_t)tw - fromskew); + this_toskew = (int32_t)(toskew + (int32_t)(uint32_t)fromskew); } tmsize_t roffset = (tmsize_t)y * w + tocol; (*put)(img, raster + roffset, tocol, y, this_tw, nrow, fromskew, @@ -1097,7 +1109,8 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, this_toskew = toskew; } - y += ((flip & FLIP_VERTICALLY) ? -(int32_t)nrow : (int32_t)nrow); + y += (uint32_t)((flip & FLIP_VERTICALLY) ? -(int32_t)nrow + : (int32_t)nrow); } if (flip & FLIP_HORIZONTALLY) @@ -1106,7 +1119,7 @@ static int gtTileSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, for (line = 0; line < h; line++) { - uint32_t *left = raster + (line * w); + uint32_t *left = raster + (size_t)line * w; /* Use wmin to only flip horizontally data in place and not complete * raster-row. */ uint32_t *right = left + wmin - 1; @@ -1155,13 +1168,13 @@ static int gtStripContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, uint32_t wmin; if (0 <= img->col_offset && (uint32_t)img->col_offset < imagewidth) { - wmin = TIFFmin(w, imagewidth - img->col_offset); + wmin = TIFFmin(w, imagewidth - (uint32_t)img->col_offset); } else { TIFFErrorExtR(tif, TIFFFileName(tif), "Error in gtStripContig: column offset %d exceeds " - "image width %d", + "image width %u", img->col_offset, imagewidth); return 0; } @@ -1195,7 +1208,7 @@ static int gtStripContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, y = 0; /* Skew forward to the end of the raster width of the row currently * copied. */ - toskew = w - wmin; + toskew = (int32_t)(w - wmin); } TIFFGetFieldDefaulted(tif, TIFFTAG_ROWSPERSTRIP, &rowsperstrip); @@ -1206,16 +1219,18 @@ static int gtStripContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, } scanline = TIFFScanlineSize(tif); - fromskew = (w < imagewidth ? imagewidth - w : 0); + fromskew = (int32_t)(w < imagewidth ? imagewidth - w : 0); for (row = 0; row < h; row += nrow) { uint32_t temp; - rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip; + rowstoread = rowsperstrip - + (uint32_t)((int32_t)row + img->row_offset) % rowsperstrip; nrow = (row + rowstoread > h ? h - row : rowstoread); nrowsub = nrow; if ((nrowsub % subsamplingver) != 0) nrowsub += subsamplingver - nrowsub % subsamplingver; - temp = (row + img->row_offset) % rowsperstrip + nrowsub; + temp = + (uint32_t)((int32_t)row + img->row_offset) % rowsperstrip + nrowsub; if (scanline > 0 && temp > (size_t)(TIFF_TMSIZE_T_MAX / scanline)) { TIFFErrorExtR(tif, TIFFFileName(tif), @@ -1223,21 +1238,27 @@ static int gtStripContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, return 0; } if (_TIFFReadEncodedStripAndAllocBuffer( - tif, TIFFComputeStrip(tif, row + img->row_offset, 0), + tif, + TIFFComputeStrip(tif, + (uint32_t)((int32_t)row + img->row_offset), 0), (void **)(&buf), maxstripsize, - temp * scanline) == (tmsize_t)(-1) && + (tmsize_t)((size_t)temp * (size_t)scanline)) == + (tmsize_t)(-1) && (buf == NULL || img->stoponerr)) { ret = 0; break; } - pos = ((row + img->row_offset) % rowsperstrip) * scanline + + pos = (tmsize_t)((uint32_t)((int32_t)row + img->row_offset) % + rowsperstrip) * + scanline + ((tmsize_t)img->col_offset * img->samplesperpixel); tmsize_t roffset = (tmsize_t)y * w; (*put)(img, raster + roffset, 0, y, wmin, nrow, fromskew, toskew, buf + pos); - y += ((flip & FLIP_VERTICALLY) ? -(int32_t)nrow : (int32_t)nrow); + y += (uint32_t)((flip & FLIP_VERTICALLY) ? -(int32_t)nrow + : (int32_t)nrow); } if (flip & FLIP_HORIZONTALLY) @@ -1250,7 +1271,7 @@ static int gtStripContig(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, for (line = 0; line < h; line++) { - uint32_t *left = raster + (line * w); + uint32_t *left = raster + (size_t)line * w; /* Use wmin to only flip horizontally data in place and not complete * raster-row. */ uint32_t *right = left + wmin - 1; @@ -1300,13 +1321,13 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, uint32_t wmin; if (0 <= img->col_offset && (uint32_t)img->col_offset < imagewidth) { - wmin = TIFFmin(w, imagewidth - img->col_offset); + wmin = TIFFmin(w, imagewidth - (uint32_t)img->col_offset); } else { TIFFErrorExtR(tif, TIFFFileName(tif), "Error in gtStripSeparate: column offset %d exceeds " - "image width %d", + "image width %u", img->col_offset, imagewidth); return 0; } @@ -1337,7 +1358,7 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, y = 0; /* Skew forward to the end of the raster width of the row currently * written. */ - toskew = w - wmin; + toskew = (int32_t)(w - wmin); } switch (img->photometric) @@ -1361,14 +1382,16 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, } scanline = TIFFScanlineSize(tif); - fromskew = (w < imagewidth ? imagewidth - w : 0); + fromskew = (int32_t)(w < imagewidth ? imagewidth - w : 0); for (row = 0; row < h; row += nrow) { uint32_t temp; - rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip; + rowstoread = + rowsperstrip - + ((uint32_t)((int32_t)row + img->row_offset) % rowsperstrip); nrow = (row + rowstoread > h ? h - row : rowstoread); - offset_row = row + img->row_offset; - temp = (row + img->row_offset) % rowsperstrip + nrow; + offset_row = (uint32_t)((int32_t)row + img->row_offset); + temp = (uint32_t)((int32_t)row + img->row_offset) % rowsperstrip + nrow; if (scanline > 0 && temp > (size_t)(TIFF_TMSIZE_T_MAX / scanline)) { TIFFErrorExtR(tif, TIFFFileName(tif), @@ -1435,12 +1458,15 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, /* For SEPARATE the pos-offset is per sample and should not be * multiplied by img->samplesperpixel. */ - pos = ((row + img->row_offset) % rowsperstrip) * scanline + + pos = (tmsize_t)((uint32_t)((int32_t)row + img->row_offset) % + rowsperstrip) * + scanline + (tmsize_t)img->col_offset; tmsize_t roffset = (tmsize_t)y * w; (*put)(img, raster + roffset, 0, y, wmin, nrow, fromskew, toskew, p0 + pos, p1 + pos, p2 + pos, (alpha ? (pa + pos) : NULL)); - y += ((flip & FLIP_VERTICALLY) ? -(int32_t)nrow : (int32_t)nrow); + y += (uint32_t)((flip & FLIP_VERTICALLY) ? -(int32_t)nrow + : (int32_t)nrow); } if (flip & FLIP_HORIZONTALLY) @@ -1449,7 +1475,7 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, for (line = 0; line < h; line++) { - uint32_t *left = raster + (line * w); + uint32_t *left = raster + (size_t)line * w; /* Use wmin to only flip horizontally data in place and not complete * raster-row. */ uint32_t *right = left + wmin - 1; @@ -1505,6 +1531,9 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, op; /*-fallthrough*/ \ case 1: \ op; \ + break; \ + default: \ + break; \ } #define CASE4(x, op) \ switch (x) \ @@ -1515,6 +1544,9 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, op; /*-fallthrough*/ \ case 1: \ op; \ + break; \ + default: \ + break; \ } #define NOP @@ -1581,13 +1613,6 @@ static int gtStripSeparate(TIFFRGBAImage *img, uint32_t *raster, uint32_t w, #define PACK4(r, g, b, a) \ ((uint32_t)(r) | ((uint32_t)(g) << 8) | ((uint32_t)(b) << 16) | \ ((uint32_t)(a) << 24)) -#define W2B(v) (((v) >> 8) & 0xff) -/* TODO: PACKW should have be made redundant in favor of Bitdepth16To8 LUT */ -#define PACKW(r, g, b) \ - ((uint32_t)W2B(r) | ((uint32_t)W2B(g) << 8) | ((uint32_t)W2B(b) << 16) | A1) -#define PACKW4(r, g, b, a) \ - ((uint32_t)W2B(r) | ((uint32_t)W2B(g) << 8) | ((uint32_t)W2B(b) << 16) | \ - ((uint32_t)W2B(a) << 24)) #define DECLAREContigPutFunc(name) \ static void name(TIFFRGBAImage *img, uint32_t *cp, uint32_t x, uint32_t y, \ @@ -1706,7 +1731,7 @@ DECLAREContigPutFunc(putagreytile) { for (x = w; x > 0; --x) { - *cp++ = BWmap[*pp][0] & ((uint32_t) * (pp + 1) << 24 | ~A1); + *cp++ = BWmap[*pp][0] & ((uint32_t)*(pp + 1) << 24 | ~A1); pp += samplesperpixel; } cp += toskew; @@ -1721,6 +1746,9 @@ DECLAREContigPutFunc(put16bitbwtile) { int samplesperpixel = img->samplesperpixel; uint32_t **BWmap = img->BWmap; + /* Convert pixel skew to byte skew (16-bit samples) */ + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)(2 * samplesperpixel); (void)y; for (; h > 0; --h) @@ -1736,7 +1764,7 @@ DECLAREContigPutFunc(put16bitbwtile) wp += samplesperpixel; } cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -1806,13 +1834,14 @@ DECLAREContigPutFunc(putRGBcontig8bittile) (void)x; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { UNROLL8(w, NOP, *cp++ = PACK(pp[0], pp[1], pp[2]); pp += samplesperpixel); cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -1826,13 +1855,14 @@ DECLAREContigPutFunc(putRGBAAcontig8bittile) (void)x; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { UNROLL8(w, NOP, *cp++ = PACK4(pp[0], pp[1], pp[2], pp[3]); pp += samplesperpixel); cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -1844,7 +1874,8 @@ DECLAREContigPutFunc(putRGBUAcontig8bittile) { int samplesperpixel = img->samplesperpixel; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { uint32_t r, g, b, a; @@ -1860,7 +1891,7 @@ DECLAREContigPutFunc(putRGBUAcontig8bittile) pp += samplesperpixel; } cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -1872,7 +1903,8 @@ DECLAREContigPutFunc(putRGBcontig16bittile) int samplesperpixel = img->samplesperpixel; uint16_t *wp = (uint16_t *)pp; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { for (x = w; x > 0; --x) @@ -1882,7 +1914,7 @@ DECLAREContigPutFunc(putRGBcontig16bittile) wp += samplesperpixel; } cp += toskew; - wp += fromskew; + wp += fromskewLocal; } } @@ -1895,7 +1927,8 @@ DECLAREContigPutFunc(putRGBAAcontig16bittile) int samplesperpixel = img->samplesperpixel; uint16_t *wp = (uint16_t *)pp; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { for (x = w; x > 0; --x) @@ -1905,7 +1938,7 @@ DECLAREContigPutFunc(putRGBAAcontig16bittile) wp += samplesperpixel; } cp += toskew; - wp += fromskew; + wp += fromskewLocal; } } @@ -1918,7 +1951,8 @@ DECLAREContigPutFunc(putRGBUAcontig16bittile) int samplesperpixel = img->samplesperpixel; uint16_t *wp = (uint16_t *)pp; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { uint32_t r, g, b, a; @@ -1934,7 +1968,7 @@ DECLAREContigPutFunc(putRGBUAcontig16bittile) wp += samplesperpixel; } cp += toskew; - wp += fromskew; + wp += fromskewLocal; } } @@ -1950,14 +1984,17 @@ DECLAREContigPutFunc(putRGBcontig8bitCMYKtile) (void)x; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { - UNROLL8(w, NOP, k = 255 - pp[3]; r = (k * (255 - pp[0])) / 255; - g = (k * (255 - pp[1])) / 255; b = (k * (255 - pp[2])) / 255; + UNROLL8(w, NOP, k = (uint16_t)(255 - pp[3]); + r = (uint16_t)((k * (uint16_t)(255 - pp[0])) / 255); + g = (uint16_t)((k * (uint16_t)(255 - pp[1])) / 255); + b = (uint16_t)((k * (uint16_t)(255 - pp[2])) / 255); *cp++ = PACK(r, g, b); pp += samplesperpixel); cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -1970,22 +2007,23 @@ DECLAREContigPutFunc(putRGBcontig8bitCMYKMaptile) { int samplesperpixel = img->samplesperpixel; TIFFRGBValue *Map = img->Map; - uint16_t r, g, b, k; + unsigned int r, g, b, k; (void)y; - fromskew *= samplesperpixel; + const tmsize_t fromskewLocal = + (tmsize_t)fromskew * (tmsize_t)samplesperpixel; for (; h > 0; --h) { for (x = w; x > 0; --x) { - k = 255 - pp[3]; - r = (k * (255 - pp[0])) / 255; - g = (k * (255 - pp[1])) / 255; - b = (k * (255 - pp[2])) / 255; + k = 255U - pp[3]; + r = (k * (255U - pp[0])) / 255U; + g = (k * (255U - pp[1])) / 255U; + b = (k * (255U - pp[2])) / 255U; *cp++ = PACK(Map[r], Map[g], Map[b]); pp += samplesperpixel; } - pp += fromskew; + pp += fromskewLocal; cp += toskew; } } @@ -2038,13 +2076,13 @@ DECLARESepPutFunc(putCMYKseparate8bittile) (void)y; for (; h > 0; --h) { - uint32_t rv, gv, bv, kv; + unsigned int rv, gv, bv, kv; for (x = w; x > 0; --x) { - kv = 255 - *a++; - rv = (kv * (255 - *r++)) / 255; - gv = (kv * (255 - *g++)) / 255; - bv = (kv * (255 - *b++)) / 255; + kv = 255U - *a++; + rv = (kv * (255U - *r++)) / 255U; + gv = (kv * (255U - *g++)) / 255U; + bv = (kv * (255U - *b++)) / 255U; *cp++ = PACK4(rv, gv, bv, 255); } SKEW4(r, g, b, a, fromskew); @@ -2156,7 +2194,7 @@ DECLAREContigPutFunc(putcontig8bitCIELab8) float X, Y, Z; uint32_t r, g, b; (void)y; - fromskew *= 3; + const tmsize_t fromskewLocal = (tmsize_t)fromskew * (tmsize_t)3; for (; h > 0; --h) { for (x = w; x > 0; --x) @@ -2168,7 +2206,7 @@ DECLAREContigPutFunc(putcontig8bitCIELab8) pp += 3; } cp += toskew; - pp += fromskew; + pp += fromskewLocal; } } @@ -2181,7 +2219,7 @@ DECLAREContigPutFunc(putcontig8bitCIELab16) uint32_t r, g, b; uint16_t *wp = (uint16_t *)pp; (void)y; - fromskew *= 3; + const tmsize_t fromskewLocal = (tmsize_t)fromskew * (tmsize_t)3; for (; h > 0; --h) { for (x = w; x > 0; --x) @@ -2193,7 +2231,7 @@ DECLAREContigPutFunc(putcontig8bitCIELab16) wp += 3; } cp += toskew; - wp += fromskew; + wp += fromskewLocal; } } @@ -2204,7 +2242,7 @@ DECLAREContigPutFunc(putcontig8bitCIELab16) #define YCbCrtoRGB(dst, Y) \ { \ uint32_t r, g, b; \ - TIFFYCbCrtoRGB(img->ycbcr, (Y), Cb, Cr, &r, &g, &b); \ + TIFFYCbCrtoRGB(img->ycbcr, (Y), (int32_t)Cb, (int32_t)Cr, &r, &g, &b); \ dst = PACK(r, g, b); \ } @@ -2216,11 +2254,12 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) uint32_t *cp1 = cp + w + toskew; uint32_t *cp2 = cp1 + w + toskew; uint32_t *cp3 = cp2 + w + toskew; - int32_t incr = 3 * w + 4 * toskew; + const tmsize_t incr = 3 * (tmsize_t)w + 4 * (tmsize_t)toskew; (void)y; /* adjust fromskew */ - fromskew = (fromskew / 4) * (4 * 2 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 4) * (tmsize_t)(4 * 4 + 2); if ((h & 3) == 0 && (w & 3) == 0) { for (; h >= 4; h -= 4) @@ -2258,7 +2297,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) cp1 += incr; cp2 += incr; cp3 += incr; - pp += fromskew; + pp += fromskewLocal; } } else @@ -2282,7 +2321,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) YCbCrtoRGB(cp1[3], pp[7]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[3], pp[3]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 3: switch (h) { @@ -2294,7 +2333,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) YCbCrtoRGB(cp1[2], pp[6]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[2], pp[2]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 2: switch (h) { @@ -2306,7 +2345,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) YCbCrtoRGB(cp1[1], pp[5]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[1], pp[1]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 1: switch (h) { @@ -2318,7 +2357,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) YCbCrtoRGB(cp1[0], pp[4]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[0], pp[0]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ } if (x < 4) { @@ -2345,7 +2384,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) cp1 += incr; cp2 += incr; cp3 += incr; - pp += fromskew; + pp += fromskewLocal; } } } @@ -2356,10 +2395,11 @@ DECLAREContigPutFunc(putcontig8bitYCbCr44tile) DECLAREContigPutFunc(putcontig8bitYCbCr42tile) { uint32_t *cp1 = cp + w + toskew; - int32_t incr = 2 * toskew + w; + const tmsize_t incr = 2 * (tmsize_t)toskew + (tmsize_t)w; (void)y; - fromskew = (fromskew / 4) * (4 * 2 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 4) * (tmsize_t)(4 * 2 + 2); if ((w & 3) == 0 && (h & 1) == 0) { for (; h >= 2; h -= 2) @@ -2385,7 +2425,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) } while (--x); cp += incr; cp1 += incr; - pp += fromskew; + pp += fromskewLocal; } } else @@ -2405,7 +2445,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) YCbCrtoRGB(cp1[3], pp[7]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[3], pp[3]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 3: switch (h) { @@ -2413,7 +2453,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) YCbCrtoRGB(cp1[2], pp[6]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[2], pp[2]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 2: switch (h) { @@ -2421,7 +2461,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) YCbCrtoRGB(cp1[1], pp[5]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[1], pp[1]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ case 1: switch (h) { @@ -2429,7 +2469,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) YCbCrtoRGB(cp1[0], pp[4]); /* FALLTHROUGH */ case 1: YCbCrtoRGB(cp[0], pp[0]); /* FALLTHROUGH */ - } /* FALLTHROUGH */ + } /* FALLTHROUGH */ } if (x < 4) { @@ -2450,7 +2490,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) h -= 2; cp += incr; cp1 += incr; - pp += fromskew; + pp += fromskewLocal; } } } @@ -2461,7 +2501,8 @@ DECLAREContigPutFunc(putcontig8bitYCbCr42tile) DECLAREContigPutFunc(putcontig8bitYCbCr41tile) { (void)y; - fromskew = (fromskew / 4) * (4 * 1 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 4) * (tmsize_t)(4 * 1 + 2); do { x = w >> 2; @@ -2495,6 +2536,8 @@ DECLAREContigPutFunc(putcontig8bitYCbCr41tile) YCbCrtoRGB(cp[0], pp[0]); /*-fallthrough*/ case 0: break; + default: + break; } cp += (w & 3); @@ -2502,7 +2545,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr41tile) } cp += toskew; - pp += fromskew; + pp += fromskewLocal; } while (--h); } @@ -2512,9 +2555,10 @@ DECLAREContigPutFunc(putcontig8bitYCbCr41tile) DECLAREContigPutFunc(putcontig8bitYCbCr22tile) { uint32_t *cp2; - int32_t incr = 2 * toskew + w; + const tmsize_t incr = 2 * (tmsize_t)toskew + (tmsize_t)w; (void)y; - fromskew = (fromskew / 2) * (2 * 2 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 2) * (tmsize_t)(2 * 2 + 2); cp2 = cp + w + toskew; while (h >= 2) { @@ -2544,7 +2588,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr22tile) } cp += incr; cp2 += incr; - pp += fromskew; + pp += fromskewLocal; h -= 2; } if (h == 1) @@ -2576,7 +2620,8 @@ DECLAREContigPutFunc(putcontig8bitYCbCr22tile) DECLAREContigPutFunc(putcontig8bitYCbCr21tile) { (void)y; - fromskew = (fromskew / 2) * (2 * 1 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 2) * (tmsize_t)(2 * 1 + 2); do { x = w >> 1; @@ -2605,7 +2650,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr21tile) } cp += toskew; - pp += fromskew; + pp += fromskewLocal; } while (--h); } @@ -2615,9 +2660,10 @@ DECLAREContigPutFunc(putcontig8bitYCbCr21tile) DECLAREContigPutFunc(putcontig8bitYCbCr12tile) { uint32_t *cp2; - int32_t incr = 2 * toskew + w; + const tmsize_t incr = 2 * (tmsize_t)toskew + (tmsize_t)w; (void)y; - fromskew = (fromskew / 1) * (1 * 2 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 1) * (tmsize_t)(1 * 2 + 2); cp2 = cp + w + toskew; while (h >= 2) { @@ -2634,7 +2680,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr12tile) } while (--x); cp += incr; cp2 += incr; - pp += fromskew; + pp += fromskewLocal; h -= 2; } if (h == 1) @@ -2657,7 +2703,8 @@ DECLAREContigPutFunc(putcontig8bitYCbCr12tile) DECLAREContigPutFunc(putcontig8bitYCbCr11tile) { (void)y; - fromskew = (fromskew / 1) * (1 * 1 + 2); + const tmsize_t fromskewLocal = + (tmsize_t)(fromskew / 1) * (tmsize_t)(1 * 1 + 2); do { x = w; /* was x = w>>1; patched 2000/09/25 [email protected] */ @@ -2671,7 +2718,7 @@ DECLAREContigPutFunc(putcontig8bitYCbCr11tile) pp += 3; } while (--x); cp += toskew; - pp += fromskew; + pp += fromskewLocal; } while (--h); } @@ -2730,8 +2777,8 @@ static int initYCbCrConversion(TIFFRGBAImage *img) /* Do some validation to avoid later issues. Detect NaN for now */ /* and also if lumaGreen is zero since we divide by it later */ - if (luma[0] != luma[0] || luma[1] != luma[1] || luma[1] == 0.0 || - luma[2] != luma[2]) + if (isnan(luma[0]) || isnan(luma[1]) || TIFF_FLOAT_EQ(luma[1], 0.0f) || + isnan(luma[2])) { TIFFErrorExtR(img->tif, module, "Invalid values for YCbCrCoefficients tag"); @@ -2763,7 +2810,7 @@ static tileContigRoutine initCIELabConversion(TIFFRGBAImage *img) float refWhite[3]; TIFFGetFieldDefaulted(img->tif, TIFFTAG_WHITEPOINT, &whitePoint); - if (whitePoint[1] == 0.0f) + if (TIFF_FLOAT_EQ(whitePoint[1], 0.0f)) { TIFFErrorExtR(img->tif, module, "Invalid value for WhitePoint tag."); return NULL; @@ -2781,10 +2828,10 @@ static tileContigRoutine initCIELabConversion(TIFFRGBAImage *img) } } - refWhite[1] = 100.0F; + refWhite[1] = 100.0f; refWhite[0] = whitePoint[0] / whitePoint[1] * refWhite[1]; refWhite[2] = - (1.0F - whitePoint[0] - whitePoint[1]) / whitePoint[1] * refWhite[1]; + (1.0f - whitePoint[0] - whitePoint[1]) / whitePoint[1] * refWhite[1]; if (TIFFCIELabToRGBInit(img->cielab, &display_sRGB, refWhite) < 0) { TIFFErrorExtR(img->tif, module, @@ -2820,7 +2867,8 @@ static int makebwmap(TIFFRGBAImage *img) img->BWmap = (uint32_t **)_TIFFmallocExt( img->tif, - 256 * sizeof(uint32_t *) + (256 * nsamples * sizeof(uint32_t))); + (tmsize_t)((size_t)256 * sizeof(uint32_t *) + + ((size_t)256 * (size_t)nsamples * sizeof(uint32_t)))); if (img->BWmap == NULL) { TIFFErrorExtR(img->tif, TIFFFileName(img->tif), @@ -2861,6 +2909,8 @@ static int makebwmap(TIFFRGBAImage *img) case 16: GREY(i); break; + default: + break; } #undef GREY } @@ -2876,14 +2926,14 @@ static int setupMap(TIFFRGBAImage *img) { int32_t x, range; - range = (int32_t)((1L << img->bitspersample) - 1); + range = (int32_t)((1U << img->bitspersample) - 1U); /* treat 16 bit the same as eight bit */ if (img->bitspersample == 16) - range = (int32_t)255; + range = 255; img->Map = (TIFFRGBValue *)_TIFFmallocExt( - img->tif, (range + 1) * sizeof(TIFFRGBValue)); + img->tif, (tmsize_t)((size_t)(range + 1) * sizeof(TIFFRGBValue))); if (img->Map == NULL) { TIFFErrorExtR(img->tif, TIFFFileName(img->tif), @@ -2966,7 +3016,8 @@ static int makecmap(TIFFRGBAImage *img) img->PALmap = (uint32_t **)_TIFFmallocExt( img->tif, - 256 * sizeof(uint32_t *) + (256 * nsamples * sizeof(uint32_t))); + (tmsize_t)((size_t)256 * sizeof(uint32_t *) + + ((size_t)256 * (size_t)nsamples * sizeof(uint32_t)))); if (img->PALmap == NULL) { TIFFErrorExtR(img->tif, TIFFFileName(img->tif), @@ -3006,6 +3057,8 @@ static int makecmap(TIFFRGBAImage *img) case 8: CMAP(i); break; + default: + break; } #undef CMAP } @@ -3048,6 +3101,8 @@ static int buildMap(TIFFRGBAImage *img) if (img->bitspersample <= 8 && !makecmap(img)) return (0); break; + default: + break; } return (1); } @@ -3096,6 +3151,8 @@ static int PickContigCase(TIFFRGBAImage *img) img->put.contig = putRGBcontig16bittile; } break; + default: + break; } break; case PHOTOMETRIC_SEPARATED: @@ -3127,6 +3184,8 @@ static int PickContigCase(TIFFRGBAImage *img) case 1: img->put.contig = put1bitcmaptile; break; + default: + break; } } break; @@ -3154,6 +3213,8 @@ static int PickContigCase(TIFFRGBAImage *img) case 1: img->put.contig = put1bitbwtile; break; + default: + break; } } break; @@ -3175,6 +3236,19 @@ static int PickContigCase(TIFFRGBAImage *img) uint16_t SubsamplingVer; TIFFGetFieldDefaulted(img->tif, TIFFTAG_YCBCRSUBSAMPLING, &SubsamplingHor, &SubsamplingVer); + /* Validate that the image dimensions are compatible with + the subsampling block. All putcontig8bitYCbCrXYtile routines + assume width >= X and height >= Y. */ + if (img->width < SubsamplingHor || + img->height < SubsamplingVer) + { + TIFFErrorExtR(img->tif, TIFFFileName(img->tif), + "YCbCr subsampling (%u,%u) incompatible " + "with image size %ux%u", + SubsamplingHor, SubsamplingVer, + img->width, img->height); + return (0); + } switch ((SubsamplingHor << 4) | SubsamplingVer) { case 0x44: @@ -3198,6 +3272,8 @@ static int PickContigCase(TIFFRGBAImage *img) case 0x11: img->put.contig = putcontig8bitYCbCr11tile; break; + default: + break; } } } @@ -3209,6 +3285,9 @@ static int PickContigCase(TIFFRGBAImage *img) img->put.contig = initCIELabConversion(img); break; } + break; + default: + break; } return ((img->get != NULL) && (img->put.contig != NULL)); } @@ -3260,6 +3339,8 @@ static int PickSeparateCase(TIFFRGBAImage *img) img->put.separate = putRGBseparate16bittile; } break; + default: + break; } break; case PHOTOMETRIC_SEPARATED: @@ -3284,10 +3365,14 @@ static int PickSeparateCase(TIFFRGBAImage *img) img->put.separate = putseparate8bitYCbCr11tile; break; /* TODO: add other cases here */ + default: + break; } } } break; + default: + break; } return ((img->get != NULL) && (img->put.separate != NULL)); } @@ -3298,7 +3383,7 @@ static int BuildMapUaToAa(TIFFRGBAImage *img) uint8_t *m; uint16_t na, nv; assert(img->UaToAa == NULL); - img->UaToAa = _TIFFmallocExt(img->tif, 65536); + img->UaToAa = (uint8_t *)_TIFFmallocExt(img->tif, 65536); if (img->UaToAa == NULL) { TIFFErrorExtR(img->tif, module, "Out of memory"); @@ -3319,7 +3404,7 @@ static int BuildMapBitdepth16To8(TIFFRGBAImage *img) uint8_t *m; uint32_t n; assert(img->Bitdepth16To8 == NULL); - img->Bitdepth16To8 = _TIFFmallocExt(img->tif, 65536); + img->Bitdepth16To8 = (uint8_t *)_TIFFmallocExt(img->tif, 65536); if (img->Bitdepth16To8 == NULL) { TIFFErrorExtR(img->tif, module, "Out of memory"); @@ -3386,7 +3471,7 @@ int TIFFReadRGBAStripExt(TIFF *tif, uint32_t row, uint32_t *raster, return (0); } - img.row_offset = row; + img.row_offset = (int)row; img.col_offset = 0; if (row + rowsperstrip > img.height) @@ -3497,8 +3582,8 @@ int TIFFReadRGBATileExt(TIFF *tif, uint32_t col, uint32_t row, uint32_t *raster, * Read the chunk of imagery. */ - img.row_offset = row; - img.col_offset = col; + img.row_offset = (int)row; + img.col_offset = (int)col; ok = TIFFRGBAImageGet(&img, raster, read_xsize, read_ysize); @@ -3520,15 +3605,16 @@ int TIFFReadRGBATileExt(TIFF *tif, uint32_t col, uint32_t row, uint32_t *raster, memmove(raster + (size_t)(tile_ysize - i_row - 1) * tile_xsize, raster + (size_t)(read_ysize - i_row - 1) * read_xsize, read_xsize * sizeof(uint32_t)); - _TIFFmemset(raster + (size_t)(tile_ysize - i_row - 1) * tile_xsize + - read_xsize, - 0, sizeof(uint32_t) * (tile_xsize - read_xsize)); + _TIFFmemset( + raster + (size_t)(tile_ysize - i_row - 1) * tile_xsize + read_xsize, + 0, + (tmsize_t)(sizeof(uint32_t) * (size_t)(tile_xsize - read_xsize))); } for (i_row = read_ysize; i_row < tile_ysize; i_row++) { _TIFFmemset(raster + (size_t)(tile_ysize - i_row - 1) * tile_xsize, 0, - sizeof(uint32_t) * tile_xsize); + (tmsize_t)(sizeof(uint32_t) * (size_t)tile_xsize)); } return (ok); diff --git a/contrib/libs/libtiff/tif_hash_set.c b/contrib/libs/libtiff/tif_hash_set.c index 81dea3fcf26..7c27191ceb6 100644 --- a/contrib/libs/libtiff/tif_hash_set.c +++ b/contrib/libs/libtiff/tif_hash_set.c @@ -76,7 +76,7 @@ static const int anPrimes[] = { 100663319, 201326611, 402653189, 805306457, 1610612741}; /************************************************************************/ -/* TIFFHashSetHashPointer() */ +/* TIFFHashSetHashPointer() */ /************************************************************************/ /** @@ -93,7 +93,7 @@ static unsigned long TIFFHashSetHashPointer(const void *elt) } /************************************************************************/ -/* TIFFHashSetEqualPointer() */ +/* TIFFHashSetEqualPointer() */ /************************************************************************/ /** @@ -111,7 +111,7 @@ static bool TIFFHashSetEqualPointer(const void *elt1, const void *elt2) } /************************************************************************/ -/* TIFFHashSetNew() */ +/* TIFFHashSetNew() */ /************************************************************************/ /** @@ -164,7 +164,7 @@ TIFFHashSet *TIFFHashSetNew(TIFFHashSetHashFunc fnHashFunc, } /************************************************************************/ -/* TIFFHashSetSize() */ +/* TIFFHashSetSize() */ /************************************************************************/ /** @@ -184,7 +184,7 @@ int TIFFHashSetSize(const TIFFHashSet *set) } /************************************************************************/ -/* TIFFHashSetGetNewListElt() */ +/* TIFFHashSetGetNewListElt() */ /************************************************************************/ static TIFFList *TIFFHashSetGetNewListElt(TIFFHashSet *set) @@ -202,7 +202,7 @@ static TIFFList *TIFFHashSetGetNewListElt(TIFFHashSet *set) } /************************************************************************/ -/* TIFFHashSetReturnListElt() */ +/* TIFFHashSetReturnListElt() */ /************************************************************************/ static void TIFFHashSetReturnListElt(TIFFHashSet *set, TIFFList *psList) @@ -220,7 +220,7 @@ static void TIFFHashSetReturnListElt(TIFFHashSet *set, TIFFList *psList) } /************************************************************************/ -/* TIFFHashSetClearInternal() */ +/* TIFFHashSetClearInternal() */ /************************************************************************/ static void TIFFHashSetClearInternal(TIFFHashSet *set, bool bFinalize) @@ -246,7 +246,7 @@ static void TIFFHashSetClearInternal(TIFFHashSet *set, bool bFinalize) } /************************************************************************/ -/* TIFFListDestroy() */ +/* TIFFListDestroy() */ /************************************************************************/ /** @@ -270,7 +270,7 @@ static void TIFFListDestroy(TIFFList *psList) } /************************************************************************/ -/* TIFFHashSetDestroy() */ +/* TIFFHashSetDestroy() */ /************************************************************************/ /** @@ -295,7 +295,7 @@ void TIFFHashSetDestroy(TIFFHashSet *set) #ifdef notused /************************************************************************/ -/* TIFFHashSetClear() */ +/* TIFFHashSetClear() */ /************************************************************************/ /** @@ -319,7 +319,7 @@ void TIFFHashSetClear(TIFFHashSet *set) } /************************************************************************/ -/* TIFFHashSetForeach() */ +/* TIFFHashSetForeach() */ /************************************************************************/ /** @@ -360,14 +360,14 @@ void TIFFHashSetForeach(TIFFHashSet *set, TIFFHashSetIterEltFunc fnIterFunc, #endif /************************************************************************/ -/* TIFFHashSetRehash() */ +/* TIFFHashSetRehash() */ /************************************************************************/ static bool TIFFHashSetRehash(TIFFHashSet *set) { int nNewAllocatedSize = anPrimes[set->nIndiceAllocatedSize]; TIFFList **newTabList = - (TIFFList **)(calloc(nNewAllocatedSize, sizeof(TIFFList *))); + (TIFFList **)(calloc((size_t)nNewAllocatedSize, sizeof(TIFFList *))); if (newTabList == NULL) return false; #ifdef HASH_DEBUG @@ -384,7 +384,7 @@ static bool TIFFHashSetRehash(TIFFHashSet *set) while (cur) { const unsigned long nNewHashVal = - set->fnHashFunc(cur->pData) % nNewAllocatedSize; + set->fnHashFunc(cur->pData) % (unsigned long)nNewAllocatedSize; #ifdef HASH_DEBUG if (newTabList[nNewHashVal]) set->nCollisions++; @@ -403,12 +403,13 @@ static bool TIFFHashSetRehash(TIFFHashSet *set) } /************************************************************************/ -/* TIFFHashSetFindPtr() */ +/* TIFFHashSetFindPtr() */ /************************************************************************/ static void **TIFFHashSetFindPtr(TIFFHashSet *set, const void *elt) { - const unsigned long nHashVal = set->fnHashFunc(elt) % set->nAllocatedSize; + const unsigned long nHashVal = + set->fnHashFunc(elt) % (unsigned long)set->nAllocatedSize; TIFFList *cur = set->tabList[nHashVal]; while (cur) { @@ -464,7 +465,8 @@ bool TIFFHashSetInsert(TIFFHashSet *set, void *elt) } } - const unsigned long nHashVal = set->fnHashFunc(elt) % set->nAllocatedSize; + const unsigned long nHashVal = + set->fnHashFunc(elt) % (unsigned long)set->nAllocatedSize; #ifdef HASH_DEBUG if (set->tabList[nHashVal]) set->nCollisions++; @@ -486,7 +488,7 @@ bool TIFFHashSetInsert(TIFFHashSet *set, void *elt) } /************************************************************************/ -/* TIFFHashSetLookup() */ +/* TIFFHashSetLookup() */ /************************************************************************/ /** @@ -532,7 +534,8 @@ static bool TIFFHashSetRemoveInternal(TIFFHashSet *set, const void *elt, } } - int nHashVal = (int)(set->fnHashFunc(elt) % set->nAllocatedSize); + int nHashVal = + (int)(set->fnHashFunc(elt) % (unsigned long)set->nAllocatedSize); TIFFList *cur = set->tabList[nHashVal]; TIFFList *prev = NULL; while (cur) @@ -581,7 +584,7 @@ bool TIFFHashSetRemove(TIFFHashSet *set, const void *elt) #ifdef notused /************************************************************************/ -/* TIFFHashSetRemoveDeferRehash() */ +/* TIFFHashSetRemoveDeferRehash() */ /************************************************************************/ /** diff --git a/contrib/libs/libtiff/tif_jbig.c b/contrib/libs/libtiff/tif_jbig.c index d0fae1f124a..4c35f3c0dcf 100644 --- a/contrib/libs/libtiff/tif_jbig.c +++ b/contrib/libs/libtiff/tif_jbig.c @@ -33,7 +33,14 @@ #include "tiffiop.h" #ifdef JBIG_SUPPORT +#ifdef __cplusplus +extern "C" +{ +#endif #error #include "jbig.h" +#ifdef __cplusplus +} +#endif static int JBIGSetupDecode(TIFF *tif) { @@ -100,7 +107,7 @@ static int JBIGDecode(TIFF *tif, uint8_t *buffer, tmsize_t size, uint16_t s) decodedSize = jbg_dec_getsize(&decoder); if ((tmsize_t)decodedSize < size) { - memset(buffer + decodedSize, 0, (size_t)(size - decodedSize)); + memset(buffer + decodedSize, 0, (size_t)(size - (tmsize_t)decodedSize)); TIFFWarningExtR(tif, "JBIG", "Only decoded %lu bytes, whereas %" TIFF_SSIZE_FORMAT " requested", @@ -116,7 +123,7 @@ static int JBIGDecode(TIFF *tif, uint8_t *buffer, tmsize_t size, uint16_t s) return 0; } pImage = jbg_dec_getimage(&decoder, 0); - _TIFFmemcpy(buffer, pImage, decodedSize); + _TIFFmemcpy(buffer, pImage, (tmsize_t)decodedSize); jbg_dec_free(&decoder); tif->tif_rawcp += tif->tif_rawcc; diff --git a/contrib/libs/libtiff/tif_jpeg.c b/contrib/libs/libtiff/tif_jpeg.c index aba5f99b7c0..b9a85cb18d9 100644 --- a/contrib/libs/libtiff/tif_jpeg.c +++ b/contrib/libs/libtiff/tif_jpeg.c @@ -22,9 +22,6 @@ * OF THIS SOFTWARE. */ -#define WIN32_LEAN_AND_MEAN -#define VC_EXTRALEAN - #include "tiffiop.h" #include <stdlib.h> @@ -146,20 +143,9 @@ int TIFFJPEGIsFullStripRequired_12(TIFF *tif); #define LONGJMP(jbuf, code) longjmp(jbuf, code) #define JMP_BUF jmp_buf -#ifndef TIFF_jpeg_destination_mgr_defined -#define TIFF_jpeg_destination_mgr_defined -typedef struct jpeg_destination_mgr jpeg_destination_mgr; -#endif - -#ifndef TIFF_jpeg_source_mgr_defined -#define TIFF_jpeg_source_mgr_defined -typedef struct jpeg_source_mgr jpeg_source_mgr; -#endif - -#ifndef TIFF_jpeg_error_mgr_defined -#define TIFF_jpeg_error_mgr_defined -typedef struct jpeg_error_mgr jpeg_error_mgr; -#endif +typedef struct jpeg_destination_mgr tiff_jpeg_destination_mgr; +typedef struct jpeg_source_mgr tiff_jpeg_source_mgr; +typedef struct jpeg_error_mgr tiff_jpeg_error_mgr; /* * State block for each open TIFF file using @@ -174,6 +160,10 @@ typedef struct jpeg_error_mgr jpeg_error_mgr; * so we can safely cast JPEGState* -> jpeg_xxx_struct* * and vice versa! */ +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4324) /* structure padding due to alignment */ +#endif typedef struct { union @@ -184,22 +174,24 @@ typedef struct } cinfo; /* NB: must be first */ int cinfo_initialized; - jpeg_error_mgr err; /* libjpeg error manager */ - JMP_BUF exit_jmpbuf; /* for catching libjpeg failures */ + tiff_jpeg_error_mgr err; /* libjpeg error manager */ + JMP_BUF exit_jmpbuf; /* for catching libjpeg failures */ struct jpeg_progress_mgr progress; /* * The following two members could be a union, but * they're small enough that it's not worth the effort. */ - jpeg_destination_mgr dest; /* data dest for compression */ - jpeg_source_mgr src; /* data source for decompression */ - /* private state */ - TIFF *tif; /* back link needed by some code */ - uint16_t photometric; /* copy of PhotometricInterpretation */ - uint16_t h_sampling; /* luminance sampling factors */ + tiff_jpeg_destination_mgr dest; /* data dest for compression */ + tiff_jpeg_source_mgr src; /* data source for decompression */ + /* private state */ + TIFF *tif; /* back link needed by some code */ + uint16_t photometric; /* copy of PhotometricInterpretation */ + uint16_t h_sampling; /* luminance sampling factors */ uint16_t v_sampling; tmsize_t bytesperline; /* decompressed bytes per scanline */ + uint32_t strile_width; + uint32_t strile_height; /* pointers to intermediate buffers when processing downsampled data */ TIFF_JSAMPARRAY ds_buffer[MAX_COMPONENTS]; int scancount; /* number of "scanlines" accumulated */ @@ -209,6 +201,9 @@ typedef struct int encode_raw_error; } JPEGState; +#ifdef _MSC_VER +#pragma warning(pop) +#endif #define JState(tif) ((JPEGState *)(tif)->tif_data) @@ -559,7 +554,7 @@ static boolean tables_empty_output_buffer(j_compress_ptr cinfo) /* the entire buffer has been filled; enlarge it by 1000 bytes */ newbuf = _TIFFreallocExt(sp->tif, (void *)sp->otherSettings.jpegtables, - (tmsize_t)(sp->otherSettings.jpegtables_length + 1000)); + (tmsize_t)sp->otherSettings.jpegtables_length + 1000); if (newbuf == NULL) ERREXIT1(cinfo, JERR_OUT_OF_MEMORY, 100); sp->dest.next_output_byte = @@ -912,7 +907,7 @@ JPEGFixupTagsSubsamplingSec(struct JPEGFixupTagsSubsamplingData *data) return (0); if (n < 2) return (0); - n -= 2; + n = (uint16_t)(n - 2); if (n > 0) JPEGFixupTagsSubsamplingSkip(data, n); } @@ -1016,7 +1011,7 @@ JPEGFixupTagsSubsamplingReadByte(struct JPEGFixupTagsSubsamplingData *data, assert(m < 0x80000000UL); if (TIFFReadFile(data->tif, data->buffer, (tmsize_t)m) != (tmsize_t)m) return (0); - data->buffercurrentbyte = data->buffer; + data->buffercurrentbyte = (uint8_t *)data->buffer; data->bufferbytesleft = m; data->fileoffset += m; data->filebytesleft -= m; @@ -1037,7 +1032,7 @@ JPEGFixupTagsSubsamplingReadWord(struct JPEGFixupTagsSubsamplingData *data, return (0); if (!JPEGFixupTagsSubsamplingReadByte(data, &mb)) return (0); - *result = (ma << 8) | mb; + *result = (uint16_t)((ma << 8) | mb); return (1); } @@ -1159,6 +1154,47 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) return ret; } +static int JPEGComputeStrileWidthHeightBytesPerLine(TIFF *tif, uint16_t s) +{ + JPEGState *sp = JState(tif); + TIFFDirectory *td = &tif->tif_dir; + + /* + * Check image parameters and set decompression parameters. + */ + if (isTiled(tif)) + { + sp->strile_width = td->td_tilewidth; + sp->strile_height = td->td_tilelength; + sp->bytesperline = TIFFTileRowSize(tif); + } + else + { + sp->strile_width = td->td_imagewidth; + sp->strile_height = td->td_imagelength - tif->tif_dir.td_row; + if (sp->strile_height > td->td_rowsperstrip) + sp->strile_height = td->td_rowsperstrip; + sp->bytesperline = TIFFScanlineSize(tif); + } + if (td->td_planarconfig == PLANARCONFIG_SEPARATE && s > 0) + { + /* + * For PC 2, scale down the expected strip/tile size + * to match a downsampled component + */ + if (sp->h_sampling == 0 || sp->v_sampling == 0) + { + TIFFErrorExtR(tif, "JPEGComputeStrileWidthHeightBytesPerLine", + "JPEG horizontal or vertical sampling is zero"); + return (0); + } + sp->strile_width = TIFFhowmany_32(sp->strile_width, sp->h_sampling); + sp->strile_height = TIFFhowmany_32(sp->strile_height, sp->v_sampling); + } + + return 1; +} + /* * Set up for decoding a strip or tile. */ @@ -1167,7 +1203,6 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) JPEGState *sp = JState(tif); TIFFDirectory *td = &tif->tif_dir; static const char module[] = "JPEGPreDecode"; - uint32_t segment_width, segment_height; int downsampled_output; int ci; @@ -1193,52 +1228,24 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) return (0); tif->tif_rawcp = (uint8_t *)sp->src.next_input_byte; - tif->tif_rawcc = sp->src.bytes_in_buffer; + tif->tif_rawcc = (tmsize_t)sp->src.bytes_in_buffer; - /* - * Check image parameters and set decompression parameters. - */ - if (isTiled(tif)) - { - segment_width = td->td_tilewidth; - segment_height = td->td_tilelength; - sp->bytesperline = TIFFTileRowSize(tif); - } - else - { - segment_width = td->td_imagewidth; - segment_height = td->td_imagelength - tif->tif_row; - if (segment_height > td->td_rowsperstrip) - segment_height = td->td_rowsperstrip; - sp->bytesperline = TIFFScanlineSize(tif); - } - if (td->td_planarconfig == PLANARCONFIG_SEPARATE && s > 0) - { - /* - * For PC 2, scale down the expected strip/tile size - * to match a downsampled component - */ - if (sp->h_sampling == 0 || sp->v_sampling == 0) - { - TIFFErrorExtR(tif, module, - "JPEG horizontal or vertical sampling is zero"); - return (0); - } - segment_width = TIFFhowmany_32(segment_width, sp->h_sampling); - segment_height = TIFFhowmany_32(segment_height, sp->v_sampling); - } - if (sp->cinfo.d.image_width < segment_width || - sp->cinfo.d.image_height < segment_height) + if (!JPEGComputeStrileWidthHeightBytesPerLine(tif, s)) + return 0; + + if (sp->cinfo.d.image_width < sp->strile_width || + sp->cinfo.d.image_height < sp->strile_height) { TIFFWarningExtR(tif, module, "Improper JPEG strip/tile size, " "expected %" PRIu32 "x%" PRIu32 ", got %ux%u", - segment_width, segment_height, sp->cinfo.d.image_width, - sp->cinfo.d.image_height); + sp->strile_width, sp->strile_height, + sp->cinfo.d.image_width, sp->cinfo.d.image_height); } - if (sp->cinfo.d.image_width == segment_width && - sp->cinfo.d.image_height > segment_height && - tif->tif_row + segment_height == td->td_imagelength && !isTiled(tif)) + if (sp->cinfo.d.image_width == sp->strile_width && + sp->cinfo.d.image_height > sp->strile_height && + tif->tif_dir.td_row + sp->strile_height == td->td_imagelength && + !isTiled(tif)) { /* Some files have a last strip, that should be truncated, */ /* but their JPEG codestream has still the maximum strip */ @@ -1247,11 +1254,11 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) TIFFWarningExtR(tif, module, "JPEG strip size exceeds expected dimensions," " expected %" PRIu32 "x%" PRIu32 ", got %ux%u", - segment_width, segment_height, sp->cinfo.d.image_width, - sp->cinfo.d.image_height); + sp->strile_width, sp->strile_height, + sp->cinfo.d.image_width, sp->cinfo.d.image_height); } - else if (sp->cinfo.d.image_width > segment_width || - sp->cinfo.d.image_height > segment_height) + else if (sp->cinfo.d.image_width > sp->strile_width || + sp->cinfo.d.image_height > sp->strile_height) { /* * This case could be dangerous, if the strip or tile size has @@ -1262,8 +1269,8 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) TIFFErrorExtR(tif, module, "JPEG strip/tile size exceeds expected dimensions," " expected %" PRIu32 "x%" PRIu32 ", got %ux%u", - segment_width, segment_height, sp->cinfo.d.image_width, - sp->cinfo.d.image_height); + sp->strile_width, sp->strile_height, + sp->cinfo.d.image_width, sp->cinfo.d.image_height); return (0); } if (sp->cinfo.d.num_components != @@ -1282,7 +1289,8 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) sp->cinfo.d.data_precision = td->td_bitspersample; sp->cinfo.d.bits_in_jsample = td->td_bitspersample; #else - if (sp->cinfo.d.data_precision != td->td_bitspersample) + if (td->td_bitspersample != BITS_IN_JSAMPLE || + sp->cinfo.d.data_precision != td->td_bitspersample) { TIFFErrorExtR(tif, module, "Improper JPEG data precision"); return (0); @@ -1319,11 +1327,12 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) if (compptr->h_samp_factor > 0 && compptr->v_samp_factor > 0) { nRequiredMemory += - (toff_t)(((compptr->width_in_blocks + - compptr->h_samp_factor - 1) / - compptr->h_samp_factor)) * - ((compptr->height_in_blocks + compptr->v_samp_factor - 1) / - compptr->v_samp_factor) * + (toff_t)((JDIMENSION)(((int)compptr->width_in_blocks + + compptr->h_samp_factor - 1) / + compptr->h_samp_factor)) * + (JDIMENSION)(((int)compptr->height_in_blocks + + compptr->v_samp_factor - 1) / + compptr->v_samp_factor) * sizeof(JBLOCK); } } @@ -1439,7 +1448,7 @@ int TIFFJPEGIsFullStripRequired(TIFF *tif) * Decode a chunk of pixels. * "Standard" case: returned data is not downsampled. */ -#if !JPEG_LIB_MK1_OR_12BIT +#if !defined(JPEG_LIB_MK1_OR_12BIT) static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) { JPEGState *sp = JState(tif); @@ -1464,7 +1473,18 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) TIFFWarningExtR(tif, tif->tif_name, "fractional scanline not read"); if (nrows > (tmsize_t)sp->cinfo.d.image_height) + { + /* Initialize the output buffer if the JPEG image is smaller than the + * strile height. */ + memset(buf, 0, (size_t)cc); nrows = sp->cinfo.d.image_height; + } + else if (sp->strile_width > sp->cinfo.d.image_width) + { + /* Initialize the output buffer if the JPEG image is smaller than the + * strile width. */ + memset(buf, 0, (size_t)cc); + } /* data is expected to be read in multiples of a scanline */ if (nrows) @@ -1483,7 +1503,7 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) return (0); } - ++tif->tif_row; + ++tif->tif_dir.td_row; buf += sp->bytesperline; cc -= sp->bytesperline; } while (--nrows > 0); @@ -1491,15 +1511,15 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) /* Update information on consumed data */ tif->tif_rawcp = (uint8_t *)sp->src.next_input_byte; - tif->tif_rawcc = sp->src.bytes_in_buffer; + tif->tif_rawcc = (tmsize_t)sp->src.bytes_in_buffer; /* Close down the decompressor if we've finished the strip or tile. */ return sp->cinfo.d.output_scanline < sp->cinfo.d.output_height || TIFFjpeg_finish_decompress(sp); } -#endif /* !JPEG_LIB_MK1_OR_12BIT */ +#endif /* !defined(JPEG_LIB_MK1_OR_12BIT) */ -#if JPEG_LIB_MK1_OR_12BIT +#if defined(JPEG_LIB_MK1_OR_12BIT) /*ARGSUSED*/ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) { @@ -1525,7 +1545,18 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) TIFFWarningExtR(tif, tif->tif_name, "fractional scanline not read"); if (nrows > (tmsize_t)sp->cinfo.d.image_height) + { + /* Initialize the output buffer if the JPEG image is smaller than the + * strile height. */ + memset(buf, 0, (size_t)cc); nrows = sp->cinfo.d.image_height; + } + else if (sp->strile_width > sp->cinfo.d.image_width) + { + /* Initialize the output buffer if the JPEG image is smaller than the + * strile width. */ + memset(buf, 0, (size_t)cc); + } /* data is expected to be read in multiples of a scanline */ if (nrows) @@ -1539,8 +1570,9 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) if (sp->cinfo.d.data_precision == 12) { line_work_buf = (TIFF_JSAMPROW)_TIFFmallocExt( - tif, sizeof(short) * sp->cinfo.d.output_width * - sp->cinfo.d.num_components); + tif, (tmsize_t)((size_t)sizeof(short) * + (size_t)sp->cinfo.d.output_width * + (size_t)sp->cinfo.d.num_components)); } do @@ -1556,14 +1588,16 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) if (TIFFjpeg_read_scanlines(sp, &line_work_buf, 1) != 1) { memset(buf, 0, (size_t)cc); + _TIFFfreeExt(tif, line_work_buf); return (0); } if (sp->cinfo.d.data_precision == 12) { - int value_pairs = (sp->cinfo.d.output_width * - sp->cinfo.d.num_components) / - 2; + int value_pairs = + (int)((JDIMENSION)sp->cinfo.d.output_width * + (JDIMENSION)sp->cinfo.d.num_components) / + 2; int iPair; for (iPair = 0; iPair < value_pairs; iPair++) @@ -1582,18 +1616,19 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) else if (sp->cinfo.d.data_precision == 8) { int value_count = - (sp->cinfo.d.output_width * sp->cinfo.d.num_components); + (int)((JDIMENSION)sp->cinfo.d.output_width * + (JDIMENSION)sp->cinfo.d.num_components); int iValue; for (iValue = 0; iValue < value_count; iValue++) { ((unsigned char *)buf)[iValue] = - line_work_buf[iValue] & 0xff; + (unsigned char)(line_work_buf[iValue] & 0xff); } } } - ++tif->tif_row; + ++tif->tif_dir.td_row; buf += sp->bytesperline; cc -= sp->bytesperline; } while (--nrows > 0); @@ -1604,13 +1639,13 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) /* Update information on consumed data */ tif->tif_rawcp = (uint8_t *)sp->src.next_input_byte; - tif->tif_rawcc = sp->src.bytes_in_buffer; + tif->tif_rawcc = (tmsize_t)sp->src.bytes_in_buffer; /* Close down the decompressor if we've finished the strip or tile. */ return sp->cinfo.d.output_scanline < sp->cinfo.d.output_height || TIFFjpeg_finish_decompress(sp); } -#endif /* JPEG_LIB_MK1_OR_12BIT */ +#endif /* defined(JPEG_LIB_MK1_OR_12BIT) */ /*ARGSUSED*/ static int DecodeRowError(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) @@ -1644,8 +1679,16 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) /* For last strip, limit number of rows to its truncated height */ /* even if the codestream height is larger (which is not compliant, */ /* but that we tolerate) */ - if ((uint32_t)nrows > td->td_imagelength - tif->tif_row && !isTiled(tif)) - nrows = td->td_imagelength - tif->tif_row; + if ((uint32_t)nrows > td->td_imagelength - tif->tif_dir.td_row && + !isTiled(tif)) + nrows = td->td_imagelength - tif->tif_dir.td_row; + + /* The downsampled-data decode loop below only writes up to the JPEG + * codestream dimensions (and, due to the clump stride, may not cover the + * full caller buffer even for compliant files). Zero the whole buffer so + * no uninitialised heap is ever returned, mirroring the guard JPEGDecode + * received in 65759931ab6e (#826). */ + memset(buf, 0, (size_t)cc); #if defined(JPEG_LIB_MK1_OR_12BIT) unsigned short *tmpbuf = NULL; @@ -1660,9 +1703,10 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) int samples_per_clump = sp->samplesperclump; #if defined(JPEG_LIB_MK1_OR_12BIT) - tmpbuf = _TIFFmallocExt(tif, sizeof(unsigned short) * - sp->cinfo.d.output_width * - sp->cinfo.d.num_components); + tmpbuf = (unsigned short *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)sizeof(unsigned short) * + (size_t)sp->cinfo.d.output_width * + (size_t)sp->cinfo.d.num_components)); if (tmpbuf == NULL) { TIFFErrorExtR(tif, "JPEGDecodeRaw", "Out of memory"); @@ -1755,18 +1799,20 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) if (sp->cinfo.d.data_precision == 8) { int i = 0; - int len = - sp->cinfo.d.output_width * sp->cinfo.d.num_components; + int len = (int)((JDIMENSION)sp->cinfo.d.output_width * + (JDIMENSION)sp->cinfo.d.num_components); for (i = 0; i < len; i++) { - ((unsigned char *)buf)[i] = tmpbuf[i] & 0xff; + ((unsigned char *)buf)[i] = + (unsigned char)(tmpbuf[i] & 0xff); } } else { /* 12-bit */ - int value_pairs = (sp->cinfo.d.output_width * - sp->cinfo.d.num_components) / - 2; + int value_pairs = + (int)((JDIMENSION)sp->cinfo.d.output_width * + (JDIMENSION)sp->cinfo.d.num_components) / + 2; int iPair; for (iPair = 0; iPair < value_pairs; iPair++) { @@ -1785,7 +1831,7 @@ static int JPEGDecode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) #endif sp->scancount++; - tif->tif_row += sp->v_sampling; + tif->tif_dir.td_row += sp->v_sampling; buf += sp->bytesperline; cc -= sp->bytesperline; @@ -2024,9 +2070,9 @@ static int JPEGSetupEncode(TIFF *tif) if (!TIFFGetField(tif, TIFFTAG_REFERENCEBLACKWHITE, &ref)) { float refbw[6]; - long top = 1L << td->td_bitspersample; + uint32_t top = 1U << td->td_bitspersample; refbw[0] = 0; - refbw[1] = (float)(top - 1L); + refbw[1] = (float)(top - 1); refbw[2] = (float)(top >> 1); refbw[3] = refbw[1]; refbw[4] = refbw[2]; @@ -2074,30 +2120,30 @@ static int JPEGSetupEncode(TIFF *tif) #endif if (isTiled(tif)) { - if ((td->td_tilelength % (sp->v_sampling * DCTSIZE)) != 0) + if ((td->td_tilelength % ((uint32_t)sp->v_sampling * DCTSIZE)) != 0) { TIFFErrorExtR(tif, module, "JPEG tile height must be multiple of %" PRIu32, - (uint32_t)(sp->v_sampling * DCTSIZE)); + (uint32_t)sp->v_sampling * DCTSIZE); return (0); } - if ((td->td_tilewidth % (sp->h_sampling * DCTSIZE)) != 0) + if ((td->td_tilewidth % ((uint32_t)sp->h_sampling * DCTSIZE)) != 0) { TIFFErrorExtR(tif, module, "JPEG tile width must be multiple of %" PRIu32, - (uint32_t)(sp->h_sampling * DCTSIZE)); + (uint32_t)sp->h_sampling * DCTSIZE); return (0); } } else { if (td->td_rowsperstrip < td->td_imagelength && - (td->td_rowsperstrip % (sp->v_sampling * DCTSIZE)) != 0) + (td->td_rowsperstrip % ((uint32_t)sp->v_sampling * DCTSIZE)) != 0) { TIFFErrorExtR(tif, module, "RowsPerStrip must be multiple of %" PRIu32 " for JPEG", - (uint32_t)(sp->v_sampling * DCTSIZE)); + (uint32_t)sp->v_sampling * DCTSIZE); return (0); } } @@ -2153,7 +2199,6 @@ static int JPEGPreEncode(TIFF *tif, uint16_t s) JPEGState *sp = JState(tif); TIFFDirectory *td = &tif->tif_dir; static const char module[] = "JPEGPreEncode"; - uint32_t segment_width, segment_height; int downsampled_input; assert(sp != NULL); @@ -2164,47 +2209,20 @@ static int JPEGPreEncode(TIFF *tif, uint16_t s) } assert(!sp->cinfo.comm.is_decompressor); - /* - * Set encoding parameters for this strip/tile. - */ - if (isTiled(tif)) - { - segment_width = td->td_tilewidth; - segment_height = td->td_tilelength; - sp->bytesperline = TIFFTileRowSize(tif); - } - else - { - segment_width = td->td_imagewidth; - segment_height = td->td_imagelength - tif->tif_row; - if (segment_height > td->td_rowsperstrip) - segment_height = td->td_rowsperstrip; - sp->bytesperline = TIFFScanlineSize(tif); - } - if (td->td_planarconfig == PLANARCONFIG_SEPARATE && s > 0) - { - /* for PC 2, scale down the strip/tile size - * to match a downsampled component - */ - if (sp->h_sampling == 0 || sp->v_sampling == 0) - { - TIFFErrorExtR(tif, module, - "JPEG horizontal or vertical sampling is zero"); - return (0); - } - segment_width = TIFFhowmany_32(segment_width, sp->h_sampling); - segment_height = TIFFhowmany_32(segment_height, sp->v_sampling); - } - if (segment_width > (uint32_t)JPEG_MAX_DIMENSION || - segment_height > (uint32_t)JPEG_MAX_DIMENSION) + + if (!JPEGComputeStrileWidthHeightBytesPerLine(tif, s)) + return 0; + + if (sp->strile_width > (uint32_t)JPEG_MAX_DIMENSION || + sp->strile_height > (uint32_t)JPEG_MAX_DIMENSION) { TIFFErrorExtR(tif, module, "Strip/tile too large for JPEG. Maximum dimension is %d", (int)JPEG_MAX_DIMENSION); return (0); } - sp->cinfo.c.image_width = segment_width; - sp->cinfo.c.image_height = segment_height; + sp->cinfo.c.image_width = sp->strile_width; + sp->cinfo.c.image_height = sp->strile_height; downsampled_input = FALSE; if (td->td_planarconfig == PLANARCONFIG_CONTIG) { @@ -2276,7 +2294,13 @@ static int JPEGPreEncode(TIFF *tif, uint16_t s) /* an existing file */ suppress_huff_table(sp, 0); suppress_huff_table(sp, 1); - sp->cinfo.c.optimize_coding = FALSE; + + /* We want to keep optimize_coding = TRUE for 12-bit JPEG */ + /* See lengthy explanation at + * https://gitlab.com/libtiff/libtiff/-/work_items/773#note_3009836854 + */ + if (sp->cinfo.c.data_precision == 8) + sp->cinfo.c.optimize_coding = FALSE; } else sp->cinfo.c.optimize_coding = TRUE; @@ -2332,13 +2356,15 @@ static int JPEGEncode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) TIFFWarningExtR(tif, tif->tif_name, "fractional scanline discarded"); /* The last strip will be limited to image size */ - if (!isTiled(tif) && tif->tif_row + nrows > tif->tif_dir.td_imagelength) - nrows = tif->tif_dir.td_imagelength - tif->tif_row; + if (!isTiled(tif) && + tif->tif_dir.td_row + nrows > tif->tif_dir.td_imagelength) + nrows = tif->tif_dir.td_imagelength - tif->tif_dir.td_row; if (sp->cinfo.c.data_precision == 12) { line16_count = (int)((sp->bytesperline * 2) / 3); - line16 = (short *)_TIFFmallocExt(tif, sizeof(short) * line16_count); + line16 = (short *)_TIFFmallocExt( + tif, (tmsize_t)(sizeof(short) * (size_t)line16_count)); if (!line16) { TIFFErrorExtR(tif, "JPEGEncode", "Failed to allocate memory"); @@ -2363,8 +2389,10 @@ static int JPEGEncode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) unsigned char *in_ptr = ((unsigned char *)buf) + iPair * 3; TIFF_JSAMPLE *out_ptr = (TIFF_JSAMPLE *)(line16 + iPair * 2); - out_ptr[0] = (in_ptr[0] << 4) | ((in_ptr[1] & 0xf0) >> 4); - out_ptr[1] = ((in_ptr[1] & 0x0f) << 8) | in_ptr[2]; + out_ptr[0] = (TIFF_JSAMPLE)((in_ptr[0] << 4) | + ((in_ptr[1] & 0xf0) >> 4)); + out_ptr[1] = + (TIFF_JSAMPLE)(((in_ptr[1] & 0x0f) << 8) | in_ptr[2]); } } else @@ -2374,7 +2402,7 @@ static int JPEGEncode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) if (TIFFjpeg_write_scanlines(sp, bufptr, 1) != 1) return (0); if (nrows > 0) - tif->tif_row++; + tif->tif_dir.td_row++; buf += sp->bytesperline; } @@ -2443,8 +2471,9 @@ static int JPEGEncodeRaw(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) { int hsamp = compptr->h_samp_factor; int vsamp = compptr->v_samp_factor; - int padding = (int)(compptr->width_in_blocks * DCTSIZE - - clumps_per_line * hsamp); + int padding = + (int)(compptr->width_in_blocks * DCTSIZE - + (JDIMENSION)clumps_per_line * (JDIMENSION)hsamp); for (ypos = 0; ypos < vsamp; ypos++) { inptr = ((TIFF_JSAMPLE *)buf) + clumpoffset; @@ -2488,7 +2517,7 @@ static int JPEGEncodeRaw(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) } sp->scancount = 0; } - tif->tif_row += sp->v_sampling; + tif->tif_dir.td_row += sp->v_sampling; buf += bytesperclumpline; nrows -= sp->v_sampling; } @@ -2582,10 +2611,11 @@ static void JPEGResetUpsampled(TIFF *tif) * Must recalculate cached tile size in case sampling state changed. * Should we really be doing this now if image size isn't set? */ - if (tif->tif_tilesize > 0) - tif->tif_tilesize = isTiled(tif) ? TIFFTileSize(tif) : (tmsize_t)(-1); - if (tif->tif_scanlinesize > 0) - tif->tif_scanlinesize = TIFFScanlineSize(tif); + if (tif->tif_dir.td_tilesize > 0) + tif->tif_dir.td_tilesize = + isTiled(tif) ? TIFFTileSize(tif) : (tmsize_t)(-1); + if (tif->tif_dir.td_scanlinesize > 0) + tif->tif_dir.td_scanlinesize = TIFFScanlineSize(tif); } static int JPEGVSetField(TIFF *tif, uint32_t tag, va_list ap) @@ -2793,6 +2823,33 @@ static int JPEGInitializeLibJPEG(TIFF *tif, int decompress) return 1; } +static uint64_t JPEGGetMaxCompressionRatio(TIFF *tif) +{ + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + const JPEGState *sp = JState(tif); + if ((tif->tif_dir.td_photometric == PHOTOMETRIC_YCBCR) && + (tif->tif_dir.td_planarconfig == PLANARCONFIG_CONTIG) && + (tif->tif_dir.td_samplesperpixel == 3)) + { + if (sp->h_sampling == 2 && sp->v_sampling == 2) + { + if (tif->tif_dir.td_bitspersample == 12) + return 768; + else + return 512; + } + + return 0; /* unknown */ + } + + if (tif->tif_dir.td_bitspersample == 12) + return 384; + else + return 256; +} + /* Common to tif_jpeg.c and tif_jpeg_12.c */ static void TIFFInitJPEGCommon(TIFF *tif) { @@ -2829,6 +2886,7 @@ static void TIFFInitJPEGCommon(TIFF *tif) tif->tif_encoderow = JPEGEncode; tif->tif_encodestrip = JPEGEncode; tif->tif_encodetile = JPEGEncode; + tif->tif_getmaxcompressionratio = JPEGGetMaxCompressionRatio; tif->tif_cleanup = JPEGCleanup; tif->tif_defstripsize = JPEGDefaultStripSize; diff --git a/contrib/libs/libtiff/tif_lerc.c b/contrib/libs/libtiff/tif_lerc.c index 4c7f1b210c3..cdbc6106b01 100644 --- a/contrib/libs/libtiff/tif_lerc.c +++ b/contrib/libs/libtiff/tif_lerc.c @@ -33,6 +33,7 @@ #error #include "Lerc_c_api.h" #include "zlib.h" +#include <math.h> #ifdef ZSTD_SUPPORT #include "zstd.h" #endif @@ -188,7 +189,7 @@ static int SetupBuffers(TIFF *tif, LERCState *sp, const char *module) else { sp->segment_width = td->td_imagewidth; - sp->segment_height = td->td_imagelength - tif->tif_row; + sp->segment_height = td->td_imagelength - tif->tif_dir.td_row; if (sp->segment_height > td->td_rowsperstrip) sp->segment_height = td->td_rowsperstrip; } @@ -242,7 +243,7 @@ static int SetupBuffers(TIFF *tif, LERCState *sp, const char *module) } if ((td->td_planarconfig == PLANARCONFIG_CONTIG && - td->td_extrasamples > 0 && + td->td_extrasamples > 0 && td->td_sampleinfo && td->td_sampleinfo[td->td_extrasamples - 1] == EXTRASAMPLE_UNASSALPHA && GetLercDataType(tif) == 1) || (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && @@ -343,7 +344,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) if (res != LIBDEFLATE_SUCCESS) { TIFFErrorExtR(tif, module, "Decoding error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); return 0; } assert(lerc_data_sizet == (unsigned int)lerc_data_sizet); @@ -387,7 +388,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) size_t zstd_ret; zstd_ret = ZSTD_decompress(sp->compressed_buffer, sp->compressed_size, - tif->tif_rawcp, tif->tif_rawcc); + tif->tif_rawcp, (size_t)tif->tif_rawcc); if (ZSTD_isError(zstd_ret)) { TIFFErrorExtR(tif, module, "Error in ZSTD_decompress(): %s", @@ -420,6 +421,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) /* LERC info has dim == samplesperpixel - 1, then there is a LERC */ /* mask. */ if (td->td_planarconfig == PLANARCONFIG_CONTIG && td->td_extrasamples > 0 && + td->td_sampleinfo && td->td_sampleinfo[td->td_extrasamples - 1] == EXTRASAMPLE_UNASSALPHA && GetLercDataType(tif) == 1 && infoArray[2] == td->td_samplesperpixel - 1U) @@ -432,7 +434,8 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) use_mask = 1; } - ndims = td->td_planarconfig == PLANARCONFIG_CONTIG ? nomask_bands : 1; + ndims = + (int)(td->td_planarconfig == PLANARCONFIG_CONTIG ? nomask_bands : 1); /* Info returned in infoArray is { version, dataType, nDim/nDepth, nCols, nRows, nBands, nValidPixels, blobSize, @@ -440,12 +443,12 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) if (infoArray[0] != (unsigned)sp->lerc_version) { TIFFWarningExtR(tif, module, - "Unexpected version number: %d. Expected: %d", + "Unexpected version number: %u. Expected: %d", infoArray[0], sp->lerc_version); } if (infoArray[1] != (unsigned)lerc_data_type) { - TIFFErrorExtR(tif, module, "Unexpected dataType: %d. Expected: %d", + TIFFErrorExtR(tif, module, "Unexpected dataType: %u. Expected: %d", infoArray[1], lerc_data_type); return 0; } @@ -458,7 +461,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) { if (nFoundDims != 1 && nFoundDims != (unsigned)ndims) { - TIFFErrorExtR(tif, module, "Unexpected nDim: %d. Expected: 1 or %d", + TIFFErrorExtR(tif, module, "Unexpected nDim: %u. Expected: 1 or %d", nFoundDims, ndims); return 0; } @@ -467,20 +470,20 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) #endif if (nFoundDims != (unsigned)ndims) { - TIFFErrorExtR(tif, module, "Unexpected nDim: %d. Expected: %d", + TIFFErrorExtR(tif, module, "Unexpected nDim: %u. Expected: %d", nFoundDims, ndims); return 0; } if (infoArray[3] != sp->segment_width) { - TIFFErrorExtR(tif, module, "Unexpected nCols: %d. Expected: %du", + TIFFErrorExtR(tif, module, "Unexpected nCols: %u. Expected: %u", infoArray[3], sp->segment_width); return 0; } if (infoArray[4] != sp->segment_height) { - TIFFErrorExtR(tif, module, "Unexpected nRows: %d. Expected: %u", + TIFFErrorExtR(tif, module, "Unexpected nRows: %u. Expected: %u", infoArray[4], sp->segment_height); return 0; } @@ -504,28 +507,28 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) #endif if (nFoundBands != td->td_samplesperpixel) { - TIFFErrorExtR(tif, module, "Unexpected nBands: %d. Expected: %d", + TIFFErrorExtR(tif, module, "Unexpected nBands: %u. Expected: %d", nFoundBands, td->td_samplesperpixel); return 0; } } else if (nFoundBands != 1) { - TIFFErrorExtR(tif, module, "Unexpected nBands: %d. Expected: %d", + TIFFErrorExtR(tif, module, "Unexpected nBands: %u. Expected: %d", nFoundBands, 1); return 0; } if (infoArray[7] != lerc_data_size) { - TIFFErrorExtR(tif, module, "Unexpected blobSize: %d. Expected: %u", + TIFFErrorExtR(tif, module, "Unexpected blobSize: %u. Expected: %u", infoArray[7], lerc_data_size); return 0; } int nRequestedMasks = use_mask ? 1 : 0; #if LERC_AT_LEAST_VERSION(3, 0, 0) - const int nFoundMasks = infoArray[8]; + const int nFoundMasks = (int)infoArray[8]; if (td->td_sampleformat == SAMPLEFORMAT_IEEEFP && td->td_planarconfig == PLANARCONFIG_CONTIG && td->td_samplesperpixel > 1 && nFoundDims == 1) @@ -576,21 +579,22 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) sp->uncompressed_buffer_multiband_alloc = num_bytes_needed; } lerc_ret = lerc_decode(lerc_data, lerc_data_size, nRequestedMasks, - sp->mask_buffer, nFoundDims, sp->segment_width, - sp->segment_height, nFoundBands, lerc_data_type, + sp->mask_buffer, (int)nFoundDims, + (int)sp->segment_width, (int)sp->segment_height, + (int)nFoundBands, (unsigned int)lerc_data_type, sp->uncompressed_buffer_multiband); } else #endif { - lerc_ret = - lerc_decode(lerc_data, lerc_data_size, + lerc_ret = lerc_decode( + lerc_data, lerc_data_size, #if LERC_AT_LEAST_VERSION(3, 0, 0) - nRequestedMasks, + nRequestedMasks, #endif - use_mask ? sp->mask_buffer : NULL, nFoundDims, - sp->segment_width, sp->segment_height, nFoundBands, - lerc_data_type, sp->uncompressed_buffer); + use_mask ? sp->mask_buffer : NULL, (int)nFoundDims, + (int)sp->segment_width, (int)sp->segment_height, (int)nFoundBands, + (unsigned int)lerc_data_type, sp->uncompressed_buffer); } if (lerc_ret != 0) { @@ -601,17 +605,17 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) /* Interleave alpha mask with other samples. */ if (use_mask && GetLercDataType(tif) == 1) { - unsigned src_stride = - (td->td_samplesperpixel - 1) * (td->td_bitspersample / 8); + unsigned src_stride = (unsigned int)((td->td_samplesperpixel - 1) * + (td->td_bitspersample / 8)); unsigned dst_stride = - td->td_samplesperpixel * (td->td_bitspersample / 8); + (unsigned int)(td->td_samplesperpixel * (td->td_bitspersample / 8)); unsigned i = sp->segment_width * sp->segment_height; /* Operate from end to begin to be able to move in place */ while (i > 0 && i > nomask_bands) { i--; sp->uncompressed_buffer[i * dst_stride + td->td_samplesperpixel - - 1] = 255 * sp->mask_buffer[i]; + 1] = (uint8_t)(255 * sp->mask_buffer[i]); memcpy(sp->uncompressed_buffer + i * dst_stride, sp->uncompressed_buffer + i * src_stride, src_stride); } @@ -620,7 +624,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) { i--; sp->uncompressed_buffer[i * dst_stride + td->td_samplesperpixel - - 1] = 255 * sp->mask_buffer[i]; + 1] = (uint8_t)(255 * sp->mask_buffer[i]); memmove(sp->uncompressed_buffer + i * dst_stride, sp->uncompressed_buffer + i * src_stride, src_stride); } @@ -649,7 +653,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) } else { - const double nan_float64 = nan_float32; + const double nan_float64 = (double)nan_float32; for (i = 0; i < nb_pixels; i++) { if (sp->mask_buffer[i] == 0) @@ -677,7 +681,7 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) } else { - const double nan_float64 = nan_float32; + const double nan_float64 = (double)nan_float32; for (i = 0; i < nb_pixels; i++) { for (int j = 0; j < td->td_samplesperpixel; j++) @@ -704,30 +708,32 @@ static int LERCPreDecode(TIFF *tif, uint16_t s) { for (int j = 0; j < td->td_samplesperpixel; j++) { - if (sp->mask_buffer[i + j * nb_pixels] == 0) + if (sp->mask_buffer[i + (unsigned int)j * nb_pixels] == + 0) ((float *)sp->uncompressed_buffer)[k] = nan_float32; else ((float *)sp->uncompressed_buffer)[k] = ((float *)sp->uncompressed_buffer_multiband) - [i + j * nb_pixels]; + [i + (unsigned int)j * nb_pixels]; ++k; } } } else { - const double nan_float64 = nan_float32; + const double nan_float64 = (double)nan_float32; for (i = 0; i < nb_pixels; i++) { for (int j = 0; j < td->td_samplesperpixel; j++) { - if (sp->mask_buffer[i + j * nb_pixels] == 0) + if (sp->mask_buffer[i + (unsigned int)j * nb_pixels] == + 0) ((double *)sp->uncompressed_buffer)[k] = nan_float64; else ((double *)sp->uncompressed_buffer)[k] = ((double *)sp->uncompressed_buffer_multiband) - [i + j * nb_pixels]; + [i + (unsigned int)j * nb_pixels]; ++k; } } @@ -766,8 +772,8 @@ static int LERCDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) return 0; } - memcpy(op, sp->uncompressed_buffer + sp->uncompressed_offset, occ); - sp->uncompressed_offset += (unsigned)occ; + memcpy(op, sp->uncompressed_buffer + sp->uncompressed_offset, (size_t)occ); + sp->uncompressed_offset += (unsigned int)occ; return 1; } @@ -832,8 +838,8 @@ static int LERCEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) return 0; } - memcpy(sp->uncompressed_buffer + sp->uncompressed_offset, bp, cc); - sp->uncompressed_offset += (unsigned)cc; + memcpy(sp->uncompressed_buffer + sp->uncompressed_offset, bp, (size_t)cc); + sp->uncompressed_offset += (unsigned int)cc; return 1; } @@ -863,13 +869,15 @@ static int LERCPostEncode(TIFF *tif) /* Extract alpha mask (if containing only 0 and 255 values, */ /* and compact array of regular bands */ if (td->td_planarconfig == PLANARCONFIG_CONTIG && td->td_extrasamples > 0 && + td->td_sampleinfo && td->td_sampleinfo[td->td_extrasamples - 1] == EXTRASAMPLE_UNASSALPHA && GetLercDataType(tif) == 1) { const unsigned dst_stride = - (td->td_samplesperpixel - 1) * (td->td_bitspersample / 8); + (unsigned int)((td->td_samplesperpixel - 1) * + (td->td_bitspersample / 8)); const unsigned src_stride = - td->td_samplesperpixel * (td->td_bitspersample / 8); + (unsigned int)(td->td_samplesperpixel * (td->td_bitspersample / 8)); unsigned i = 0; use_mask = 1; @@ -923,7 +931,7 @@ static int LERCPostEncode(TIFF *tif) { const float val = ((float *)sp->uncompressed_buffer)[k]; ++k; - if (val != val) + if (isnan(val)) { ++count_nan; } @@ -944,7 +952,7 @@ static int LERCPostEncode(TIFF *tif) for (i = 0; i < nb_pixels; i++) { const float val = ((float *)sp->uncompressed_buffer)[i]; - if (val != val) + if (isnan(val)) { use_mask = 1; break; @@ -965,7 +973,7 @@ static int LERCPostEncode(TIFF *tif) const double val = ((double *)sp->uncompressed_buffer)[k]; ++k; - if (val != val) + if (isnan(val)) { ++count_nan; } @@ -986,7 +994,7 @@ static int LERCPostEncode(TIFF *tif) for (i = 0; i < nb_pixels; i++) { const double val = ((double *)sp->uncompressed_buffer)[i]; - if (val != val) + if (isnan(val)) { use_mask = 1; break; @@ -1025,10 +1033,10 @@ static int LERCPostEncode(TIFF *tif) const float val = ((float *)sp->uncompressed_buffer)[k]; ((float *)sp->uncompressed_buffer_multiband) - [i + j * nb_pixels] = val; + [i + (unsigned int)j * nb_pixels] = val; ++k; - sp->mask_buffer[i + j * nb_pixels] = - (val == val) ? 255 : 0; + sp->mask_buffer[i + (unsigned int)j * nb_pixels] = + !isnan(val) ? 255 : 0; } } } @@ -1041,10 +1049,10 @@ static int LERCPostEncode(TIFF *tif) const double val = ((double *)sp->uncompressed_buffer)[k]; ((double *)sp->uncompressed_buffer_multiband) - [i + j * nb_pixels] = val; + [i + (unsigned int)j * nb_pixels] = val; ++k; - sp->mask_buffer[i + j * nb_pixels] = - (val == val) ? 255 : 0; + sp->mask_buffer[i + (unsigned int)j * nb_pixels] = + !isnan(val) ? 255 : 0; } } } @@ -1064,7 +1072,7 @@ static int LERCPostEncode(TIFF *tif) { const float val = ((float *)sp->uncompressed_buffer)[i * dst_nbands]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + sp->mask_buffer[i] = !isnan(val) ? 255 : 0; } } else @@ -1073,7 +1081,7 @@ static int LERCPostEncode(TIFF *tif) { const double val = ((double *)sp->uncompressed_buffer)[i * dst_nbands]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + sp->mask_buffer[i] = !isnan(val) ? 255 : 0; } } } @@ -1084,7 +1092,7 @@ static int LERCPostEncode(TIFF *tif) for (i = 0; i < nb_pixels; i++) { const float val = ((float *)sp->uncompressed_buffer)[i]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + sp->mask_buffer[i] = !isnan(val) ? 255 : 0; } } else @@ -1093,7 +1101,7 @@ static int LERCPostEncode(TIFF *tif) { const double val = ((double *)sp->uncompressed_buffer)[i]; - sp->mask_buffer[i] = (val == val) ? 255 : 0; + sp->mask_buffer[i] = !isnan(val) ? 255 : 0; } } } @@ -1104,7 +1112,8 @@ static int LERCPostEncode(TIFF *tif) #if LERC_AT_LEAST_VERSION(3, 0, 0) if (mask_count > 1) { - estimated_compressed_size += nb_pixels * mask_count / 8; + estimated_compressed_size += + (unsigned int)(nb_pixels * (unsigned int)mask_count / 8); } #endif @@ -1125,8 +1134,9 @@ static int LERCPostEncode(TIFF *tif) { lerc_ret = lerc_encodeForVersion( sp->uncompressed_buffer_multiband, sp->lerc_version, - GetLercDataType(tif), 1, sp->segment_width, sp->segment_height, - dst_nbands, dst_nbands, sp->mask_buffer, sp->maxzerror, + (unsigned int)GetLercDataType(tif), 1, (int)sp->segment_width, + (int)sp->segment_height, (int)dst_nbands, (int)dst_nbands, + sp->mask_buffer, sp->maxzerror, (unsigned char *)sp->compressed_buffer, sp->compressed_size, &numBytesWritten); } @@ -1134,9 +1144,10 @@ static int LERCPostEncode(TIFF *tif) #endif { lerc_ret = lerc_encodeForVersion( - sp->uncompressed_buffer, sp->lerc_version, GetLercDataType(tif), - td->td_planarconfig == PLANARCONFIG_CONTIG ? dst_nbands : 1, - sp->segment_width, sp->segment_height, 1, + sp->uncompressed_buffer, sp->lerc_version, + (unsigned int)GetLercDataType(tif), + (int)(td->td_planarconfig == PLANARCONFIG_CONTIG ? dst_nbands : 1), + (int)sp->segment_width, (int)sp->segment_height, 1, #if LERC_AT_LEAST_VERSION(3, 0, 0) use_mask ? 1 : 0, #endif @@ -1179,14 +1190,14 @@ static int LERCPostEncode(TIFF *tif) return 0; } - tif->tif_rawcc = libdeflate_zlib_compress( + tif->tif_rawcc = (tmsize_t)libdeflate_zlib_compress( sp->libdeflate_enc, sp->compressed_buffer, numBytesWritten, sp->uncompressed_buffer, sp->uncompressed_alloc); if (tif->tif_rawcc == 0) { TIFFErrorExtR(tif, module, "Encoder error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); return 0; } #else @@ -1252,7 +1263,7 @@ static int LERCPostEncode(TIFF *tif) int ret; uint8_t *tif_rawdata_backup = tif->tif_rawdata; tif->tif_rawdata = sp->uncompressed_buffer; - tif->tif_rawcc = zstd_ret; + tif->tif_rawcc = (tmsize_t)zstd_ret; ret = TIFFFlushData1(tif); tif->tif_rawdata = tif_rawdata_backup; if (!ret) @@ -1350,7 +1361,7 @@ static int LERCVSetField(TIFF *tif, uint32_t tag, va_list ap) { case TIFFTAG_LERC_PARAMETERS: { - uint32_t count = va_arg(ap, int); + uint32_t count = (uint32_t)va_arg(ap, int); int *params = va_arg(ap, int *); if (count < 2) { @@ -1529,6 +1540,13 @@ int TIFFInitLERC(TIFF *tif, int scheme) #endif tif->tif_cleanup = LERCCleanup; + /* LERC compression ratio can grow to several millions */ + /* eg. 5703725 for Lerc deflate on 16383x16383 array */ + /* or 3829644 for regular Lerc */ + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + /* so we don't define tif->tif_getmaxcompressionratio */ + /* Default values for codec-specific fields */ TIFFSetField(tif, TIFFTAG_LERC_VERSION, LERC_VERSION_2_4); TIFFSetField(tif, TIFFTAG_LERC_ADD_COMPRESSION, LERC_ADD_COMPRESSION_NONE); diff --git a/contrib/libs/libtiff/tif_luv.c b/contrib/libs/libtiff/tif_luv.c index ecc32252659..d9bc8fe3105 100644 --- a/contrib/libs/libtiff/tif_luv.c +++ b/contrib/libs/libtiff/tif_luv.c @@ -212,7 +212,7 @@ static int LogL16Decode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } tp = (int16_t *)sp->tbuf; } - _TIFFmemset((void *)tp, 0, npixels * sizeof(tp[0])); + _TIFFmemset((void *)tp, 0, (tmsize_t)((size_t)npixels * sizeof(tp[0]))); bp = (unsigned char *)tif->tif_rawcp; cc = tif->tif_rawcc; @@ -235,7 +235,7 @@ static int LogL16Decode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { /* non-run */ rc = *bp++; /* nul is noop */ while (--cc && rc-- && i < npixels) - tp[i++] |= (int16_t)*bp++ << shft; + tp[i++] |= (int16_t)(*bp++ << shft); } } if (i != npixels) @@ -243,7 +243,7 @@ static int LogL16Decode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at row %" PRIu32 " (short %" TIFF_SSIZE_FORMAT " pixels)", - tif->tif_row, npixels - i); + tif->tif_dir.td_row, npixels - i); tif->tif_rawcp = (uint8_t *)bp; tif->tif_rawcc = cc; return (0); @@ -290,7 +290,7 @@ static int LogLuvDecode24(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) cc = tif->tif_rawcc; for (i = 0; i < npixels && cc >= 3; i++) { - tp[i] = bp[0] << 16 | bp[1] << 8 | bp[2]; + tp[i] = (uint32_t)bp[0] << 16 | (uint32_t)bp[1] << 8 | bp[2]; bp += 3; cc -= 3; } @@ -301,7 +301,7 @@ static int LogLuvDecode24(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at row %" PRIu32 " (short %" TIFF_SSIZE_FORMAT " pixels)", - tif->tif_row, npixels - i); + tif->tif_dir.td_row, npixels - i); return (0); } (*sp->tfunc)(sp, op, npixels); @@ -342,7 +342,7 @@ static int LogLuvDecode32(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } tp = (uint32_t *)sp->tbuf; } - _TIFFmemset((void *)tp, 0, npixels * sizeof(tp[0])); + _TIFFmemset((void *)tp, 0, (tmsize_t)((size_t)npixels * sizeof(tp[0]))); bp = (unsigned char *)tif->tif_rawcp; cc = tif->tif_rawcc; @@ -373,7 +373,7 @@ static int LogLuvDecode32(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at row %" PRIu32 " (short %" TIFF_SSIZE_FORMAT " pixels)", - tif->tif_row, npixels - i); + tif->tif_dir.td_row, npixels - i); tif->tif_rawcp = (uint8_t *)bp; tif->tif_rawcc = cc; return (0); @@ -767,25 +767,21 @@ static int LogLuvEncodeTile(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) #ifndef M_PI #define M_PI 3.14159265358979323846 #endif -#undef log2 /* Conflict with C'99 function */ -#define log2(x) ((1. / M_LN2) * log(x)) -#undef exp2 /* Conflict with C'99 function */ -#define exp2(x) exp(M_LN2 *(x)) #define TIFF_RAND_MAX 32767 // From POSIX.1-2001 as an example of an implementation of rand() -static uint32_t _TIFFRand() +static uint32_t _TIFFRand(void) { static uint32_t nCounter = 0; if (!nCounter) nCounter = (uint32_t)(time(NULL) & UINT32_MAX); ++nCounter; uint32_t nCounterLocal = - (uint32_t)(((uint64_t)(nCounter)*1103515245U + 12345U) & UINT32_MAX); + (uint32_t)(((uint64_t)(nCounter) * 1103515245U + 12345U) & UINT32_MAX); nCounter = nCounterLocal; return (nCounterLocal / 65536U) % (TIFF_RAND_MAX + 1); -}; +} static int tiff_itrunc(double x, int m) { @@ -797,8 +793,7 @@ static int tiff_itrunc(double x, int m) #if !LOGLUV_PUBLIC static #endif - double - LogL16toY(int p16) /* compute luminance from 16-bit LogL */ + double LogL16toY(int p16) /* compute luminance from 16-bit LogL */ { int Le = p16 & 0x7fff; double Y; @@ -812,8 +807,7 @@ static #if !LOGLUV_PUBLIC static #endif - int - LogL16fromY(double Y, int em) /* get 16-bit LogL from Y */ + int LogL16fromY(double Y, int em) /* get 16-bit LogL from Y */ { if (Y >= 1.8371976e19) return (0x7fff); @@ -855,20 +849,22 @@ static void L16fromY(LogLuvState *sp, uint8_t *op, tmsize_t n) float *yp = (float *)op; while (n-- > 0) - *l16++ = (int16_t)(LogL16fromY(*yp++, sp->encode_meth)); + *l16++ = (int16_t)(LogL16fromY((double)*yp++, sp->encode_meth)); } #if !LOGLUV_PUBLIC static #endif - void - XYZtoRGB24(float *xyz, uint8_t *rgb) + void XYZtoRGB24(float *xyz, uint8_t *rgb) { double r, g, b; /* assume CCIR-709 primaries */ - r = 2.690 * xyz[0] + -1.276 * xyz[1] + -0.414 * xyz[2]; - g = -1.022 * xyz[0] + 1.978 * xyz[1] + 0.044 * xyz[2]; - b = 0.061 * xyz[0] + -0.224 * xyz[1] + 1.163 * xyz[2]; + r = 2.690 * (double)xyz[0] + -1.276 * (double)xyz[1] + + -0.414 * (double)xyz[2]; + g = -1.022 * (double)xyz[0] + 1.978 * (double)xyz[1] + + 0.044 * (double)xyz[2]; + b = 0.061 * (double)xyz[0] + -0.224 * (double)xyz[1] + + 1.163 * (double)xyz[2]; /* assume 2.0 gamma for speed */ /* could use integer sqrt approx., but this is probably faster */ rgb[0] = (uint8_t)((r <= 0.) ? 0 : (r >= 1.) ? 255 : (int)(256. * sqrt(r))); @@ -879,8 +875,7 @@ static #if !LOGLUV_PUBLIC static #endif - double - LogL10toY(int p10) /* compute luminance from 10-bit LogL */ + double LogL10toY(int p10) /* compute luminance from 10-bit LogL */ { if (p10 == 0) return (0.); @@ -890,8 +885,7 @@ static #if !LOGLUV_PUBLIC static #endif - int - LogL10fromY(double Y, int em) /* get 10-bit LogL from Y */ + int LogL10fromY(double Y, int em) /* get 10-bit LogL from Y */ { if (Y >= 15.742) return (0x3ff); @@ -903,13 +897,14 @@ static #define NANGLES 100 #define uv2ang(u, v) \ - ((NANGLES * .499999999 / M_PI) * atan2((v)-V_NEU, (u)-U_NEU) + .5 * NANGLES) + ((NANGLES * .499999999 / M_PI) * atan2((v) - V_NEU, (u) - U_NEU) + \ + .5 * NANGLES) static int oog_encode(double u, double v) /* encode out-of-gamut chroma */ { static int oog_table[NANGLES]; static int initialized = 0; - register int i; + int i; if (!initialized) { /* set up perimeter table */ @@ -919,13 +914,14 @@ static int oog_encode(double u, double v) /* encode out-of-gamut chroma */ eps[i] = 2.; for (vi = UV_NVS; vi--;) { - va = UV_VSTART + (vi + .5) * UV_SQSIZ; + va = (double)UV_VSTART + ((double)vi + .5) * (double)UV_SQSIZ; ustep = uv_row[vi].nus - 1; if (vi == UV_NVS - 1 || vi == 0 || ustep <= 0) ustep = 1; for (ui = uv_row[vi].nus - 1; ui >= 0; ui -= ustep) { - ua = uv_row[vi].ustart + (ui + .5) * UV_SQSIZ; + ua = (double)uv_row[vi].ustart + + ((double)ui + .5) * (double)UV_SQSIZ; ang = uv2ang(ua, va); i = (int)ang; epsa = fabs(ang - (i + .5)); @@ -963,27 +959,28 @@ static int oog_encode(double u, double v) /* encode out-of-gamut chroma */ #if !LOGLUV_PUBLIC static #endif - int - uv_encode(double u, double v, int em) /* encode (u',v') coordinates */ + int uv_encode(double u, double v, int em) /* encode (u',v') coordinates */ { unsigned int vi; int ui; /* check for NaN */ - if (u != u || v != v) + if (isnan(u) || isnan(v)) { u = U_NEU; v = V_NEU; } - if (v < UV_VSTART) + if ((double)v < (double)UV_VSTART) return oog_encode(u, v); - vi = tiff_itrunc((v - UV_VSTART) * (1. / UV_SQSIZ), em); + vi = (unsigned int)tiff_itrunc( + ((double)v - (double)UV_VSTART) * (1. / (double)UV_SQSIZ), em); if (vi >= UV_NVS) return oog_encode(u, v); - if (u < uv_row[vi].ustart) + if ((double)u < (double)uv_row[vi].ustart) return oog_encode(u, v); - ui = tiff_itrunc((u - uv_row[vi].ustart) * (1. / UV_SQSIZ), em); + ui = tiff_itrunc( + ((double)u - (double)uv_row[vi].ustart) * (1. / (double)UV_SQSIZ), em); if (ui >= uv_row[vi].nus) return oog_encode(u, v); @@ -993,8 +990,7 @@ static #if !LOGLUV_PUBLIC static #endif - int - uv_decode(double *up, double *vp, int c) /* decode (u',v') index */ + int uv_decode(double *up, double *vp, int c) /* decode (u',v') index */ { unsigned int upper, lower; int ui; @@ -1020,16 +1016,15 @@ static } vi = lower; ui = c - uv_row[vi].ncum; - *up = uv_row[vi].ustart + (ui + .5) * UV_SQSIZ; - *vp = UV_VSTART + (vi + .5) * UV_SQSIZ; + *up = (double)uv_row[vi].ustart + ((double)ui + .5) * (double)UV_SQSIZ; + *vp = (double)UV_VSTART + ((double)vi + .5) * (double)UV_SQSIZ; return (0); } #if !LOGLUV_PUBLIC static #endif - void - LogLuv24toXYZ(uint32_t p, float *XYZ) + void LogLuv24toXYZ(uint32_t p, float *XYZ) { int Ce; double L, u, v, s, x, y; @@ -1059,15 +1054,14 @@ static #if !LOGLUV_PUBLIC static #endif - uint32_t - LogLuv24fromXYZ(float *XYZ, int em) + uint32_t LogLuv24fromXYZ(float *XYZ, int em) { int Le, Ce; double u, v, s; /* encode luminance */ - Le = LogL10fromY(XYZ[1], em); + Le = LogL10fromY((double)XYZ[1], em); /* encode color */ - s = XYZ[0] + 15. * XYZ[1] + 3. * XYZ[2]; + s = (double)XYZ[0] + 15. * (double)XYZ[1] + 3. * (double)XYZ[2]; if (!Le || s <= 0.) { u = U_NEU; @@ -1075,14 +1069,14 @@ static } else { - u = 4. * XYZ[0] / s; - v = 9. * XYZ[1] / s; + u = 4. * (double)XYZ[0] / s; + v = 9. * (double)XYZ[1] / s; } Ce = uv_encode(u, v, em); if (Ce < 0) /* never happens */ Ce = uv_encode(U_NEU, V_NEU, SGILOGENCODE_NODITHER); /* combine encodings */ - return (Le << 14 | Ce); + return (uint32_t)Le << 14 | (uint32_t)Ce; } static void Luv24toXYZ(LogLuvState *sp, uint8_t *op, tmsize_t n) @@ -1113,8 +1107,8 @@ static void Luv24toLuv48(LogLuvState *sp, uint8_t *op, tmsize_t n) u = U_NEU; v = V_NEU; } - *luv3++ = (int16_t)(u * (1L << 15)); - *luv3++ = (int16_t)(v * (1L << 15)); + *luv3++ = (int16_t)(u * (1 << 15)); + *luv3++ = (int16_t)(v * (1 << 15)); luv++; } } @@ -1168,7 +1162,7 @@ static void Luv24fromLuv48(LogLuvState *sp, uint8_t *op, tmsize_t n) sp->encode_meth); if (Ce < 0) /* never happens */ Ce = uv_encode(U_NEU, V_NEU, SGILOGENCODE_NODITHER); - *luv++ = (uint32_t)Le << 14 | Ce; + *luv++ = (uint32_t)Le << 14 | (uint32_t)Ce; luv3 += 3; } } @@ -1176,8 +1170,7 @@ static void Luv24fromLuv48(LogLuvState *sp, uint8_t *op, tmsize_t n) #if !LOGLUV_PUBLIC static #endif - void - LogLuv32toXYZ(uint32_t p, float *XYZ) + void LogLuv32toXYZ(uint32_t p, float *XYZ) { double L, u, v, s, x, y; /* decode luminance */ @@ -1202,15 +1195,14 @@ static #if !LOGLUV_PUBLIC static #endif - uint32_t - LogLuv32fromXYZ(float *XYZ, int em) + uint32_t LogLuv32fromXYZ(float *XYZ, int em) { unsigned int Le, ue, ve; double u, v, s; /* encode luminance */ - Le = (unsigned int)LogL16fromY(XYZ[1], em); + Le = (unsigned int)LogL16fromY((double)XYZ[1], em); /* encode color */ - s = XYZ[0] + 15. * XYZ[1] + 3. * XYZ[2]; + s = (double)XYZ[0] + 15. * (double)XYZ[1] + 3. * (double)XYZ[2]; if (!Le || s <= 0.) { u = U_NEU; @@ -1218,19 +1210,19 @@ static } else { - u = 4. * XYZ[0] / s; - v = 9. * XYZ[1] / s; + u = 4. * (double)XYZ[0] / s; + v = 9. * (double)XYZ[1] / s; } if (u <= 0.) ue = 0; else - ue = tiff_itrunc(UVSCALE * u, em); + ue = (unsigned int)tiff_itrunc(UVSCALE * u, em); if (ue > 255) ue = 255; if (v <= 0.) ve = 0; else - ve = tiff_itrunc(UVSCALE * v, em); + ve = (unsigned int)tiff_itrunc(UVSCALE * v, em); if (ve > 255) ve = 255; /* combine encodings */ @@ -1261,8 +1253,8 @@ static void Luv32toLuv48(LogLuvState *sp, uint8_t *op, tmsize_t n) *luv3++ = (int16_t)(*luv >> 16); u = 1. / UVSCALE * ((*luv >> 8 & 0xff) + .5); v = 1. / UVSCALE * ((*luv & 0xff) + .5); - *luv3++ = (int16_t)(u * (1L << 15)); - *luv3++ = (int16_t)(v * (1L << 15)); + *luv3++ = (int16_t)(u * (1 << 15)); + *luv3++ = (int16_t)(v * (1 << 15)); luv++; } } @@ -1303,22 +1295,24 @@ static void Luv32fromLuv48(LogLuvState *sp, uint8_t *op, tmsize_t n) { while (n-- > 0) { - *luv++ = (uint32_t)luv3[0] << 16 | - (luv3[1] * (uint32_t)(UVSCALE + .5) >> 7 & 0xff00) | - (luv3[2] * (uint32_t)(UVSCALE + .5) >> 15 & 0xff); + *luv++ = + (uint32_t)luv3[0] << 16 | + ((uint32_t)luv3[1] * (uint32_t)(UVSCALE + .5) >> 7 & 0xff00) | + ((uint32_t)luv3[2] * (uint32_t)(UVSCALE + .5) >> 15 & 0xff); luv3 += 3; } return; } while (n-- > 0) { - *luv++ = - (uint32_t)luv3[0] << 16 | - (tiff_itrunc(luv3[1] * (UVSCALE / (1 << 15)), sp->encode_meth) - << 8 & - 0xff00) | - (tiff_itrunc(luv3[2] * (UVSCALE / (1 << 15)), sp->encode_meth) & - 0xff); + *luv++ = (uint32_t)luv3[0] << 16 | + ((uint32_t)tiff_itrunc(luv3[1] * (UVSCALE / (1 << 15)), + sp->encode_meth) + << 8 & + 0xff00) | + ((uint32_t)tiff_itrunc(luv3[2] * (UVSCALE / (1 << 15)), + sp->encode_meth) & + 0xff); luv3 += 3; } } @@ -1345,6 +1339,8 @@ static int LogL16GuessDataFmt(TIFFDirectory *td) case PACK(1, 8, SAMPLEFORMAT_VOID): case PACK(1, 8, SAMPLEFORMAT_UINT): return (SGILOGDATAFMT_8BIT); + default: + break; } #undef PACK return (SGILOGDATAFMT_UNKNOWN); @@ -1399,7 +1395,7 @@ static int LogL16InitState(TIFF *tif) sp->tbuflen = multiply_ms(td->td_imagewidth, td->td_imagelength); if (multiply_ms(sp->tbuflen, sizeof(int16_t)) == 0 || (sp->tbuf = (uint8_t *)_TIFFmallocExt( - tif, sp->tbuflen * sizeof(int16_t))) == NULL) + tif, (tmsize_t)((size_t)sp->tbuflen * sizeof(int16_t)))) == NULL) { TIFFErrorExtR(tif, module, "No space for SGILog translation buffer"); return (0); @@ -1506,7 +1502,7 @@ static int LogLuvInitState(TIFF *tif) sp->tbuflen = multiply_ms(td->td_imagewidth, td->td_imagelength); if (multiply_ms(sp->tbuflen, sizeof(uint32_t)) == 0 || (sp->tbuf = (uint8_t *)_TIFFmallocExt( - tif, sp->tbuflen * sizeof(uint32_t))) == NULL) + tif, (tmsize_t)((size_t)sp->tbuflen * sizeof(uint32_t)))) == NULL) { TIFFErrorExtR(tif, module, "No space for SGILog translation buffer"); return (0); @@ -1546,6 +1542,8 @@ static int LogLuvSetupDecode(TIFF *tif) case SGILOGDATAFMT_8BIT: sp->tfunc = Luv24toRGB; break; + default: + break; } } else @@ -1562,6 +1560,8 @@ static int LogLuvSetupDecode(TIFF *tif) case SGILOGDATAFMT_8BIT: sp->tfunc = Luv32toRGB; break; + default: + break; } } return (1); @@ -1577,6 +1577,8 @@ static int LogLuvSetupDecode(TIFF *tif) case SGILOGDATAFMT_8BIT: sp->tfunc = L16toGry; break; + default: + break; } return (1); default: @@ -1755,8 +1757,9 @@ static int LogLuvVSetField(TIFF *tif, uint32_t tag, va_list ap) /* * Must recalculate sizes should bits/sample change. */ - tif->tif_tilesize = isTiled(tif) ? TIFFTileSize(tif) : (tmsize_t)-1; - tif->tif_scanlinesize = TIFFScanlineSize(tif); + tif->tif_dir.td_tilesize = + isTiled(tif) ? TIFFTileSize(tif) : (tmsize_t)-1; + tif->tif_dir.td_scanlinesize = TIFFScanlineSize(tif); return (1); case TIFFTAG_SGILOGENCODE: sp->encode_meth = (int)va_arg(ap, int); diff --git a/contrib/libs/libtiff/tif_lzma.c b/contrib/libs/libtiff/tif_lzma.c index 9834ae2c2c5..7763f61acd9 100644 --- a/contrib/libs/libtiff/tif_lzma.c +++ b/contrib/libs/libtiff/tif_lzma.c @@ -95,6 +95,17 @@ static const char *LZMAStrerror(lzma_ret ret) return "no progress is possible (stream is truncated or corrupt)"; case LZMA_PROG_ERROR: return "programming error"; +#if LZMA_VERSION >= 50040000 /* 5.4.0 */ + case LZMA_SEEK_NEEDED: + case LZMA_RET_INTERNAL1: + case LZMA_RET_INTERNAL2: + case LZMA_RET_INTERNAL3: + case LZMA_RET_INTERNAL4: + case LZMA_RET_INTERNAL5: + case LZMA_RET_INTERNAL6: + case LZMA_RET_INTERNAL7: + case LZMA_RET_INTERNAL8: +#endif default: return "unidentified liblzma error"; } @@ -179,7 +190,7 @@ static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "LZMADecode: Scanline %" PRIu32 " cannot be read due to " "previous error", - tif->tif_row); + tif->tif_dir.td_row); return 0; } @@ -231,7 +242,7 @@ static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { TIFFErrorExtR(tif, module, "Decoding error at scanline %" PRIu32 ", %s", - tif->tif_row, LZMAStrerror(ret)); + tif->tif_dir.td_row, LZMAStrerror(ret)); break; } } while (sp->stream.avail_out > 0); @@ -242,12 +253,12 @@ static int LZMADecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at scanline %" PRIu32 " (short %" TIFF_SIZE_FORMAT " bytes)", - tif->tif_row, sp->stream.avail_out); + tif->tif_dir.td_row, sp->stream.avail_out); return 0; } tif->tif_rawcp = (uint8_t *)sp->stream.next_in; /* cast away const */ - tif->tif_rawcc = sp->stream.avail_in; + tif->tif_rawcc = (tmsize_t)sp->stream.avail_in; return 1; } @@ -326,7 +337,7 @@ static int LZMAEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) { TIFFErrorExtR(tif, module, "Encoding error at scanline %" PRIu32 ", %s", - tif->tif_row, LZMAStrerror(ret)); + tif->tif_dir.td_row, LZMAStrerror(ret)); return 0; } if (sp->stream.avail_out == 0) @@ -365,7 +376,7 @@ static int LZMAPostEncode(TIFF *tif) if ((tmsize_t)sp->stream.avail_out != tif->tif_rawdatasize) { tif->tif_rawcc = - tif->tif_rawdatasize - sp->stream.avail_out; + tif->tif_rawdatasize - (tmsize_t)sp->stream.avail_out; if (!TIFFFlushData1(tif)) return 0; sp->stream.next_out = tif->tif_rawdata; @@ -376,6 +387,27 @@ static int LZMAPostEncode(TIFF *tif) ZIPPreEncode */ } break; + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_BUF_ERROR: + case LZMA_PROG_ERROR: +#if LZMA_VERSION >= 50040000 /* 5.4.0 */ + case LZMA_SEEK_NEEDED: + case LZMA_RET_INTERNAL1: + case LZMA_RET_INTERNAL2: + case LZMA_RET_INTERNAL3: + case LZMA_RET_INTERNAL4: + case LZMA_RET_INTERNAL5: + case LZMA_RET_INTERNAL6: + case LZMA_RET_INTERNAL7: + case LZMA_RET_INTERNAL8: +#endif default: TIFFErrorExtR(tif, module, "Liblzma error: %s", LZMAStrerror(ret)); @@ -416,7 +448,7 @@ static int LZMAVSetField(TIFF *tif, uint32_t tag, va_list ap) { case TIFFTAG_LZMAPRESET: sp->preset = (int)va_arg(ap, int); - lzma_lzma_preset(&sp->opt_lzma, sp->preset); + lzma_lzma_preset(&sp->opt_lzma, (uint32_t)sp->preset); if (sp->state & LSTATE_INIT_ENCODE) { lzma_ret ret = @@ -454,6 +486,21 @@ static const TIFFField lzmaFields[] = { FALSE, "LZMA2 Compression Preset", NULL}, }; +static uint64_t LZMAGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + /* 1024x1024: 3800 */ + /* 4096x4096: 6534 */ + /* 16383x16383: 6846 */ + /* 65536x65536: 6874 */ + + return 7000; +} + int TIFFInitLZMA(TIFF *tif, int scheme) { static const char module[] = "TIFFInitLZMA"; @@ -506,7 +553,7 @@ int TIFFInitLZMA(TIFF *tif, int scheme) sp->filters[0].id = LZMA_FILTER_DELTA; sp->filters[0].options = &sp->opt_delta; - lzma_lzma_preset(&sp->opt_lzma, sp->preset); + lzma_lzma_preset(&sp->opt_lzma, (uint32_t)sp->preset); sp->filters[1].id = LZMA_FILTER_LZMA2; sp->filters[1].options = &sp->opt_lzma; @@ -529,6 +576,8 @@ int TIFFInitLZMA(TIFF *tif, int scheme) tif->tif_encodestrip = LZMAEncode; tif->tif_encodetile = LZMAEncode; tif->tif_cleanup = LZMACleanup; + tif->tif_getmaxcompressionratio = LZMAGetMaxCompressionRatio; + /* * Setup predictor setup. */ diff --git a/contrib/libs/libtiff/tif_lzw.c b/contrib/libs/libtiff/tif_lzw.c index 4bf845b393d..d9f56a427ba 100644 --- a/contrib/libs/libtiff/tif_lzw.c +++ b/contrib/libs/libtiff/tif_lzw.c @@ -60,7 +60,7 @@ typedef size_t WordType; */ #define LZW_COMPAT /* include backwards compatibility code */ -#define MAXCODE(n) ((1L << (n)) - 1) +#define MAXCODE(n) ((1 << (n)) - 1) /* * The TIFF spec specifies that encoded bit * strings range from 9 to 12 bits. @@ -72,7 +72,7 @@ typedef size_t WordType; #define CODE_EOI 257 /* end-of-information code */ #define CODE_FIRST 258 /* first free code entry */ #define CODE_MAX MAXCODE(BITS_MAX) -#define HSIZE 9001L /* 91% occupancy */ +#define HSIZE 9001 /* 91% occupancy */ #define HSHIFT (13 - 8) #ifdef LZW_COMPAT /* NB: +1024 is for compatibility with old files */ @@ -325,7 +325,7 @@ static int LZWPreDecode(TIFF *tif, uint16_t s) */ /* Get the next 32 or 64-bit from the input data */ -#ifdef WORDS_BIGENDIAN +#if WORDS_BIGENDIAN #define GetNextData(nextdata, bp) memcpy(&nextdata, bp, sizeof(nextdata)) #elif SIZEOF_WORDTYPE == 8 #if defined(_M_X64) @@ -372,7 +372,7 @@ static int LZWPreDecode(TIFF *tif, uint16_t s) nextbits += 8 * SIZEOF_WORDTYPE; \ dec_bitsleft -= 8 * SIZEOF_WORDTYPE; \ code = (WordType)((codetmp | (nextdata >> nextbits)) & \ - nbitsmask); \ + (WordType)nbitsmask); \ break; \ } \ else \ @@ -396,7 +396,7 @@ static int LZWPreDecode(TIFF *tif, uint16_t s) } \ } \ } \ - code = (WordType)((nextdata >> nextbits) & nbitsmask); \ + code = (WordType)((nextdata >> nextbits) & (WordType)nbitsmask); \ } while (0) static int LZWDecode(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) @@ -420,7 +420,7 @@ static int LZWDecode(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) TIFFErrorExtR(tif, module, "LZWDecode: Scanline %" PRIu32 " cannot be read due to " "previous error", - tif->tif_row); + tif->tif_dir.td_row); return 0; } @@ -472,7 +472,8 @@ static int LZWDecode(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) } bp = (uint8_t *)tif->tif_rawcp; - sp->dec_bitsleft += (((uint64_t)tif->tif_rawcc - sp->old_tif_rawcc) << 3); + sp->dec_bitsleft += + (((uint64_t)tif->tif_rawcc - (uint64_t)sp->old_tif_rawcc) << 3); uint64_t dec_bitsleft = sp->dec_bitsleft; nbits = sp->lzw_nbits; nextdata = sp->lzw_nextdata; @@ -493,7 +494,7 @@ begin: { WordType code; GetNextCodeLZW(); - codep = dec_codetab + code; + codep = dec_codetab + (unsigned long)code; if (code >= CODE_FIRST) goto code_above_or_equal_to_258; if (code < 256) @@ -508,7 +509,7 @@ code_below_256: goto error_code; free_entp->next = oldcodep; free_entp->firstchar = oldcodep->firstchar; - free_entp->length = oldcodep->length + 1; + free_entp->length = (uint16_t)(oldcodep->length + 1); free_entp->value = (uint8_t)code; free_entp->repeated = (bool)(oldcodep->repeated & (oldcodep->value == code)); @@ -556,7 +557,7 @@ code_above_or_equal_to_258: free_entp->next = oldcodep; free_entp->firstchar = oldcodep->firstchar; - free_entp->length = oldcodep->length + 1; + free_entp->length = (uint16_t)(oldcodep->length + 1); if (++free_entp > maxcodep) { if (++nbits > BITS_MAX) /* should not happen for a conformant encoder */ @@ -673,7 +674,7 @@ code_clear: free_entp = dec_codetab + CODE_FIRST; nbits = BITS_MIN; nbitsmask = MAXCODE(BITS_MIN); - maxcodep = dec_codetab + nbitsmask - 1; + maxcodep = dec_codetab + (unsigned long)nbitsmask - 1; do { GetNextCodeLZW(); @@ -736,7 +737,7 @@ after_loop: TIFFErrorExtR(tif, module, "Not enough data at scanline %" PRIu32 " (short %" PRIu64 " bytes)", - tif->tif_row, (uint64_t)occ); + tif->tif_dir.td_row, (uint64_t)occ); return (0); } return (1); @@ -746,7 +747,7 @@ no_eoi: sp->read_error = 1; TIFFErrorExtR(tif, module, "LZWDecode: Strip %" PRIu32 " not terminated with EOI code", - tif->tif_curstrip); + tif->tif_dir.td_curstrip); return 0; error_code: memset(op, 0, (size_t)occ); @@ -768,13 +769,13 @@ error_code: TIFFWarningExtR(_tif, module, \ "LZWDecode: Strip %" PRIu32 \ " not terminated with EOI code", \ - _tif->tif_curstrip); \ + _tif->tif_dir.td_curstrip); \ _code = CODE_EOI; \ } \ else \ { \ _get(_sp, _bp, _code); \ - dec_bitsleft -= nbits; \ + dec_bitsleft -= (uint64_t)nbits; \ } \ } @@ -790,7 +791,7 @@ error_code: nextdata |= (unsigned long)*(bp)++ << nextbits; \ nextbits += 8; \ } \ - code = (hcode_t)(nextdata & nbitsmask); \ + code = (hcode_t)(nextdata & (unsigned long)nbitsmask); \ nextdata >>= nbits; \ nextbits -= nbits; \ } @@ -858,7 +859,8 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) bp = (uint8_t *)tif->tif_rawcp; - sp->dec_bitsleft += (((uint64_t)tif->tif_rawcc - sp->old_tif_rawcc) << 3); + sp->dec_bitsleft += + (((uint64_t)tif->tif_rawcc - (uint64_t)sp->old_tif_rawcc) << 3); uint64_t dec_bitsleft = sp->dec_bitsleft; nbits = sp->lzw_nbits; @@ -893,7 +895,7 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) TIFFErrorExtR( tif, tif->tif_name, "LZWDecode: Corrupted LZW table at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } *op++ = (uint8_t)code; @@ -911,7 +913,7 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) { TIFFErrorExtR(tif, module, "Corrupted LZW table at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } @@ -921,11 +923,11 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) { TIFFErrorExtR(tif, module, "Corrupted LZW table at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } free_entp->firstchar = free_entp->next->firstchar; - free_entp->length = free_entp->next->length + 1; + free_entp->length = (uint16_t)(free_entp->next->length + 1); free_entp->value = (codep < free_entp) ? codep->firstchar : free_entp->firstchar; if (++free_entp > maxcodep) @@ -948,7 +950,7 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) tif, module, "Wrong length of decoded " "string: data probably corrupted at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } if (codep->length > occ) @@ -1010,7 +1012,7 @@ static int LZWDecodeCompat(TIFF *tif, uint8_t *op0, tmsize_t occ0, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at scanline %" PRIu32 " (short %" PRIu64 " bytes)", - tif->tif_row, (uint64_t)occ); + tif->tif_dir.td_row, (uint64_t)occ); return (0); } return (1); @@ -1116,10 +1118,10 @@ static int LZWPreEncode(TIFF *tif, uint16_t s) */ static int LZWEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) { - register LZWCodecState *sp = LZWEncoderState(tif); - register long fcode; - register hash_t *hp; - register int h, c; + LZWCodecState *sp = LZWEncoderState(tif); + long fcode; + hash_t *hp; + int h, c; hcode_t ent; long disp; tmsize_t incount, outcount, checkpoint; @@ -1196,7 +1198,7 @@ static int LZWEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) * Avoid pointer arithmetic because of * wraparound problems with segments. */ - if ((h -= disp) < 0) + if ((h -= (int)disp) < 0) h += HSIZE; hp = &sp->enc_hashtab[h]; if (hp->hash == fcode) @@ -1301,7 +1303,7 @@ static int LZWEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) */ static int LZWPostEncode(TIFF *tif) { - register LZWCodecState *sp = LZWEncoderState(tif); + LZWCodecState *sp = LZWEncoderState(tif); uint8_t *op = tif->tif_rawcp; long nextbits = sp->lzw_nextbits; WordType nextdata = sp->lzw_nextdata; @@ -1319,7 +1321,7 @@ static int LZWPostEncode(TIFF *tif) { int free_ent = sp->lzw_free_ent; - PutNextCode(op, sp->enc_oldcode); + PutNextCode(op, (WordType)sp->enc_oldcode); sp->enc_oldcode = (hcode_t)-1; free_ent++; @@ -1357,8 +1359,8 @@ static int LZWPostEncode(TIFF *tif) */ static void cl_hash(LZWCodecState *sp) { - register hash_t *hp = &sp->enc_hashtab[HSIZE - 1]; - register long i = HSIZE - 8; + hash_t *hp = &sp->enc_hashtab[HSIZE - 1]; + long i = HSIZE - 8; do { @@ -1397,6 +1399,21 @@ static void LZWCleanup(TIFF *tif) _TIFFSetDefaultCompressionState(tif); } +static uint64_t LZWGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + /* 1024x1024: 562 */ + /* 4096x4096: 1243 */ + /* 16383x16383: 1353 */ + /* 65536x65536: 1362 */ + + return 1400; +} + int TIFFInitLZW(TIFF *tif, int scheme) { static const char module[] = "TIFFInitLZW"; @@ -1430,6 +1447,7 @@ int TIFFInitLZW(TIFF *tif, int scheme) tif->tif_encodestrip = LZWEncode; tif->tif_encodetile = LZWEncode; #endif + tif->tif_getmaxcompressionratio = LZWGetMaxCompressionRatio; tif->tif_cleanup = LZWCleanup; /* * Setup predictor setup. diff --git a/contrib/libs/libtiff/tif_next.c b/contrib/libs/libtiff/tif_next.c index f000574ee74..04ae474b95b 100644 --- a/contrib/libs/libtiff/tif_next.c +++ b/contrib/libs/libtiff/tif_next.c @@ -38,21 +38,22 @@ op[0] = (unsigned char)((v) << 6); \ break; \ case 1: \ - op[0] |= (v) << 4; \ + op[0] |= (unsigned char)((v) << 4); \ break; \ case 2: \ - op[0] |= (v) << 2; \ + op[0] |= (unsigned char)((v) << 2); \ break; \ case 3: \ - *op++ |= (v); \ + *op++ |= (unsigned char)(v); \ op_offset++; \ break; \ + default: \ + break; \ } \ } #define LITERALROW 0x00 #define LITERALSPAN 0x40 -#define WHITE ((1 << 2) - 1) static int NeXTDecode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) { @@ -73,7 +74,7 @@ static int NeXTDecode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) bp = (unsigned char *)tif->tif_rawcp; cc = tif->tif_rawcc; - scanline = tif->tif_scanlinesize; + scanline = tif->tif_dir.td_scanlinesize; if (occ % scanline) { TIFFErrorExtR(tif, module, "Fractional scanlines cannot be read"); @@ -146,7 +147,7 @@ static int NeXTDecode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) { TIFFErrorExtR(tif, module, "Invalid data for scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } if (cc == 0) @@ -163,7 +164,7 @@ static int NeXTDecode(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) return (1); bad: TIFFErrorExtR(tif, module, "Not enough data for scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } diff --git a/contrib/libs/libtiff/tif_ojpeg.c b/contrib/libs/libtiff/tif_ojpeg.c index 5ef7a6af6fe..e62ec9ea72c 100644 --- a/contrib/libs/libtiff/tif_ojpeg.c +++ b/contrib/libs/libtiff/tif_ojpeg.c @@ -122,9 +122,6 @@ restarting of LibJpeg decoding session. */ -#define WIN32_LEAN_AND_MEAN -#define VC_EXTRALEAN - #include "tiffiop.h" #ifdef OJPEG_SUPPORT @@ -209,18 +206,12 @@ static const TIFFField ojpegFields[] = { #include "jerror.h" #include "jpeglib.h" -#ifndef TIFF_jpeg_source_mgr_defined -#define TIFF_jpeg_source_mgr_defined -typedef struct jpeg_source_mgr jpeg_source_mgr; -#endif +typedef struct jpeg_source_mgr tiff_ojpeg_source_mgr; -#ifndef TIFF_jpeg_error_mgr_defined -#define TIFF_jpeg_error_mgr_defined -typedef struct jpeg_error_mgr jpeg_error_mgr; -#endif +typedef struct jpeg_error_mgr tiff_ojpeg_error_mgr; -typedef struct jpeg_common_struct jpeg_common_struct; -typedef struct jpeg_decompress_struct jpeg_decompress_struct; +typedef struct jpeg_common_struct tiff_ojpeg_common_struct; +typedef struct jpeg_decompress_struct tiff_ojpeg_decompress_struct; typedef enum { @@ -316,9 +307,9 @@ typedef struct uint32_t write_curstrile; uint8_t libjpeg_session_active; uint8_t libjpeg_jpeg_query_style; - jpeg_error_mgr libjpeg_jpeg_error_mgr; - jpeg_decompress_struct libjpeg_jpeg_decompress_struct; - jpeg_source_mgr libjpeg_jpeg_source_mgr; + tiff_ojpeg_error_mgr libjpeg_jpeg_error_mgr; + tiff_ojpeg_decompress_struct libjpeg_jpeg_decompress_struct; + tiff_ojpeg_source_mgr libjpeg_jpeg_source_mgr; uint8_t subsampling_convert_log; uint32_t subsampling_convert_ylinelen; uint32_t subsampling_convert_ylines; @@ -413,46 +404,51 @@ static void OJPEGWriteStreamEoi(TIFF *tif, void **mem, uint32_t *len); #ifdef LIBJPEG_ENCAP_EXTERNAL extern int jpeg_create_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo); -extern int jpeg_read_header_encap(OJPEGState *sp, jpeg_decompress_struct *cinfo, + tiff_ojpeg_decompress_struct *cinfo); +extern int jpeg_read_header_encap(OJPEGState *sp, + tiff_ojpeg_decompress_struct *cinfo, uint8_t require_image); extern int jpeg_start_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo); + tiff_ojpeg_decompress_struct *cinfo); extern int jpeg_read_scanlines_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, + tiff_ojpeg_decompress_struct *cinfo, void *scanlines, uint32_t max_lines); extern int jpeg_read_raw_data_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, void *data, - uint32_t max_lines); + tiff_ojpeg_decompress_struct *cinfo, + void *data, uint32_t max_lines); extern void jpeg_encap_unwind(TIFF *tif); #else static int jpeg_create_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *j); -static int jpeg_read_header_encap(OJPEGState *sp, jpeg_decompress_struct *cinfo, + tiff_ojpeg_decompress_struct *j); +static int jpeg_read_header_encap(OJPEGState *sp, + tiff_ojpeg_decompress_struct *cinfo, uint8_t require_image); static int jpeg_start_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo); + tiff_ojpeg_decompress_struct *cinfo); static int jpeg_read_scanlines_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, + tiff_ojpeg_decompress_struct *cinfo, void *scanlines, uint32_t max_lines); static int jpeg_read_raw_data_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, void *data, - uint32_t max_lines); + tiff_ojpeg_decompress_struct *cinfo, + void *data, uint32_t max_lines); static void jpeg_encap_unwind(TIFF *tif); #endif -static void OJPEGLibjpegJpegErrorMgrOutputMessage(jpeg_common_struct *cinfo); -static void OJPEGLibjpegJpegErrorMgrErrorExit(jpeg_common_struct *cinfo); -static void OJPEGLibjpegJpegSourceMgrInitSource(jpeg_decompress_struct *cinfo); +static void +OJPEGLibjpegJpegErrorMgrOutputMessage(tiff_ojpeg_common_struct *cinfo); +static void OJPEGLibjpegJpegErrorMgrErrorExit(tiff_ojpeg_common_struct *cinfo); +static void +OJPEGLibjpegJpegSourceMgrInitSource(tiff_ojpeg_decompress_struct *cinfo); static boolean -OJPEGLibjpegJpegSourceMgrFillInputBuffer(jpeg_decompress_struct *cinfo); +OJPEGLibjpegJpegSourceMgrFillInputBuffer(tiff_ojpeg_decompress_struct *cinfo); static void -OJPEGLibjpegJpegSourceMgrSkipInputData(jpeg_decompress_struct *cinfo, +OJPEGLibjpegJpegSourceMgrSkipInputData(tiff_ojpeg_decompress_struct *cinfo, long num_bytes); static boolean -OJPEGLibjpegJpegSourceMgrResyncToRestart(jpeg_decompress_struct *cinfo, +OJPEGLibjpegJpegSourceMgrResyncToRestart(tiff_ojpeg_decompress_struct *cinfo, int desired); -static void OJPEGLibjpegJpegSourceMgrTermSource(jpeg_decompress_struct *cinfo); +static void +OJPEGLibjpegJpegSourceMgrTermSource(tiff_ojpeg_decompress_struct *cinfo); int TIFFInitOJPEG(TIFF *tif, int scheme) { @@ -473,7 +469,7 @@ int TIFFInitOJPEG(TIFF *tif, int scheme) } /* state block */ - sp = _TIFFmallocExt(tif, sizeof(OJPEGState)); + sp = (OJPEGState *)_TIFFmallocExt(tif, sizeof(OJPEGState)); if (sp == NULL) { TIFFErrorExtR(tif, module, "No space for OJPEG state block"); @@ -724,9 +720,9 @@ static int OJPEGPreDecode(TIFF *tif, uint16_t s) return (0); } if (isTiled(tif)) - m = tif->tif_curtile; + m = tif->tif_dir.td_curtile; else - m = tif->tif_curstrip; + m = tif->tif_dir.td_curstrip; if ((sp->writeheader_done != 0) && ((sp->write_cursample != s) || (sp->write_curstrile > m))) { @@ -797,7 +793,7 @@ static int OJPEGPreDecodeSkipRaw(TIFF *tif) { if (jpeg_read_raw_data_encap(sp, &(sp->libjpeg_jpeg_decompress_struct), sp->subsampling_convert_ycbcrimage, - sp->subsampling_ver * 8) == 0) + (uint32_t)sp->subsampling_ver * 8) == 0) return (0); m -= sp->subsampling_convert_clines; } @@ -805,7 +801,7 @@ static int OJPEGPreDecodeSkipRaw(TIFF *tif) { if (jpeg_read_raw_data_encap(sp, &(sp->libjpeg_jpeg_decompress_struct), sp->subsampling_convert_ycbcrimage, - sp->subsampling_ver * 8) == 0) + (uint32_t)sp->subsampling_ver * 8) == 0) return (0); sp->subsampling_convert_state = m; } @@ -819,7 +815,7 @@ static int OJPEGPreDecodeSkipScanlines(TIFF *tif) uint32_t m; if (sp->skip_buffer == NULL) { - sp->skip_buffer = _TIFFmallocExt(tif, sp->bytes_per_line); + sp->skip_buffer = (uint8_t *)_TIFFmallocExt(tif, sp->bytes_per_line); if (sp->skip_buffer == NULL) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -910,10 +906,10 @@ static int OJPEGDecodeRaw(TIFF *tif, uint8_t *buf, tmsize_t cc) { if (sp->subsampling_convert_state == 0) { - if (jpeg_read_raw_data_encap(sp, - &(sp->libjpeg_jpeg_decompress_struct), - sp->subsampling_convert_ycbcrimage, - sp->subsampling_ver * 8) == 0) + if (jpeg_read_raw_data_encap( + sp, &(sp->libjpeg_jpeg_decompress_struct), + sp->subsampling_convert_ycbcrimage, + (uint32_t)sp->subsampling_ver * 8) == 0) { sp->error_in_raw_data_decoding = 1; return (0); @@ -1225,7 +1221,7 @@ static int OJPEGReadHeaderInfo(TIFF *tif) TIFFErrorExtR(tif, module, "Invalid subsampling values"); return (0); } - if (sp->strile_length % (sp->subsampling_ver * 8) != 0) + if (sp->strile_length % ((uint32_t)sp->subsampling_ver * 8) != 0) { TIFFErrorExtR(tif, module, "Incompatible vertical subsampling and image " @@ -1233,9 +1229,11 @@ static int OJPEGReadHeaderInfo(TIFF *tif) return (0); } sp->restart_interval = - (uint16_t)(((sp->strile_width + sp->subsampling_hor * 8 - 1) / - (sp->subsampling_hor * 8)) * - (sp->strile_length / (sp->subsampling_ver * 8))); + (uint16_t)(((sp->strile_width + (uint32_t)sp->subsampling_hor * 8 - + 1) / + ((uint32_t)sp->subsampling_hor * 8)) * + (sp->strile_length / + ((uint32_t)sp->subsampling_ver * 8))); } if (OJPEGReadHeaderInfoSec(tif) == 0) return (0); @@ -1358,20 +1356,43 @@ static int OJPEGWriteHeaderInfo(TIFF *tif) /* Check for division by zero. */ if (sp->subsampling_hor == 0 || sp->subsampling_ver == 0) return (0); + /* Check for potential overflow in subsampling_convert_ylinelen + * computation. + */ + if (sp->strile_width > + UINT32_MAX - ((uint32_t)sp->subsampling_hor * 8 - 1)) + return (0); sp->subsampling_convert_ylinelen = - ((sp->strile_width + sp->subsampling_hor * 8 - 1) / - (sp->subsampling_hor * 8) * sp->subsampling_hor * 8); - sp->subsampling_convert_ylines = sp->subsampling_ver * 8; + ((sp->strile_width + (uint32_t)sp->subsampling_hor * 8 - 1) / + ((uint32_t)sp->subsampling_hor * 8) * + ((uint32_t)sp->subsampling_hor * 8)); + sp->subsampling_convert_ylines = (uint32_t)sp->subsampling_ver * 8; sp->subsampling_convert_clinelen = sp->subsampling_convert_ylinelen / sp->subsampling_hor; sp->subsampling_convert_clines = 8; - sp->subsampling_convert_ybuflen = sp->subsampling_convert_ylinelen * - sp->subsampling_convert_ylines; - sp->subsampling_convert_cbuflen = sp->subsampling_convert_clinelen * - sp->subsampling_convert_clines; - sp->subsampling_convert_ycbcrbuflen = - sp->subsampling_convert_ybuflen + - 2 * sp->subsampling_convert_cbuflen; + /* Check for potential overflow in buffer length computations. + * Use 64-bit intermediates to detect uint32_t overflow in + * ylinelen * ylines, clinelen * clines, and their sum. + */ + { + uint64_t ybuflen64 = + (uint64_t)sp->subsampling_convert_ylinelen * + sp->subsampling_convert_ylines; + uint64_t cbuflen64 = + (uint64_t)sp->subsampling_convert_clinelen * + sp->subsampling_convert_clines; + uint64_t ycbcrbuflen64 = ybuflen64 + 2 * cbuflen64; + if (ybuflen64 > UINT32_MAX || cbuflen64 > UINT32_MAX || + ycbcrbuflen64 > UINT32_MAX) + { + TIFFErrorExtR(tif, module, + "Integer overflow in OJPEG buffer size"); + return (0); + } + sp->subsampling_convert_ybuflen = (uint32_t)ybuflen64; + sp->subsampling_convert_cbuflen = (uint32_t)cbuflen64; + sp->subsampling_convert_ycbcrbuflen = (uint32_t)ycbcrbuflen64; + } /* The calloc is not normally necessary, except in some edge/broken * cases */ /* for example for a tiled image of height 1 with a tile height of 1 @@ -1383,8 +1404,8 @@ static int OJPEGWriteHeaderInfo(TIFF *tif) /* Even if this case is allowed (?), its handling is broken because * OJPEGPreDecode() should also likely */ /* reset subsampling_convert_state to 0 when changing tile. */ - sp->subsampling_convert_ycbcrbuf = - _TIFFcallocExt(tif, 1, sp->subsampling_convert_ycbcrbuflen); + sp->subsampling_convert_ycbcrbuf = (uint8_t *)_TIFFcallocExt( + tif, 1, sp->subsampling_convert_ycbcrbuflen); if (sp->subsampling_convert_ycbcrbuf == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1398,8 +1419,9 @@ static int OJPEGWriteHeaderInfo(TIFF *tif) sp->subsampling_convert_ycbcrimagelen = 3 + sp->subsampling_convert_ylines + 2 * sp->subsampling_convert_clines; - sp->subsampling_convert_ycbcrimage = _TIFFmallocExt( - tif, sp->subsampling_convert_ycbcrimagelen * sizeof(uint8_t *)); + sp->subsampling_convert_ycbcrimage = (uint8_t **)_TIFFmallocExt( + tif, (tmsize_t)((size_t)sp->subsampling_convert_ycbcrimagelen * + sizeof(uint8_t *))); if (sp->subsampling_convert_ycbcrimage == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1426,9 +1448,14 @@ static int OJPEGWriteHeaderInfo(TIFF *tif) ((sp->strile_width % sp->subsampling_hor) != 0 ? 1 : 0); sp->subsampling_convert_state = 0; sp->error_in_raw_data_decoding = 0; - sp->bytes_per_line = - sp->subsampling_convert_clinelenout * - (sp->subsampling_ver * sp->subsampling_hor + 2); + + const uint64_t bpl = + (uint64_t)sp->subsampling_convert_clinelenout * + ((uint64_t)sp->subsampling_ver * sp->subsampling_hor + 2); + if (bpl > UINT32_MAX) + return (0); + sp->bytes_per_line = (uint32_t)bpl; + sp->lines_per_strile = sp->strile_length / sp->subsampling_ver + ((sp->strile_length % sp->subsampling_ver) != 0 ? 1 : 0); @@ -1478,7 +1505,8 @@ static void OJPEGLibjpegSessionAbort(TIFF *tif) { OJPEGState *sp = (OJPEGState *)tif->tif_data; assert(sp->libjpeg_session_active != 0); - jpeg_destroy((jpeg_common_struct *)(&(sp->libjpeg_jpeg_decompress_struct))); + jpeg_destroy( + (tiff_ojpeg_common_struct *)(&(sp->libjpeg_jpeg_decompress_struct))); sp->libjpeg_session_active = 0; } @@ -1560,7 +1588,7 @@ static int OJPEGReadHeaderInfoSec(TIFF *tif) return (0); } if (n > 2) - OJPEGReadSkip(sp, n - 2); + OJPEGReadSkip(sp, (uint16_t)(n - 2)); break; case JPEG_MARKER_DRI: if (OJPEGReadHeaderInfoSecStreamDri(tif) == 0) @@ -1603,8 +1631,9 @@ static int OJPEGReadHeaderInfoSec(TIFF *tif) return (0); sp->sof_marker_id = JPEG_MARKER_SOF0; for (o = 0; o < sp->samples_per_pixel; o++) - sp->sof_c[o] = o; - sp->sof_hv[0] = ((sp->subsampling_hor << 4) | sp->subsampling_ver); + sp->sof_c[o] = (uint8_t)o; + sp->sof_hv[0] = + (uint8_t)((sp->subsampling_hor << 4) | sp->subsampling_ver); for (o = 1; o < sp->samples_per_pixel; o++) sp->sof_hv[o] = 17; sp->sof_x = sp->strile_width; @@ -1659,10 +1688,10 @@ static int OJPEGReadHeaderInfoSecStreamDqt(TIFF *tif) return (0); } if (sp->subsamplingcorrect != 0) - OJPEGReadSkip(sp, m - 2); + OJPEGReadSkip(sp, (uint16_t)(m - 2)); else { - m -= 2; + m = (uint16_t)(m - 2); do { if (m < 65) @@ -1671,7 +1700,7 @@ static int OJPEGReadHeaderInfoSecStreamDqt(TIFF *tif) return (0); } na = sizeof(uint32_t) + 69; - nb = _TIFFmallocExt(tif, na); + nb = (uint8_t *)_TIFFmallocExt(tif, na); if (nb == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1697,7 +1726,7 @@ static int OJPEGReadHeaderInfoSecStreamDqt(TIFF *tif) if (sp->qtable[o] != 0) _TIFFfreeExt(tif, sp->qtable[o]); sp->qtable[o] = nb; - m -= 65; + m = (uint16_t)(m - 65); } while (m > 0); } return (1); @@ -1725,12 +1754,12 @@ static int OJPEGReadHeaderInfoSecStreamDht(TIFF *tif) } if (sp->subsamplingcorrect != 0) { - OJPEGReadSkip(sp, m - 2); + OJPEGReadSkip(sp, (uint16_t)(m - 2)); } else { - na = sizeof(uint32_t) + 2 + m; - nb = _TIFFmallocExt(tif, na); + na = (uint32_t)(sizeof(uint32_t) + 2 + m); + nb = (uint8_t *)_TIFFmallocExt(tif, na); if (nb == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -1739,9 +1768,10 @@ static int OJPEGReadHeaderInfoSecStreamDht(TIFF *tif) *(uint32_t *)nb = na; nb[sizeof(uint32_t)] = 255; nb[sizeof(uint32_t) + 1] = JPEG_MARKER_DHT; - nb[sizeof(uint32_t) + 2] = (m >> 8); - nb[sizeof(uint32_t) + 3] = (m & 255); - if (OJPEGReadBlock(sp, m - 2, &nb[sizeof(uint32_t) + 4]) == 0) + nb[sizeof(uint32_t) + 2] = (uint8_t)(m >> 8); + nb[sizeof(uint32_t) + 3] = (uint8_t)(m & 255); + if (OJPEGReadBlock(sp, (uint16_t)(m - 2), &nb[sizeof(uint32_t) + 4]) == + 0) { _TIFFfreeExt(tif, nb); return (0); @@ -1809,7 +1839,7 @@ static int OJPEGReadHeaderInfoSecStreamSof(TIFF *tif, uint8_t marker_id) TIFFErrorExtR(tif, module, "Corrupt SOF marker in JPEG data"); return (0); } - m -= 8; + m = (uint16_t)(m - 8); if (m % 3 != 0) { if (sp->subsamplingcorrect == 0) @@ -2030,7 +2060,7 @@ static int OJPEGReadHeaderInfoSecTablesQTable(TIFF *tif) } } oa = sizeof(uint32_t) + 69; - ob = _TIFFmallocExt(tif, oa); + ob = (uint8_t *)_TIFFmallocExt(tif, oa); if (ob == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2098,8 +2128,8 @@ static int OJPEGReadHeaderInfoSecTablesDcTable(TIFF *tif) q = 0; for (n = 0; n < 16; n++) q += o[n]; - ra = sizeof(uint32_t) + 21 + q; - rb = _TIFFmallocExt(tif, ra); + ra = (uint32_t)(sizeof(uint32_t) + 21 + q); + rb = (uint8_t *)_TIFFmallocExt(tif, ra); if (rb == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2122,7 +2152,7 @@ static int OJPEGReadHeaderInfoSecTablesDcTable(TIFF *tif) if (sp->dctable[m] != 0) _TIFFfreeExt(tif, sp->dctable[m]); sp->dctable[m] = rb; - sp->sos_tda[m] = (m << 4); + sp->sos_tda[m] = (uint8_t)(m << 4); } else sp->sos_tda[m] = sp->sos_tda[m - 1]; @@ -2168,8 +2198,8 @@ static int OJPEGReadHeaderInfoSecTablesAcTable(TIFF *tif) q = 0; for (n = 0; n < 16; n++) q += o[n]; - ra = sizeof(uint32_t) + 21 + q; - rb = _TIFFmallocExt(tif, ra); + ra = (uint32_t)(sizeof(uint32_t) + 21 + q); + rb = (uint8_t *)_TIFFmallocExt(tif, ra); if (rb == 0) { TIFFErrorExtR(tif, module, "Out of memory"); @@ -2220,7 +2250,7 @@ static int OJPEGReadBufferFill(OJPEGState *sp) if ((uint64_t)m > sp->in_buffer_file_togo) m = (uint16_t)sp->in_buffer_file_togo; n = TIFFReadFile(sp->tif, sp->in_buffer, (tmsize_t)m); - if (n == 0) + if (n <= 0) return (0); assert(n > 0); assert(n <= OJPEG_BUFFER); @@ -2286,6 +2316,7 @@ static int OJPEGReadBufferFill(OJPEGState *sp) sp->in_buffer_next_strile++; } break; + case osibsEof: default: return (0); } @@ -2331,7 +2362,7 @@ static int OJPEGReadWord(OJPEGState *sp, uint16_t *word) uint8_t m; if (OJPEGReadByte(sp, &m) == 0) return (0); - *word = (m << 8); + *word = (uint16_t)(m << 8); if (OJPEGReadByte(sp, &m) == 0) return (0); *word |= m; @@ -2345,7 +2376,7 @@ static int OJPEGReadBlock(OJPEGState *sp, uint16_t len, void *mem) uint16_t n; assert(len > 0); mlen = len; - mmem = mem; + mmem = (uint8_t *)mem; do { if (sp->in_buffer_togo == 0) @@ -2359,8 +2390,8 @@ static int OJPEGReadBlock(OJPEGState *sp, uint16_t len, void *mem) n = sp->in_buffer_togo; _TIFFmemcpy(mmem, sp->in_buffer_cur, n); sp->in_buffer_cur += n; - sp->in_buffer_togo -= n; - mlen -= n; + sp->in_buffer_togo = (uint16_t)(sp->in_buffer_togo - n); + mlen = (uint16_t)(mlen - n); mmem += n; } while (mlen > 0); return (1); @@ -2375,8 +2406,8 @@ static void OJPEGReadSkip(OJPEGState *sp, uint16_t len) if (n > sp->in_buffer_togo) n = sp->in_buffer_togo; sp->in_buffer_cur += n; - sp->in_buffer_togo -= n; - m -= n; + sp->in_buffer_togo = (uint16_t)(sp->in_buffer_togo - n); + m = (uint16_t)(m - n); if (m > 0) { assert(sp->in_buffer_togo == 0); @@ -2462,6 +2493,8 @@ static int OJPEGWriteStream(TIFF *tif, void **mem, uint32_t *len) case ososEoi: OJPEGWriteStreamEoi(tif, mem, len); break; + default: + break; } } while (*len == 0); return (1); @@ -2475,7 +2508,7 @@ static void OJPEGWriteStreamSoi(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[1] = JPEG_MARKER_SOI; *len = 2; *mem = (void *)sp->out_buffer; - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamQTable(TIFF *tif, uint8_t table_index, void **mem, @@ -2485,9 +2518,10 @@ static void OJPEGWriteStreamQTable(TIFF *tif, uint8_t table_index, void **mem, if (sp->qtable[table_index] != 0) { *mem = (void *)(sp->qtable[table_index] + sizeof(uint32_t)); - *len = *((uint32_t *)sp->qtable[table_index]) - sizeof(uint32_t); + *len = (uint32_t)(*((uint32_t *)sp->qtable[table_index]) - + sizeof(uint32_t)); } - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamDcTable(TIFF *tif, uint8_t table_index, void **mem, @@ -2497,9 +2531,10 @@ static void OJPEGWriteStreamDcTable(TIFF *tif, uint8_t table_index, void **mem, if (sp->dctable[table_index] != 0) { *mem = (void *)(sp->dctable[table_index] + sizeof(uint32_t)); - *len = *((uint32_t *)sp->dctable[table_index]) - sizeof(uint32_t); + *len = (uint32_t)(*((uint32_t *)sp->dctable[table_index]) - + sizeof(uint32_t)); } - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamAcTable(TIFF *tif, uint8_t table_index, void **mem, @@ -2509,9 +2544,10 @@ static void OJPEGWriteStreamAcTable(TIFF *tif, uint8_t table_index, void **mem, if (sp->actable[table_index] != 0) { *mem = (void *)(sp->actable[table_index] + sizeof(uint32_t)); - *len = *((uint32_t *)sp->actable[table_index]) - sizeof(uint32_t); + *len = (uint32_t)(*((uint32_t *)sp->actable[table_index]) - + sizeof(uint32_t)); } - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamDri(TIFF *tif, void **mem, uint32_t *len) @@ -2524,12 +2560,12 @@ static void OJPEGWriteStreamDri(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[1] = JPEG_MARKER_DRI; sp->out_buffer[2] = 0; sp->out_buffer[3] = 4; - sp->out_buffer[4] = (sp->restart_interval >> 8); - sp->out_buffer[5] = (sp->restart_interval & 255); + sp->out_buffer[4] = (uint8_t)(sp->restart_interval >> 8); + sp->out_buffer[5] = (uint8_t)(sp->restart_interval & 255); *len = 6; *mem = (void *)sp->out_buffer; } - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamSof(TIFF *tif, void **mem, uint32_t *len) @@ -2542,15 +2578,15 @@ static void OJPEGWriteStreamSof(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[1] = sp->sof_marker_id; /* Lf */ sp->out_buffer[2] = 0; - sp->out_buffer[3] = 8 + sp->samples_per_pixel_per_plane * 3; + sp->out_buffer[3] = (uint8_t)(8 + sp->samples_per_pixel_per_plane * 3); /* P */ sp->out_buffer[4] = 8; /* Y */ sp->out_buffer[5] = (uint8_t)(sp->sof_y >> 8); - sp->out_buffer[6] = (sp->sof_y & 255); + sp->out_buffer[6] = (uint8_t)(sp->sof_y & 255); /* X */ sp->out_buffer[7] = (uint8_t)(sp->sof_x >> 8); - sp->out_buffer[8] = (sp->sof_x & 255); + sp->out_buffer[8] = (uint8_t)(sp->sof_x & 255); /* Nf */ sp->out_buffer[9] = sp->samples_per_pixel_per_plane; for (m = 0; m < sp->samples_per_pixel_per_plane; m++) @@ -2564,9 +2600,9 @@ static void OJPEGWriteStreamSof(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[10 + m * 3 + 2] = sp->sof_tq[sp->plane_sample_offset + m]; } - *len = 10 + sp->samples_per_pixel_per_plane * 3; + *len = 10 + (uint32_t)sp->samples_per_pixel_per_plane * 3; *mem = (void *)sp->out_buffer; - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static void OJPEGWriteStreamSos(TIFF *tif, void **mem, uint32_t *len) @@ -2579,7 +2615,7 @@ static void OJPEGWriteStreamSos(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[1] = JPEG_MARKER_SOS; /* Ls */ sp->out_buffer[2] = 0; - sp->out_buffer[3] = 6 + sp->samples_per_pixel_per_plane * 2; + sp->out_buffer[3] = (uint8_t)(6 + sp->samples_per_pixel_per_plane * 2); /* Ns */ sp->out_buffer[4] = sp->samples_per_pixel_per_plane; for (m = 0; m < sp->samples_per_pixel_per_plane; m++) @@ -2596,9 +2632,9 @@ static void OJPEGWriteStreamSos(TIFF *tif, void **mem, uint32_t *len) sp->out_buffer[5 + sp->samples_per_pixel_per_plane * 2 + 1] = 63; /* Ah and Al */ sp->out_buffer[5 + sp->samples_per_pixel_per_plane * 2 + 2] = 0; - *len = 8 + sp->samples_per_pixel_per_plane * 2; + *len = 8 + (uint32_t)sp->samples_per_pixel_per_plane * 2; *mem = (void *)sp->out_buffer; - sp->out_state++; + sp->out_state = (OJPEGStateOutState)(sp->out_state + 1); } static int OJPEGWriteStreamCompressed(TIFF *tif, void **mem, uint32_t *len) @@ -2626,6 +2662,8 @@ static int OJPEGWriteStreamCompressed(TIFF *tif, void **mem, uint32_t *len) case osibsEof: sp->out_state = ososEoi; break; + case osibsNotSetYet: + case osibsJpegInterchangeFormat: default: break; } @@ -2638,7 +2676,7 @@ static void OJPEGWriteStreamRst(TIFF *tif, void **mem, uint32_t *len) OJPEGState *sp = (OJPEGState *)tif->tif_data; assert(OJPEG_BUFFER >= 2); sp->out_buffer[0] = 255; - sp->out_buffer[1] = JPEG_MARKER_RST0 + sp->restart_index; + sp->out_buffer[1] = (uint8_t)(JPEG_MARKER_RST0 + sp->restart_index); sp->restart_index++; if (sp->restart_index == 8) sp->restart_index = 0; @@ -2659,7 +2697,7 @@ static void OJPEGWriteStreamEoi(TIFF *tif, void **mem, uint32_t *len) #ifndef LIBJPEG_ENCAP_EXTERNAL static int jpeg_create_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo) + tiff_ojpeg_decompress_struct *cinfo) { if (SETJMP(sp->exit_jmpbuf)) return 0; @@ -2672,7 +2710,8 @@ static int jpeg_create_decompress_encap(OJPEGState *sp, #endif #ifndef LIBJPEG_ENCAP_EXTERNAL -static int jpeg_read_header_encap(OJPEGState *sp, jpeg_decompress_struct *cinfo, +static int jpeg_read_header_encap(OJPEGState *sp, + tiff_ojpeg_decompress_struct *cinfo, uint8_t require_image) { if (SETJMP(sp->exit_jmpbuf)) @@ -2687,7 +2726,7 @@ static int jpeg_read_header_encap(OJPEGState *sp, jpeg_decompress_struct *cinfo, #ifndef LIBJPEG_ENCAP_EXTERNAL static int jpeg_start_decompress_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo) + tiff_ojpeg_decompress_struct *cinfo) { if (SETJMP(sp->exit_jmpbuf)) return 0; @@ -2701,14 +2740,14 @@ static int jpeg_start_decompress_encap(OJPEGState *sp, #ifndef LIBJPEG_ENCAP_EXTERNAL static int jpeg_read_scanlines_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, + tiff_ojpeg_decompress_struct *cinfo, void *scanlines, uint32_t max_lines) { if (SETJMP(sp->exit_jmpbuf)) return 0; else { - jpeg_read_scanlines(cinfo, scanlines, max_lines); + jpeg_read_scanlines(cinfo, (JSAMPARRAY)scanlines, max_lines); return 1; } } @@ -2716,14 +2755,14 @@ static int jpeg_read_scanlines_encap(OJPEGState *sp, #ifndef LIBJPEG_ENCAP_EXTERNAL static int jpeg_read_raw_data_encap(OJPEGState *sp, - jpeg_decompress_struct *cinfo, void *data, - uint32_t max_lines) + tiff_ojpeg_decompress_struct *cinfo, + void *data, uint32_t max_lines) { if (SETJMP(sp->exit_jmpbuf)) return 0; else { - jpeg_read_raw_data(cinfo, data, max_lines); + jpeg_read_raw_data(cinfo, (JSAMPIMAGE)data, max_lines); return 1; } } @@ -2737,14 +2776,15 @@ static void jpeg_encap_unwind(TIFF *tif) } #endif -static void OJPEGLibjpegJpegErrorMgrOutputMessage(jpeg_common_struct *cinfo) +static void +OJPEGLibjpegJpegErrorMgrOutputMessage(tiff_ojpeg_common_struct *cinfo) { char buffer[JMSG_LENGTH_MAX]; (*cinfo->err->format_message)(cinfo, buffer); TIFFWarningExtR(((TIFF *)(cinfo->client_data)), "LibJpeg", "%s", buffer); } -static void OJPEGLibjpegJpegErrorMgrErrorExit(jpeg_common_struct *cinfo) +static void OJPEGLibjpegJpegErrorMgrErrorExit(tiff_ojpeg_common_struct *cinfo) { char buffer[JMSG_LENGTH_MAX]; (*cinfo->err->format_message)(cinfo, buffer); @@ -2752,13 +2792,14 @@ static void OJPEGLibjpegJpegErrorMgrErrorExit(jpeg_common_struct *cinfo) jpeg_encap_unwind((TIFF *)(cinfo->client_data)); } -static void OJPEGLibjpegJpegSourceMgrInitSource(jpeg_decompress_struct *cinfo) +static void +OJPEGLibjpegJpegSourceMgrInitSource(tiff_ojpeg_decompress_struct *cinfo) { (void)cinfo; } static boolean -OJPEGLibjpegJpegSourceMgrFillInputBuffer(jpeg_decompress_struct *cinfo) +OJPEGLibjpegJpegSourceMgrFillInputBuffer(tiff_ojpeg_decompress_struct *cinfo) { TIFF *tif = (TIFF *)cinfo->client_data; OJPEGState *sp = (OJPEGState *)tif->tif_data; @@ -2770,12 +2811,12 @@ OJPEGLibjpegJpegSourceMgrFillInputBuffer(jpeg_decompress_struct *cinfo) jpeg_encap_unwind(tif); } sp->libjpeg_jpeg_source_mgr.bytes_in_buffer = len; - sp->libjpeg_jpeg_source_mgr.next_input_byte = mem; + sp->libjpeg_jpeg_source_mgr.next_input_byte = (const JOCTET *)mem; return (1); } static void -OJPEGLibjpegJpegSourceMgrSkipInputData(jpeg_decompress_struct *cinfo, +OJPEGLibjpegJpegSourceMgrSkipInputData(tiff_ojpeg_decompress_struct *cinfo, long num_bytes) { TIFF *tif = (TIFF *)cinfo->client_data; @@ -2789,7 +2830,7 @@ OJPEGLibjpegJpegSourceMgrSkipInputData(jpeg_decompress_struct *cinfo, #pragma warning(disable : 4702) /* unreachable code */ #endif static boolean -OJPEGLibjpegJpegSourceMgrResyncToRestart(jpeg_decompress_struct *cinfo, +OJPEGLibjpegJpegSourceMgrResyncToRestart(tiff_ojpeg_decompress_struct *cinfo, int desired) { TIFF *tif = (TIFF *)cinfo->client_data; @@ -2802,7 +2843,8 @@ OJPEGLibjpegJpegSourceMgrResyncToRestart(jpeg_decompress_struct *cinfo, #pragma warning(pop) #endif -static void OJPEGLibjpegJpegSourceMgrTermSource(jpeg_decompress_struct *cinfo) +static void +OJPEGLibjpegJpegSourceMgrTermSource(tiff_ojpeg_decompress_struct *cinfo) { (void)cinfo; } diff --git a/contrib/libs/libtiff/tif_open.c b/contrib/libs/libtiff/tif_open.c index 565d13cf694..43cb170286d 100644 --- a/contrib/libs/libtiff/tif_open.c +++ b/contrib/libs/libtiff/tif_open.c @@ -77,7 +77,7 @@ int _TIFFgetMode(TIFFOpenOptions *opts, thandle_t clientdata, const char *mode, return (m); } -TIFFOpenOptions *TIFFOpenOptionsAlloc() +TIFFOpenOptions *TIFFOpenOptionsAlloc(void) { TIFFOpenOptions *opts = (TIFFOpenOptions *)_TIFFcalloc(1, sizeof(TIFFOpenOptions)); @@ -308,6 +308,7 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, TIFF *tif; int m; const char *cp; + tmsize_t size_to_alloc; /* The following are configuration checks. They should be redundant, but * should not compile to any actual code in an optimised release build @@ -330,7 +331,7 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, n.a8[0] = 1; n.a8[1] = 0; (void)n; -#ifdef WORDS_BIGENDIAN +#if WORDS_BIGENDIAN assert(n.a16 == 256); #else assert(n.a16 == 1); @@ -340,7 +341,7 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, m = _TIFFgetMode(opts, clientdata, mode, module); if (m == -1) goto bad2; - tmsize_t size_to_alloc = (tmsize_t)(sizeof(TIFF) + strlen(name) + 1); + size_to_alloc = (tmsize_t)(sizeof(TIFF) + strlen(name) + 1); if (opts && opts->max_single_mem_alloc > 0 && size_to_alloc > opts->max_single_mem_alloc) { @@ -377,8 +378,6 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, tif->tif_curdir = TIFF_NON_EXISTENT_DIR_NUMBER; /* non-existent directory */ tif->tif_curdircount = TIFF_NON_EXISTENT_DIR_NUMBER; tif->tif_curoff = 0; - tif->tif_curstrip = (uint32_t)-1; /* invalid strip */ - tif->tif_row = (uint32_t)-1; /* read/write pre-increment */ tif->tif_clientdata = clientdata; tif->tif_readproc = readproc; tif->tif_writeproc = writeproc; @@ -398,6 +397,10 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, tif->tif_warn_about_unknown_tags = opts->warn_about_unknown_tags; } + /* Reset tif->tif_dir structure to zero and + * initialize some IFD strile counter and index parameters. */ + _TIFFResetTifDirAndInitStrileCounters(&tif->tif_dir); + if (!readproc || !writeproc || !seekproc || !closeproc || !sizeproc) { TIFFErrorExtR(tif, module, @@ -480,13 +483,13 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, switch (*cp) { case 'b': -#ifndef WORDS_BIGENDIAN +#if !WORDS_BIGENDIAN if (m & O_CREAT) tif->tif_flags |= TIFF_SWAB; #endif break; case 'l': -#ifdef WORDS_BIGENDIAN +#if WORDS_BIGENDIAN if ((m & O_CREAT)) tif->tif_flags |= TIFF_SWAB; #endif @@ -536,7 +539,9 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, case 'O': if (m == O_RDONLY) tif->tif_flags |= - (TIFF_LAZYSTRILELOAD | TIFF_DEFERSTRILELOAD); + (TIFF_LAZYSTRILELOAD_ASKED | TIFF_DEFERSTRILELOAD); + break; + default: break; } @@ -561,7 +566,7 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, /* * Setup header and write. */ -#ifdef WORDS_BIGENDIAN +#if WORDS_BIGENDIAN tif->tif_header.common.tiff_magic = (tif->tif_flags & TIFF_SWAB) ? TIFF_LITTLEENDIAN : TIFF_BIGENDIAN; #else @@ -654,13 +659,13 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, } if (tif->tif_header.common.tiff_magic == TIFF_BIGENDIAN) { -#ifndef WORDS_BIGENDIAN +#if !WORDS_BIGENDIAN tif->tif_flags |= TIFF_SWAB; #endif } else { -#ifdef WORDS_BIGENDIAN +#if WORDS_BIGENDIAN tif->tif_flags |= TIFF_SWAB; #endif } @@ -780,6 +785,8 @@ TIFF *TIFFClientOpenExt(const char *name, const char *mode, if (!TIFFDefaultDirectory(tif)) goto bad; return (tif); + default: + break; } bad: tif->tif_mode = O_RDONLY; /* XXX avoid flush */ @@ -861,7 +868,7 @@ int TIFFIsTiled(TIFF *tif) { return (isTiled(tif)); } /* * Return current row being read/written. */ -uint32_t TIFFCurrentRow(TIFF *tif) { return (tif->tif_row); } +uint32_t TIFFCurrentRow(TIFF *tif) { return (tif->tif_dir.td_row); } /* * Return index of the current directory. @@ -871,12 +878,12 @@ tdir_t TIFFCurrentDirectory(TIFF *tif) { return (tif->tif_curdir); } /* * Return current strip. */ -uint32_t TIFFCurrentStrip(TIFF *tif) { return (tif->tif_curstrip); } +uint32_t TIFFCurrentStrip(TIFF *tif) { return (tif->tif_dir.td_curstrip); } /* * Return current tile. */ -uint32_t TIFFCurrentTile(TIFF *tif) { return (tif->tif_curtile); } +uint32_t TIFFCurrentTile(TIFF *tif) { return (tif->tif_dir.td_curtile); } /* * Return nonzero if the file has byte-swapped data. diff --git a/contrib/libs/libtiff/tif_packbits.c b/contrib/libs/libtiff/tif_packbits.c index d7db9b64ea3..13d0f1d3908 100644 --- a/contrib/libs/libtiff/tif_packbits.c +++ b/contrib/libs/libtiff/tif_packbits.c @@ -194,12 +194,17 @@ static int PackBitsEncode(TIFF *tif, uint8_t *buf, tmsize_t cc, uint16_t s) */ if (n == 1 && op[-2] == (uint8_t)-1 && *lastliteral < 126) { - state = (((*lastliteral) += 2) == 127 ? BASE : LITERAL); + state = + (((*lastliteral) = (uint8_t)(*lastliteral + 2)) == 127 + ? BASE + : LITERAL); op[-2] = op[-1]; /* replicate */ } else state = RUN; goto again; + default: + break; } } tif->tif_rawcc += (tmsize_t)(op - tif->tif_rawcp); @@ -306,12 +311,22 @@ static int PackBitsDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Not enough data for scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); return (0); } return (1); } +static uint64_t PackBitsGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + return 64; +} + int TIFFInitPackBits(TIFF *tif, int scheme) { (void)scheme; @@ -325,6 +340,8 @@ int TIFFInitPackBits(TIFF *tif, int scheme) tif->tif_encodestrip = PackBitsEncodeChunk; tif->tif_encodetile = PackBitsEncodeChunk; #endif + tif->tif_getmaxcompressionratio = PackBitsGetMaxCompressionRatio; + return (1); } #endif /* PACKBITS_SUPPORT */ diff --git a/contrib/libs/libtiff/tif_pixarlog.c b/contrib/libs/libtiff/tif_pixarlog.c index 309054804d3..fa267dfbff2 100644 --- a/contrib/libs/libtiff/tif_pixarlog.c +++ b/contrib/libs/libtiff/tif_pixarlog.c @@ -117,11 +117,11 @@ static float LogK1, LogK2; } while (i > 0); \ } -static void horizontalAccumulateF(uint16_t *wp, int n, int stride, float *op, - float *ToLinearF) +static void horizontalAccumulateF(uint16_t *wp, tmsize_t n, int stride, + float *op, float *ToLinearF) { - register unsigned int cr, cg, cb, ca, mask; - register float t0, t1, t2, t3; + unsigned int cr, cg, cb, ca, mask; + float t0, t1, t2, t3; if (n >= stride) { @@ -180,22 +180,22 @@ static void horizontalAccumulateF(uint16_t *wp, int n, int stride, float *op, n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; *op = ToLinearF[*wp & mask]; - wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + *op = ToLinearF[*wp & mask]; wp++; op++) n -= stride; } } } } -static void horizontalAccumulate12(uint16_t *wp, int n, int stride, int16_t *op, - float *ToLinearF) +static void horizontalAccumulate12(uint16_t *wp, tmsize_t n, int stride, + int16_t *op, float *ToLinearF) { - register unsigned int cr, cg, cb, ca, mask; - register float t0, t1, t2, t3; + unsigned int cr, cg, cb, ca, mask; + float t0, t1, t2, t3; -#define SCALE12 2048.0F -#define CLAMP12(t) (((t) < 3071) ? (uint16_t)(t) : 3071) +#define SCALE12 2048.0f +#define CLAMP12(t) (((t) < 3071) ? (int16_t)(uint16_t)(t) : (int16_t)3071) if (n >= stride) { @@ -255,19 +255,19 @@ static void horizontalAccumulate12(uint16_t *wp, int n, int stride, int16_t *op, n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; - t0 = ToLinearF[wp[stride] & mask] * SCALE12; - *op = CLAMP12(t0); wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + t0 = ToLinearF[*wp & mask] * SCALE12; *op = CLAMP12(t0); + wp++; op++) n -= stride; } } } } -static void horizontalAccumulate16(uint16_t *wp, int n, int stride, +static void horizontalAccumulate16(uint16_t *wp, tmsize_t n, int stride, uint16_t *op, uint16_t *ToLinear16) { - register unsigned int cr, cg, cb, ca, mask; + unsigned int cr, cg, cb, ca, mask; if (n >= stride) { @@ -312,8 +312,8 @@ static void horizontalAccumulate16(uint16_t *wp, int n, int stride, n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; *op = ToLinear16[*wp & mask]; - wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + *op = ToLinear16[*wp & mask]; wp++; op++) n -= stride; } } @@ -324,10 +324,10 @@ static void horizontalAccumulate16(uint16_t *wp, int n, int stride, * Returns the log encoded 11-bit values with the horizontal * differencing undone. */ -static void horizontalAccumulate11(uint16_t *wp, int n, int stride, +static void horizontalAccumulate11(uint16_t *wp, tmsize_t n, int stride, uint16_t *op) { - register unsigned int cr, cg, cb, ca, mask; + unsigned int cr, cg, cb, ca, mask; if (n >= stride) { @@ -375,21 +375,22 @@ static void horizontalAccumulate11(uint16_t *wp, int n, int stride, } else { - REPEAT(stride, *op = *wp & mask; wp++; op++) + REPEAT(stride, *op = (uint16_t)(*wp & mask); wp++; op++) n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; *op = *wp & mask; wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + *op = (uint16_t)(*wp & mask); wp++; op++) n -= stride; } } } } -static void horizontalAccumulate8(uint16_t *wp, int n, int stride, +static void horizontalAccumulate8(uint16_t *wp, tmsize_t n, int stride, unsigned char *op, unsigned char *ToLinear8) { - register unsigned int cr, cg, cb, ca, mask; + unsigned int cr, cg, cb, ca, mask; if (n >= stride) { @@ -434,20 +435,20 @@ static void horizontalAccumulate8(uint16_t *wp, int n, int stride, n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; *op = ToLinear8[*wp & mask]; - wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + *op = ToLinear8[*wp & mask]; wp++; op++) n -= stride; } } } } -static void horizontalAccumulate8abgr(uint16_t *wp, int n, int stride, +static void horizontalAccumulate8abgr(uint16_t *wp, tmsize_t n, int stride, unsigned char *op, unsigned char *ToLinear8) { - register unsigned int cr, cg, cb, ca, mask; - register unsigned char t0, t1, t2, t3; + unsigned int cr, cg, cb, ca, mask; + unsigned char t0, t1, t2, t3; if (n >= stride) { @@ -508,8 +509,8 @@ static void horizontalAccumulate8abgr(uint16_t *wp, int n, int stride, n -= stride; while (n > 0) { - REPEAT(stride, wp[stride] += *wp; *op = ToLinear8[*wp & mask]; - wp++; op++) + REPEAT(stride, *wp = (uint16_t)(*wp + wp[-stride]); + *op = ToLinear8[*wp & mask]; wp++; op++) n -= stride; } } @@ -577,7 +578,8 @@ static int PixarLogMakeTables(TIFF *tif, PixarLogState *sp) LogK1 = (float)(1. / c); /* if (v >= 2) token = k1*log(v*k2) */ LogK2 = (float)(1. / b); lt2size = (int)(2. / linstep) + 1; - FromLT2 = (uint16_t *)_TIFFmallocExt(tif, lt2size * sizeof(uint16_t)); + FromLT2 = (uint16_t *)_TIFFmallocExt( + tif, (tmsize_t)((size_t)lt2size * sizeof(uint16_t))); From14 = (uint16_t *)_TIFFmallocExt(tif, 16384 * sizeof(uint16_t)); From8 = (uint16_t *)_TIFFmallocExt(tif, 256 * sizeof(uint16_t)); ToLinearF = (float *)_TIFFmallocExt(tif, TSIZEP1 * sizeof(float)); @@ -623,16 +625,17 @@ static int PixarLogMakeTables(TIFF *tif, PixarLogState *sp) for (i = 0; i < TSIZEP1; i++) { - v = ToLinearF[i] * 65535.0 + 0.5; + v = (double)ToLinearF[i] * 65535.0 + 0.5; ToLinear16[i] = (v > 65535.0) ? 65535 : (uint16_t)v; - v = ToLinearF[i] * 255.0 + 0.5; + v = (double)ToLinearF[i] * 255.0 + 0.5; ToLinear8[i] = (v > 255.0) ? 255 : (unsigned char)v; } j = 0; for (i = 0; i < lt2size; i++) { - if ((i * linstep) * (i * linstep) > ToLinearF[j] * ToLinearF[j + 1]) + if ((i * linstep) * (i * linstep) > + (double)ToLinearF[j] * (double)ToLinearF[j + 1]) j++; FromLT2[i] = (uint16_t)j; } @@ -645,7 +648,8 @@ static int PixarLogMakeTables(TIFF *tif, PixarLogState *sp) j = 0; for (i = 0; i < 16384; i++) { - while ((i / 16383.) * (i / 16383.) > ToLinearF[j] * ToLinearF[j + 1]) + while ((i / 16383.) * (i / 16383.) > + (double)ToLinearF[j] * (double)ToLinearF[j + 1]) j++; From14[i] = (uint16_t)j; } @@ -653,7 +657,8 @@ static int PixarLogMakeTables(TIFF *tif, PixarLogState *sp) j = 0; for (i = 0; i < 256; i++) { - while ((i / 255.) * (i / 255.) > ToLinearF[j] * ToLinearF[j + 1]) + while ((i / 255.) * (i / 255.) > + (double)ToLinearF[j] * (double)ToLinearF[j + 1]) j++; From8[i] = (uint16_t)j; } @@ -708,6 +713,8 @@ static int PixarLogGuessDataFmt(TIFFDirectory *td) if (format == SAMPLEFORMAT_VOID || format == SAMPLEFORMAT_UINT) guess = PIXARLOGDATAFMT_8BIT; break; + default: + break; } return guess; @@ -769,7 +776,8 @@ static int PixarLogSetupDecode(TIFF *tif) multiply_ms(multiply_ms(sp->stride, td->td_imagewidth), strip_height), sizeof(uint16_t)); /* add one more stride in case input ends mid-stride */ - tbuf_size = add_ms(tbuf_size, sizeof(uint16_t) * sp->stride); + tbuf_size = + add_ms(tbuf_size, (tmsize_t)(sizeof(uint16_t) * (size_t)sp->stride)); if (tbuf_size == 0) return (0); /* TODO: this is an error return without error report through TIFFErrorExt */ @@ -838,18 +846,21 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) PixarLogState *sp = PixarLogDecoderState(tif); tmsize_t i; tmsize_t nsamples; - int llen; + tmsize_t llen; uint16_t *up; switch (sp->user_datafmt) { case PIXARLOGDATAFMT_FLOAT: - nsamples = occ / sizeof(float); /* XXX float == 32 bits */ + nsamples = (tmsize_t)((uint64_t)occ / + sizeof(float)); /* XXX float == 32 bits */ break; case PIXARLOGDATAFMT_16BIT: case PIXARLOGDATAFMT_12BITPICIO: case PIXARLOGDATAFMT_11BITLOG: - nsamples = occ / sizeof(uint16_t); /* XXX uint16_t == 16 bits */ + nsamples = + (tmsize_t)((uint64_t)occ / + sizeof(uint16_t)); /* XXX uint16_t == 16 bits */ break; case PIXARLOGDATAFMT_8BIT: case PIXARLOGDATAFMT_8BITABGR: @@ -863,7 +874,55 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) return 0; } - llen = sp->stride * td->td_imagewidth; + llen = (tmsize_t)sp->stride * td->td_imagewidth; + + /* Fix: ABGR with stride=3 expands 3 samples to 4 output bytes per pixel */ + if (sp->user_datafmt == PIXARLOGDATAFMT_8BITABGR && sp->stride == 3) + { + tmsize_t required = (tmsize_t)td->td_imagewidth * 4; + tmsize_t max_rows; + tmsize_t max_nsamples; + + /* + * Ensure at least one expanded output row fits. + */ + if (occ < required) + { + TIFFErrorExtR(tif, module, + "Output buffer too small for PixarLog ABGR data"); + memset(op, 0, (size_t)occ); + return (0); + } + + /* + * The caller-provided output buffer size must represent a whole + * number of expanded ABGR scanlines. + */ + if (occ % required) + { + TIFFErrorExtR( + tif, module, + "Fractional scanline not supported for PixarLog ABGR data"); + memset(op, 0, (size_t)occ); + return (0); + } + + /* + * PixarLogDecode() may process multiple rows per call + * (e.g. strip decoding). Limit nsamples so the total + * output written by the loop below never exceeds occ. + */ + max_rows = occ / required; + max_nsamples = max_rows * llen; + + if (nsamples > max_nsamples) + { + TIFFErrorExtR(tif, module, + "Output buffer too small for PixarLog ABGR data"); + memset(op, 0, (size_t)occ); + return (0); + } + } (void)s; assert(sp != NULL); @@ -876,8 +935,8 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) we need to simplify this code to reflect a ZLib that is likely updated to deal with 8byte memory sizes, though this code will respond appropriately even before we simplify it */ - sp->stream.avail_out = (uInt)(nsamples * sizeof(uint16_t)); - if (sp->stream.avail_out != nsamples * sizeof(uint16_t)) + sp->stream.avail_out = (uInt)((unsigned long)nsamples * sizeof(uint16_t)); + if (sp->stream.avail_out != (unsigned long)nsamples * sizeof(uint16_t)) { TIFFErrorExtR(tif, module, "ZLib cannot deal with buffers this size"); memset(op, 0, (size_t)occ); @@ -899,9 +958,10 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } if (state == Z_DATA_ERROR) { - TIFFErrorExtR( - tif, module, "Decoding error at scanline %" PRIu32 ", %s", - tif->tif_row, sp->stream.msg ? sp->stream.msg : "(null)"); + TIFFErrorExtR(tif, module, + "Decoding error at scanline %" PRIu32 ", %s", + tif->tif_dir.td_row, + sp->stream.msg ? sp->stream.msg : "(null)"); memset(op, 0, (size_t)occ); return (0); } @@ -920,7 +980,7 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at scanline %" PRIu32 " (short %u bytes)", - tif->tif_row, sp->stream.avail_out); + tif->tif_dir.td_row, sp->stream.avail_out); memset(op, 0, (size_t)occ); return (0); } @@ -941,7 +1001,8 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) if (nsamples % llen) { TIFFWarningExtR(tif, module, - "stride %d is not a multiple of sample count, " + "stride %" TIFF_SSIZE_FORMAT + " is not a multiple of sample count, " "%" TIFF_SSIZE_FORMAT ", data truncated.", llen, nsamples); nsamples -= nsamples % llen; @@ -954,31 +1015,37 @@ static int PixarLogDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) case PIXARLOGDATAFMT_FLOAT: horizontalAccumulateF(up, llen, sp->stride, (float *)op, sp->ToLinearF); - op += llen * sizeof(float); + op += (unsigned long)llen * sizeof(float); break; case PIXARLOGDATAFMT_16BIT: horizontalAccumulate16(up, llen, sp->stride, (uint16_t *)op, sp->ToLinear16); - op += llen * sizeof(uint16_t); + op += (unsigned long)llen * sizeof(uint16_t); break; case PIXARLOGDATAFMT_12BITPICIO: horizontalAccumulate12(up, llen, sp->stride, (int16_t *)op, sp->ToLinearF); - op += llen * sizeof(int16_t); + op += (unsigned long)llen * sizeof(int16_t); break; case PIXARLOGDATAFMT_11BITLOG: horizontalAccumulate11(up, llen, sp->stride, (uint16_t *)op); - op += llen * sizeof(uint16_t); + op += (unsigned long)llen * sizeof(uint16_t); break; case PIXARLOGDATAFMT_8BIT: horizontalAccumulate8(up, llen, sp->stride, (unsigned char *)op, sp->ToLinear8); - op += llen * sizeof(unsigned char); + op += (unsigned long)llen * sizeof(unsigned char); break; case PIXARLOGDATAFMT_8BITABGR: horizontalAccumulate8abgr(up, llen, sp->stride, (unsigned char *)op, sp->ToLinear8); - op += llen * sizeof(unsigned char); + + /* For stride == 3 (RGB), horizontalAccumulate8abgr expands to 4 + * bytes/pixel (ABGR) */ + if (sp->stride == 3) + op += (unsigned long)td->td_imagewidth * 4; + else + op += (unsigned long)llen * sizeof(unsigned char); break; default: TIFFErrorExtR(tif, module, "Unsupported bits/sample: %" PRIu16, @@ -1063,17 +1130,18 @@ static int PixarLogPreEncode(TIFF *tif, uint16_t s) return (deflateReset(&sp->stream) == Z_OK); } -static void horizontalDifferenceF(float *ip, int n, int stride, uint16_t *wp, - uint16_t *FromLT2) +static void horizontalDifferenceF(float *ip, tmsize_t n, int stride, + uint16_t *wp, uint16_t *FromLT2) { int32_t r1, g1, b1, a1, r2, g2, b2, a2, mask; float fltsize = Fltsize; #define CLAMP(v) \ - ((v < (float)0.) ? 0 \ - : (v < (float)2.) ? FromLT2[(int)(v * fltsize)] \ - : (v > (float)24.2) ? 2047 \ - : LogK1 * log(v * LogK2) + 0.5) + ((v < (float)0.) ? 0 \ + : (v < (float)2.) ? FromLT2[(int)(v * fltsize)] \ + : (v > (float)24.2) \ + ? 2047 \ + : (double)LogK1 * log((double)v * (double)LogK2) + 0.5) mask = CODE_MASK; if (n >= stride) @@ -1143,10 +1211,10 @@ static void horizontalDifferenceF(float *ip, int n, int stride, uint16_t *wp, } } -static void horizontalDifference16(unsigned short *ip, int n, int stride, +static void horizontalDifference16(unsigned short *ip, tmsize_t n, int stride, unsigned short *wp, uint16_t *From14) { - register int r1, g1, b1, a1, r2, g2, b2, a2, mask; + int r1, g1, b1, a1, r2, g2, b2, a2, mask; /* assumption is unsigned pixel values */ #undef CLAMP @@ -1219,10 +1287,10 @@ static void horizontalDifference16(unsigned short *ip, int n, int stride, } } -static void horizontalDifference8(unsigned char *ip, int n, int stride, +static void horizontalDifference8(unsigned char *ip, tmsize_t n, int stride, unsigned short *wp, uint16_t *From8) { - register int r1, g1, b1, a1, r2, g2, b2, a2, mask; + int r1, g1, b1, a1, r2, g2, b2, a2, mask; #undef CLAMP #define CLAMP(v) (From8[(v)]) @@ -1304,7 +1372,7 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) PixarLogState *sp = PixarLogEncoderState(tif); tmsize_t i; tmsize_t n; - int llen; + tmsize_t llen; unsigned short *up; (void)s; @@ -1312,12 +1380,14 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) switch (sp->user_datafmt) { case PIXARLOGDATAFMT_FLOAT: - n = cc / sizeof(float); /* XXX float == 32 bits */ + n = (tmsize_t)((unsigned long)cc / + sizeof(float)); /* XXX float == 32 bits */ break; case PIXARLOGDATAFMT_16BIT: case PIXARLOGDATAFMT_12BITPICIO: case PIXARLOGDATAFMT_11BITLOG: - n = cc / sizeof(uint16_t); /* XXX uint16_t == 16 bits */ + n = (tmsize_t)((unsigned long)cc / + sizeof(uint16_t)); /* XXX uint16_t == 16 bits */ break; case PIXARLOGDATAFMT_8BIT: case PIXARLOGDATAFMT_8BITABGR: @@ -1330,7 +1400,7 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) return 0; } - llen = sp->stride * td->td_imagewidth; + llen = (tmsize_t)sp->stride * td->td_imagewidth; /* Check against the number of elements (of size uint16_t) of sp->tbuf */ if (n > ((tmsize_t)td->td_rowsperstrip * llen)) { @@ -1345,17 +1415,17 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) case PIXARLOGDATAFMT_FLOAT: horizontalDifferenceF((float *)bp, llen, sp->stride, up, sp->FromLT2); - bp += llen * sizeof(float); + bp += (unsigned long)llen * sizeof(float); break; case PIXARLOGDATAFMT_16BIT: horizontalDifference16((uint16_t *)bp, llen, sp->stride, up, sp->From14); - bp += llen * sizeof(uint16_t); + bp += (unsigned long)llen * sizeof(uint16_t); break; case PIXARLOGDATAFMT_8BIT: horizontalDifference8((unsigned char *)bp, llen, sp->stride, up, sp->From8); - bp += llen * sizeof(unsigned char); + bp += (unsigned long)llen * sizeof(unsigned char); break; default: TIFFErrorExtR(tif, module, @@ -1370,8 +1440,8 @@ static int PixarLogEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) we need to simplify this code to reflect a ZLib that is likely updated to deal with 8byte memory sizes, though this code will respond appropriately even before we simplify it */ - sp->stream.avail_in = (uInt)(n * sizeof(uint16_t)); - if ((sp->stream.avail_in / sizeof(uint16_t)) != (uInt)n) + sp->stream.avail_in = (uInt)((unsigned long)n * sizeof(uint16_t)); + if ((sp->stream.avail_in / sizeof(uint16_t)) != (unsigned long)n) { TIFFErrorExtR(tif, module, "ZLib cannot deal with buffers this size"); return (0); @@ -1562,13 +1632,15 @@ static int PixarLogVSetField(TIFF *tif, uint32_t tag, va_list ap) TIFFSetField(tif, TIFFTAG_SAMPLEFORMAT, SAMPLEFORMAT_IEEEFP); break; + default: + break; } /* * Must recalculate sizes should bits/sample change. */ - tif->tif_tilesize = + tif->tif_dir.td_tilesize = isTiled(tif) ? TIFFTileSize(tif) : (tmsize_t)(-1); - tif->tif_scanlinesize = TIFFScanlineSize(tif); + tif->tif_dir.td_scanlinesize = TIFFScanlineSize(tif); result = 1; /* NB: pseudo tag */ break; default: @@ -1601,6 +1673,16 @@ static const TIFFField pixarlogFields[] = { {TIFFTAG_PIXARLOGQUALITY, 0, 0, TIFF_ANY, 0, TIFF_SETGET_INT, FIELD_PSEUDO, FALSE, FALSE, "", NULL}}; +static uint64_t PixarLogGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + /* cf https://zlib.net/zlib_tech.html */ + const uint64_t MAX_DEFLATE_RATIO = 1032; + + /* security margin as I don't understand what this codec does */ + return MAX_DEFLATE_RATIO * (uint64_t)4; +} + int TIFFInitPixarLog(TIFF *tif, int scheme) { static const char module[] = "TIFFInitPixarLog"; @@ -1648,6 +1730,7 @@ int TIFFInitPixarLog(TIFF *tif, int scheme) tif->tif_encodetile = PixarLogEncode; tif->tif_close = PixarLogClose; tif->tif_cleanup = PixarLogCleanup; + tif->tif_getmaxcompressionratio = PixarLogGetMaxCompressionRatio; /* Override SetField so we can handle our private pseudo-tag */ sp->vgetparent = tif->tif_tagmethods.vgetfield; diff --git a/contrib/libs/libtiff/tif_predict.c b/contrib/libs/libtiff/tif_predict.c index 7a6fc4af8b7..18b90c211e1 100644 --- a/contrib/libs/libtiff/tif_predict.c +++ b/contrib/libs/libtiff/tif_predict.c @@ -30,7 +30,7 @@ #include "tif_predict.h" #include "tiffiop.h" -#if defined(__x86_64__) || defined(_M_X64) +#if defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC)) #include <emmintrin.h> #endif @@ -151,6 +151,8 @@ static int PredictorSetupDecode(TIFF *tif) case 64: sp->decodepfunc = horAcc64; break; + default: + break; } /* * Override default decoding method with one that does the @@ -247,6 +249,8 @@ static int PredictorSetupEncode(TIFF *tif) case 64: sp->encodepfunc = horDiff64; break; + default: + break; } /* * Override default encoding method with one that does the @@ -590,7 +594,7 @@ static int fpAcc(TIFF *tif, uint8_t *cp0, tmsize_t cc) cp = (uint8_t *)cp0; count = 0; -#if defined(__x86_64__) || defined(_M_X64) +#if defined(__x86_64__) || (defined(_M_X64) && !defined(_M_ARM64EC)) if (bps == 4) { /* Optimization of general case */ @@ -972,7 +976,7 @@ static int PredictorEncodeRow(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) (int64_t)cc); return 0; } - memcpy(working_copy, bp, cc); + memcpy(working_copy, bp, (size_t)cc); if (!(*sp->encodepfunc)(tif, working_copy, cc)) { @@ -1010,7 +1014,7 @@ static int PredictorEncodeTile(TIFF *tif, uint8_t *bp0, tmsize_t cc0, (int64_t)cc0); return 0; } - memcpy(working_copy, bp0, cc0); + memcpy(working_copy, bp0, (size_t)cc0); bp = working_copy; rowsize = sp->rowsize; @@ -1098,8 +1102,10 @@ static void PredictorPrintDir(TIFF *tif, FILE *fd, long flags) case 3: fprintf(fd, "floating point predictor "); break; + default: + break; } - fprintf(fd, "%d (0x%x)\n", sp->predictor, sp->predictor); + fprintf(fd, "%d (0x%x)\n", sp->predictor, (unsigned)sp->predictor); } if (sp->printdir) (*sp->printdir)(tif, fd, flags); diff --git a/contrib/libs/libtiff/tif_print.c b/contrib/libs/libtiff/tif_print.c index addc03ab078..1ea450db95b 100644 --- a/contrib/libs/libtiff/tif_print.c +++ b/contrib/libs/libtiff/tif_print.c @@ -118,10 +118,10 @@ static void _TIFFPrintField(FILE *fd, const TIFFField *fip, if (tv_size == 8) fprintf(fd, "%lf", ((double *)raw_data)[j]); else - fprintf(fd, "%f", ((float *)raw_data)[j]); + fprintf(fd, "%f", (double)((float *)raw_data)[j]); } else if (fip->field_type == TIFF_FLOAT) - fprintf(fd, "%f", ((float *)raw_data)[j]); + fprintf(fd, "%f", (double)((float *)raw_data)[j]); else if (fip->field_type == TIFF_LONG8) fprintf(fd, "%" PRIu64, ((uint64_t *)raw_data)[j]); else if (fip->field_type == TIFF_SLONG8) @@ -193,8 +193,9 @@ static int _TIFFPrettyPrintField(TIFF *tif, const TIFFField *fip, FILE *fd, case TIFFTAG_WHITEPOINT: if (value_count == 2 && fip->field_type == TIFF_RATIONAL) { - fprintf(fd, " White Point: %g-%g\n", ((float *)raw_data)[0], - ((float *)raw_data)[1]); + fprintf(fd, " White Point: %g-%g\n", + (double)((float *)raw_data)[0], + (double)((float *)raw_data)[1]); return 1; } return 0; @@ -232,6 +233,9 @@ static int _TIFFPrettyPrintField(TIFF *tif, const TIFFField *fip, FILE *fd, return 1; } return 0; + + default: + break; } return 0; @@ -244,8 +248,7 @@ static int _TIFFPrettyPrintField(TIFF *tif, const TIFFField *fip, FILE *fd, void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) { TIFFDirectory *td = &tif->tif_dir; - char *sep; - long l, n; + const char *sep; fprintf(fd, "TIFF Directory at offset 0x%" PRIx64 " (%" PRIu64 ")\n", tif->tif_diroff, tif->tif_diroff); @@ -286,8 +289,8 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) } if (TIFFFieldSet(tif, FIELD_RESOLUTION)) { - fprintf(fd, " Resolution: %g, %g", td->td_xresolution, - td->td_yresolution); + fprintf(fd, " Resolution: %g, %g", (double)td->td_xresolution, + (double)td->td_yresolution); if (TIFFFieldSet(tif, FIELD_RESOLUTIONUNIT)) { switch (td->td_resolutionunit) @@ -310,7 +313,8 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) fprintf(fd, "\n"); } if (TIFFFieldSet(tif, FIELD_POSITION)) - fprintf(fd, " Position: %g, %g\n", td->td_xposition, td->td_yposition); + fprintf(fd, " Position: %g, %g\n", (double)td->td_xposition, + (double)td->td_yposition); if (TIFFFieldSet(tif, FIELD_BITSPERSAMPLE)) fprintf(fd, " Bits/Sample: %" PRIu16 "\n", td->td_bitspersample); if (TIFFFieldSet(tif, FIELD_SAMPLEFORMAT)) @@ -374,7 +378,8 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) } } } - if (TIFFFieldSet(tif, FIELD_EXTRASAMPLES) && td->td_extrasamples) + if (TIFFFieldSet(tif, FIELD_EXTRASAMPLES) && td->td_extrasamples && + td->td_sampleinfo) { uint16_t i; fprintf(fd, " Extra Samples: %" PRIu16 "<", td->td_extrasamples); @@ -412,7 +417,8 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) i > 0 && cp < td->td_inknames + td->td_inknameslen; cp = strchr(cp, '\0') + 1, i--) { - size_t max_chars = td->td_inknameslen - (cp - td->td_inknames); + size_t max_chars = + (size_t)(td->td_inknameslen - (cp - td->td_inknames)); fputs(sep, fd); _TIFFprintAsciiBounded(fd, cp, max_chars); sep = ", "; @@ -548,17 +554,25 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) if (TIFFFieldSet(tif, FIELD_PAGENUMBER)) fprintf(fd, " Page Number: %" PRIu16 "-%" PRIu16 "\n", td->td_pagenumber[0], td->td_pagenumber[1]); - if (TIFFFieldSet(tif, FIELD_COLORMAP)) + if (TIFFFieldSet(tif, FIELD_COLORMAP) && td->td_colormap[0] && + td->td_colormap[1] && td->td_colormap[2]) { fprintf(fd, " Color Map: "); if (flags & TIFFPRINT_COLORMAP) { fprintf(fd, "\n"); - n = 1L << td->td_bitspersample; - for (l = 0; l < n; l++) - fprintf(fd, " %5ld: %5" PRIu16 " %5" PRIu16 " %5" PRIu16 "\n", - l, td->td_colormap[0][l], td->td_colormap[1][l], - td->td_colormap[2][l]); + if (td->td_bitspersample >= 64) + fprintf(fd, " (BitsPerSample too large to print safely)\n"); + else + { + uint64_t n = 1ULL << td->td_bitspersample; + for (uint64_t l = 0u; l < n; l++) + fprintf(fd, + " %5" PRIu64 ": %5" PRIu16 " %5" PRIu16 + " %5" PRIu16 "\n", + l, td->td_colormap[0][l], td->td_colormap[1][l], + td->td_colormap[2][l]); + } } else fprintf(fd, "(present)\n"); @@ -569,26 +583,37 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) fprintf(fd, " Reference Black/White:\n"); for (i = 0; i < 3; i++) fprintf(fd, " %2d: %5g %5g\n", i, - td->td_refblackwhite[2 * i + 0], - td->td_refblackwhite[2 * i + 1]); + (double)td->td_refblackwhite[2 * i + 0], + (double)td->td_refblackwhite[2 * i + 1]); } - if (TIFFFieldSet(tif, FIELD_TRANSFERFUNCTION)) + if (TIFFFieldSet(tif, FIELD_TRANSFERFUNCTION) && + td->td_transferfunction[0] && + ((td->td_samplesperpixel - td->td_extrasamples > 1 && + td->td_transferfunction[1] && td->td_transferfunction[2]) || + td->td_samplesperpixel - td->td_extrasamples <= 1)) { fprintf(fd, " Transfer Function: "); if (flags & TIFFPRINT_CURVES) { fprintf(fd, "\n"); - n = 1L << td->td_bitspersample; - for (l = 0; l < n; l++) + if (td->td_bitspersample >= 64) + fprintf(fd, " (BitsPerSample too large to print safely)\n"); + else { - uint16_t i; - fprintf(fd, " %2ld: %5" PRIu16, l, - td->td_transferfunction[0][l]); - for (i = 1; - i < td->td_samplesperpixel - td->td_extrasamples && i < 3; - i++) - fprintf(fd, " %5" PRIu16, td->td_transferfunction[i][l]); - fputc('\n', fd); + uint64_t n = 1ULL << td->td_bitspersample; + for (uint64_t l = 0; l < n; l++) + { + uint16_t i; + fprintf(fd, " %2" PRIu64 ": %5" PRIu16, l, + td->td_transferfunction[0][l]); + for (i = 1; + i < td->td_samplesperpixel - td->td_extrasamples && + i < 3; + i++) + fprintf(fd, " %5" PRIu16, + td->td_transferfunction[i][l]); + fputc('\n', fd); + } } } else @@ -655,7 +680,7 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) else if (fip->field_readcount == TIFF_SPP) value_count = td->td_samplesperpixel; else - value_count = fip->field_readcount; + value_count = (uint32_t)fip->field_readcount; if (fip->field_tag == TIFFTAG_DOTRANGE && strcmp(fip->field_name, "DotRange") == 0) { @@ -680,7 +705,8 @@ void TIFFPrintDirectory(TIFF *tif, FILE *fd, long flags) * "set_get_field_type" to determine internal storage size. */ int tv_size = TIFFFieldSetGetSize(fip); - raw_data = _TIFFmallocExt(tif, tv_size * value_count); + raw_data = _TIFFCheckMalloc(tif, value_count, tv_size, + "for tag data"); mem_alloc = 1; if (TIFFGetField(tif, tag, raw_data) != 1) { @@ -744,7 +770,7 @@ static void _TIFFprintAsciiBounded(FILE *fd, const char *cp, size_t max_chars) if (*tp) fprintf(fd, "\\%c", *tp); else - fprintf(fd, "\\%03o", *cp & 0xff); + fprintf(fd, "\\%03o", (unsigned int)(*cp & 0xff)); } } diff --git a/contrib/libs/libtiff/tif_read.c b/contrib/libs/libtiff/tif_read.c index a2bb3046237..d1a4e892168 100644 --- a/contrib/libs/libtiff/tif_read.c +++ b/contrib/libs/libtiff/tif_read.c @@ -27,6 +27,7 @@ * Scanline-oriented Read Support */ #include "tiffiop.h" +#include <limits.h> #include <stdio.h> int TIFFFillStrip(TIFF *tif, uint32_t strip); @@ -39,9 +40,6 @@ static tmsize_t TIFFReadRawStrip1(TIFF *tif, uint32_t strip, void *buf, static tmsize_t TIFFReadRawTile1(TIFF *tif, uint32_t tile, void *buf, tmsize_t size, const char *module); -#define NOSTRIP ((uint32_t)(-1)) /* undefined state */ -#define NOTILE ((uint32_t)(-1)) /* undefined state */ - #define INITIAL_THRESHOLD (1024 * 1024) #define THRESHOLD_MULTIPLIER 10 #define MAX_THRESHOLD \ @@ -99,7 +97,9 @@ static int TIFFReadAndRealloc(TIFF *tif, tmsize_t size, tmsize_t rawdata_offset, uint8_t *new_rawdata; assert((tif->tif_flags & TIFF_MYBUFFER) != 0); tif->tif_rawdatasize = (tmsize_t)TIFFroundup_64( - (uint64_t)already_read + to_read + rawdata_offset, 1024); + (uint64_t)already_read + (uint64_t)to_read + + (uint64_t)rawdata_offset, + 1024); if (tif->tif_rawdatasize == 0) { TIFFErrorExtR(tif, module, "Invalid buffer size"); @@ -111,7 +111,7 @@ static int TIFFReadAndRealloc(TIFF *tif, tmsize_t size, tmsize_t rawdata_offset, { TIFFErrorExtR(tif, module, "No space for data buffer at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); _TIFFfreeExt(tif, tif->tif_rawdata); tif->tif_rawdata = 0; tif->tif_rawdatasize = 0; @@ -127,18 +127,22 @@ static int TIFFReadAndRealloc(TIFF *tif, tmsize_t size, tmsize_t rawdata_offset, bytes_read = TIFFReadFile( tif, tif->tif_rawdata + rawdata_offset + already_read, to_read); + if (bytes_read < 0) + /* Treat read errors as short reads before updating offsets. */ + bytes_read = 0; already_read += bytes_read; if (bytes_read != to_read) { - memset(tif->tif_rawdata + rawdata_offset + already_read, 0, - tif->tif_rawdatasize - rawdata_offset - already_read); + memset( + tif->tif_rawdata + rawdata_offset + already_read, 0, + (size_t)(tif->tif_rawdatasize - rawdata_offset - already_read)); if (is_strip) { TIFFErrorExtR(tif, module, "Read error at scanline %" PRIu32 "; got %" TIFF_SSIZE_FORMAT " bytes, " "expected %" TIFF_SSIZE_FORMAT, - tif->tif_row, already_read, size); + tif->tif_dir.td_row, already_read, size); } else { @@ -147,8 +151,8 @@ static int TIFFReadAndRealloc(TIFF *tif, tmsize_t size, tmsize_t rawdata_offset, ", tile %" PRIu32 "; " "got %" TIFF_SSIZE_FORMAT " bytes, expected %" TIFF_SSIZE_FORMAT "", - tif->tif_row, tif->tif_col, strip_or_tile, - already_read, size); + tif->tif_dir.td_row, tif->tif_dir.td_col, + strip_or_tile, already_read, size); } return 0; } @@ -160,7 +164,7 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, int restart) { static const char module[] = "TIFFFillStripPartial"; - register TIFFDirectory *td = &tif->tif_dir; + TIFFDirectory *td = &tif->tif_dir; tmsize_t unused_data; uint64_t read_offset; tmsize_t to_read; @@ -185,7 +189,7 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, { assert(restart); - tif->tif_curstrip = NOSTRIP; + tif->tif_dir.td_curstrip = NOSTRIP; if ((tif->tif_flags & TIFF_MYBUFFER) == 0) { TIFFErrorExtR(tif, module, @@ -214,20 +218,30 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, if (unused_data > 0) { assert((tif->tif_flags & TIFF_BUFFERMMAP) == 0); - memmove(tif->tif_rawdata, tif->tif_rawcp, unused_data); + memmove(tif->tif_rawdata, tif->tif_rawcp, (size_t)unused_data); } /* ** Seek to the point in the file where more data should be read. */ - read_offset = TIFFGetStrileOffset(tif, strip) + tif->tif_rawdataoff + - tif->tif_rawdataloaded; + read_offset = TIFFGetStrileOffset(tif, (uint32_t)strip); + if (read_offset > UINT64_MAX - (uint64_t)tif->tif_rawdataoff || + read_offset + (uint64_t)tif->tif_rawdataoff > + UINT64_MAX - (uint64_t)tif->tif_rawdataloaded) + { + TIFFErrorExtR(tif, module, + "Seek error at scanline %" PRIu32 ", strip %d", + tif->tif_dir.td_row, strip); + return 0; + } + read_offset += + (uint64_t)tif->tif_rawdataoff + (uint64_t)tif->tif_rawdataloaded; if (!SeekOK(tif, read_offset)) { TIFFErrorExtR(tif, module, "Seek error at scanline %" PRIu32 ", strip %d", - tif->tif_row, strip); + tif->tif_dir.td_row, strip); return 0; } @@ -238,11 +252,13 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, to_read = read_ahead_mod - unused_data; else to_read = tif->tif_rawdatasize - unused_data; - if ((uint64_t)to_read > TIFFGetStrileByteCount(tif, strip) - - tif->tif_rawdataoff - tif->tif_rawdataloaded) + if ((uint64_t)to_read > TIFFGetStrileByteCount(tif, (uint32_t)strip) - + (uint64_t)tif->tif_rawdataoff - + (uint64_t)tif->tif_rawdataloaded) { - to_read = (tmsize_t)TIFFGetStrileByteCount(tif, strip) - - tif->tif_rawdataoff - tif->tif_rawdataloaded; + to_read = (tmsize_t)(TIFFGetStrileByteCount(tif, (uint32_t)strip) - + (uint64_t)tif->tif_rawdataoff - + (uint64_t)tif->tif_rawdataloaded); } assert((tif->tif_flags & TIFF_BUFFERMMAP) == 0); @@ -281,16 +297,17 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, /* For JPEG, if there are multiple scans (can generally be known */ /* with the read_ahead used), we need to read the whole strip */ if (tif->tif_dir.td_compression == COMPRESSION_JPEG && - (uint64_t)tif->tif_rawcc < TIFFGetStrileByteCount(tif, strip)) + (uint64_t)tif->tif_rawcc < + TIFFGetStrileByteCount(tif, (uint32_t)strip)) { if (TIFFJPEGIsFullStripRequired(tif)) { - return TIFFFillStrip(tif, strip); + return TIFFFillStrip(tif, (uint32_t)strip); } } #endif - return TIFFStartStrip(tif, strip); + return TIFFStartStrip(tif, (uint32_t)strip); } else { @@ -308,7 +325,7 @@ static int TIFFFillStripPartial(TIFF *tif, int strip, tmsize_t read_ahead, */ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) { - register TIFFDirectory *td = &tif->tif_dir; + TIFFDirectory *td = &tif->tif_dir; uint32_t strip; int whole_strip; tmsize_t read_ahead = 0; @@ -323,8 +340,16 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) td->td_imagelength); return (0); } + if (td->td_rowsperstrip == 0) + { + TIFFErrorExtR(tif, tif->tif_name, + "Cannot compute strip: RowsPerStrip is zero"); + return (0); + } if (td->td_planarconfig == PLANARCONFIG_SEPARATE) { + uint64_t sample_offset; + uint64_t strip64; if (sample >= td->td_samplesperpixel) { TIFFErrorExtR(tif, tif->tif_name, @@ -332,16 +357,26 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) sample, td->td_samplesperpixel); return (0); } - strip = (uint32_t)sample * td->td_stripsperimage + - row / td->td_rowsperstrip; + sample_offset = + _TIFFMultiply64(tif, sample, td->td_stripsperimage, "TIFFSeek"); + if (sample_offset == 0 && sample != 0 && td->td_stripsperimage != 0) + return (0); + strip64 = _TIFFAdd64(tif, sample_offset, row / td->td_rowsperstrip, + "TIFFSeek"); + if (strip64 == 0 && + (sample_offset != 0 || (row / td->td_rowsperstrip) != 0)) + return (0); + strip = _TIFFCastUInt64ToUInt32(tif, strip64, "TIFFSeek"); + if (strip == 0 && strip64 != 0) + return (0); } else strip = row / td->td_rowsperstrip; - /* - * Do we want to treat this strip as one whole chunk or - * read it a few lines at a time? - */ + /* + * Do we want to treat this strip as one whole chunk or + * read it a few lines at a time? + */ #if defined(CHUNKY_STRIP_READ_SUPPORT) whole_strip = TIFFGetStrileByteCount(tif, strip) < 10 || isMapped(tif); if (td->td_compression == COMPRESSION_LERC || @@ -359,14 +394,14 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) /* is some constant value, for example for JPEG tables */ /* coverity[dead_error_line:SUPPRESS] */ - if (tif->tif_scanlinesize < TIFF_TMSIZE_T_MAX / 16 && - tif->tif_scanlinesize * 16 < TIFF_TMSIZE_T_MAX - 5000) + if (tif->tif_dir.td_scanlinesize < TIFF_TMSIZE_T_MAX / 16 && + tif->tif_dir.td_scanlinesize * 16 < TIFF_TMSIZE_T_MAX - 5000) { - read_ahead = tif->tif_scanlinesize * 16 + 5000; + read_ahead = tif->tif_dir.td_scanlinesize * 16 + 5000; } else { - read_ahead = tif->tif_scanlinesize; + read_ahead = tif->tif_dir.td_scanlinesize; } } #else @@ -377,7 +412,7 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) * If we haven't loaded this strip, do so now, possibly * only reading the first part. */ - if (strip != tif->tif_curstrip) + if (strip != tif->tif_dir.td_curstrip) { /* different strip, refill */ if (whole_strip) @@ -412,7 +447,7 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) } #endif - if (row < tif->tif_row) + if (row < tif->tif_dir.td_row) { /* * Moving backwards within the same strip: backup @@ -425,7 +460,7 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) if (tif->tif_rawdataoff != 0) { - if (!TIFFFillStripPartial(tif, strip, read_ahead, 1)) + if (!TIFFFillStripPartial(tif, (int)strip, read_ahead, 1)) return 0; } else @@ -435,7 +470,7 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) } } - if (row != tif->tif_row) + if (row != tif->tif_dir.td_row) { /* * Seek forward to the desired row. @@ -443,9 +478,9 @@ static int TIFFSeek(TIFF *tif, uint32_t row, uint16_t sample) /* TODO: Will this really work with partial buffers? */ - if (!(*tif->tif_seek)(tif, row - tif->tif_row)) + if (!(*tif->tif_seek)(tif, row - tif->tif_dir.td_row)) return (0); - tif->tif_row = row; + tif->tif_dir.td_row = row; } return (1); @@ -462,20 +497,21 @@ int TIFFReadScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) /* * Decompress desired row into user buffer. */ - e = (*tif->tif_decoderow)(tif, (uint8_t *)buf, tif->tif_scanlinesize, - sample); + e = (*tif->tif_decoderow)(tif, (uint8_t *)buf, + tif->tif_dir.td_scanlinesize, sample); /* we are now poised at the beginning of the next row */ - tif->tif_row = row + 1; + tif->tif_dir.td_row = row + 1; if (e) - (*tif->tif_postdecode)(tif, (uint8_t *)buf, tif->tif_scanlinesize); + (*tif->tif_postdecode)(tif, (uint8_t *)buf, + tif->tif_dir.td_scanlinesize); } else { /* See TIFFReadEncodedStrip comment regarding TIFFTAG_FAXFILLFUNC. */ if (buf) - memset(buf, 0, (size_t)tif->tif_scanlinesize); + memset(buf, 0, (size_t)tif->tif_dir.td_scanlinesize); } return (e > 0 ? 1 : -1); } @@ -553,9 +589,9 @@ tmsize_t TIFFReadEncodedStrip(TIFF *tif, uint32_t strip, void *buf, if (!isFillOrder(tif, td->td_fillorder) && (tif->tif_flags & TIFF_NOBITREV) == 0) - TIFFReverseBits(buf, stripsize); + TIFFReverseBits((uint8_t *)buf, stripsize); - (*tif->tif_postdecode)(tif, buf, stripsize); + (*tif->tif_postdecode)(tif, (uint8_t *)buf, stripsize); return (stripsize); } @@ -569,9 +605,9 @@ tmsize_t TIFFReadEncodedStrip(TIFF *tif, uint32_t strip, void *buf, memset(buf, 0, (size_t)stripsize); return ((tmsize_t)(-1)); } - if ((*tif->tif_decodestrip)(tif, buf, stripsize, plane) <= 0) + if ((*tif->tif_decodestrip)(tif, (uint8_t *)buf, stripsize, plane) <= 0) return ((tmsize_t)(-1)); - (*tif->tif_postdecode)(tif, buf, stripsize); + (*tif->tif_postdecode)(tif, (uint8_t *)buf, stripsize); return (stripsize); } @@ -603,17 +639,39 @@ tmsize_t _TIFFReadEncodedStripAndAllocBuffer(TIFF *tif, uint32_t strip, if (!TIFFFillStrip(tif, strip)) return ((tmsize_t)(-1)); - *buf = _TIFFmallocExt(tif, bufsizetoalloc); + /* Sanity checks to avoid excessive memory allocation */ + /* Max compression ratio experimentally determined. Might be fragile... + * Only apply this heuristics to situations where the memory allocation + * would be big, to avoid breaking nominal use cases. + */ + if (bufsizetoalloc > 100 * 1024 * 1024) + { + const uint64_t maxCompressionRatio = TIFFGetMaxCompressionRatio(tif); + if (maxCompressionRatio > 0 && + (uint64_t)tif->tif_rawdatasize < + (uint64_t)this_stripsize / maxCompressionRatio) + { + TIFFErrorExtR(tif, TIFFFileName(tif), + "Likely invalid strip byte count for strip %u. " + "Uncompressed strip size is %" PRIu64 ", " + "compressed one is %" PRIu64, + strip, (uint64_t)this_stripsize, + (uint64_t)tif->tif_rawdatasize); + return ((tmsize_t)(-1)); + } + } + + *buf = _TIFFcallocExt(tif, 1, bufsizetoalloc); if (*buf == NULL) { TIFFErrorExtR(tif, TIFFFileName(tif), "No space for strip buffer"); return ((tmsize_t)(-1)); } - _TIFFmemset(*buf, 0, bufsizetoalloc); - if ((*tif->tif_decodestrip)(tif, *buf, this_stripsize, plane) <= 0) + if ((*tif->tif_decodestrip)(tif, (uint8_t *)*buf, this_stripsize, plane) <= + 0) return ((tmsize_t)(-1)); - (*tif->tif_postdecode)(tif, *buf, this_stripsize); + (*tif->tif_postdecode)(tif, (uint8_t *)*buf, this_stripsize); return (this_stripsize); } @@ -629,7 +687,7 @@ static tmsize_t TIFFReadRawStrip1(TIFF *tif, uint32_t strip, void *buf, { TIFFErrorExtR(tif, module, "Seek error at scanline %" PRIu32 ", strip %" PRIu32, - tif->tif_row, strip); + tif->tif_dir.td_row, strip); return ((tmsize_t)(-1)); } cc = TIFFReadFile(tif, buf, size); @@ -639,7 +697,7 @@ static tmsize_t TIFFReadRawStrip1(TIFF *tif, uint32_t strip, void *buf, "Read error at scanline %" PRIu32 "; got %" TIFF_SSIZE_FORMAT " bytes, expected %" TIFF_SSIZE_FORMAT, - tif->tif_row, cc, size); + tif->tif_dir.td_row, cc, size); return ((tmsize_t)(-1)); } } @@ -670,7 +728,7 @@ static tmsize_t TIFFReadRawStrip1(TIFF *tif, uint32_t strip, void *buf, "Read error at scanline %" PRIu32 ", strip %" PRIu32 "; got %" TIFF_SSIZE_FORMAT " bytes, expected %" TIFF_SSIZE_FORMAT, - tif->tif_row, strip, n, size); + tif->tif_dir.td_row, strip, n, size); return ((tmsize_t)(-1)); } _TIFFmemcpy(buf, tif->tif_base + ma, size); @@ -691,14 +749,14 @@ static tmsize_t TIFFReadRawStripOrTile2(TIFF *tif, uint32_t strip_or_tile, { TIFFErrorExtR(tif, module, "Seek error at scanline %" PRIu32 ", strip %" PRIu32, - tif->tif_row, strip_or_tile); + tif->tif_dir.td_row, strip_or_tile); } else { - TIFFErrorExtR(tif, module, - "Seek error at row %" PRIu32 ", col %" PRIu32 - ", tile %" PRIu32, - tif->tif_row, tif->tif_col, strip_or_tile); + TIFFErrorExtR( + tif, module, + "Seek error at row %" PRIu32 ", col %" PRIu32 ", tile %" PRIu32, + tif->tif_dir.td_row, tif->tif_dir.td_col, strip_or_tile); } return ((tmsize_t)(-1)); } @@ -774,22 +832,50 @@ int TIFFFillStrip(TIFF *tif, uint32_t strip) } /* To avoid excessive memory allocations: */ - /* Byte count should normally not be larger than a number of */ - /* times the uncompressed size plus some margin */ - if (bytecount > 1024 * 1024) + const tmsize_t stripsize = TIFFStripSize(tif); + if (stripsize > 0) { - /* 10 and 4096 are just values that could be adjusted. */ - /* Hopefully they are safe enough for all codecs */ - tmsize_t stripsize = TIFFStripSize(tif); - if (stripsize != 0 && (bytecount - 4096) / 10 > (uint64_t)stripsize) + if (bytecount > 1024 * 1024 && + (bytecount - 4096) / 10 > (uint64_t)stripsize) { + /* Byte count should normally not be larger than a number of */ + /* times the uncompressed size plus some margin */ + /* 10 and 4096 are just values that could be adjusted. */ + /* Hopefully they are safe enough for all codecs */ + /* What happens next will depend on whether only the bytecount + */ + /* was corrupted to a large value but the strip/tile data is */ + /* fine. In that situation most codecs should work fine and */ + /* only used part of the tile/strip data. If the strip/tile */ + /* data is corrupted too, then codecs will later error out. */ uint64_t newbytecount = (uint64_t)stripsize * 10 + 4096; - TIFFErrorExtR(tif, module, - "Too large strip byte count %" PRIu64 - ", strip %" PRIu32 ". Limiting to %" PRIu64, - bytecount, strip, newbytecount); + TIFFWarningExtR(tif, module, + "Too large strip byte count %" PRIu64 + ", strip %" PRIu32 ". Limiting to %" PRIu64, + bytecount, strip, newbytecount); bytecount = newbytecount; } + else if (stripsize > 100 * 1024 * 1024) + { + /* Max compression ratio experimentally determined. Might be + * fragile... Only apply this heuristics to situations where the + * memory allocation would be big, to avoid breaking nominal use + * cases. + */ + const uint64_t maxCompressionRatio = + TIFFGetMaxCompressionRatio(tif); + if (maxCompressionRatio > 0 && + bytecount < (uint64_t)stripsize / maxCompressionRatio) + { + TIFFErrorExtR( + tif, module, + "Likely invalid strip byte count for strip %u. " + "Uncompressed strip size is %" PRIu64 ", " + "compressed one is %" PRIu64, + strip, (uint64_t)stripsize, bytecount); + return 0; + } + } } if (isMapped(tif)) @@ -818,10 +904,10 @@ int TIFFFillStrip(TIFF *tif, uint32_t strip) "Read error on strip %" PRIu32 "; " "got %" PRIu64 " bytes, expected %" PRIu64, strip, - NoSanitizeSubUInt64(tif->tif_size, + NoSanitizeSubUInt64((uint64_t)tif->tif_size, TIFFGetStrileOffset(tif, strip)), bytecount); - tif->tif_curstrip = NOSTRIP; + tif->tif_dir.td_curstrip = NOSTRIP; return (0); } } @@ -878,7 +964,7 @@ int TIFFFillStrip(TIFF *tif, uint32_t strip) } if (bytecountm > tif->tif_rawdatasize) { - tif->tif_curstrip = NOSTRIP; + tif->tif_dir.td_curstrip = NOSTRIP; if ((tif->tif_flags & TIFF_MYBUFFER) == 0) { TIFFErrorExtR( @@ -889,7 +975,7 @@ int TIFFFillStrip(TIFF *tif, uint32_t strip) } if (tif->tif_flags & TIFF_BUFFERMMAP) { - tif->tif_curstrip = NOSTRIP; + tif->tif_dir.td_curstrip = NOSTRIP; tif->tif_rawdata = NULL; tif->tif_rawdatasize = 0; tif->tif_flags &= ~TIFF_BUFFERMMAP; @@ -954,7 +1040,7 @@ tmsize_t TIFFReadEncodedTile(TIFF *tif, uint32_t tile, void *buf, tmsize_t size) { static const char module[] = "TIFFReadEncodedTile"; TIFFDirectory *td = &tif->tif_dir; - tmsize_t tilesize = tif->tif_tilesize; + tmsize_t tilesize = tif->tif_dir.td_tilesize; if (!TIFFCheckRead(tif, 1)) return ((tmsize_t)(-1)); @@ -976,9 +1062,9 @@ tmsize_t TIFFReadEncodedTile(TIFF *tif, uint32_t tile, void *buf, tmsize_t size) if (!isFillOrder(tif, td->td_fillorder) && (tif->tif_flags & TIFF_NOBITREV) == 0) - TIFFReverseBits(buf, tilesize); + TIFFReverseBits((uint8_t *)buf, tilesize); - (*tif->tif_postdecode)(tif, buf, tilesize); + (*tif->tif_postdecode)(tif, (uint8_t *)buf, tilesize); return (tilesize); } @@ -1032,7 +1118,7 @@ tmsize_t _TIFFReadEncodedTileAndAllocBuffer(TIFF *tif, uint32_t tile, { static const char module[] = "_TIFFReadEncodedTileAndAllocBuffer"; TIFFDirectory *td = &tif->tif_dir; - tmsize_t tilesize = tif->tif_tilesize; + tmsize_t tilesize = tif->tif_dir.td_tilesize; if (*buf != NULL) { @@ -1072,25 +1158,22 @@ tmsize_t _TIFFReadEncodedTileAndAllocBuffer(TIFF *tif, uint32_t tile, * Only apply this heuristics to situations where the memory allocation * would be big, to avoid breaking nominal use cases. */ - const int maxCompressionRatio = - td->td_compression == COMPRESSION_ZSTD ? 33000 - : td->td_compression == COMPRESSION_JXL - ? - /* Evaluated on a 8000x8000 tile */ - 25000 * (td->td_planarconfig == PLANARCONFIG_CONTIG - ? td->td_samplesperpixel - : 1) - : td->td_compression == COMPRESSION_LZMA ? 7000 : 1000; - if (bufsizetoalloc > 100 * 1000 * 1000 && - tif->tif_rawdatasize < tilesize / maxCompressionRatio) + if (bufsizetoalloc > 100 * 1024 * 1024) { - TIFFErrorExtR(tif, TIFFFileName(tif), - "Likely invalid tile byte count for tile %u. " - "Uncompressed tile size is %" PRIu64 ", " - "compressed one is %" PRIu64, - tile, (uint64_t)tilesize, - (uint64_t)tif->tif_rawdatasize); - return ((tmsize_t)(-1)); + const uint64_t maxCompressionRatio = + TIFFGetMaxCompressionRatio(tif); + if (maxCompressionRatio > 0 && + (uint64_t)tif->tif_rawdatasize < + (uint64_t)tilesize / maxCompressionRatio) + { + TIFFErrorExtR(tif, TIFFFileName(tif), + "Likely invalid tile byte count for tile %u. " + "Uncompressed tile size is %" PRIu64 ", " + "compressed one is %" PRIu64, + tile, (uint64_t)tilesize, + (uint64_t)tif->tif_rawdatasize); + return ((tmsize_t)(-1)); + } } } @@ -1129,7 +1212,7 @@ static tmsize_t TIFFReadRawTile1(TIFF *tif, uint32_t tile, void *buf, TIFFErrorExtR(tif, module, "Seek error at row %" PRIu32 ", col %" PRIu32 ", tile %" PRIu32, - tif->tif_row, tif->tif_col, tile); + tif->tif_dir.td_row, tif->tif_dir.td_col, tile); return ((tmsize_t)(-1)); } cc = TIFFReadFile(tif, buf, size); @@ -1139,7 +1222,7 @@ static tmsize_t TIFFReadRawTile1(TIFF *tif, uint32_t tile, void *buf, "Read error at row %" PRIu32 ", col %" PRIu32 "; got %" TIFF_SSIZE_FORMAT " bytes, expected %" TIFF_SSIZE_FORMAT, - tif->tif_row, tif->tif_col, cc, size); + tif->tif_dir.td_row, tif->tif_dir.td_col, cc, size); return ((tmsize_t)(-1)); } } @@ -1162,7 +1245,8 @@ static tmsize_t TIFFReadRawTile1(TIFF *tif, uint32_t tile, void *buf, "Read error at row %" PRIu32 ", col %" PRIu32 ", tile %" PRIu32 "; got %" TIFF_SSIZE_FORMAT " bytes, expected %" TIFF_SSIZE_FORMAT, - tif->tif_row, tif->tif_col, tile, n, size); + tif->tif_dir.td_row, tif->tif_dir.td_col, tile, n, + size); return ((tmsize_t)(-1)); } _TIFFmemcpy(buf, tif->tif_base + ma, size); @@ -1229,22 +1313,49 @@ int TIFFFillTile(TIFF *tif, uint32_t tile) } /* To avoid excessive memory allocations: */ - /* Byte count should normally not be larger than a number of */ - /* times the uncompressed size plus some margin */ - if (bytecount > 1024 * 1024) + const tmsize_t tilesize = TIFFTileSize(tif); + if (tilesize > 0) { - /* 10 and 4096 are just values that could be adjusted. */ - /* Hopefully they are safe enough for all codecs */ - tmsize_t stripsize = TIFFTileSize(tif); - if (stripsize != 0 && (bytecount - 4096) / 10 > (uint64_t)stripsize) + if (bytecount > 1024 * 1024 && + (bytecount - 4096) / 10 > (uint64_t)tilesize) { - uint64_t newbytecount = (uint64_t)stripsize * 10 + 4096; - TIFFErrorExtR(tif, module, - "Too large tile byte count %" PRIu64 - ", tile %" PRIu32 ". Limiting to %" PRIu64, - bytecount, tile, newbytecount); + /* Byte count should normally not be larger than a number of */ + /* times the uncompressed size plus some margin */ + /* 10 and 4096 are just values that could be adjusted. */ + /* Hopefully they are safe enough for all codecs */ + /* What happens next will depend on whether only the bytecount + */ + /* was corrupted to a large value but the strip/tile data is */ + /* fine. In that situation most codecs should work fine and */ + /* only used part of the tile/strip data. If the strip/tile */ + /* data is corrupted too, then codecs will later error out. */ + uint64_t newbytecount = (uint64_t)tilesize * 10 + 4096; + TIFFWarningExtR(tif, module, + "Too large tile byte count %" PRIu64 + ", tile %" PRIu32 ". Limiting to %" PRIu64, + bytecount, tile, newbytecount); bytecount = newbytecount; } + else if (tilesize > 100 * 1024 * 1024) + { + /* Max compression ratio experimentally determined. Might be + * fragile... Only apply this heuristics to situations where the + * memory allocation would be big, to avoid breaking nominal use + * cases. + */ + const uint64_t maxCompressionRatio = + TIFFGetMaxCompressionRatio(tif); + if (maxCompressionRatio > 0 && + bytecount < (uint64_t)tilesize / maxCompressionRatio) + { + TIFFErrorExtR(tif, module, + "Likely invalid tile byte count for tile %u. " + "Uncompressed tile size is %" PRIu64 ", " + "compressed one is %" PRIu64, + tile, (uint64_t)tilesize, bytecount); + return 0; + } + } } if (isMapped(tif)) @@ -1262,7 +1373,7 @@ int TIFFFillTile(TIFF *tif, uint32_t tile) TIFFGetStrileOffset(tif, tile) > (uint64_t)tif->tif_size - bytecount) { - tif->tif_curtile = NOTILE; + tif->tif_dir.td_curtile = NOTILE; return (0); } } @@ -1312,7 +1423,7 @@ int TIFFFillTile(TIFF *tif, uint32_t tile) } if (bytecountm > tif->tif_rawdatasize) { - tif->tif_curtile = NOTILE; + tif->tif_dir.td_curtile = NOTILE; if ((tif->tif_flags & TIFF_MYBUFFER) == 0) { TIFFErrorExtR(tif, module, @@ -1323,7 +1434,7 @@ int TIFFFillTile(TIFF *tif, uint32_t tile) } if (tif->tif_flags & TIFF_BUFFERMMAP) { - tif->tif_curtile = NOTILE; + tif->tif_dir.td_curtile = NOTILE; tif->tif_rawdata = NULL; tif->tif_rawdatasize = 0; tif->tif_flags &= ~TIFF_BUFFERMMAP; @@ -1410,7 +1521,7 @@ int TIFFReadBufferSetup(TIFF *tif, void *bp, tmsize_t size) { TIFFErrorExtR(tif, module, "No space for data buffer at scanline %" PRIu32, - tif->tif_row); + tif->tif_dir.td_row); tif->tif_rawdatasize = 0; return (0); } @@ -1431,8 +1542,13 @@ static int TIFFStartStrip(TIFF *tif, uint32_t strip) return (0); tif->tif_flags |= TIFF_CODERSETUP; } - tif->tif_curstrip = strip; - tif->tif_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; + if (td->td_stripsperimage == 0) + { + TIFFErrorExtR(tif, "TIFFStartStrip", "Zero strips per image"); + return 0; + } + tif->tif_dir.td_curstrip = strip; + tif->tif_dir.td_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; tif->tif_flags &= ~TIFF_BUF4WRITE; if (tif->tif_flags & TIFF_NOREADRAW) @@ -1456,7 +1572,7 @@ static int TIFFStartStrip(TIFF *tif, uint32_t strip) */ /* tif_curstrip, we'd call tif_decoderow() on a possibly invalid */ /* codec state. */ - tif->tif_curstrip = NOSTRIP; + tif->tif_dir.td_curstrip = NOSTRIP; return 0; } return 1; @@ -1478,7 +1594,7 @@ static int TIFFStartTile(TIFF *tif, uint32_t tile) return (0); tif->tif_flags |= TIFF_CODERSETUP; } - tif->tif_curtile = tile; + tif->tif_dir.td_curtile = tile; if (td->td_tilewidth == 0) { TIFFErrorExtR(tif, module, "Zero tilewidth"); @@ -1490,14 +1606,14 @@ static int TIFFStartTile(TIFF *tif, uint32_t tile) TIFFErrorExtR(tif, module, "Zero tiles"); return 0; } - tif->tif_row = (tile % howmany32) * td->td_tilelength; + tif->tif_dir.td_row = (tile % howmany32) * td->td_tilelength; howmany32 = TIFFhowmany_32(td->td_imagelength, td->td_tilelength); if (howmany32 == 0) { TIFFErrorExtR(tif, module, "Zero tiles"); return 0; } - tif->tif_col = (tile % howmany32) * td->td_tilewidth; + tif->tif_dir.td_col = (tile % howmany32) * td->td_tilewidth; tif->tif_flags &= ~TIFF_BUF4WRITE; if (tif->tif_flags & TIFF_NOREADRAW) { @@ -1568,14 +1684,14 @@ int TIFFReadFromUserBuffer(TIFF *tif, uint32_t strile, void *inbuf, tif->tif_flags &= ~TIFF_MYBUFFER; tif->tif_flags |= TIFF_BUFFERMMAP; tif->tif_rawdatasize = insize; - tif->tif_rawdata = inbuf; + tif->tif_rawdata = (uint8_t *)inbuf; tif->tif_rawdataoff = 0; tif->tif_rawdataloaded = insize; if (!isFillOrder(tif, td->td_fillorder) && (tif->tif_flags & TIFF_NOBITREV) == 0) { - TIFFReverseBits(inbuf, insize); + TIFFReverseBits((uint8_t *)inbuf, insize); } if (TIFFIsTiled(tif)) @@ -1632,13 +1748,13 @@ int TIFFReadFromUserBuffer(TIFF *tif, uint32_t strile, void *inbuf, if (!isFillOrder(tif, td->td_fillorder) && (tif->tif_flags & TIFF_NOBITREV) == 0) { - TIFFReverseBits(inbuf, insize); + TIFFReverseBits((uint8_t *)inbuf, insize); } tif->tif_flags = (old_tif_flags & (TIFF_MYBUFFER | TIFF_BUFFERMMAP)) | (tif->tif_flags & ~(TIFF_MYBUFFER | TIFF_BUFFERMMAP)); tif->tif_rawdatasize = old_rawdatasize; - tif->tif_rawdata = old_rawdata; + tif->tif_rawdata = (uint8_t *)old_rawdata; tif->tif_rawdataoff = 0; tif->tif_rawdataloaded = 0; diff --git a/contrib/libs/libtiff/tif_strip.c b/contrib/libs/libtiff/tif_strip.c index f13c2b33fba..740de68a21e 100644 --- a/contrib/libs/libtiff/tif_strip.c +++ b/contrib/libs/libtiff/tif_strip.c @@ -47,6 +47,8 @@ uint32_t TIFFComputeStrip(TIFF *tif, uint32_t row, uint16_t sample) strip = row / td->td_rowsperstrip; if (td->td_planarconfig == PLANARCONFIG_SEPARATE) { + uint64_t sample_offset; + uint64_t strip64; if (sample >= td->td_samplesperpixel) { TIFFErrorExtR(tif, module, "%lu: Sample out of range, max %lu", @@ -54,7 +56,16 @@ uint32_t TIFFComputeStrip(TIFF *tif, uint32_t row, uint16_t sample) (unsigned long)td->td_samplesperpixel); return (0); } - strip += (uint32_t)sample * td->td_stripsperimage; + sample_offset = _TIFFMultiply64(tif, sample, td->td_stripsperimage, + "TIFFComputeStrip"); + if (sample_offset == 0 && sample != 0 && td->td_stripsperimage != 0) + return (0); + strip64 = _TIFFAdd64(tif, sample_offset, strip, "TIFFComputeStrip"); + if (strip64 == 0 && (sample_offset != 0 || strip != 0)) + return (0); + strip = _TIFFCastUInt64ToUInt32(tif, strip64, "TIFFComputeStrip"); + if (strip == 0 && strip64 != 0) + return (0); } return (strip); } @@ -83,14 +94,24 @@ uint32_t TIFFNumberOfStrips(TIFF *tif) } /* - * Compute the # bytes in a variable height, row-aligned strip. + * Compute the # bytes in a variable height, row-aligned strip if isStrip is + * TRUE, or in a tile if isStrip is FALSE */ -uint64_t TIFFVStripSize64(TIFF *tif, uint32_t nrows) +uint64_t _TIFFStrileSize64(TIFF *tif, uint32_t nrows, int isStrip) { - static const char module[] = "TIFFVStripSize64"; + static const char module[] = "_TIFFStrileSize64"; TIFFDirectory *td = &tif->tif_dir; - if (nrows == (uint32_t)(-1)) - nrows = td->td_imagelength; + if (isStrip) + { + if (nrows == (uint32_t)(-1)) + nrows = td->td_imagelength; + } + else + { + if (td->td_tilelength == 0 || td->td_tilewidth == 0 || + td->td_tiledepth == 0) + return (0); + } if ((td->td_planarconfig == PLANARCONFIG_CONTIG) && (td->td_photometric == PHOTOMETRIC_YCBCR) && (!isUpSampled(tif))) { @@ -125,9 +146,10 @@ uint64_t TIFFVStripSize64(TIFF *tif, uint32_t nrows) ycbcrsubsampling[0], ycbcrsubsampling[1]); return 0; } - samplingblock_samples = ycbcrsubsampling[0] * ycbcrsubsampling[1] + 2; - samplingblocks_hor = - TIFFhowmany_32(td->td_imagewidth, ycbcrsubsampling[0]); + samplingblock_samples = + (uint16_t)(ycbcrsubsampling[0] * ycbcrsubsampling[1] + 2); + const uint32_t width = isStrip ? td->td_imagewidth : td->td_tilewidth; + samplingblocks_hor = TIFFhowmany_32(width, ycbcrsubsampling[0]); samplingblocks_ver = TIFFhowmany_32(nrows, ycbcrsubsampling[1]); samplingrow_samples = _TIFFMultiply64(tif, samplingblocks_hor, samplingblock_samples, module); @@ -137,8 +159,20 @@ uint64_t TIFFVStripSize64(TIFF *tif, uint32_t nrows) _TIFFMultiply64(tif, samplingrow_size, samplingblocks_ver, module)); } else - return (_TIFFMultiply64(tif, nrows, TIFFScanlineSize64(tif), module)); + return (_TIFFMultiply64(tif, nrows, + isStrip ? TIFFScanlineSize64(tif) + : TIFFTileRowSize64(tif), + module)); } + +/* + * Compute the # bytes in a variable height, row-aligned strip. + */ +uint64_t TIFFVStripSize64(TIFF *tif, uint32_t nrows) +{ + return _TIFFStrileSize64(tif, nrows, /* isStrip = */ TRUE); +} + tmsize_t TIFFVStripSize(TIFF *tif, uint32_t nrows) { static const char module[] = "TIFFVStripSize"; @@ -286,7 +320,7 @@ uint64_t TIFFScanlineSize64(TIFF *tif) return 0; } samplingblock_samples = - ycbcrsubsampling[0] * ycbcrsubsampling[1] + 2; + (uint16_t)(ycbcrsubsampling[0] * ycbcrsubsampling[1] + 2); samplingblocks_hor = TIFFhowmany_32(td->td_imagewidth, ycbcrsubsampling[0]); samplingrow_samples = _TIFFMultiply64( diff --git a/contrib/libs/libtiff/tif_swab.c b/contrib/libs/libtiff/tif_swab.c index 827b025ce7a..50fe635fbc0 100644 --- a/contrib/libs/libtiff/tif_swab.c +++ b/contrib/libs/libtiff/tif_swab.c @@ -32,7 +32,7 @@ #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabShort) void TIFFSwabShort(uint16_t *wp) { - register unsigned char *cp = (unsigned char *)wp; + unsigned char *cp = (unsigned char *)wp; unsigned char t; assert(sizeof(uint16_t) == 2); t = cp[1]; @@ -44,7 +44,7 @@ void TIFFSwabShort(uint16_t *wp) #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabLong) void TIFFSwabLong(uint32_t *lp) { - register unsigned char *cp = (unsigned char *)lp; + unsigned char *cp = (unsigned char *)lp; unsigned char t; assert(sizeof(uint32_t) == 4); t = cp[3]; @@ -59,7 +59,7 @@ void TIFFSwabLong(uint32_t *lp) #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabLong8) void TIFFSwabLong8(uint64_t *lp) { - register unsigned char *cp = (unsigned char *)lp; + unsigned char *cp = (unsigned char *)lp; unsigned char t; assert(sizeof(uint64_t) == 8); t = cp[7]; @@ -78,10 +78,10 @@ void TIFFSwabLong8(uint64_t *lp) #endif #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfShort) -void TIFFSwabArrayOfShort(register uint16_t *wp, tmsize_t n) +void TIFFSwabArrayOfShort(uint16_t *wp, tmsize_t n) { - register unsigned char *cp; - register unsigned char t; + unsigned char *cp; + unsigned char t; assert(sizeof(uint16_t) == 2); /* XXX unroll loop some */ while (n-- > 0) @@ -96,7 +96,7 @@ void TIFFSwabArrayOfShort(register uint16_t *wp, tmsize_t n) #endif #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfTriples) -void TIFFSwabArrayOfTriples(register uint8_t *tp, tmsize_t n) +void TIFFSwabArrayOfTriples(uint8_t *tp, tmsize_t n) { unsigned char *cp; unsigned char t; @@ -114,10 +114,10 @@ void TIFFSwabArrayOfTriples(register uint8_t *tp, tmsize_t n) #endif #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfLong) -void TIFFSwabArrayOfLong(register uint32_t *lp, tmsize_t n) +void TIFFSwabArrayOfLong(uint32_t *lp, tmsize_t n) { - register unsigned char *cp; - register unsigned char t; + unsigned char *cp; + unsigned char t; assert(sizeof(uint32_t) == 4); /* XXX unroll loop some */ while (n-- > 0) @@ -135,10 +135,10 @@ void TIFFSwabArrayOfLong(register uint32_t *lp, tmsize_t n) #endif #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfLong8) -void TIFFSwabArrayOfLong8(register uint64_t *lp, tmsize_t n) +void TIFFSwabArrayOfLong8(uint64_t *lp, tmsize_t n) { - register unsigned char *cp; - register unsigned char t; + unsigned char *cp; + unsigned char t; assert(sizeof(uint64_t) == 8); /* XXX unroll loop some */ while (n-- > 0) @@ -164,7 +164,7 @@ void TIFFSwabArrayOfLong8(register uint64_t *lp, tmsize_t n) #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabFloat) void TIFFSwabFloat(float *fp) { - register unsigned char *cp = (unsigned char *)fp; + unsigned char *cp = (unsigned char *)fp; unsigned char t; assert(sizeof(float) == 4); t = cp[3]; @@ -177,10 +177,10 @@ void TIFFSwabFloat(float *fp) #endif #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfFloat) -void TIFFSwabArrayOfFloat(register float *fp, tmsize_t n) +void TIFFSwabArrayOfFloat(float *fp, tmsize_t n) { - register unsigned char *cp; - register unsigned char t; + unsigned char *cp; + unsigned char t; assert(sizeof(float) == 4); /* XXX unroll loop some */ while (n-- > 0) @@ -200,7 +200,7 @@ void TIFFSwabArrayOfFloat(register float *fp, tmsize_t n) #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabDouble) void TIFFSwabDouble(double *dp) { - register unsigned char *cp = (unsigned char *)dp; + unsigned char *cp = (unsigned char *)dp; unsigned char t; assert(sizeof(double) == 8); t = cp[7]; @@ -221,8 +221,8 @@ void TIFFSwabDouble(double *dp) #if defined(DISABLE_CHECK_TIFFSWABMACROS) || !defined(TIFFSwabArrayOfDouble) void TIFFSwabArrayOfDouble(double *dp, tmsize_t n) { - register unsigned char *cp; - register unsigned char t; + unsigned char *cp; + unsigned char t; assert(sizeof(double) == 8); /* XXX unroll loop some */ while (n-- > 0) diff --git a/contrib/libs/libtiff/tif_thunder.c b/contrib/libs/libtiff/tif_thunder.c index bac0607dbe5..0c688db1ee4 100644 --- a/contrib/libs/libtiff/tif_thunder.c +++ b/contrib/libs/libtiff/tif_thunder.c @@ -41,7 +41,7 @@ * or 3-bit delta values are used, with the deltas packed * into a single byte. */ -#define THUNDER_DATA 0x3f /* mask for 6-bit data */ +// #define THUNDER_DATA 0x3f /* mask for 6-bit data */ #define THUNDER_CODE 0xc0 /* mask for 2-bit code word */ /* code values */ #define THUNDER_RUN 0x00 /* run of pixels w/ encoded count */ @@ -56,11 +56,11 @@ static const int threebitdeltas[8] = {0, 1, 2, 3, 0, -3, -2, -1}; #define SETPIXEL(op, v) \ { \ - lastpixel = (v)&0xf; \ + lastpixel = (v) & 0xf; \ if (npixels < maxpixels) \ { \ if (npixels++ & 1) \ - *op++ |= lastpixel; \ + *op++ |= (uint8_t)lastpixel; \ else \ op[0] = (uint8_t)(lastpixel << 4); \ } \ @@ -85,8 +85,8 @@ static int ThunderSetupDecode(TIFF *tif) static int ThunderDecode(TIFF *tif, uint8_t *op0, tmsize_t maxpixels) { static const char module[] = "ThunderDecode"; - register unsigned char *bp; - register tmsize_t cc; + unsigned char *bp; + tmsize_t cc; unsigned int lastpixel; tmsize_t npixels; uint8_t *op = op0; @@ -112,7 +112,7 @@ static int ThunderDecode(TIFF *tif, uint8_t *op0, tmsize_t maxpixels) break; if (npixels & 1) { - op[0] |= lastpixel; + op[0] |= (uint8_t)lastpixel; lastpixel = *op++; npixels++; n--; @@ -150,6 +150,8 @@ static int ThunderDecode(TIFF *tif, uint8_t *op0, tmsize_t maxpixels) case THUNDER_RAW: /* raw data */ SETPIXEL(op, n); break; + default: + break; } } tif->tif_rawcp = (uint8_t *)bp; @@ -161,7 +163,7 @@ static int ThunderDecode(TIFF *tif, uint8_t *op0, tmsize_t maxpixels) TIFFErrorExtR(tif, module, "%s data at scanline %lu (%" PRIu64 " != %" PRIu64 ")", npixels < maxpixels ? "Not enough" : "Too much", - (unsigned long)tif->tif_row, (uint64_t)npixels, + (unsigned long)tif->tif_dir.td_row, (uint64_t)npixels, (uint64_t)maxpixels); return (0); } @@ -175,7 +177,7 @@ static int ThunderDecodeRow(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) uint8_t *row = buf; (void)s; - if (occ % tif->tif_scanlinesize) + if (occ % tif->tif_dir.td_scanlinesize) { TIFFErrorExtR(tif, module, "Fractional scanlines cannot be read"); return (0); @@ -184,8 +186,8 @@ static int ThunderDecodeRow(TIFF *tif, uint8_t *buf, tmsize_t occ, uint16_t s) { if (!ThunderDecode(tif, row, tif->tif_dir.td_imagewidth)) return (0); - occ -= tif->tif_scanlinesize; - row += tif->tif_scanlinesize; + occ -= tif->tif_dir.td_scanlinesize; + row += tif->tif_dir.td_scanlinesize; } return (1); } diff --git a/contrib/libs/libtiff/tif_tile.c b/contrib/libs/libtiff/tif_tile.c index f07032f731f..f9076094009 100644 --- a/contrib/libs/libtiff/tif_tile.c +++ b/contrib/libs/libtiff/tif_tile.c @@ -54,12 +54,50 @@ uint32_t TIFFComputeTile(TIFF *tif, uint32_t x, uint32_t y, uint32_t z, uint32_t xpt = TIFFhowmany_32(td->td_imagewidth, dx); uint32_t ypt = TIFFhowmany_32(td->td_imagelength, dy); uint32_t zpt = TIFFhowmany_32(td->td_imagedepth, dz); + uint32_t xpt_ypt = _TIFFMultiply32(tif, xpt, ypt, "TIFFComputeTile"); + uint32_t xpt_ypt_zpt = + _TIFFMultiply32(tif, xpt_ypt, zpt, "TIFFComputeTile"); + uint64_t z_offset; + uint64_t y_offset; + uint64_t tile64; + if ((xpt_ypt == 0 && xpt != 0 && ypt != 0) || + (xpt_ypt_zpt == 0 && xpt_ypt != 0 && zpt != 0)) + return (0); + + z_offset = _TIFFMultiply64(tif, xpt_ypt, z / dz, "TIFFComputeTile"); + y_offset = _TIFFMultiply64(tif, xpt, y / dy, "TIFFComputeTile"); + if ((z_offset == 0 && xpt_ypt != 0 && (z / dz) != 0) || + (y_offset == 0 && xpt != 0 && (y / dy) != 0)) + return (0); + tile64 = _TIFFAdd64(tif, z_offset, y_offset, "TIFFComputeTile"); + if (tile64 == 0 && (z_offset != 0 || y_offset != 0)) + return (0); + tile64 = _TIFFAdd64(tif, tile64, x / dx, "TIFFComputeTile"); + if (tile64 == 0 && (z_offset != 0 || y_offset != 0 || (x / dx) != 0)) + return (0); if (td->td_planarconfig == PLANARCONFIG_SEPARATE) - tile = (xpt * ypt * zpt) * s + (xpt * ypt) * (z / dz) + - xpt * (y / dy) + x / dx; - else - tile = (xpt * ypt) * (z / dz) + xpt * (y / dy) + x / dx; + { + uint64_t sample_offset; + if (s >= td->td_samplesperpixel) + { + TIFFErrorExtR( + tif, "TIFFComputeTile", "%lu: Sample out of range, max %lu", + (unsigned long)s, (unsigned long)td->td_samplesperpixel); + return (0); + } + sample_offset = + _TIFFMultiply64(tif, xpt_ypt_zpt, s, "TIFFComputeTile"); + if (sample_offset == 0 && xpt_ypt_zpt != 0 && s != 0) + return (0); + tile64 = _TIFFAdd64(tif, sample_offset, tile64, "TIFFComputeTile"); + if (tile64 == 0 && (sample_offset != 0 || z_offset != 0 || + y_offset != 0 || (x / dx) != 0)) + return (0); + } + tile = _TIFFCastUInt64ToUInt32(tif, tile64, "TIFFComputeTile"); + if (tile == 0 && tile64 != 0) + return (0); } return (tile); } @@ -187,54 +225,9 @@ tmsize_t TIFFTileRowSize(TIFF *tif) */ uint64_t TIFFVTileSize64(TIFF *tif, uint32_t nrows) { - static const char module[] = "TIFFVTileSize64"; - TIFFDirectory *td = &tif->tif_dir; - if (td->td_tilelength == 0 || td->td_tilewidth == 0 || - td->td_tiledepth == 0) - return (0); - if ((td->td_planarconfig == PLANARCONFIG_CONTIG) && - (td->td_photometric == PHOTOMETRIC_YCBCR) && - (td->td_samplesperpixel == 3) && (!isUpSampled(tif))) - { - /* - * Packed YCbCr data contain one Cb+Cr for every - * HorizontalSampling*VerticalSampling Y values. - * Must also roundup width and height when calculating - * since images that are not a multiple of the - * horizontal/vertical subsampling area include - * YCbCr data for the extended image. - */ - uint16_t ycbcrsubsampling[2]; - uint16_t samplingblock_samples; - uint32_t samplingblocks_hor; - uint32_t samplingblocks_ver; - uint64_t samplingrow_samples; - uint64_t samplingrow_size; - TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING, - ycbcrsubsampling + 0, ycbcrsubsampling + 1); - if ((ycbcrsubsampling[0] != 1 && ycbcrsubsampling[0] != 2 && - ycbcrsubsampling[0] != 4) || - (ycbcrsubsampling[1] != 1 && ycbcrsubsampling[1] != 2 && - ycbcrsubsampling[1] != 4)) - { - TIFFErrorExtR(tif, module, "Invalid YCbCr subsampling (%dx%d)", - ycbcrsubsampling[0], ycbcrsubsampling[1]); - return 0; - } - samplingblock_samples = ycbcrsubsampling[0] * ycbcrsubsampling[1] + 2; - samplingblocks_hor = - TIFFhowmany_32(td->td_tilewidth, ycbcrsubsampling[0]); - samplingblocks_ver = TIFFhowmany_32(nrows, ycbcrsubsampling[1]); - samplingrow_samples = _TIFFMultiply64(tif, samplingblocks_hor, - samplingblock_samples, module); - samplingrow_size = TIFFhowmany8_64(_TIFFMultiply64( - tif, samplingrow_samples, td->td_bitspersample, module)); - return ( - _TIFFMultiply64(tif, samplingrow_size, samplingblocks_ver, module)); - } - else - return (_TIFFMultiply64(tif, nrows, TIFFTileRowSize64(tif), module)); + return _TIFFStrileSize64(tif, nrows, /* isStrip = */ FALSE); } + tmsize_t TIFFVTileSize(TIFF *tif, uint32_t nrows) { static const char module[] = "TIFFVTileSize"; diff --git a/contrib/libs/libtiff/tif_unix.c b/contrib/libs/libtiff/tif_unix.c index 9e6fc75b080..b02fe294eb3 100644 --- a/contrib/libs/libtiff/tif_unix.c +++ b/contrib/libs/libtiff/tif_unix.c @@ -77,10 +77,10 @@ static tmsize_t _tiffReadProc(thandle_t fd, void *buf, tmsize_t size) return (tmsize_t)-1; } fdh.h = fd; - for (bytes_read = 0; bytes_read < bytes_total; bytes_read += count) + for (bytes_read = 0; bytes_read < bytes_total; bytes_read += (size_t)count) { char *buf_offset = (char *)buf + bytes_read; - size_t io_size = bytes_total - bytes_read; + size_t io_size = (size_t)(bytes_total - bytes_read); if (io_size > TIFF_IO_MAX) io_size = TIFF_IO_MAX; /* Below is an obvious false positive of Coverity Scan */ @@ -108,10 +108,11 @@ static tmsize_t _tiffWriteProc(thandle_t fd, void *buf, tmsize_t size) return (tmsize_t)-1; } fdh.h = fd; - for (bytes_written = 0; bytes_written < bytes_total; bytes_written += count) + for (bytes_written = 0; bytes_written < bytes_total; + bytes_written += (size_t)count) { const char *buf_offset = (char *)buf + bytes_written; - size_t io_size = bytes_total - bytes_written; + size_t io_size = (size_t)(bytes_total - bytes_written); if (io_size > TIFF_IO_MAX) io_size = TIFF_IO_MAX; /* Below is an obvious false positive of Coverity Scan */ @@ -174,7 +175,7 @@ static int _tiffMapProc(thandle_t fd, void **pbase, toff_t *psize) (void *)mmap(0, (size_t)sizem, PROT_READ, MAP_SHARED, fdh.fd, 0); if (*pbase != (void *)-1) { - *psize = (tmsize_t)sizem; + *psize = (toff_t)sizem; return (1); } } @@ -184,7 +185,7 @@ static int _tiffMapProc(thandle_t fd, void **pbase, toff_t *psize) static void _tiffUnmapProc(thandle_t fd, void *base, toff_t size) { (void)fd; - (void)munmap(base, (off_t)size); + (void)munmap(base, (size_t)size); } #else /* !HAVE_MMAP */ static int _tiffMapProc(thandle_t fd, void **pbase, toff_t *psize) @@ -362,7 +363,8 @@ int _TIFFmemcmp(const void *p1, const void *p2, tmsize_t c) return (memcmp(p1, p2, (size_t)c)); } -static void unixWarningHandler(const char *module, const char *fmt, va_list ap) +static void TIFF_ATTRIBUTE((__format__(__printf__, 2, 0))) + unixWarningHandler(const char *module, const char *fmt, va_list ap) { if (module != NULL) fprintf(stderr, "%s: ", module); @@ -372,7 +374,8 @@ static void unixWarningHandler(const char *module, const char *fmt, va_list ap) } TIFFErrorHandler _TIFFwarningHandler = unixWarningHandler; -static void unixErrorHandler(const char *module, const char *fmt, va_list ap) +static void TIFF_ATTRIBUTE((__format__(__printf__, 2, 0))) + unixErrorHandler(const char *module, const char *fmt, va_list ap) { if (module != NULL) fprintf(stderr, "%s: ", module); diff --git a/contrib/libs/libtiff/tif_webp.c b/contrib/libs/libtiff/tif_webp.c index c06ce163b71..cc8b680c52b 100644 --- a/contrib/libs/libtiff/tif_webp.c +++ b/contrib/libs/libtiff/tif_webp.c @@ -84,15 +84,16 @@ static int TWebPDatasetWriter(const uint8_t *data, size_t data_size, if ((tif->tif_rawcc + (tmsize_t)data_size) > tif->tif_rawdatasize) { - TIFFErrorExtR( - tif, module, "Buffer too small by %" TIFF_SIZE_FORMAT " bytes.", - (size_t)(tif->tif_rawcc + data_size - tif->tif_rawdatasize)); + TIFFErrorExtR(tif, module, + "Buffer too small by %" TIFF_SIZE_FORMAT " bytes.", + (size_t)((uint64_t)tif->tif_rawcc + (uint64_t)data_size - + (uint64_t)tif->tif_rawdatasize)); return 0; } else { - _TIFFmemcpy(tif->tif_rawcp, data, data_size); - tif->tif_rawcc += data_size; + _TIFFmemcpy(tif->tif_rawcp, data, (tmsize_t)data_size); + tif->tif_rawcc += (tmsize_t)data_size; tif->tif_rawcp += data_size; return 1; } @@ -116,7 +117,7 @@ static int TWebPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) return 0; } - memcpy(sp->pBuffer + sp->buffer_offset, bp, cc); + memcpy(sp->pBuffer + sp->buffer_offset, bp, (size_t)cc); sp->buffer_offset += (unsigned)cc; return 1; @@ -141,7 +142,7 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "ZIPDecode: Scanline %" PRIu32 " cannot be read due to " "previous error", - tif->tif_row); + tif->tif_dir.td_row); return 0; } @@ -158,7 +159,7 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) else { segment_width = td->td_imagewidth; - segment_height = td->td_imagelength - tif->tif_row; + segment_height = td->td_imagelength - tif->tif_dir.td_row; if (segment_height > td->td_rowsperstrip) segment_height = td->td_rowsperstrip; } @@ -246,7 +247,7 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) sp->pBuffer = NULL; } - sp->pBuffer = _TIFFmallocExt(tif, buffer_size); + sp->pBuffer = (uint8_t *)_TIFFmallocExt(tif, buffer_size); if (!sp->pBuffer) { TIFFErrorExtR(tif, module, "Cannot allocate buffer"); @@ -262,10 +263,10 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) WebPInitDecBuffer(&sp->sDecBuffer); sp->sDecBuffer.is_external_memory = 1; - sp->sDecBuffer.width = segment_width; - sp->sDecBuffer.height = segment_height; + sp->sDecBuffer.width = (int)segment_width; + sp->sDecBuffer.height = (int)segment_height; sp->sDecBuffer.u.RGBA.rgba = decode_whole_strile ? op : sp->pBuffer; - sp->sDecBuffer.u.RGBA.stride = segment_width * sp->nSamples; + sp->sDecBuffer.u.RGBA.stride = (int)(segment_width * sp->nSamples); sp->sDecBuffer.u.RGBA.size = buffer_size; if (sp->nSamples > 3) @@ -298,7 +299,7 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) return 0; } - status = WebPIAppend(sp->psDecoder, tif->tif_rawcp, tif->tif_rawcc); + status = WebPIAppend(sp->psDecoder, tif->tif_rawcp, (size_t)tif->tif_rawcc); if (status != VP8_STATUS_OK && status != VP8_STATUS_SUSPENDED) { @@ -345,7 +346,7 @@ static int TWebPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } else { - memcpy(op, buf + (sp->last_y * stride), occ); + memcpy(op, buf + (sp->last_y * stride), (size_t)occ); } tif->tif_rawcp += tif->tif_rawcc; @@ -465,7 +466,7 @@ static int TWebPPreDecode(TIFF *tif, uint16_t s) else { segment_width = td->td_imagewidth; - segment_height = td->td_imagelength - tif->tif_row; + segment_height = td->td_imagelength - tif->tif_dir.td_row; if (segment_height > td->td_rowsperstrip) segment_height = td->td_rowsperstrip; } @@ -601,7 +602,7 @@ static int TWebPPreEncode(TIFF *tif, uint16_t s) else { segment_width = td->td_imagewidth; - segment_height = td->td_imagelength - tif->tif_row; + segment_height = td->td_imagelength - tif->tif_dir.td_row; if (segment_height > td->td_rowsperstrip) segment_height = td->td_rowsperstrip; } @@ -623,7 +624,7 @@ static int TWebPPreEncode(TIFF *tif, uint16_t s) sp->pBuffer = NULL; } - sp->pBuffer = _TIFFmallocExt(tif, sp->buffer_size); + sp->pBuffer = (uint8_t *)_TIFFmallocExt(tif, sp->buffer_size); if (!sp->pBuffer) { TIFFErrorExtR(tif, module, "Cannot allocate buffer"); @@ -631,8 +632,8 @@ static int TWebPPreEncode(TIFF *tif, uint16_t s) } sp->buffer_offset = 0; - sp->sPicture.width = segment_width; - sp->sPicture.height = segment_height; + sp->sPicture.width = (int)segment_width; + sp->sPicture.height = (int)segment_height; sp->sPicture.writer = TWebPDatasetWriter; sp->sPicture.custom_ptr = tif; @@ -708,9 +709,11 @@ static int TWebPPostEncode(TIFF *tif) case VP8_ENC_ERROR_USER_ABORT: pszErrorMsg = "User interrupted"; break; + case VP8_ENC_OK: + case VP8_ENC_ERROR_LAST: default: TIFFErrorExtR(tif, module, - "WebPEncode returned an unknown error code: %d", + "WebPEncode returned an unknown error code: %u", sp->sPicture.error_code); pszErrorMsg = "Unknown WebP error type."; break; @@ -776,7 +779,7 @@ static int TWebPVSetField(TIFF *tif, uint32_t tag, va_list ap) { case TIFFTAG_WEBP_LEVEL: sp->quality_level = (int)va_arg(ap, int); - if (sp->quality_level <= 0 || sp->quality_level > 100.0f) + if (sp->quality_level <= 0 || sp->quality_level > 100) { TIFFWarningExtR(tif, module, "WEBP_LEVEL should be between 1 and 100"); @@ -844,6 +847,17 @@ static const TIFFField TWebPFields[] = { FIELD_PSEUDO, TRUE, FALSE, "WEBP exact lossless", NULL}, }; +static uint64_t TWebPGetMaxCompressionRatio(TIFF *tif) +{ + /* See README_for_libtiff_developpers.md for raw data used to estimate + * the maximum compression rate. */ + + /* lossy compression: */ + /* return (tif->tif_dir.td_samplesperpixel == 4) ? 2199 : 1685; */ + /* lossless compression: */ + return (tif->tif_dir.td_samplesperpixel == 4) ? 104194 : 78146; +} + int TIFFInitWebP(TIFF *tif, int scheme) { static const char module[] = "TIFFInitWebP"; @@ -907,6 +921,7 @@ int TIFFInitWebP(TIFF *tif, int scheme) tif->tif_encodestrip = TWebPEncode; tif->tif_encodetile = TWebPEncode; tif->tif_cleanup = TWebPCleanup; + tif->tif_getmaxcompressionratio = TWebPGetMaxCompressionRatio; return 1; bad: diff --git a/contrib/libs/libtiff/tif_win32.c b/contrib/libs/libtiff/tif_win32.c index 525cebec76d..4b21ab7d482 100644 --- a/contrib/libs/libtiff/tif_win32.c +++ b/contrib/libs/libtiff/tif_win32.c @@ -5,6 +5,7 @@ * Permission to use, copy, modify, distribute, and sell this software and * its documentation for any purpose is hereby granted without fee, provided * that (i) the above copyright notices and this permission notice appear in + * that (i) the above copyright notices and this permission notice appear in * all copies of the software and related documentation, and (ii) the names of * Sam Leffler and Silicon Graphics may not be used in any advertising or * publicity relating to the software without the specific, prior written @@ -72,7 +73,7 @@ static tmsize_t _tiffReadProc(thandle_t fd, void *buf, tmsize_t size) DWORD o; tmsize_t p; ma = (uint8_t *)buf; - mb = size; + mb = (uint64_t)size; p = 0; while (mb > 0) { @@ -101,7 +102,7 @@ static tmsize_t _tiffWriteProc(thandle_t fd, void *buf, tmsize_t size) DWORD o; tmsize_t p; ma = (uint8_t *)buf; - mb = size; + mb = (uint64_t)size; p = 0; while (mb > 0) { @@ -123,7 +124,7 @@ static uint64_t _tiffSeekProc(thandle_t fd, uint64_t off, int whence) { LARGE_INTEGER offli; DWORD dwMoveMethod; - offli.QuadPart = off; + offli.QuadPart = (LONGLONG)off; switch (whence) { case SEEK_SET: @@ -140,11 +141,11 @@ static uint64_t _tiffSeekProc(thandle_t fd, uint64_t off, int whence) break; } offli.LowPart = - SetFilePointer(fd, offli.LowPart, &offli.HighPart, dwMoveMethod); + SetFilePointer(fd, (LONG)offli.LowPart, &offli.HighPart, dwMoveMethod); if ((offli.LowPart == INVALID_SET_FILE_POINTER) && (GetLastError() != NO_ERROR)) offli.QuadPart = 0; - return (offli.QuadPart); + return ((uint64_t)offli.QuadPart); } static int _tiffCloseProc(thandle_t fd) { return (CloseHandle(fd) ? 0 : -1); } @@ -153,7 +154,7 @@ static uint64_t _tiffSizeProc(thandle_t fd) { LARGE_INTEGER m; if (GetFileSizeEx(fd, &m)) - return (m.QuadPart); + return ((uint64_t)m.QuadPart); else return (0); } @@ -343,7 +344,7 @@ TIFF *TIFFOpenWExt(const wchar_t *name, const char *mode, TIFFOpenOptions *opts) NULL); if (fd == INVALID_HANDLE_VALUE) { - _TIFFErrorEarly(opts, NULL, module, "%S: Cannot open", name); + _TIFFErrorEarly(opts, NULL, module, "%ls: Cannot open", name); return ((TIFF *)0); } diff --git a/contrib/libs/libtiff/tif_write.c b/contrib/libs/libtiff/tif_write.c index 3263853cc0f..614a8ea85a4 100644 --- a/contrib/libs/libtiff/tif_write.c +++ b/contrib/libs/libtiff/tif_write.c @@ -30,7 +30,7 @@ #include "tiffiop.h" #include <stdio.h> -#define STRIPINCR 20 /* expansion factor on strip array */ +#define NOSTRIP ((uint32_t)(-1)) /* undefined state */ #define WRITECHECKSTRIPS(tif, module) \ (((tif)->tif_flags & TIFF_BEENWRITING) || TIFFWriteCheck((tif), 0, module)) @@ -38,7 +38,7 @@ (((tif)->tif_flags & TIFF_BEENWRITING) || TIFFWriteCheck((tif), 1, module)) #define BUFFERCHECK(tif) \ ((((tif)->tif_flags & TIFF_BUFFERSETUP) && tif->tif_rawdata) || \ - TIFFWriteBufferSetup((tif), NULL, (tmsize_t)-1)) + TIFFWriteBufferSetup((tif), NULL, (tmsize_t)(-1))) static int TIFFGrowStrips(TIFF *tif, uint32_t delta, const char *module); static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, @@ -47,7 +47,7 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) { static const char module[] = "TIFFWriteScanline"; - register TIFFDirectory *td; + TIFFDirectory *td; int status, imagegrew = 0; uint32_t strip; @@ -82,8 +82,16 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) /* * Calculate strip and check for crossings. */ + if (td->td_rowsperstrip == 0) + { + TIFFErrorExtR(tif, module, + "Cannot compute strip: RowsPerStrip is zero"); + return (-1); + } if (td->td_planarconfig == PLANARCONFIG_SEPARATE) { + uint64_t sample_offset; + uint64_t strip64; if (sample >= td->td_samplesperpixel) { TIFFErrorExtR(tif, module, "%lu: Sample out of range, max %lu", @@ -91,7 +99,18 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) (unsigned long)td->td_samplesperpixel); return (-1); } - strip = sample * td->td_stripsperimage + row / td->td_rowsperstrip; + sample_offset = + _TIFFMultiply64(tif, sample, td->td_stripsperimage, module); + if (sample_offset == 0 && sample != 0 && td->td_stripsperimage != 0) + return (-1); + strip64 = + _TIFFAdd64(tif, sample_offset, row / td->td_rowsperstrip, module); + if (strip64 == 0 && + (sample_offset != 0 || (row / td->td_rowsperstrip) != 0)) + return (-1); + strip = _TIFFCastUInt64ToUInt32(tif, strip64, module); + if (strip == 0 && strip64 != 0) + return (-1); } else strip = row / td->td_rowsperstrip; @@ -104,14 +123,14 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) */ if (strip >= td->td_nstrips && !TIFFGrowStrips(tif, 1, module)) return (-1); - if (strip != tif->tif_curstrip) + if (strip != tif->tif_dir.td_curstrip) { /* * Changing strips -- flush any data present. */ if (!TIFFFlushData(tif)) return (-1); - tif->tif_curstrip = strip; + tif->tif_dir.td_curstrip = strip; /* * Watch out for a growing image. The value of strips/image * will initially be 1 (since it can't be deduced until the @@ -125,7 +144,8 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) TIFFErrorExtR(tif, module, "Zero strips per image"); return (-1); } - tif->tif_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; + tif->tif_dir.td_row = + (strip % td->td_stripsperimage) * td->td_rowsperstrip; if ((tif->tif_flags & TIFF_CODERSETUP) == 0) { if (!(*tif->tif_setupencode)(tif)) @@ -148,35 +168,35 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) * beginning of a strip (or that we can randomly * access the data -- i.e. no encoding). */ - if (row != tif->tif_row) + if (row != tif->tif_dir.td_row) { - if (row < tif->tif_row) + if (row < tif->tif_dir.td_row) { /* * Moving backwards within the same strip: * backup to the start and then decode * forward (below). */ - tif->tif_row = + tif->tif_dir.td_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; tif->tif_rawcp = tif->tif_rawdata; } /* * Seek forward to the desired row. */ - if (!(*tif->tif_seek)(tif, row - tif->tif_row)) + if (!(*tif->tif_seek)(tif, row - tif->tif_dir.td_row)) return (-1); - tif->tif_row = row; + tif->tif_dir.td_row = row; } /* swab if needed - note that source buffer will be altered */ - tif->tif_postdecode(tif, (uint8_t *)buf, tif->tif_scanlinesize); + tif->tif_postdecode(tif, (uint8_t *)buf, tif->tif_dir.td_scanlinesize); - status = (*tif->tif_encoderow)(tif, (uint8_t *)buf, tif->tif_scanlinesize, - sample); + status = (*tif->tif_encoderow)(tif, (uint8_t *)buf, + tif->tif_dir.td_scanlinesize, sample); /* we are now poised at the beginning of the next row */ - tif->tif_row = row + 1; + tif->tif_dir.td_row = row + 1; return (status); } @@ -188,7 +208,21 @@ int TIFFWriteScanline(TIFF *tif, void *buf, uint32_t row, uint16_t sample) /* time if the new compressed tile is bigger than the older one. (GDAL #4771) */ static int _TIFFReserveLargeEnoughWriteBuffer(TIFF *tif, uint32_t strip_or_tile) { + static const char module[] = "_TIFFReserveLargeEnoughWriteBuffer"; TIFFDirectory *td = &tif->tif_dir; + + if (td->td_stripbytecount_p == NULL) + { + TIFFErrorExtR(tif, module, "Strip bytecount array pointer is NULL"); + return 0; + } + + if (strip_or_tile == NOSTRIP || strip_or_tile >= td->td_nstrips) + { + TIFFErrorExtR(tif, module, "Strip/tile number not valid"); + return 0; + } + if (td->td_stripbytecount_p[strip_or_tile] > 0) { /* The +1 is to ensure at least one extra bytes */ @@ -254,7 +288,7 @@ tmsize_t TIFFWriteEncodedStrip(TIFF *tif, uint32_t strip, void *data, tif->tif_flags |= TIFF_BUF4WRITE; - tif->tif_curstrip = strip; + tif->tif_dir.td_curstrip = strip; /* this informs TIFFAppendToStrip() we have changed or reset strip */ tif->tif_curoff = 0; @@ -273,7 +307,7 @@ tmsize_t TIFFWriteEncodedStrip(TIFF *tif, uint32_t strip, void *data, return ((tmsize_t)-1); } - tif->tif_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; + tif->tif_dir.td_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; if ((tif->tif_flags & TIFF_CODERSETUP) == 0) { if (!(*tif->tif_setupencode)(tif)) @@ -362,9 +396,9 @@ tmsize_t TIFFWriteRawStrip(TIFF *tif, uint32_t strip, void *data, tmsize_t cc) return ((tmsize_t)-1); } - if (tif->tif_curstrip != strip) + if (tif->tif_dir.td_curstrip != strip) { - tif->tif_curstrip = strip; + tif->tif_dir.td_curstrip = strip; /* this informs TIFFAppendToStrip() we have changed or reset strip */ tif->tif_curoff = 0; @@ -375,7 +409,7 @@ tmsize_t TIFFWriteRawStrip(TIFF *tif, uint32_t strip, void *data, tmsize_t cc) TIFFErrorExtR(tif, module, "Zero strips per image"); return ((tmsize_t)-1); } - tif->tif_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; + tif->tif_dir.td_row = (strip % td->td_stripsperimage) * td->td_rowsperstrip; return (TIFFAppendToStrip(tif, strip, (uint8_t *)data, cc) ? cc : (tmsize_t)-1); } @@ -437,7 +471,7 @@ tmsize_t TIFFWriteEncodedTile(TIFF *tif, uint32_t tile, void *data, tmsize_t cc) tif->tif_flags |= TIFF_BUF4WRITE; - tif->tif_curtile = tile; + tif->tif_dir.td_curtile = tile; /* this informs TIFFAppendToStrip() we have changed or reset tile */ tif->tif_curoff = 0; @@ -460,14 +494,14 @@ tmsize_t TIFFWriteEncodedTile(TIFF *tif, uint32_t tile, void *data, tmsize_t cc) TIFFErrorExtR(tif, module, "Zero tiles"); return ((tmsize_t)(-1)); } - tif->tif_row = (tile % howmany32) * td->td_tilelength; + tif->tif_dir.td_row = (tile % howmany32) * td->td_tilelength; howmany32 = TIFFhowmany_32(td->td_imagewidth, td->td_tilewidth); if (howmany32 == 0) { TIFFErrorExtR(tif, module, "Zero tiles"); return ((tmsize_t)(-1)); } - tif->tif_col = (tile % howmany32) * td->td_tilewidth; + tif->tif_dir.td_col = (tile % howmany32) * td->td_tilewidth; if ((tif->tif_flags & TIFF_CODERSETUP) == 0) { @@ -482,8 +516,8 @@ tmsize_t TIFFWriteEncodedTile(TIFF *tif, uint32_t tile, void *data, tmsize_t cc) * done so that callers can pass in some large number * (e.g. -1) and have the tile size used instead. */ - if (cc < 1 || cc > tif->tif_tilesize) - cc = tif->tif_tilesize; + if (cc < 1 || cc > tif->tif_dir.td_tilesize) + cc = tif->tif_dir.td_tilesize; /* shortcut to avoid an extra memcpy() */ if (td->td_compression == COMPRESSION_NONE) @@ -588,8 +622,10 @@ int TIFFSetupStrips(TIFF *tif) * Place data at the end-of-file * (by setting offsets to zero). */ - _TIFFmemset(td->td_stripoffset_p, 0, td->td_nstrips * sizeof(uint64_t)); - _TIFFmemset(td->td_stripbytecount_p, 0, td->td_nstrips * sizeof(uint64_t)); + _TIFFmemset(td->td_stripoffset_p, 0, + (tmsize_t)((size_t)td->td_nstrips * sizeof(uint64_t))); + _TIFFmemset(td->td_stripbytecount_p, 0, + (tmsize_t)((size_t)td->td_nstrips * sizeof(uint64_t))); TIFFSetFieldBit(tif, FIELD_STRIPOFFSETS); TIFFSetFieldBit(tif, FIELD_STRIPBYTECOUNTS); return (1); @@ -644,14 +680,14 @@ int TIFFWriteCheck(TIFF *tif, int tiles, const char *module) } if (isTiled(tif)) { - tif->tif_tilesize = TIFFTileSize(tif); - if (tif->tif_tilesize == 0) + tif->tif_dir.td_tilesize = TIFFTileSize(tif); + if (tif->tif_dir.td_tilesize == 0) return (0); } else - tif->tif_tilesize = (tmsize_t)(-1); - tif->tif_scanlinesize = TIFFScanlineSize(tif); - if (tif->tif_scanlinesize == 0) + tif->tif_dir.td_tilesize = (tmsize_t)(-1); + tif->tif_dir.td_scanlinesize = TIFFScanlineSize(tif); + if (tif->tif_dir.td_scanlinesize == 0) return (0); tif->tif_flags |= TIFF_BEENWRITING; @@ -689,7 +725,7 @@ int TIFFWriteBufferSetup(TIFF *tif, void *bp, tmsize_t size) } if (size == (tmsize_t)(-1)) { - size = (isTiled(tif) ? tif->tif_tilesize : TIFFStripSize(tif)); + size = (isTiled(tif) ? tif->tif_dir.td_tilesize : TIFFStripSize(tif)); /* Adds 10% margin for cases where compression would expand a bit */ if (size < TIFF_TMSIZE_T_MAX - size / 10) @@ -732,26 +768,31 @@ static int TIFFGrowStrips(TIFF *tif, uint32_t delta, const char *module) assert(td->td_planarconfig == PLANARCONFIG_CONTIG); new_stripoffset = (uint64_t *)_TIFFreallocExt( - tif, td->td_stripoffset_p, (td->td_nstrips + delta) * sizeof(uint64_t)); + tif, td->td_stripoffset_p, + (tmsize_t)(((size_t)td->td_nstrips + (size_t)delta) * + sizeof(uint64_t))); + /* + * Update td_stripoffset_p immediately so the old pointer is not left + * dangling if the second realloc fails. + */ + if (new_stripoffset) + td->td_stripoffset_p = new_stripoffset; new_stripbytecount = (uint64_t *)_TIFFreallocExt( tif, td->td_stripbytecount_p, - (td->td_nstrips + delta) * sizeof(uint64_t)); + (tmsize_t)(((size_t)td->td_nstrips + (size_t)delta) * + sizeof(uint64_t))); + if (new_stripbytecount) + td->td_stripbytecount_p = new_stripbytecount; if (new_stripoffset == NULL || new_stripbytecount == NULL) { - if (new_stripoffset) - _TIFFfreeExt(tif, new_stripoffset); - if (new_stripbytecount) - _TIFFfreeExt(tif, new_stripbytecount); td->td_nstrips = 0; TIFFErrorExtR(tif, module, "No space to expand strip arrays"); return (0); } - td->td_stripoffset_p = new_stripoffset; - td->td_stripbytecount_p = new_stripbytecount; _TIFFmemset(td->td_stripoffset_p + td->td_nstrips, 0, - delta * sizeof(uint64_t)); + (tmsize_t)((size_t)delta * sizeof(uint64_t))); _TIFFmemset(td->td_stripbytecount_p + td->td_nstrips, 0, - delta * sizeof(uint64_t)); + (tmsize_t)((size_t)delta * sizeof(uint64_t))); td->td_nstrips += delta; tif->tif_flags |= TIFF_DIRTYDIRECT; @@ -769,6 +810,23 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, uint64_t m; int64_t old_byte_count = -1; + /* Some security checks */ + if (td->td_stripoffset_p == NULL) + { + TIFFErrorExtR(tif, module, "Strip offset array pointer is NULL"); + return (0); + } + if (td->td_stripbytecount_p == NULL) + { + TIFFErrorExtR(tif, module, "Strip bytecount array pointer is NULL"); + return (0); + } + if (strip == NOSTRIP) + { + TIFFErrorExtR(tif, module, "Strip number not valid (NOSTRIP)"); + return (0); + } + if (tif->tif_curoff == 0) tif->tif_lastvalidoff = 0; @@ -790,7 +848,7 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, if (!SeekOK(tif, td->td_stripoffset_p[strip])) { TIFFErrorExtR(tif, module, "Seek error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); return (0); } @@ -812,11 +870,11 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, /* * We are starting a fresh strip/tile, so set the size to zero. */ - old_byte_count = td->td_stripbytecount_p[strip]; + old_byte_count = (int64_t)td->td_stripbytecount_p[strip]; td->td_stripbytecount_p[strip] = 0; } - m = tif->tif_curoff + cc; + m = tif->tif_curoff + (uint64_t)cc; if (!(tif->tif_flags & TIFF_BIGTIFF)) m = (uint32_t)m; if ((m < tif->tif_curoff) || (m < (uint64_t)cc)) @@ -850,7 +908,7 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, offsetRead = td->td_stripoffset_p[strip]; offsetWrite = TIFFSeekFile(tif, 0, SEEK_END); - m = offsetWrite + toCopy + cc; + m = offsetWrite + (uint64_t)toCopy + (uint64_t)cc; if (!(tif->tif_flags & TIFF_BIGTIFF) && m != (uint32_t)m) { TIFFErrorExtR(tif, module, "Maximum TIFF file size exceeded"); @@ -872,13 +930,15 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, /* Move data written by previous calls to us at end of file */ while (toCopy > 0) { + tmsize_t chunkSize = + toCopy < (uint64_t)tempSize ? (tmsize_t)toCopy : tempSize; if (!SeekOK(tif, offsetRead)) { TIFFErrorExtR(tif, module, "Seek error"); _TIFFfreeExt(tif, temp); return (0); } - if (!ReadOK(tif, temp, tempSize)) + if (!ReadOK(tif, temp, chunkSize)) { TIFFErrorExtR(tif, module, "Cannot read"); _TIFFfreeExt(tif, temp); @@ -890,32 +950,32 @@ static int TIFFAppendToStrip(TIFF *tif, uint32_t strip, uint8_t *data, _TIFFfreeExt(tif, temp); return (0); } - if (!WriteOK(tif, temp, tempSize)) + if (!WriteOK(tif, temp, chunkSize)) { TIFFErrorExtR(tif, module, "Cannot write"); _TIFFfreeExt(tif, temp); return (0); } - offsetRead += tempSize; - offsetWrite += tempSize; - td->td_stripbytecount_p[strip] += tempSize; - toCopy -= tempSize; + offsetRead += (uint64_t)chunkSize; + offsetWrite += (uint64_t)chunkSize; + td->td_stripbytecount_p[strip] += (uint64_t)chunkSize; + toCopy -= (uint64_t)chunkSize; } _TIFFfreeExt(tif, temp); /* Append the data of this call */ - offsetWrite += cc; + offsetWrite += (uint64_t)cc; m = offsetWrite; } if (!WriteOK(tif, data, cc)) { TIFFErrorExtR(tif, module, "Write error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); return (0); } tif->tif_curoff = m; - td->td_stripbytecount_p[strip] += cc; + td->td_stripbytecount_p[strip] += (uint64_t)cc; if ((int64_t)td->td_stripbytecount_p[strip] != old_byte_count) tif->tif_flags |= TIFF_DIRTYSTRIP; @@ -935,9 +995,10 @@ int TIFFFlushData1(TIFF *tif) if (!isFillOrder(tif, tif->tif_dir.td_fillorder) && (tif->tif_flags & TIFF_NOBITREV) == 0) TIFFReverseBits((uint8_t *)tif->tif_rawdata, tif->tif_rawcc); - if (!TIFFAppendToStrip( - tif, isTiled(tif) ? tif->tif_curtile : tif->tif_curstrip, - tif->tif_rawdata, tif->tif_rawcc)) + if (!TIFFAppendToStrip(tif, + isTiled(tif) ? tif->tif_dir.td_curtile + : tif->tif_dir.td_curstrip, + tif->tif_rawdata, tif->tif_rawcc)) { /* We update those variables even in case of error since there's */ /* code that doesn't really check the return code of this */ diff --git a/contrib/libs/libtiff/tif_zip.c b/contrib/libs/libtiff/tif_zip.c index 441c7fe57c3..ac55bf0795e 100644 --- a/contrib/libs/libtiff/tif_zip.c +++ b/contrib/libs/libtiff/tif_zip.c @@ -159,7 +159,8 @@ static int ZIPPreDecode(TIFF *tif, uint16_t s) we need to simplify this code to reflect a ZLib that is likely updated to deal with 8byte memory sizes, though this code will respond appropriately even before we simplify it */ - sp->stream.avail_in = TIFF_CLAMP_UINT64_TO_INT32_MAX(tif->tif_rawcc); + sp->stream.avail_in = + TIFF_CLAMP_UINT64_TO_INT32_MAX((uint64_t)tif->tif_rawcc); if (inflateReset(&sp->stream) == Z_OK) { sp->read_error = 0; @@ -183,7 +184,7 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "ZIPDecode: Scanline %" PRIu32 " cannot be read due to " "previous error", - tif->tif_row); + tif->tif_dir.td_row); return 0; } @@ -210,7 +211,7 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) } else { - uint32_t strip_height = td->td_imagelength - tif->tif_row; + uint32_t strip_height = td->td_imagelength - tif->tif_dir.td_row; if (strip_height > td->td_rowsperstrip) strip_height = td->td_rowsperstrip; if (TIFFVStripSize64(tif, strip_height) != (uint64_t)occ) @@ -254,7 +255,7 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { memset(op, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Decoding error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); sp->read_error = 1; return 0; } @@ -275,8 +276,9 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) do { int state; - uInt avail_in_before = TIFF_CLAMP_UINT64_TO_INT32_MAX(tif->tif_rawcc); - uInt avail_out_before = TIFF_CLAMP_UINT64_TO_INT32_MAX(occ); + uInt avail_in_before = + TIFF_CLAMP_UINT64_TO_INT32_MAX((uint64_t)tif->tif_rawcc); + uInt avail_out_before = TIFF_CLAMP_UINT64_TO_INT32_MAX((uint64_t)occ); sp->stream.avail_in = avail_in_before; sp->stream.avail_out = avail_out_before; state = inflate(&sp->stream, Z_PARTIAL_FLUSH); @@ -288,7 +290,7 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) { memset(sp->stream.next_out, 0, (size_t)occ); TIFFErrorExtR(tif, module, "Decoding error at scanline %lu, %s", - (unsigned long)tif->tif_row, SAFE_MSG(sp)); + (unsigned long)tif->tif_dir.td_row, SAFE_MSG(sp)); sp->read_error = 1; return (0); } @@ -305,7 +307,7 @@ static int ZIPDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) TIFFErrorExtR(tif, module, "Not enough data at scanline %lu (short %" PRIu64 " bytes)", - (unsigned long)tif->tif_row, (uint64_t)occ); + (unsigned long)tif->tif_dir.td_row, (uint64_t)occ); memset(sp->stream.next_out, 0, (size_t)occ); sp->read_error = 1; return (0); @@ -411,7 +413,7 @@ static int ZIPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) } else { - uint32_t strip_height = td->td_imagelength - tif->tif_row; + uint32_t strip_height = td->td_imagelength - tif->tif_dir.td_row; if (strip_height > td->td_rowsperstrip) strip_height = td->td_rowsperstrip; if (TIFFVStripSize64(tif, strip_height) != (uint64_t)cc) @@ -463,11 +465,11 @@ static int ZIPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) if (nCompressedBytes == 0) { TIFFErrorExtR(tif, module, "Encoder error at scanline %lu", - (unsigned long)tif->tif_row); + (unsigned long)tif->tif_dir.td_row); return 0; } - tif->tif_rawcc = nCompressedBytes; + tif->tif_rawcc = (tmsize_t)nCompressedBytes; if (!TIFFFlushData1(tif)) return 0; @@ -485,7 +487,7 @@ static int ZIPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) appropriately even before we simplify it */ do { - uInt avail_in_before = TIFF_CLAMP_UINT64_TO_INT32_MAX(cc); + uInt avail_in_before = TIFF_CLAMP_UINT64_TO_INT32_MAX((uint64_t)cc); sp->stream.avail_in = avail_in_before; if (deflate(&sp->stream, Z_NO_FLUSH) != Z_OK) { @@ -499,7 +501,7 @@ static int ZIPEncode(TIFF *tif, uint8_t *bp, tmsize_t cc, uint16_t s) return 0; sp->stream.next_out = tif->tif_rawdata; sp->stream.avail_out = - TIFF_CLAMP_UINT64_TO_INT32_MAX(tif->tif_rawdatasize); + TIFF_CLAMP_UINT64_TO_INT32_MAX((uint64_t)tif->tif_rawdatasize); } cc -= (avail_in_before - sp->stream.avail_in); } while (cc > 0); @@ -682,7 +684,7 @@ static const TIFFField zipFields[] = { static void *TIFF_zalloc(void *opaque, unsigned int items, unsigned int size) { static const char module[] = "TIFF_zalloc"; - TIFF *tif = opaque; + TIFF *tif = (TIFF *)opaque; if (items > ~(size_t)0 / size) { @@ -698,6 +700,13 @@ static void TIFF_zfree(void *opaque, void *ptr) _TIFFfreeExt((TIFF *)opaque, ptr); } +static uint64_t ZIPGetMaxCompressionRatio(TIFF *tif) +{ + (void)tif; + /* cf https://zlib.net/zlib_tech.html */ + return 1032; +} + int TIFFInitZIP(TIFF *tif, int scheme) { static const char module[] = "TIFFInitZIP"; @@ -764,6 +773,7 @@ int TIFFInitZIP(TIFF *tif, int scheme) tif->tif_encodestrip = ZIPEncode; tif->tif_encodetile = ZIPEncode; tif->tif_cleanup = ZIPCleanup; + tif->tif_getmaxcompressionratio = ZIPGetMaxCompressionRatio; /* * Setup predictor setup. */ diff --git a/contrib/libs/libtiff/tif_zstd.c b/contrib/libs/libtiff/tif_zstd.c index c828dbf7432..9a0b969e31a 100644 --- a/contrib/libs/libtiff/tif_zstd.c +++ b/contrib/libs/libtiff/tif_zstd.c @@ -159,13 +159,13 @@ static int ZSTDDecode(TIFF *tif, uint8_t *op, tmsize_t occ, uint16_t s) memset(op + out_buffer.pos, 0, out_buffer.size - out_buffer.pos); TIFFErrorExtR(tif, module, "Not enough data at scanline %lu (short %lu bytes)", - (unsigned long)tif->tif_row, + (unsigned long)tif->tif_dir.td_row, (unsigned long)((size_t)occ - out_buffer.pos)); return 0; } tif->tif_rawcp += in_buffer.pos; - tif->tif_rawcc -= in_buffer.pos; + tif->tif_rawcc -= (tmsize_t)in_buffer.pos; return 1; } @@ -287,7 +287,7 @@ static int ZSTDPostEncode(TIFF *tif) } if (sp->out_buffer.pos > 0) { - tif->tif_rawcc = sp->out_buffer.pos; + tif->tif_rawcc = (tmsize_t)sp->out_buffer.pos; if (!TIFFFlushData1(tif)) return 0; sp->out_buffer.dst = tif->tif_rawcp; diff --git a/contrib/libs/libtiff/tiffio.h b/contrib/libs/libtiff/tiffio.h index f9c206e32a8..bb8fbde248b 100644 --- a/contrib/libs/libtiff/tiffio.h +++ b/contrib/libs/libtiff/tiffio.h @@ -125,13 +125,13 @@ typedef void *thandle_t; /* client data handle */ */ /* reference white */ -#define D65_X0 (95.0470F) -#define D65_Y0 (100.0F) -#define D65_Z0 (108.8827F) +#define D65_X0 (95.0470f) +#define D65_Y0 (100.0f) +#define D65_Z0 (108.8827f) -#define D50_X0 (96.4250F) -#define D50_Y0 (100.0F) -#define D50_Z0 (82.4680F) +#define D50_X0 (96.4250f) +#define D50_Y0 (100.0f) +#define D50_Z0 (82.4680f) /* Structure for holding information about a display device. */ @@ -254,7 +254,7 @@ struct _TIFFRGBAImage * Macros for extracting components from the * packed ABGR form returned by TIFFReadRGBAImage. */ -#define TIFFGetR(abgr) ((abgr)&0xff) +#define TIFFGetR(abgr) ((abgr) & 0xff) #define TIFFGetG(abgr) (((abgr) >> 8) & 0xff) #define TIFFGetB(abgr) (((abgr) >> 16) & 0xff) #define TIFFGetA(abgr) (((abgr) >> 24) & 0xff) @@ -269,7 +269,7 @@ struct _TIFFRGBAImage typedef int (*TIFFInitMethod)(TIFF *, int); typedef struct { - char *name; + const char *name; uint16_t scheme; TIFFInitMethod init; } TIFFCodec; @@ -584,6 +584,7 @@ extern int TIFFReadRGBAImageOriented(TIFF *, uint32_t, uint32_t, uint32_t *, tmsize_t cc); extern tmsize_t TIFFWriteRawTile(TIFF *tif, uint32_t tile, void *data, tmsize_t cc); + extern uint64_t TIFFGetMaxCompressionRatio(TIFF *tif); extern int TIFFDataWidth( TIFFDataType); /* table of tag datatype widths within TIFF file. */ extern void TIFFSetWriteOffset(TIFF *tif, toff_t off); @@ -659,7 +660,7 @@ extern int TIFFReadRGBAImageOriented(TIFF *, uint32_t, uint32_t, uint32_t *, unsigned short field_bit; /* bit in fieldsset bit vector */ unsigned char field_oktochange; /* if true, can change while writing */ unsigned char field_passcount; /* if true, pass dir count on set */ - char *field_name; /* ASCII name */ + const char *field_name; /* ASCII name */ } TIFFFieldInfo; extern int TIFFMergeFieldInfo(TIFF *, const TIFFFieldInfo[], uint32_t); diff --git a/contrib/libs/libtiff/tiffiop.h b/contrib/libs/libtiff/tiffiop.h index 80c04c279be..4f4a29cfa5f 100644 --- a/contrib/libs/libtiff/tiffiop.h +++ b/contrib/libs/libtiff/tiffiop.h @@ -38,6 +38,7 @@ #include <sys/types.h> #endif +#include <math.h> #include <string.h> #ifdef HAVE_ASSERT_H @@ -94,6 +95,7 @@ typedef int (*TIFFSeekMethod)(TIFF *, uint32_t); typedef void (*TIFFPostMethod)(TIFF *tif, uint8_t *buf, tmsize_t size); typedef uint32_t (*TIFFStripMethod)(TIFF *, uint32_t); typedef void (*TIFFTileMethod)(TIFF *, uint32_t *, uint32_t *); +typedef uint64_t (*TIFFGetMaxCompressionRatioMethod)(TIFF *); struct TIFFOffsetAndDirNumber { @@ -143,13 +145,18 @@ struct tiff 0x800000U /* read buffer (tif_rawdata) points into mmap() memory */ #define TIFF_DEFERSTRILELOAD \ 0x1000000U /* defer strip/tile offset/bytecount array loading. */ -#define TIFF_LAZYSTRILELOAD \ - 0x2000000U /* lazy/ondemand loading of strip/tile offset/bytecount values. \ - Only used if TIFF_DEFERSTRILELOAD is set and in read-only \ - mode */ +#define TIFF_LAZYSTRILELOAD_DONE \ + 0x2000000U /* set when lazy/ondemand loading of strip/tile \ + offset/bytecount values has been done. Only used if \ + TIFF_DEFERSTRILELOAD is set and in read-only mode */ #define TIFF_CHOPPEDUPARRAYS \ 0x4000000U /* set when allocChoppedUpStripArrays() has modified strip \ array */ +#define TIFF_LAZYSTRILELOAD_ASKED \ + 0x8000000U /* set when lazy/ondemand loading of strip/tile \ + offset/bytecount values has been requested on opening ('O' \ + flag) */ + uint64_t tif_diroff; /* file offset of current directory */ uint64_t tif_nextdiroff; /* file offset of following directory */ uint64_t tif_lastdiroff; /* file offset of last directory written so far */ @@ -162,7 +169,6 @@ struct tiff tif_customdir; /* custom IFDs are separated from the main ones */ TIFFHeaderUnion tif_header; /* file's header block Classic/BigTIFF union */ uint16_t tif_header_size; /* file's header block and its length */ - uint32_t tif_row; /* current scanline */ /* There are IFDs in the file and an "active" IFD in memory, * from which fields are "set" and "get". @@ -184,7 +190,6 @@ struct tiff * - TIFF_NON_EXISTENT_DIR_NUMBER means 'dont know number of IFDs'. * - 0 means 'empty file opened for writing, but no IFD written yet' */ tdir_t tif_curdircount; - uint32_t tif_curstrip; /* current strip for read/write */ uint64_t tif_curoff; /* current offset for read/write */ uint64_t tif_lastvalidoff; /* last valid offset allowed for rewrite in place. Used only by TIFFAppendToStrip() */ @@ -192,10 +197,6 @@ struct tiff /* SubIFD support */ uint16_t tif_nsubifd; /* remaining subifds to write */ uint64_t tif_subifdoff; /* offset for patching SubIFD link */ - /* tiling support */ - uint32_t tif_col; /* current column (offset by row too) */ - uint32_t tif_curtile; /* current tile for read/write */ - tmsize_t tif_tilesize; /* # of bytes in a tile */ /* compression scheme hooks */ int tif_decodestatus; TIFFBoolMethod tif_fixuptags; /* called in TIFFReadDirectory */ @@ -216,10 +217,10 @@ struct tiff TIFFVoidMethod tif_cleanup; /* cleanup state routine */ TIFFStripMethod tif_defstripsize; /* calculate/constrain strip size */ TIFFTileMethod tif_deftilesize; /* calculate/constrain tile size */ - uint8_t *tif_data; /* compression scheme private data */ + /* returns maximum compression ratio for current compression method */ + TIFFGetMaxCompressionRatioMethod tif_getmaxcompressionratio; + uint8_t *tif_data; /* compression scheme private data */ /* input/output buffering */ - tmsize_t tif_scanlinesize; /* # of bytes in a scanline */ - tmsize_t tif_scanlineskew; /* scanline skew for reading strips */ uint8_t *tif_rawdata; /* raw data buffer */ tmsize_t tif_rawdatasize; /* # of bytes in raw data buffer */ tmsize_t tif_rawdataoff; /* rawdata offset within strip */ @@ -295,13 +296,15 @@ struct TIFFOpenOptions * Default Read/Seek/Write definitions. */ #ifndef ReadOK -#define ReadOK(tif, buf, size) (TIFFReadFile((tif), (buf), (size)) == (size)) +#define ReadOK(tif, buf, size) \ + (TIFFReadFile((tif), (buf), (size)) == (tmsize_t)(size)) #endif #ifndef SeekOK #define SeekOK(tif, off) _TIFFSeekOK(tif, off) #endif #ifndef WriteOK -#define WriteOK(tif, buf, size) (TIFFWriteFile((tif), (buf), (size)) == (size)) +#define WriteOK(tif, buf, size) \ + (TIFFWriteFile((tif), (buf), (size)) == (tmsize_t)(size)) #endif /* NB: the uint32_t casts are to silence certain ANSI-C compilers */ @@ -315,19 +318,19 @@ struct TIFFOpenOptions (((uint32_t)(x) / (uint32_t)(y)) + \ ((((uint32_t)(x) % (uint32_t)(y)) != 0) ? 1 : 0)) #define TIFFhowmany8_32(x) \ - (((x)&0x07) ? ((uint32_t)(x) >> 3) + 1 : (uint32_t)(x) >> 3) -#define TIFFroundup_32(x, y) (TIFFhowmany_32(x, y) * (y)) + (((x) & 0x07) ? ((uint32_t)(x) >> 3) + 1 : (uint32_t)(x) >> 3) +#define TIFFroundup_32(x, y) ((uint32_t)(TIFFhowmany_32(x, y) * (uint32_t)(y))) #define TIFFhowmany_64(x, y) \ ((((uint64_t)(x)) + (((uint64_t)(y)) - 1)) / ((uint64_t)(y))) #define TIFFhowmany8_64(x) \ - (((x)&0x07) ? ((uint64_t)(x) >> 3) + 1 : (uint64_t)(x) >> 3) -#define TIFFroundup_64(x, y) (TIFFhowmany_64(x, y) * (y)) + (((x) & 0x07) ? ((uint64_t)(x) >> 3) + 1 : (uint64_t)(x) >> 3) +#define TIFFroundup_64(x, y) ((uint64_t)(TIFFhowmany_64(x, y) * (uint64_t)(y))) /* Safe multiply which returns zero if there is an *unsigned* integer overflow. * This macro is not safe for *signed* integer types */ #define TIFFSafeMultiply(t, v, m) \ - ((((t)(m) != (t)0) && (((t)(((v) * (m)) / (m))) == (t)(v))) \ - ? (t)((v) * (m)) \ + ((((t)(m) != (t)0) && (((t)((((t)(v)) * ((t)(m))) / ((t)(m)))) == (t)(v))) \ + ? (t)(((t)(v)) * ((t)(m))) \ : (t)0) #define TIFFmax(A, B) ((A) > (B) ? (A) : (B)) @@ -335,6 +338,10 @@ struct TIFFOpenOptions #define TIFFArrayCount(a) (sizeof(a) / sizeof((a)[0])) +/* Float/double equality macros that suppress -Wfloat-equal warnings */ +#define TIFF_FLOAT_EQ(x, y) (!(fabsf((x) - (y)) > 0.0f)) +#define TIFF_DOUBLE_EQ(x, y) (!(fabs((x) - (y)) > 0.0)) + /* Support for large files. @@ -355,7 +362,7 @@ struct TIFFOpenOptions */ #if defined(HAVE_FSEEKO) #define fseek(stream, offset, whence) fseeko(stream, offset, whence) -#define ftell(stream, offset, whence) ftello(stream, offset, whence) +#define ftell(stream) ftello(stream) #endif #endif #if defined(_WIN32) && \ @@ -459,9 +466,18 @@ extern "C" extern uint32_t _TIFFMultiply32(TIFF *, uint32_t, uint32_t, const char *); extern uint64_t _TIFFMultiply64(TIFF *, uint64_t, uint64_t, const char *); + extern uint64_t _TIFFAdd64(TIFF *, uint64_t, uint64_t, const char *); extern tmsize_t _TIFFMultiplySSize(TIFF *, tmsize_t, tmsize_t, const char *); + extern tmsize_t _TIFFAddSSize(TIFF *, tmsize_t, tmsize_t, const char *); extern tmsize_t _TIFFCastUInt64ToSSize(TIFF *, uint64_t, const char *); + extern uint32_t _TIFFCastUInt64ToUInt32(TIFF *, uint64_t, const char *); + extern uint64_t _TIFFComputeRowSize64(TIFF *, uint32_t, uint16_t, uint16_t, + const char *); + extern tmsize_t _TIFFComputeRowOffset(TIFF *, tmsize_t rowstride, uint32_t, + const char *); + extern uint64_t _TIFFComputeBitOffset(TIFF *, uint32_t, uint16_t, uint16_t, + const char *); extern void *_TIFFCheckMalloc(TIFF *, tmsize_t, tmsize_t, const char *); extern void *_TIFFCheckRealloc(TIFF *, void *, tmsize_t, tmsize_t, const char *); @@ -471,6 +487,8 @@ extern "C" extern void _TIFFCleanupIFDOffsetAndNumberMaps(TIFF *tif); + extern uint64_t _TIFFStrileSize64(TIFF *tif, uint32_t nrows, int isStrip); + extern tmsize_t _TIFFReadEncodedStripAndAllocBuffer(TIFF *tif, uint32_t strip, void **buf, diff --git a/contrib/libs/libtiff/tiffvers.h b/contrib/libs/libtiff/tiffvers.h index 5b86ac165d2..57821df8320 100644 --- a/contrib/libs/libtiff/tiffvers.h +++ b/contrib/libs/libtiff/tiffvers.h @@ -8,7 +8,7 @@ * Furthermore, configure_file variables of type "" are * modified by clang-format and won't be substituted by CMake. */ -#define TIFFLIB_VERSION_STR "LIBTIFF, Version 4.7.1\nCopyright (c) 1988-1996 Sam Leffler\nCopyright (c) 1991-1996 Silicon Graphics, Inc." +#define TIFFLIB_VERSION_STR "LIBTIFF, Version 4.7.2\nCopyright (c) 1988-1996 Sam Leffler\nCopyright (c) 1991-1996 Silicon Graphics, Inc." /* * This define can be used in code that requires * compilation-related definitions specific to a @@ -16,13 +16,13 @@ * version checking should be done based on the * string returned by TIFFGetVersion. */ -#define TIFFLIB_VERSION 20250911 +#define TIFFLIB_VERSION 20260627 /* The following defines have been added in 4.5.0 */ #define TIFFLIB_MAJOR_VERSION 4 #define TIFFLIB_MINOR_VERSION 7 -#define TIFFLIB_MICRO_VERSION 1 -#define TIFFLIB_VERSION_STR_MAJ_MIN_MIC "4.7.1" +#define TIFFLIB_MICRO_VERSION 2 +#define TIFFLIB_VERSION_STR_MAJ_MIN_MIC "4.7.2" /* Macro added in 4.5.0. Returns TRUE if the current libtiff version is * greater or equal to major.minor.micro diff --git a/contrib/libs/libtiff/ya.make b/contrib/libs/libtiff/ya.make index 3cf767876b6..b1fdf7bf784 100644 --- a/contrib/libs/libtiff/ya.make +++ b/contrib/libs/libtiff/ya.make @@ -10,9 +10,9 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -VERSION(4.7.1) +VERSION(4.7.2) -ORIGINAL_SOURCE(https://gitlab.com/api/v4/projects/libtiff%2Flibtiff/repository/archive.tar.gz?sha=v4.7.1) +ORIGINAL_SOURCE(https://gitlab.com/api/v4/projects/libtiff%2Flibtiff/repository/archive.tar.gz?sha=v4.7.2) PEERDIR( contrib/libs/libjpeg-turbo diff --git a/contrib/libs/opentelemetry-proto/.yandex_meta/__init__.py b/contrib/libs/opentelemetry-proto/.yandex_meta/__init__.py index 34a3f3fd550..d07e4060761 100644 --- a/contrib/libs/opentelemetry-proto/.yandex_meta/__init__.py +++ b/contrib/libs/opentelemetry-proto/.yandex_meta/__init__.py @@ -13,6 +13,7 @@ def post_install(self): GRPC=True, PROTO_NAMESPACE=ArcPath(self.arcdir, GLOBAL=True), PY_NAMESPACE=".", + INCLUDE_TAGS=["TS_PROTO", "TS_PREPARE_DEPS"], ) diff --git a/contrib/libs/opentelemetry-proto/ya.make b/contrib/libs/opentelemetry-proto/ya.make index d1a48612fac..cea01da4ebf 100644 --- a/contrib/libs/opentelemetry-proto/ya.make +++ b/contrib/libs/opentelemetry-proto/ya.make @@ -17,6 +17,11 @@ PROTO_NAMESPACE( contrib/libs/opentelemetry-proto ) +INCLUDE_TAGS( + TS_PREPARE_DEPS + TS_PROTO +) + GRPC() SRCS( diff --git a/contrib/libs/zstd06/common/bitstream.h b/contrib/libs/zstd06/common/bitstream.h index 97fc621579c..d6ff81bfdf4 100644 --- a/contrib/libs/zstd06/common/bitstream.h +++ b/contrib/libs/zstd06/common/bitstream.h @@ -52,6 +52,7 @@ extern "C" { ******************************************/ #include "mem.h" /* unaligned access routines */ #include "error_private.h" /* error codes and messages */ +#include <assert.h> /*========================================= @@ -154,6 +155,8 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); ****************************************************************/ MEM_STATIC unsigned BIT_highbit32 (register U32 val) { + assert(val != 0); + { # if defined(_MSC_VER) /* Visual */ unsigned long r=0; _BitScanReverse ( &r, val ); @@ -172,6 +175,7 @@ MEM_STATIC unsigned BIT_highbit32 (register U32 val) r = DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; return r; # endif + } } /*===== Local Constants =====*/ diff --git a/contrib/libs/zstd06/common/zstd_internal.h b/contrib/libs/zstd06/common/zstd_internal.h index 2ce2b18cf16..0f73e12dd7f 100644 --- a/contrib/libs/zstd06/common/zstd_internal.h +++ b/contrib/libs/zstd06/common/zstd_internal.h @@ -39,6 +39,7 @@ #include "mem.h" #include "error_private.h" #include "zstd_static.h" +#include <assert.h> /*-************************************* @@ -163,6 +164,8 @@ MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length) MEM_STATIC unsigned ZSTD_highbit(U32 val) { + assert(val != 0); + { # if defined(_MSC_VER) /* Visual */ unsigned long r=0; _BitScanReverse(&r, val); @@ -181,6 +184,7 @@ MEM_STATIC unsigned ZSTD_highbit(U32 val) r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; return r; # endif + } } diff --git a/contrib/libs/zstd06/compress/zstd_compress.c b/contrib/libs/zstd06/compress/zstd_compress.c index 1bb75c68ccf..d61227ef6c1 100644 --- a/contrib/libs/zstd06/compress/zstd_compress.c +++ b/contrib/libs/zstd06/compress/zstd_compress.c @@ -183,7 +183,8 @@ void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) { - U32 const srcLog = ZSTD_highbit((U32)(rSize)-1) + 1; + U32 const srcLog = + MAX(ZSTD_HASHLOG_MIN, (rSize==1) ? 1 : ZSTD_highbit((U32)(rSize)-1) + 1); if (params->windowLog > srcLog) params->windowLog = srcLog; } } if (params->hashLog > params->windowLog) params->hashLog = params->windowLog; diff --git a/contrib/python/scramp/.dist-info/METADATA b/contrib/python/scramp/.dist-info/METADATA index 3551c560d9d..354f0631bb7 100644 --- a/contrib/python/scramp/.dist-info/METADATA +++ b/contrib/python/scramp/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: scramp -Version: 1.4.11 +Version: 1.4.12 Summary: An implementation of the SCRAM protocol. Project-URL: Homepage, https://codeberg.org/tlocke/scramp Author: The Contributors @@ -54,6 +54,7 @@ Scramp supports the following mechanisms: * [OpenSSF Scorecard](#openssf-scorecard) * [Doing A Release Of Scramp](#doing-a-release-of-scramp) * [Release Notes](#release-notes) + * [Version 1.4.12, 2026-07-05](#version-1412-2026-07-05) * [Version 1.4.11, 2026-07-04](#version-1411-2026-07-04) * [Version 1.4.10, 2026-06-27](#version-1410-2026-06-27) * [Version 1.4.9, 2026-06-19](#version-149-2026-06-19) @@ -386,6 +387,11 @@ Run `tox` to make sure all tests pass, then update the release notes, then do: ## Release Notes +### Version 1.4.12, 2026-07-05 + +- Check for invalid escape chars in username. + + ### Version 1.4.11, 2026-07-04 - Make sure all errors detected by the server get returned to the client as valid errors. diff --git a/contrib/python/scramp/README.md b/contrib/python/scramp/README.md index 63260c0f265..abd5f1ebf18 100644 --- a/contrib/python/scramp/README.md +++ b/contrib/python/scramp/README.md @@ -33,6 +33,7 @@ Scramp supports the following mechanisms: * [OpenSSF Scorecard](#openssf-scorecard) * [Doing A Release Of Scramp](#doing-a-release-of-scramp) * [Release Notes](#release-notes) + * [Version 1.4.12, 2026-07-05](#version-1412-2026-07-05) * [Version 1.4.11, 2026-07-04](#version-1411-2026-07-04) * [Version 1.4.10, 2026-06-27](#version-1410-2026-06-27) * [Version 1.4.9, 2026-06-19](#version-149-2026-06-19) @@ -365,6 +366,11 @@ Run `tox` to make sure all tests pass, then update the release notes, then do: ## Release Notes +### Version 1.4.12, 2026-07-05 + +- Check for invalid escape chars in username. + + ### Version 1.4.11, 2026-07-04 - Make sure all errors detected by the server get returned to the client as valid errors. diff --git a/contrib/python/scramp/scramp/core.py b/contrib/python/scramp/scramp/core.py index ba53424634f..3518975852f 100644 --- a/contrib/python/scramp/scramp/core.py +++ b/contrib/python/scramp/scramp/core.py @@ -445,13 +445,31 @@ def _parse_message(msg, desc, *expected_attr_sets): ) +ESCAPE_EQUALS = "=3D" +ESCAPE_COMMA = "=2C" + + +def _username_escape(username): + return username.replace("=", ESCAPE_EQUALS).replace(",", ESCAPE_COMMA) + + +def _username_unescape(username): + for i in (i for i, c in enumerate(username) if c == "="): + if username[i : i + 3] not in (ESCAPE_EQUALS, ESCAPE_COMMA): + raise ScramException( + "An '=' in a username must be followed by '3D', or '2C'", + SERVER_ERROR_INVALID_USERNAME_ENCODING, + ) + return username.replace(ESCAPE_COMMA, ",").replace(ESCAPE_EQUALS, "=") + + def _get_client_first(username, c_nonce, channel_binding, use_binding): try: prepped_u = saslprep(username) except ScramException as e: raise ScramException(e.args[0], SERVER_ERROR_INVALID_USERNAME_ENCODING) - u = prepped_u.replace("=", "=3D").replace(",", "=2C") + u = _username_escape(prepped_u) bare = ",".join((f"n={u}", f"r={c_nonce}")) _, gs2_header = _make_gs2_header(channel_binding, use_binding) @@ -538,7 +556,7 @@ def _set_client_first(client_first, s_nonce, channel_binding, use_binding): c_nonce = msg["r"] nonce = c_nonce + s_nonce - user = msg["n"].replace("=2C", ",").replace("=3D", "=") + user = _username_unescape(msg["n"]) return nonce, user, client_first_bare, upgrade_mechanism diff --git a/contrib/python/scramp/ya.make b/contrib/python/scramp/ya.make index cf5785c2ea7..5b612dfca89 100644 --- a/contrib/python/scramp/ya.make +++ b/contrib/python/scramp/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.4.11) +VERSION(1.4.12) LICENSE(MIT-0) diff --git a/contrib/tools/cython/.dist-info/METADATA b/contrib/tools/cython/.dist-info/METADATA index 7ee86c1f0ca..76194acc38d 100644 --- a/contrib/tools/cython/.dist-info/METADATA +++ b/contrib/tools/cython/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: Cython -Version: 3.2.5 +Version: 3.2.8 Summary: The Cython compiler for writing C extensions in the Python language. Home-page: https://cython.org/ Author: Robert Bradshaw, Stefan Behnel, David Woods, Greg Ewing, et al. @@ -64,82 +64,21 @@ to install an uncompiled (slower) version of Cython with:: .. _Pyrex: https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ -3.2.5 (2026-05-23) +3.2.8 (2026-06-30) ================== Bugs fixed ---------- -* A compile failure was fixed when using the walrus operator inside of try-except. - (Github issue https://github.com/cython/cython/issues/7462) +* Assigning a Python 3.14+ ``.__annotate__`` function to a Cython compiled function no longer + evaluates annotations eagerly. Fixes a regression with ``@functools.wraps()`` in Cython 3.2.6. + Patch by Jelle Zijlstra. (Github issue https://github.com/cython/cython/issues/7767) -* Expressions with side-effects as object argument to ``isinstance()`` could get - evaluated multiple times, e.g. when they use the walrus operator. - (Github issue https://github.com/cython/cython/issues/7670) +* In freethreading Python, the necessary object keep-alive while executing user code in + ``.__dealloc__()`` uses a safer scheme. + Patch by Yaxing Cai. (Github issue https://github.com/cython/cython/issues/7769) -* Several problems generating the shared utility module were resolved, including - a performance regression with memory views. - (Github issues https://github.com/cython/cython/issues/7487, https://github.com/cython/cython/issues/7497, https://github.com/cython/cython/issues/7504, https://github.com/cython/cython/issues/7558) +* The local function state used by ``sys.monitoring`` read from uninitialised memory. + (Github issue https://github.com/cython/cython/issues/7774) -* Some GC and refcounting issues were resolved for Cython functions in the Limited API. - (Github issue https://github.com/cython/cython/issues/7594) - -* Refcounting errors and error handling issues were resolved in some rare error handling cases. - (Github issues https://github.com/cython/cython/issues/7597, https://github.com/cython/cython/issues/7599, https://github.com/cython/cython/issues/7612, https://github.com/cython/cython/issues/7673) - -* Using ``cython.pymutex`` in an extension type with ``cdef`` methods generated - invalid C code missing the required ``PyMutex`` declarations. - (Github issue https://github.com/cython/cython/issues/6995) - -* Calling ``.get_frame()`` on Cython coroutines could crash in freethreading Python. - (Github issue https://github.com/cython/cython/issues/7632) - -* The vectorcall protocol was not used correctly in ``.throw()`` of Cython coroutines - when raising the exception only by type (without value or traceback). - (Github issue https://github.com/cython/cython/issues/7677) - -* A problem with cpdef enums in the Limited API of Python 3.11+ was resolved. - (Github issue https://github.com/cython/cython/issues/7503) - -* Unicode predicates like ``.isdigit()`` are now allowed to fail in the Limited API. - (Github issue https://github.com/cython/cython/issues/7602) - -* Conditional expressions mixing Python float and int object types could accidentally - infer float as the common result type, instead of treating both independently. - -* Using ``sizeof()`` in the size declarations of ``extern`` arrays failed. - (Github issue https://github.com/cython/cython/issues/7451) - -* Enabling profiling generated invalid C code for non-Python return tuples. - (Github issue https://github.com/cython/cython/issues/7580) - -* ``abs()`` on C ``long long`` values could generate invalid C code. - -* When the ``CYTHON_AVOID_BORROWED_REFS`` C macro is enabled, e.g. in GraalPython, - dict iteration could fail to compile. - Patch by Michael Šimáček. (Github issue https://github.com/cython/cython/issues/7631) - -* A C compiler warning about unused functions was resolved. - (Github issue https://github.com/cython/cython/issues/7560) - -* The ``py_safe_call_once()`` C++ mutex helper function in ``libcpp.mutex`` - failed to compile. - (Github issue https://github.com/cython/cython/issues/7585) - -* The ``cpython.array`` declarations were adapted for Python 3.15. Direct access - to the C struct of the ``array.array`` data type might require user code changes. - (Github issue https://github.com/cython/cython/issues/7659) - -* Spaces in file paths written to the ``depfile`` were not escaped. - Patch by Loïc Estève. (Github issue https://github.com/cython/cython/issues/7423) - -* Cython's cache failed to restore multi-file results. - (Github issue https://github.com/cython/cython/issues/7559) - -* Using Tempita from its command line failed with a name error. - (Github issue https://github.com/cython/cython/issues/7567) - -Other changes -------------- - -* The known builtin types were updated for Py3.15. +* The ``.ag_running`` flag attribute of async generators was always false on big-endian systems. diff --git a/contrib/tools/cython/.yandex_meta/override.nix b/contrib/tools/cython/.yandex_meta/override.nix index aea78547bf9..fd901b25082 100644 --- a/contrib/tools/cython/.yandex_meta/override.nix +++ b/contrib/tools/cython/.yandex_meta/override.nix @@ -1,10 +1,10 @@ pkgs: attrs: with pkgs; with pkgs.python311.pkgs; with attrs; rec { - version = "3.2.5"; + version = "3.2.8"; src = fetchPypi { pname = "cython"; inherit version; - hash = "sha256-PdQuTPNq0V8mW9/sIzfMAMaIyOttN0/9E7sZQ3wnu6E="; + hash = "sha256-9PI6VrJSIaBvkYF/6PMRSri0ik+scxh9u2S8LEqHlh8="; }; patches = []; diff --git a/contrib/tools/cython/CHANGES.rst b/contrib/tools/cython/CHANGES.rst index a495a987b4b..b97a2aeb4dc 100644 --- a/contrib/tools/cython/CHANGES.rst +++ b/contrib/tools/cython/CHANGES.rst @@ -2,6 +2,55 @@ Cython Changelog ================ +3.2.8 (2026-06-30) +================== + +Bugs fixed +---------- + +* Assigning a Python 3.14+ ``.__annotate__`` function to a Cython compiled function no longer + evaluates annotations eagerly. Fixes a regression with ``@functools.wraps()`` in Cython 3.2.6. + Patch by Jelle Zijlstra. (Github issue :issue:`7767`) + +* In freethreading Python, the necessary object keep-alive while executing user code in + ``.__dealloc__()`` uses a safer scheme. + Patch by Yaxing Cai. (Github issue :issue:`7769`) + +* The local function state used by ``sys.monitoring`` read from uninitialised memory. + (Github issue :issue:`7774`) + +* The ``.ag_running`` flag attribute of async generators was always false on big-endian systems. + + +3.2.6 (2026-06-24) +================== + +Bugs fixed +---------- + +* ``@functools.wraps()`` was broken in Py3.14+ for Cython compiled functions. + (Github issue :issue:`7675`) + +* A double-free in the t-string code was fixed. + (Github issue :issue:`7712`) + +* The ``-`` operator declarations for iterators in ``libcpp.vector`` we corrected. + Patch by Vadim Markovtsev. (Github issue :issue:`7717`) + +* The shared utility code module no longer uses a temporary file path that + changed the C code on each generation. + (Github issue :issue:`7723`) + +* On 32 bit platforms, cached constants are no longer made immortal during module import. + (Github issue :issue:`7744`) + +Other changes +------------- + +* Binary wheels are now built with ``-DNDEBUG`` to discard runtime assertions from CPython's + inline functions. + + 3.2.5 (2026-05-23) ================== diff --git a/contrib/tools/cython/Cython/Build/Dependencies.py b/contrib/tools/cython/Cython/Build/Dependencies.py index 3726a72b529..d45e53d832b 100644 --- a/contrib/tools/cython/Cython/Build/Dependencies.py +++ b/contrib/tools/cython/Cython/Build/Dependencies.py @@ -52,7 +52,7 @@ def extended_iglob(pattern): # because '/' is generally common for relative paths. if '**/' in pattern or os.sep == '\\' and '**\\' in pattern: seen = set() - first, rest = re.split(r'\*\*[%s]' % ('/\\\\' if os.sep == '\\' else '/'), pattern, 1) + first, rest = re.split(r'\*\*[%s]' % ('/\\\\' if os.sep == '\\' else '/'), pattern, maxsplit=1) if first: first = iglob(first + os.sep) else: diff --git a/contrib/tools/cython/Cython/Build/SharedModule.py b/contrib/tools/cython/Cython/Build/SharedModule.py index fed6263cd89..45da4b46238 100644 --- a/contrib/tools/cython/Cython/Build/SharedModule.py +++ b/contrib/tools/cython/Cython/Build/SharedModule.py @@ -1,13 +1,10 @@ import os -import re -import shutil -import tempfile from Cython.Compiler import ( MemoryView, Code, Options, Pipeline, Errors, Main, Symtab ) from Cython.Compiler.StringEncoding import EncodedString -from Cython.Compiler.Scanning import FileSourceDescriptor +from Cython.Compiler.Scanning import SharedUtilitySourceDescriptor def create_shared_library_pipeline(context, scope, options, result): @@ -72,23 +69,17 @@ def generate_shared_module(options): Errors.open_listing_file(None) dest_c_file = options.shared_c_file_path + pyx_file = os.path.splitext(dest_c_file)[0] + '.pyx' module_name = os.path.splitext(os.path.basename(dest_c_file))[0] context = Main.Context.from_options(options) scope = Symtab.ModuleScope('MemoryView', parent_module = None, context = context, is_package=False) - with tempfile.TemporaryDirectory() as tmpdirname: - pyx_file = os.path.join(tmpdirname, f'{module_name}.pyx') - c_file = os.path.join(tmpdirname, f'{module_name}.c') - with open(pyx_file, 'w'): - pass - source_desc = FileSourceDescriptor(pyx_file) - comp_src = Main.CompilationSource(source_desc, EncodedString(module_name), os.getcwd()) - result = Main.create_default_resultobj(comp_src, options) + source_desc = SharedUtilitySourceDescriptor(pyx_file) + comp_src = Main.CompilationSource(source_desc, EncodedString(module_name), os.getcwd()) + result = Main.create_default_resultobj(comp_src, options) - pipeline = create_shared_library_pipeline(context, scope, options, result) - err, enddata = Pipeline.run_pipeline(pipeline, comp_src) - if err is None: - shutil.copy(c_file, dest_c_file) + pipeline = create_shared_library_pipeline(context, scope, options, result) + err, enddata = Pipeline.run_pipeline(pipeline, comp_src) return err, enddata diff --git a/contrib/tools/cython/Cython/Compiler/Code.py b/contrib/tools/cython/Cython/Compiler/Code.py index 630683bf166..ed783164ff8 100644 --- a/contrib/tools/cython/Cython/Compiler/Code.py +++ b/contrib/tools/cython/Cython/Compiler/Code.py @@ -2429,19 +2429,26 @@ class GlobalState: writer.putln("{") writer.putln(f"PyObject **table = {array_cname};") writer.putln(f"for (Py_ssize_t i=0; i<{constant_count}; ++i) {{") - writer.putln("#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING") + writer.putln("#if PY_VERSION_HEX >= 0x030F0000") + writer.putln("PyUnstable_SetImmortal(table[i]);") + writer.putln("#elif CYTHON_COMPILING_IN_CPYTHON_FREETHREADING") # We don't want to set the refcount on shared constants (e.g. cached integers) # because setting the refcount isn't thread-safe. The chances are that most of the constants # that this applies to are already immortal though so that isn't a great loss. + # Overflow, e.g. on 32 bit systems. + writer.putln("if ((PY_SSIZE_T_MAX <= _Py_IMMORTAL_REFCNT_LOCAL)) break;") writer.putln("#if PY_VERSION_HEX < 0x030E0000") writer.putln("if (_Py_IsOwnedByCurrentThread(table[i]) && Py_REFCNT(table[i]) == 1)") writer.putln("#else") writer.putln("if (PyUnstable_Object_IsUniquelyReferenced(table[i]))") writer.putln("#endif") writer.putln("{") - writer.putln("Py_SET_REFCNT(table[i], _Py_IMMORTAL_REFCNT_LOCAL);") + # Go one higher than we think we need to because of a bug in SET_REFCNT check in CPython + writer.putln("Py_SET_REFCNT(table[i], ((Py_ssize_t)_Py_IMMORTAL_REFCNT_LOCAL + 1));") writer.putln("}") writer.putln("#else") + # Overflow, e.g. on 32 bit systems. + writer.putln("if ((PY_SSIZE_T_MAX < _Py_IMMORTAL_INITIAL_REFCNT)) break;") writer.putln("Py_SET_REFCNT(table[i], _Py_IMMORTAL_INITIAL_REFCNT);") writer.putln("#endif") writer.putln("}") # for() diff --git a/contrib/tools/cython/Cython/Compiler/ModuleNode.py b/contrib/tools/cython/Cython/Compiler/ModuleNode.py index eb683f8249d..a14f3e9e2ee 100644 --- a/contrib/tools/cython/Cython/Compiler/ModuleNode.py +++ b/contrib/tools/cython/Cython/Compiler/ModuleNode.py @@ -2020,14 +2020,17 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not entry or not entry.is_special: return + code.globalstate.use_utility_code( + UtilityCode.load_cached("DeallocKeepAlive", "ExtensionTypes.c")) code.putln("{") code.putln("PyObject *etype, *eval, *etb;") code.putln("PyErr_Fetch(&etype, &eval, &etb);") - # increase the refcount while we are calling into user code - # to prevent recursive deallocation - code.putln("__Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1);") + # Keep the object alive while we are calling into user code, to prevent + # the user code from triggering recursive deallocation. On free-threading + # this must not use Py_SET_REFCNT() (see DeallocKeepAlive in ExtensionTypes.c). + code.putln("__Pyx_DeallocKeepAliveBegin(o);") code.putln("%s(o);" % entry.func_cname) - code.putln("__Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1);") + code.putln("__Pyx_DeallocKeepAliveEnd(o);") code.putln("PyErr_Restore(etype, eval, etb);") code.putln("}") diff --git a/contrib/tools/cython/Cython/Compiler/Scanning.py b/contrib/tools/cython/Cython/Compiler/Scanning.py index d63c1263f27..d843960dd17 100644 --- a/contrib/tools/cython/Cython/Compiler/Scanning.py +++ b/contrib/tools/cython/Cython/Compiler/Scanning.py @@ -278,6 +278,15 @@ class StringSourceDescriptor(SourceDescriptor): return "<StringSourceDescriptor:%s>" % self.name +class SharedUtilitySourceDescriptor(FileSourceDescriptor): + """ + A specialized source descriptor for shared utility code only. Not part of public API. + """ + + def get_file_object(self, encoding=None, error_handling=None): + from io import StringIO + return StringIO('') + #------------------------------------------------------------------ class PyrexScanner(Scanner): diff --git a/contrib/tools/cython/Cython/Includes/libcpp/vector.pxd b/contrib/tools/cython/Cython/Includes/libcpp/vector.pxd index f7f75928640..14f453ba682 100644 --- a/contrib/tools/cython/Cython/Includes/libcpp/vector.pxd +++ b/contrib/tools/cython/Cython/Includes/libcpp/vector.pxd @@ -79,8 +79,8 @@ cdef extern from "<vector>" namespace "std" nogil: reverse_iterator operator--(int) reverse_iterator operator+(size_type) reverse_iterator operator-(size_type) - difference_type operator-(iterator) - difference_type operator-(const_iterator) + difference_type operator-(reverse_iterator) + difference_type operator-(const_reverse_iterator) bint operator==(reverse_iterator) bint operator==(const_reverse_iterator) bint operator!=(reverse_iterator) @@ -104,8 +104,8 @@ cdef extern from "<vector>" namespace "std" nogil: const_reverse_iterator operator--(int) const_reverse_iterator operator+(size_type) const_reverse_iterator operator-(size_type) - difference_type operator-(iterator) - difference_type operator-(const_iterator) + difference_type operator-(reverse_iterator) + difference_type operator-(const_reverse_iterator) bint operator==(reverse_iterator) bint operator==(const_reverse_iterator) bint operator!=(reverse_iterator) diff --git a/contrib/tools/cython/Cython/Shadow.py b/contrib/tools/cython/Cython/Shadow.py index 8339642e91b..1ec77baac3a 100644 --- a/contrib/tools/cython/Cython/Shadow.py +++ b/contrib/tools/cython/Cython/Shadow.py @@ -1,7 +1,7 @@ # cython.* namespace for pure mode. # Possible version formats: "3.1.0", "3.1.0a1", "3.1.0a1.dev0" -__version__ = "3.2.5" +__version__ = "3.2.8" # BEGIN shameless copy from Cython/minivect/minitypes.py diff --git a/contrib/tools/cython/Cython/Utility/AsyncGen.c b/contrib/tools/cython/Cython/Utility/AsyncGen.c index 4999487d91d..c2b9fe1d449 100644 --- a/contrib/tools/cython/Cython/Utility/AsyncGen.c +++ b/contrib/tools/cython/Cython/Utility/AsyncGen.c @@ -346,11 +346,19 @@ __Pyx_async_gen_self_method(PyObject *g, PyObject *arg) { } +static PyObject * +__Pyx_async_gen_get_running(__pyx_PyAsyncGenObject *o, void *context) { + CYTHON_UNUSED_VAR(context); + return __Pyx_NewRef(o->ag_running_async ? Py_True : Py_False); +} + + static PyGetSetDef __Pyx_async_gen_getsetlist[] = { {"__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name, PyDoc_STR("name of the async generator"), 0}, {"__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, PyDoc_STR("qualified name of the async generator"), 0}, + {"ag_running", (getter)__Pyx_async_gen_get_running, NULL, NULL, 0}, //REMOVED: {(char*) "ag_await", (getter)coro_get_cr_await, NULL, //REMOVED: (char*) PyDoc_STR("object being awaited on, or None")}, {0, 0, 0, 0, 0} /* Sentinel */ @@ -358,7 +366,6 @@ static PyGetSetDef __Pyx_async_gen_getsetlist[] = { static PyMemberDef __Pyx_async_gen_memberlist[] = { //REMOVED: {(char*) "ag_frame", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_frame), READONLY}, - {"ag_running", T_BOOL, offsetof(__pyx_PyAsyncGenObject, ag_running_async), READONLY, NULL}, //REMOVED: {(char*) "ag_code", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_code), READONLY}, //ADDED: "ag_await" {"ag_await", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, diff --git a/contrib/tools/cython/Cython/Utility/Coroutine.c b/contrib/tools/cython/Cython/Utility/Coroutine.c index 7d65bc73fe9..a9ee81d4f38 100644 --- a/contrib/tools/cython/Cython/Utility/Coroutine.c +++ b/contrib/tools/cython/Cython/Utility/Coroutine.c @@ -1654,6 +1654,10 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( Py_XINCREF(code); gen->gi_code = code; gen->gi_frame = NULL; +#if CYTHON_USE_SYS_MONITORING && (CYTHON_PROFILE || CYTHON_TRACE) + memset(gen->$monitoring_states_cname, 0, sizeof(gen->$monitoring_states_cname)); + gen->$monitoring_version_cname = 0; +#endif PyObject_GC_Track(gen); return gen; diff --git a/contrib/tools/cython/Cython/Utility/CythonFunction.c b/contrib/tools/cython/Cython/Utility/CythonFunction.c index 38e2f1cfc75..cbb7f743197 100644 --- a/contrib/tools/cython/Cython/Utility/CythonFunction.c +++ b/contrib/tools/cython/Cython/Utility/CythonFunction.c @@ -134,6 +134,7 @@ static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject //@requires: ModuleSetupCode.c::IncludeStructmemberH //@requires: ObjectHandling.c::PyObjectGetAttrStr //@requires: ObjectHandling.c::CachedMethodType +//@requires: ObjectHandling.c::PyObjectCallOneArg //@requires: ExtensionTypes.c::CallTypeTraverse //@requires: Synchronization.c::CriticalSections //@substitute: naming @@ -461,6 +462,8 @@ __Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { return result; } +static int __Pyx_CyFunction_set_annotate_in_dict_if_exists(PyObject *op_in, PyObject *value); /*proto*/ + static int __Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { CYTHON_UNUSED_VAR(context); @@ -475,9 +478,59 @@ __Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, vo __Pyx_BEGIN_CRITICAL_SECTION(op); __Pyx_Py_XDECREF_SET(op->func_annotations, value); __Pyx_END_CRITICAL_SECTION(); + if (unlikely(__Pyx_CyFunction_set_annotate_in_dict_if_exists((PyObject*) op, Py_None) < 0)) return -1; return 0; } +static int +__Pyx_CyFunction_get_dict_if_exists(PyObject *op_in, PyObject **dict) { + /* Return 1 if the function dict exists, 0 otherwise. This cannot fail: + * _PyObject_GetDictPtr() may clear errors internally, but never reports them. */ +#if CYTHON_COMPILING_IN_PYPY + *dict = PyObject_GenericGetDict(op_in, NULL); +#elif CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030C0000 + *dict = ((__pyx_CyFunctionObject*) op_in)->func_dict; +#else + PyObject **dictptr = _PyObject_GetDictPtr(op_in); + *dict = likely(dictptr) ? *dictptr : NULL; +#endif + return *dict ? 1 : 0; +} + +static int +__Pyx_CyFunction_get_annotate_from_dict_if_exists(PyObject *op_in, PyObject **annotate) { + PyObject *dict; + int dict_found; + *annotate = NULL; + dict_found = __Pyx_CyFunction_get_dict_if_exists(op_in, &dict); + if (!dict_found) return 0; + return __Pyx_PyDict_GetItemRef(dict, PYIDENT("__annotate__"), annotate); +} + +static int +__Pyx_CyFunction_set_annotate_in_dict_if_exists(PyObject *op_in, PyObject *value) { + PyObject *dict; + int dict_found; + dict_found = __Pyx_CyFunction_get_dict_if_exists(op_in, &dict); + if (!dict_found) return 0; + return PyDict_SetItem(dict, PYIDENT("__annotate__"), value); +} + +static int +__Pyx_CyFunction_set_annotate_in_dict(PyObject *op_in, PyObject *value) { + PyObject *dict; + int result; +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030A0000 + dict = __Pyx_CyFunction_get_dict((__pyx_CyFunctionObject*) op_in, NULL); +#else + dict = PyObject_GenericGetDict(op_in, NULL); +#endif + if (unlikely(!dict)) return -1; + result = PyDict_SetItem(dict, PYIDENT("__annotate__"), value); + Py_DECREF(dict); + return result; +} + static PyObject * __Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { PyObject* result = op->func_annotations; @@ -491,15 +544,100 @@ __Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { } static PyObject * -__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { - PyObject *result; +__Pyx_CyFunction_get_annotations(PyObject *op_in, void *context) { + PyObject *annotate = NULL; + PyObject *result = NULL; + __pyx_CyFunctionObject *op = (__pyx_CyFunctionObject*) op_in; CYTHON_UNUSED_VAR(context); - __Pyx_BEGIN_CRITICAL_SECTION(op); - result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_BEGIN_CRITICAL_SECTION(op_in); + result = __Pyx_XNewRef(op->func_annotations); + __Pyx_END_CRITICAL_SECTION(); + if (result) return result; + + if (unlikely(__Pyx_CyFunction_get_annotate_from_dict_if_exists(op_in, &annotate) < 0)) return NULL; + if (!annotate || annotate == Py_None) { + Py_XDECREF(annotate); + __Pyx_BEGIN_CRITICAL_SECTION(op_in); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; + } + + PyObject *format = PyLong_FromLong(1L); // annotationlib.Format.VALUE + if (likely(format)) { + result = __Pyx_PyObject_CallOneArg(annotate, format); + Py_DECREF(format); + } + Py_DECREF(annotate); + if (unlikely(!result)) return NULL; + if (unlikely(!PyDict_Check(result))) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must return a dict"); + Py_DECREF(result); + return NULL; + } + __Pyx_BEGIN_CRITICAL_SECTION(op_in); + __Pyx_Py_XDECREF_SET(op->func_annotations, __Pyx_NewRef(result)); __Pyx_END_CRITICAL_SECTION(); return result; } +static PyObject *__Pyx_CyFunction_annotate_impl(PyObject *self, PyObject *args) { + // The signature of this function is 0 or 1 args, where arg should be an int (if passed). + // However, for this basic placeholder implementation, we don't care and don't check it. + CYTHON_UNUSED_VAR(args); + if (unlikely(!self)) { + PyErr_SetString(PyExc_SystemError, "cython __annotate__ called without 'self' argument"); + } + Py_XINCREF(self); + return self; +} + +static PyMethodDef __Pyx_CyFunction_annotate_method = { + "__annotate__", + (PyCFunction)(void (*)(void))__Pyx_CyFunction_annotate_impl, + METH_VARARGS, + "Placeholder __annotate__ function to allow 'functools.wraps' to work " + "on Cython functions." +}; + +// We don't yet support PEP649 properly, so __annotate__ is +// implemented in a minimal way, just so that functools.wraps works. +static PyObject * +__Pyx_CyFunction_get_annotate(PyObject *op_in, void *context) { + PyObject *annotate = NULL; + CYTHON_UNUSED_VAR(context); + if (unlikely(__Pyx_CyFunction_get_annotate_from_dict_if_exists(op_in, &annotate) < 0)) return NULL; + if (annotate) return annotate; + + PyObject *annotations = __Pyx_CyFunction_get_annotations(op_in, NULL); + if (unlikely(!annotations)) return NULL; + PyObject *method = PyCFunction_New( + &__Pyx_CyFunction_annotate_method, + annotations); + Py_DECREF(annotations); + return method; +} + +static int +__Pyx_CyFunction_set_annotate(PyObject *op_in, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "__annotate__ cannot be deleted"); + return -1; + } + if (unlikely(value != Py_None && !PyCallable_Check(value))) { + PyErr_SetString(PyExc_TypeError, "__annotate__ must be callable or None"); + return -1; + } + if (value != Py_None) { + __pyx_CyFunctionObject *op = (__pyx_CyFunctionObject*) op_in; + __Pyx_BEGIN_CRITICAL_SECTION(op_in); + Py_CLEAR(op->func_annotations); + __Pyx_END_CRITICAL_SECTION(); + } + return __Pyx_CyFunction_set_annotate_in_dict(op_in, value); +} + static PyObject * __Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; @@ -649,6 +787,7 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"__annotate__", (getter)__Pyx_CyFunction_get_annotate, (setter)__Pyx_CyFunction_set_annotate, 0, 0}, {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, // {"__signature__", (getter)__Pyx_CyFunction_get_signature, 0, 0, 0}, #if CYTHON_COMPILING_IN_LIMITED_API diff --git a/contrib/tools/cython/Cython/Utility/ExtensionTypes.c b/contrib/tools/cython/Cython/Utility/ExtensionTypes.c index 1e89372fd71..131adb6432c 100644 --- a/contrib/tools/cython/Cython/Utility/ExtensionTypes.c +++ b/contrib/tools/cython/Cython/Utility/ExtensionTypes.c @@ -402,6 +402,34 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { #define __Pyx_TRASHCAN_END #endif +/////////////// DeallocKeepAlive.proto /////////////// + +// Keep `o` alive across a call into user __dealloc__ code, to prevent the user +// code from triggering recursive deallocation, then drop the temporary reference. +// +// On the GIL build this is a plain refcount bump. On free-threading we must NOT +// use Py_SET_REFCNT(o, 1): the object reaching tp_dealloc is unowned and merged +// (ob_ref_shared == _Py_REF_MERGED), so Py_SET_REFCNT takes the unowned branch and +// writes ob_ref_shared = _Py_REF_SHARED(1, _Py_REF_MERGED), i.e. "merged, shared +// count 1" -- indistinguishable from a live object. A concurrent +// PyUnstable_TryIncRef() would then succeed and resurrect the dying object +// (use-after-free). Instead we mirror CPython's _PyObject_ResurrectStart/End: +// re-take ownership and keep the *shared* refcount at the refuse sentinel (0), so +// the bump is invisible to other threads and TryIncRef keeps refusing. + +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_DeallocKeepAliveBegin(o) do { \ + _Py_atomic_store_uintptr_relaxed(&(o)->ob_tid, _Py_ThreadId()); \ + _Py_atomic_store_uint32_relaxed(&(o)->ob_ref_local, 1); \ + _Py_atomic_store_ssize_relaxed(&(o)->ob_ref_shared, 0); \ + } while (0) +#define __Pyx_DeallocKeepAliveEnd(o) \ + _Py_atomic_store_uint32_relaxed(&(o)->ob_ref_local, 0) +#else +#define __Pyx_DeallocKeepAliveBegin(o) __Pyx_SET_REFCNT(o, Py_REFCNT(o) + 1) +#define __Pyx_DeallocKeepAliveEnd(o) __Pyx_SET_REFCNT(o, Py_REFCNT(o) - 1) +#endif + /////////////// CallNextTpDealloc.proto /////////////// static void __Pyx_call_next_tp_dealloc(PyObject* obj, destructor current_tp_dealloc); diff --git a/contrib/tools/cython/Cython/Utility/TString.c b/contrib/tools/cython/Cython/Utility/TString.c index a18cf4ac01c..e27c755d22f 100644 --- a/contrib/tools/cython/Cython/Utility/TString.c +++ b/contrib/tools/cython/Cython/Utility/TString.c @@ -301,8 +301,8 @@ static PyObject* __Pyx_MakeTemplateLibTemplate(PyObject *strings, PyObject *inte if (unlikely(__Pyx_VectorcallBuilder_AddArg(PYIDENT("interpolations"), interpolations, kwargs_builder, args, 1)<0)) goto failed_shortcut; result = __Pyx_Object_Vectorcall_CallFromBuilder(tp, args, 0, kwargs_builder); - Py_DECREF(kwargs_builder); if (result) { + Py_DECREF(kwargs_builder); Py_DECREF(tp); return result; } diff --git a/contrib/tools/cython/cython.py b/contrib/tools/cython/cython.py index 6f4880a93cc..159014bd193 100755 --- a/contrib/tools/cython/cython.py +++ b/contrib/tools/cython/cython.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Change content of this file to change uids for cython programs - cython 3.2.5 r0 +# Change content of this file to change uids for cython programs - cython 3.2.8 r0 # # Cython -- Main Program, generic diff --git a/contrib/tools/cython/ya.make b/contrib/tools/cython/ya.make index 61e66dbe8d3..fd08615635d 100644 --- a/contrib/tools/cython/ya.make +++ b/contrib/tools/cython/ya.make @@ -11,9 +11,9 @@ LICENSE_TEXTS(.yandex_meta/licenses.list.txt) SUBSCRIBER(g:python-contrib) -VERSION(3.2.5) +VERSION(3.2.8) -ORIGINAL_SOURCE(mirror://pypi/c/cython/cython-3.2.5.tar.gz) +ORIGINAL_SOURCE(mirror://pypi/c/cython/cython-3.2.8.tar.gz) NO_LINT() diff --git a/library/cpp/unified_agent_client/client_impl.cpp b/library/cpp/unified_agent_client/client_impl.cpp index ffe16d8da37..dc4241344ff 100644 --- a/library/cpp/unified_agent_client/client_impl.cpp +++ b/library/cpp/unified_agent_client/client_impl.cpp @@ -8,6 +8,7 @@ #include <contrib/libs/grpc/src/core/lib/iomgr/executor.h> #include <util/charset/utf8.h> +#include <util/generic/scope.h> #include <util/generic/size_literals.h> #include <util/system/env.h> @@ -513,14 +514,22 @@ namespace NUnifiedAgent::NPrivate { } void TClientSession::CheckGrpcCallInactivity() { - with_lock(Lock) { - if (Closed || !Started) { - return; - } - const auto timeout = Client->GetParameters().GrpcCallInactivityTimeout; - if (timeout == TDuration::Zero()) { - return; + if (!Lock.TryAcquire()) { + TSpinWait sw; + + while (Lock.IsLocked() || !Lock.TryAcquire()) { + if (ForkInProgress.load()) { + return; + } + sw.Sleep(); } + } + Y_DEFER { + Lock.Release(); + }; + + const auto timeout = Client->GetParameters().GrpcCallInactivityTimeout; + if (!Closed && Started && timeout != TDuration::Zero()) { if (NegotiatedProtocol.Defined() && *NegotiatedProtocol > 0 && ActiveGrpcCall && !CloseStarted && diff --git a/util/string/strip_ut.cpp b/util/string/strip_ut.cpp index 1d47e21e23c..7236ca2ebf8 100644 --- a/util/string/strip_ut.cpp +++ b/util/string/strip_ut.cpp @@ -8,19 +8,19 @@ Y_UNIT_TEST_SUITE(TStripStringTest) { Y_UNIT_TEST(TestConstexprStrip) { - static constexpr std::string_view stripped = StripString<std::string_view>(" \n\t abc def \r\n "); + constexpr std::string_view stripped = StripString<std::string_view>(" \n\t abc def \r\n "); static_assert(stripped == "abc def"); - static constexpr std::string_view strippedLeft = StripStringLeft(std::string_view(" \n abc \n ")); + constexpr std::string_view strippedLeft = StripStringLeft(std::string_view(" \n abc \n ")); static_assert(strippedLeft == "abc \n "); - static constexpr std::string_view strippedRight = StripStringRight(std::string_view(" \n abc \n ")); + constexpr std::string_view strippedRight = StripStringRight(std::string_view(" \n abc \n ")); static_assert(strippedRight == " \n abc"); - static constexpr std::string_view empty = StripString<std::string_view>(" \n\t\r "); + constexpr std::string_view empty = StripString<std::string_view>(" \n\t\r "); static_assert(empty.empty()); - static constexpr std::string_view unchanged = StripString<std::string_view>("abc"); + constexpr std::string_view unchanged = StripString<std::string_view>("abc"); static_assert(unchanged == "abc"); UNIT_ASSERT_VALUES_EQUAL(stripped, "abc def"); @@ -29,13 +29,13 @@ Y_UNIT_TEST_SUITE(TStripStringTest) { } Y_UNIT_TEST(TestConstexprCustomStrip) { - static constexpr std::string_view stripped = StripString(std::string_view("//abc//"), EqualsStripAdapter('/')); + constexpr std::string_view stripped = StripString(std::string_view("//abc//"), EqualsStripAdapter('/')); static_assert(stripped == "abc"); - static constexpr std::string_view strippedLeft = StripStringLeft(std::string_view("//abc//"), EqualsStripAdapter('/')); + constexpr std::string_view strippedLeft = StripStringLeft(std::string_view("//abc//"), EqualsStripAdapter('/')); static_assert(strippedLeft == "abc//"); - static constexpr std::string_view strippedRight = StripStringRight(std::string_view("//abc//"), EqualsStripAdapter('/')); + constexpr std::string_view strippedRight = StripStringRight(std::string_view("//abc//"), EqualsStripAdapter('/')); static_assert(strippedRight == "//abc"); UNIT_ASSERT_VALUES_EQUAL(stripped, "abc"); @@ -47,7 +47,7 @@ Y_UNIT_TEST_SUITE(TStripStringTest) { TStringBuf StripRightRes; TStringBuf StripRes; }; - static constexpr TStripTest StripTests[] = { + constexpr TStripTest StripTests[] = { {" 012 ", "012 ", " 012", "012"}, {" 012", "012", " 012", "012"}, {"012\t\t", "012\t\t", "012", "012"}, diff --git a/vendor/golang.org/x/crypto/pbkdf2/pbkdf2.go b/vendor/golang.org/x/crypto/pbkdf2/pbkdf2.go index 28cd99c7f3f..b332122033b 100644 --- a/vendor/golang.org/x/crypto/pbkdf2/pbkdf2.go +++ b/vendor/golang.org/x/crypto/pbkdf2/pbkdf2.go @@ -2,24 +2,17 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -/* -Package pbkdf2 implements the key derivation function PBKDF2 as defined in RFC -2898 / PKCS #5 v2.0. - -A key derivation function is useful when encrypting data based on a password -or any other not-fully-random data. It uses a pseudorandom function to derive -a secure encryption key based on the password. - -While v2.0 of the standard defines only one pseudorandom function to use, -HMAC-SHA1, the drafted v2.1 specification allows use of all five FIPS Approved -Hash Functions SHA-1, SHA-224, SHA-256, SHA-384 and SHA-512 for HMAC. To -choose, you can pass the `New` functions from the different SHA packages to -pbkdf2.Key. -*/ +// Package pbkdf2 implements the key derivation function PBKDF2 as defined in +// RFC 8018 (PKCS #5 v2.1). +// +// This package is a wrapper for the PBKDF2 implementation in the +// [crypto/pbkdf2] package. It is [frozen] and is not accepting new features. +// +// [frozen]: https://go.dev/wiki/Frozen package pbkdf2 import ( - "crypto/hmac" + "crypto/pbkdf2" "hash" ) @@ -27,51 +20,11 @@ import ( // []byte of length keylen that can be used as cryptographic key. The key is // derived based on the method described as PBKDF2 with the HMAC variant using // the supplied hash function. -// -// For example, to use a HMAC-SHA-1 based PBKDF2 key derivation function, you -// can get a derived key for e.g. AES-256 (which needs a 32-byte key) by -// doing: -// -// dk := pbkdf2.Key([]byte("some password"), salt, 4096, 32, sha1.New) -// -// Remember to get a good random salt. At least 8 bytes is recommended by the -// RFC. -// -// Using a higher iteration count will increase the cost of an exhaustive -// search but will also make derivation proportionally slower. func Key(password, salt []byte, iter, keyLen int, h func() hash.Hash) []byte { - prf := hmac.New(h, password) - hashLen := prf.Size() - numBlocks := (keyLen + hashLen - 1) / hashLen - - var buf [4]byte - dk := make([]byte, 0, numBlocks*hashLen) - U := make([]byte, hashLen) - for block := 1; block <= numBlocks; block++ { - // N.B.: || means concatenation, ^ means XOR - // for each block T_i = U_1 ^ U_2 ^ ... ^ U_iter - // U_1 = PRF(password, salt || uint(i)) - prf.Reset() - prf.Write(salt) - buf[0] = byte(block >> 24) - buf[1] = byte(block >> 16) - buf[2] = byte(block >> 8) - buf[3] = byte(block) - prf.Write(buf[:4]) - dk = prf.Sum(dk) - T := dk[len(dk)-hashLen:] - copy(U, T) - - // U_n = PRF(password, U_(n-1)) - for n := 2; n <= iter; n++ { - prf.Reset() - prf.Write(U) - U = U[:0] - U = prf.Sum(U) - for x := range U { - T[x] ^= U[x] - } - } + out, err := pbkdf2.Key(h, string(password), salt, iter, keyLen) + if err != nil { + // FIPS 140 enforcement, or an invalid key length. + panic(err) } - return dk[:keyLen] + return out } diff --git a/vendor/golang.org/x/crypto/pbkdf2/ya.make b/vendor/golang.org/x/crypto/pbkdf2/ya.make index 93efe6e8d08..bf485573593 100644 --- a/vendor/golang.org/x/crypto/pbkdf2/ya.make +++ b/vendor/golang.org/x/crypto/pbkdf2/ya.make @@ -2,7 +2,7 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) -VERSION(v0.49.0) +VERSION(v0.53.0) SRCS( pbkdf2.go diff --git a/vendor/golang.org/x/crypto/scrypt/ya.make b/vendor/golang.org/x/crypto/scrypt/ya.make index e5ef97d77b2..aab6270377c 100644 --- a/vendor/golang.org/x/crypto/scrypt/ya.make +++ b/vendor/golang.org/x/crypto/scrypt/ya.make @@ -2,7 +2,7 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) -VERSION(v0.49.0) +VERSION(v0.53.0) SRCS( scrypt.go @@ -46,33 +46,33 @@ REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://devtools-reg PLATFORM_MAP = { "data": { "win32": { - "md5": "9dee86473a0e4b0c0a0d963f419dbf33", + "md5": "91a336f13e306b42992ec43c721762ac", "urls": [ - f"{REGISTRY_ENDPOINT}/12807581301" + f"{REGISTRY_ENDPOINT}/12845376268" ] }, "darwin": { - "md5": "e5067a1d83c2dd418be0faf32c2ab354", + "md5": "7113969185c3d148b03d485d7a28b44e", "urls": [ - f"{REGISTRY_ENDPOINT}/12807578337" + f"{REGISTRY_ENDPOINT}/12845375549" ] }, "darwin-arm64": { - "md5": "732315791641f6d8dec06aad3ac70427", + "md5": "55cc64b49e2fbf9a6075661ff9d1285e", "urls": [ - f"{REGISTRY_ENDPOINT}/12807575686" + f"{REGISTRY_ENDPOINT}/12845374717" ] }, "linux-aarch64": { - "md5": "9ac4edc8c328aff5bbb941668634ab70", + "md5": "dc46e90abbe7bf7c29a460ac5d09fd54", "urls": [ - f"{REGISTRY_ENDPOINT}/12807572366" + f"{REGISTRY_ENDPOINT}/12845373877" ] }, "linux": { - "md5": "01654ceac32bcf11ad928ce31db2cc42", + "md5": "c87e561b968477340705066c5b22370b", "urls": [ - f"{REGISTRY_ENDPOINT}/12807584185" + f"{REGISTRY_ENDPOINT}/12845377052" ] } } diff --git a/ydb/core/blobstorage/ddisk/ddisk.h b/ydb/core/blobstorage/ddisk/ddisk.h index f55dcc5c91d..25e9c7642fa 100644 --- a/ydb/core/blobstorage/ddisk/ddisk.h +++ b/ydb/core/blobstorage/ddisk/ddisk.h @@ -639,7 +639,7 @@ struct TPersistentBufferFormat { TEvReadPersistentBufferResult(NKikimrBlobStorage::NDDisk::TReplyStatus::E status, const std::optional<TString>& errorReason = std::nullopt, ui64 vChunkIndex = 0, ui32 offsetInBytes = 0, ui32 sizeInBytes = 0, - TRope data = {}) { + TRope data = {}, const std::vector<ui64>& checksums = {}) { Record.SetStatus(status); if (errorReason) { Record.SetErrorReason(*errorReason); @@ -649,6 +649,12 @@ struct TPersistentBufferFormat { Record.SetOffsetInBytes(offsetInBytes); Record.SetSizeInBytes(sizeInBytes); TReadResult(AddPayload(std::move(data))).Serialize(Record.MutableReadResult()); + // Opt-in, mirrors TEvWritePersistentBuffer.Checksums: only attached when the persisted + // record actually carries sender-supplied payload checksums (see + // TPersistentBuffer::TRecord::PayloadChecksums). + for (ui64 checksum : checksums) { + Record.AddChecksums(checksum); + } } } }; diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor.h b/ydb/core/blobstorage/ddisk/ddisk_actor.h index e780c0f362b..f3d0606c8b6 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor.h +++ b/ydb/core/blobstorage/ddisk/ddisk_actor.h @@ -753,6 +753,9 @@ namespace NKikimr::NDDisk { std::map<ui64, TRope> DataParts; ui32 PartsCount; std::vector<TPersistentBufferSectorInfo> Sectors; + // Sender-supplied per-MinSectorSize-block payload checksums for this record, in order. + // Empty when the write carried no checksums. See TPersistentBuffer::TRecord::PayloadChecksums. + std::vector<ui64> PayloadChecksums; TRope JoinData(ui32 sectorSize); }; @@ -813,7 +816,7 @@ namespace NKikimr::NDDisk { void IssuePersistentBufferChunkAllocation(); void ProcessDeallocatePersistentBufferChunk(bool forceToNextChunk = false); void ProcessPersistentBufferQueue(); - std::vector<std::tuple<ui32, ui32, TRope>> SlicePersistentBuffer(ui64 tabletId, ui32 generation, ui64 vchunkIndex, ui64 lsn, ui32 offsetInBytes, ui32 size, TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors); + std::vector<std::tuple<ui32, ui32, TRope>> SlicePersistentBuffer(ui64 tabletId, ui32 generation, ui64 vchunkIndex, ui64 lsn, ui32 offsetInBytes, ui32 size, TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors, const std::vector<ui64>& payloadChecksums); std::vector<std::tuple<ui32, ui32, TRope>> SlicePersistentBufferData(TRope& data, std::vector<TPersistentBufferSectorInfo>& sectors); void StartRestorePersistentBuffer(); void RestorePersistentBufferChunk(TEvPrivate::TEvReadPersistentBufferPart::TPtr ev); @@ -822,7 +825,12 @@ namespace NKikimr::NDDisk { bool PreprocessPersistentBufferWrite(NActors::TEventHandle<TEvWritePersistentBuffer>& ev); void ProcessPersistentBufferWrite(TEvWritePersistentBuffer::TPtr ev); - bool ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr ev); + // ev is taken by reference (not TPtr by value, unlike its sibling above): TPtr is a TAutoPtr + // with ownership-transferring copy semantics, so a by-value parameter here would null out the + // caller's ev as soon as this is invoked -- including on the "doesn't fit, fall back" (false) + // return path, where Handle(TEvWritePersistentBuffer) still needs a valid ev afterwards to retry + // via ProcessPersistentBufferWrite. + bool ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr& ev); void ProcessPersistentBufferBatchWrite(); double GetPersistentBufferFreeSpace(); void ErasePersistentBuffer(IEventHandle& queryEv, const TQueryCredentials& creds, const std::vector<TEraseLsnId>& erases); diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp b/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp index 611a01228a1..3660a33cf16 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp +++ b/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp @@ -15,6 +15,20 @@ namespace NKikimr::NDDisk { static constexpr ui32 MaxLsnsPerPack = (DataAlignment - sizeof(TPersistentBufferHeader)) / (sizeof(TPersistentBufferLsnRecordHeader) + TPersistentBufferLsnRecordHeader::MaxSectorsPerPackBufferRecord * sizeof(TPersistentBufferSectorInfo)); + // Size, in bytes, of one LSN record's metadata block within the shared 4 KiB batch header sector: + // TPersistentBufferLsnRecordHeader + its TPersistentBufferSectorInfo[] location array, plus an + // optional trailing ui64 PayloadChecksums[] (one entry per data sector) when the record carries + // sender-supplied checksums. Used both to lay out the header sector (SlicePersistentBuffer / + // ProcessPersistentBufferBatchWrite) and to decide whether one more record still fits in an + // already-open batch (ProcessPersistentBufferBatchWriteData) - MaxLsnsPerPack above only bounds + // the pre-allocated data-sector budget and is deliberately not tightened for checksums, since + // checksums add header-sector bytes, not data sectors. + static ui32 LsnRecordMetadataSize(ui32 dataSectorsCnt, bool hasPayloadChecksums) { + return sizeof(TPersistentBufferLsnRecordHeader) + + dataSectorsCnt * sizeof(TPersistentBufferSectorInfo) + + (hasPayloadChecksums ? dataSectorsCnt * sizeof(ui64) : 0); + } + void TDDiskActor::IssuePersistentBufferChunkAllocation() { Y_ABORT_UNLESS(IsPersistentBufferActor); if (!IssuePersistentBufferChunkAllocationInflight) { @@ -229,7 +243,8 @@ namespace NKikimr::NDDisk { std::vector<std::tuple<ui32, ui32, TRope>> TDDiskActor::SlicePersistentBuffer( ui64 tabletId, ui32 generation, ui64 vchunkIndex, ui64 lsn, ui32 offsetInBytes, ui32 sizeInBytes, - TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors) + TRcBuf&& payloadWithHeader, std::vector<TPersistentBufferSectorInfo>& sectors, + const std::vector<ui64>& payloadChecksums) { TRope fullData(std::move(payloadWithHeader)); @@ -263,7 +278,6 @@ namespace NKikimr::NDDisk { header->Signature, TPersistentBufferHeader::PersistentBufferHeaderSignature, sizeof(TPersistentBufferHeader::PersistentBufferHeaderSignature)); - header->Version = 0; header->PersistentBufferUniqueId = PersistentBufferUniqueId; header->NodeId = BaseInfo.PDiskActorID.NodeId(); header->PDiskId = BaseInfo.PDiskId; @@ -281,6 +295,9 @@ namespace NKikimr::NDDisk { lsnRecordHeader->OffsetInBytes = offsetInBytes; lsnRecordHeader->Size = sizeInBytes; lsnRecordHeader->Lsn = lsn; + const bool hasPayloadChecksums = !payloadChecksums.empty(); + lsnRecordHeader->Flags = hasPayloadChecksums ? TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS : TPersistentBufferLsnRecordHeader::NONE; + header->Version = hasPayloadChecksums ? 1 : 0; auto* locations = reinterpret_cast<TPersistentBufferSectorInfo*>(fullData.Begin().UnsafeContiguousDataMut() + sizeof(TPersistentBufferHeader) @@ -295,6 +312,16 @@ namespace NKikimr::NDDisk { sectors[i].Checksum = loc.Checksum; } + if (hasPayloadChecksums) { + Y_ABORT_UNLESS(payloadChecksums.size() == sectors.size() - 1); + auto* checksums = reinterpret_cast<ui64*>(fullData.Begin().UnsafeContiguousDataMut() + + sizeof(TPersistentBufferHeader) + + sizeof(TPersistentBufferLsnRecordHeader) + + (sectors.size() - 1) * sizeof(TPersistentBufferSectorInfo) + ); + memcpy(checksums, payloadChecksums.data(), payloadChecksums.size() * sizeof(ui64)); + } + header->Checksum = CalculateChecksum(fullData.Begin(), SectorSize); // Slice fullData ([header | payload], a single contiguous, SectorSize-aligned buffer) into one part @@ -587,6 +614,12 @@ namespace NKikimr::NDDisk { pr.Sectors.push_back(locations[i]); } pos += sectorsCnt * sizeof(TPersistentBufferSectorInfo); + + if (recordHeader->Flags & TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS) { + const ui64* checksums = reinterpret_cast<const ui64*>(pos); + pr.PayloadChecksums.assign(checksums, checksums + sectorsCnt); + pos += sectorsCnt * sizeof(ui64); + } } } else { PersistentBufferSectorsChecksum[chunkIdx][sectorIdx] = CalculateChecksum(dataPos); @@ -721,7 +754,8 @@ namespace NKikimr::NDDisk { .Size = (ui32)record.Size, .Sectors = std::move(record.Sectors), .VChunkIndex = record.VChunkIndex, - .Timestamp = TInstant::Now() + .Timestamp = TInstant::Now(), + .PayloadChecksums = std::move(record.PayloadChecksums), }; auto& pbh = PersistentBufferHeaders[{pr.Sectors[0].ChunkIdx, pr.Sectors[0].SectorIdx}]; @@ -898,6 +932,7 @@ namespace NKikimr::NDDisk { if (instr.PayloadId) { payload = ev->Get()->GetPayload(*instr.PayloadId); } + std::vector<ui64> payloadChecksums(record.GetChecksums().begin(), record.GetChecksums().end()); YDB_LOG_TRACE_COMP(NKikimrServices::BS_PERSISTENT_BUFFER, "TDDiskActor::ProcessPersistentBufferWrite", {"marker", "BSPB"}, {"PBufferId", SelfId()}, @@ -959,7 +994,8 @@ namespace NKikimr::NDDisk { TRcBuf payloadWithHeader = ev->Get()->GetPayloadWithHeader(*instr.PayloadId); auto parts = SlicePersistentBuffer(creds.TabletId, creds.Generation, - selector.VChunkIndex, lsn, selector.OffsetInBytes, selector.Size, std::move(payloadWithHeader), sectors); + selector.VChunkIndex, lsn, selector.OffsetInBytes, selector.Size, std::move(payloadWithHeader), sectors, + payloadChecksums); auto opCookie = NextCookie++; auto& inflightRecord = PersistentBufferDiskOperationInflight[opCookie]; @@ -982,6 +1018,9 @@ namespace NKikimr::NDDisk { .DataParts = {{0, std::move(payload)}}, .PartsCount = 1, .Sectors = std::move(sectors), + // payloadChecksums was only read (never mutated) above by SlicePersistentBuffer; safe to + // move into the inflight record now that this is its last use. + .PayloadChecksums = std::move(payloadChecksums), }); PersistentBufferWriteInflightsByRecord[TPersistentBufferRecordId{creds.TabletId, creds.Generation, lsn}].emplace_back( opCookie, @@ -1008,7 +1047,7 @@ namespace NKikimr::NDDisk { } } - bool TDDiskActor::ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr ev) { + bool TDDiskActor::ProcessPersistentBufferBatchWriteData(TEvWritePersistentBuffer::TPtr& ev) { const auto& record = ev->Get()->Record; const TQueryCredentials creds(record.GetCredentials()); const TBlockSelector selector(record.GetSelector()); @@ -1020,6 +1059,9 @@ namespace NKikimr::NDDisk { const TWriteInstruction instr(record.GetInstruction()); Y_ABORT_UNLESS(instr.PayloadId, "WritePersistentBuffer without a payload"); TRope payload = ev->Get()->GetPayload(*instr.PayloadId); + std::vector<ui64> payloadChecksums(record.GetChecksums().begin(), record.GetChecksums().end()); + const ui32 sectorsCnt = selector.Size / SectorSize; + Y_ABORT_UNLESS(sectorsCnt <= TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord && sectorsCnt > 0); YDB_LOG_TRACE_COMP(NKikimrServices::BS_PERSISTENT_BUFFER, "TDDiskActor::ProcessPersistentBufferBatchWriteData", {"marker", "BSPB"}, @@ -1043,6 +1085,22 @@ namespace NKikimr::NDDisk { }); Y_ABORT_UNLESS(inserted); Schedule(TDuration::MicroSeconds(PersistentBufferFormat.WritesBatchingPeriodMicroseconds), new TEvents::TEvWakeup(EWakeupTag::WakeupProcessPersistentBufferBatchWrite)); + } else { + // Reusing an already-open batch: this record's metadata (TPersistentBufferLsnRecordHeader + + // sector locations + optional payload checksums) must still fit in the shared 4 KiB header + // sector alongside every record already queued. A brand-new batch (handled above) always + // fits a single record trivially, so this check only matters here. Bail out (fall back to + // the direct, non-batched write path in the caller) rather than overflow the header sector. + const auto it = PersistentBufferDiskOperationInflight.find(PersistentBufferBatchWriteCookie); + Y_ABORT_UNLESS(it != PersistentBufferDiskOperationInflight.end()); + const auto& existingRecords = it->second.Records; + ui32 usedBytes = sizeof(TPersistentBufferHeader); + for (auto& r : existingRecords) { + usedBytes += LsnRecordMetadataSize(r.Sectors.size() - 1, !r.PayloadChecksums.empty()); + } + if (usedBytes + LsnRecordMetadataSize(sectorsCnt, !payloadChecksums.empty()) > SectorSize) { + return false; + } } auto& inflight = PersistentBufferDiskOperationInflight[PersistentBufferBatchWriteCookie]; @@ -1071,11 +1129,10 @@ namespace NKikimr::NDDisk { .Size = selector.Size, .DataParts = {{0, payload}}, .PartsCount = 1, + .PayloadChecksums = std::move(payloadChecksums), }); auto& r = inflight.Records.back(); - ui32 sectorsCnt = selector.Size / SectorSize; - Y_ABORT_UNLESS(sectorsCnt <= TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord && sectorsCnt > 0); r.Sectors.push_back(inflight.OccupiedSectors[0]); auto sectorsIt = inflight.OccupiedSectors.begin() + inflight.DataToWrite.size() / SectorSize; @@ -1119,7 +1176,6 @@ namespace NKikimr::NDDisk { header->Signature, TPersistentBufferHeader::PersistentBufferHeaderSignature, sizeof(TPersistentBufferHeader::PersistentBufferHeaderSignature)); - header->Version = 0; header->PersistentBufferUniqueId = PersistentBufferUniqueId; header->NodeId = BaseInfo.PDiskActorID.NodeId(); header->PDiskId = BaseInfo.PDiskId; @@ -1129,6 +1185,7 @@ namespace NKikimr::NDDisk { header->Flags = 0; header->BatchSize = inflight.Records.size(); + bool anyPayloadChecksums = false; auto* pos = inflight.DataToWrite.Begin().UnsafeContiguousDataMut() + sizeof(TPersistentBufferHeader); for (auto& record : inflight.Records) { auto* lsnRecordHeader = reinterpret_cast<TPersistentBufferLsnRecordHeader*>(pos); @@ -1139,6 +1196,9 @@ namespace NKikimr::NDDisk { lsnRecordHeader->OffsetInBytes = record.OffsetInBytes; lsnRecordHeader->Size = record.Size; lsnRecordHeader->Lsn = record.Lsn; + const bool hasPayloadChecksums = !record.PayloadChecksums.empty(); + lsnRecordHeader->Flags = hasPayloadChecksums ? TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS : TPersistentBufferLsnRecordHeader::NONE; + anyPayloadChecksums = anyPayloadChecksums || hasPayloadChecksums; pos += sizeof(TPersistentBufferLsnRecordHeader); auto* locations = reinterpret_cast<TPersistentBufferSectorInfo*>(pos); @@ -1148,7 +1208,16 @@ namespace NKikimr::NDDisk { loc = record.Sectors[i]; } pos += sizeof(TPersistentBufferSectorInfo) * (record.Sectors.size() - 1); + + if (hasPayloadChecksums) { + Y_ABORT_UNLESS(record.PayloadChecksums.size() == record.Sectors.size() - 1); + memcpy(pos, record.PayloadChecksums.data(), record.PayloadChecksums.size() * sizeof(ui64)); + pos += record.PayloadChecksums.size() * sizeof(ui64); + } } + Y_ABORT_UNLESS(static_cast<size_t>(pos - inflight.DataToWrite.Begin().UnsafeContiguousDataMut()) <= SectorSize, + "persistent buffer batch header overflow"); + header->Version = anyPayloadChecksums ? 1 : 0; header->Checksum = CalculateChecksum(inflight.DataToWrite.Begin(), SectorSize); auto parts = SlicePersistentBufferData(inflight.DataToWrite, inflight.OccupiedSectors); @@ -1407,11 +1476,23 @@ namespace NKikimr::NDDisk { Y_ABORT_UNLESS(inflight.Records.size() == 1); auto& inflightRecord = inflight.Records[0]; TRope data; + std::vector<ui64> checksums; if (pr.Data) { data = std::move(TrimData(pr.Data, pr.OffsetInBytes, pr.Size, inflightRecord.OffsetInBytes, inflightRecord.Size)); + if (!pr.PayloadChecksums.empty()) { + // Persisted checksums cover [pr.OffsetInBytes, pr.OffsetInBytes + pr.Size) one entry + // per MinSectorSize block, same order as the trimmed data above - slice out the + // sub-range the selector actually asked for. Returned as-is (never recomputed from + // on-disk bytes), matching the write path's opt-in, pre-signature-correction values. + Y_ABORT_UNLESS(pr.PayloadChecksums.size() == pr.Size / SectorSize); + const ui32 firstBlock = (inflightRecord.OffsetInBytes - pr.OffsetInBytes) / SectorSize; + const ui32 blockCount = inflightRecord.Size / SectorSize; + Y_ABORT_UNLESS(firstBlock + blockCount <= pr.PayloadChecksums.size()); + checksums.assign(pr.PayloadChecksums.begin() + firstBlock, pr.PayloadChecksums.begin() + firstBlock + blockCount); + } } auto replyEv = std::make_unique<TEvReadPersistentBufferResult>(status, errorMessage, - pr.VChunkIndex, pr.OffsetInBytes, pr.Size, std::move(data)); + pr.VChunkIndex, pr.OffsetInBytes, pr.Size, std::move(data), checksums); auto h = std::make_unique<IEventHandle>(inflightRecord.Sender, SelfId(), replyEv.release(), 0, inflightRecord.Cookie); if (inflightRecord.Session) { h->Rewrite(TEvInterconnect::EvForward, inflightRecord.Session); diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp b/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp index a1db18c0ae6..0c4c1071914 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp +++ b/ydb/core/blobstorage/ddisk/ddisk_actor_read_write.cpp @@ -89,13 +89,14 @@ namespace NKikimr::NDDisk { } } + // Unlike TEvWritePersistentBuffer, TEvWrite's checksums (if any) are validated here but never + // persisted: TEvWrite goes straight to the final DDisk chunk, which has no per-record metadata + // slot to carry them (see RFC 006). Note this validation intentionally runs again for events + // that get parked in PendingEventsForChunk below and re-dispatched to this same handler once + // their chunk allocation completes -- harmless (checksums don't change), just redundant. if (record.ChecksumsSize() > 0) { - // Checksum validation is opt-in (see TEvWritePersistentBuffer for details); checked here, - // before any chunk allocation queueing or disk I/O, and the checksums are not persisted. - // Note: an event parked in PendingEventsForChunk below re-enters this handler in full once - // the chunk is allocated, so its checksums get validated twice. This is harmless (validation - // stays right next to the actual write) and intentional, not an oversight. Y_ABORT_UNLESS(instr.PayloadId, "TEvWrite without a payload, but with checksums"); + const TRope& payload = ev->Get()->GetPayload(*instr.PayloadId); if (const auto result = ValidatePayloadChecksums(record, payload)) { const bool isCorrupted = result->Status == NKikimrBlobStorage::NDDisk::TReplyStatus::CORRUPTED; diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor_sync.cpp b/ydb/core/blobstorage/ddisk/ddisk_actor_sync.cpp index b3194ec154f..ec1ac876e97 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor_sync.cpp +++ b/ydb/core/blobstorage/ddisk/ddisk_actor_sync.cpp @@ -199,7 +199,8 @@ namespace NKikimr::NDDisk { } } - Y_DEBUG_ABORT_UNLESS(SyncReadCookiesInFlight.emplace(requestId).second); + const bool inserted = SyncReadCookiesInFlight.emplace(requestId).second; + Y_DEBUG_ABORT_UNLESS(inserted); const TActorId& sourceActorId = fromPersistentBuffer ? sourceInfo.PersistentBufferActorId : sourceInfo.DDiskActorId; diff --git a/ydb/core/blobstorage/ddisk/persistent_buffer.h b/ydb/core/blobstorage/ddisk/persistent_buffer.h index 9c94108d446..1710bb0554c 100644 --- a/ydb/core/blobstorage/ddisk/persistent_buffer.h +++ b/ydb/core/blobstorage/ddisk/persistent_buffer.h @@ -43,6 +43,10 @@ namespace NKikimr::NDDisk { ui64 VChunkIndex; TInstant Timestamp; std::unordered_set<ui64> ReadInflight; + // Sender-supplied per-MinSectorSize-block payload checksums, in order, covering + // [OffsetInBytes, OffsetInBytes + Size). Empty when the record was written without + // checksums (legacy / internal writes) - opt-in, mirrors the wire-level semantics. + std::vector<ui64> PayloadChecksums; }; std::map<ui64, TRecord> Records; diff --git a/ydb/core/blobstorage/ddisk/persistent_buffer_header.h b/ydb/core/blobstorage/ddisk/persistent_buffer_header.h index 8f83c4d1142..7949fdfbbfd 100644 --- a/ydb/core/blobstorage/ddisk/persistent_buffer_header.h +++ b/ydb/core/blobstorage/ddisk/persistent_buffer_header.h @@ -73,9 +73,21 @@ namespace NKikimr::NDDisk { static constexpr ui32 MaxSectorsPerBufferRecord = 128; static constexpr ui32 MaxSectorsPerPackBufferRecord = 8; + enum EFlags : ui32 { + NONE = 0, + // When set, a ui64 payload checksum (sender-computed, unsalted XXH3-64 per MinSectorSize + // block - see CalculatePayloadChecksums) immediately follows this record's + // TPersistentBufferSectorInfo[] location array, one entry per data sector, same order. + // Opt-in per record: only present when the write that produced it carried sender-supplied + // checksums (TEvWritePersistentBuffer.Checksums). Legacy/internal records without checksums + // have this flag clear (the field was always zero-initialized via the full-sector memset + // that precedes header assembly, so old on-disk records are unaffected by this rename). + HAS_PAYLOAD_CHECKSUMS = 1, + }; + ui64 TabletId; ui32 Generation; - ui32 Reserved1; + ui32 Flags; ui64 VChunkIndex; ui32 OffsetInBytes; ui32 Size; @@ -83,8 +95,10 @@ namespace NKikimr::NDDisk { }; static_assert(sizeof(TPersistentBufferLsnRecordHeader) <= DataAlignment); + // Worst case: a single record spanning MaxSectorsPerBufferRecord sectors, all carrying a + // persisted payload checksum. static_assert(DataAlignment >= sizeof(TPersistentBufferHeader) + sizeof(TPersistentBufferLsnRecordHeader) - + TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord * sizeof(TPersistentBufferSectorInfo) + + TPersistentBufferLsnRecordHeader::MaxSectorsPerBufferRecord * (sizeof(TPersistentBufferSectorInfo) + sizeof(ui64)) ); #pragma pack(pop) } diff --git a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp index bea5e6eb01d..d7c92eb19ae 100644 --- a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp +++ b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_batch_write_ut.cpp @@ -1129,12 +1129,16 @@ Y_UNIT_TEST_SUITE(TDDiskActorBatchWriteTest) { // TPersistentBufferHeader // [record 0] TPersistentBufferLsnRecordHeader // (Sectors.size() - 1) × TPersistentBufferSectorInfo + // [if Flags & HAS_PAYLOAD_CHECKSUMS] (Sectors.size() - 1) × ui64 // [record 1] TPersistentBufferLsnRecordHeader // (Sectors.size() - 1) × TPersistentBufferSectorInfo + // [if Flags & HAS_PAYLOAD_CHECKSUMS] (Sectors.size() - 1) × ui64 // ... // Each record here has exactly 1 data sector, so Sectors.size() == 2 // (header sector + 1 data sector), meaning (Sectors.size()-1) == 1 - // TPersistentBufferSectorInfo entry follows each LsnRecordHeader. + // TPersistentBufferSectorInfo entry follows each LsnRecordHeader. Both A and B call + // ChecksumPayload(), so recB also carries 1 persisted payload checksum (ui64) after its + // TPersistentBufferSectorInfo entry. const ui32 headerChunkIdx = rawBC_combined->Get()->ChunkIdx; const ui32 headerOffset = rawBC_combined->Get()->Offset; const ui64 actualUniqueId = [&]() -> ui64 { @@ -1152,10 +1156,14 @@ Y_UNIT_TEST_SUITE(TDDiskActorBatchWriteTest) { UNIT_ASSERT_VALUES_EQUAL_C(recB->Generation, credsB.Generation, "First batched record generation must match tablet B"); - // Advance past recB's LsnRecordHeader and its 1 TPersistentBufferSectorInfo - // (Sectors.size()-1 == 1 for a single-sector data write). + // Advance past recB's LsnRecordHeader, its 1 TPersistentBufferSectorInfo + // (Sectors.size()-1 == 1 for a single-sector data write), and (since B carries a + // sender checksum) its 1 persisted payload checksum. pos += sizeof(NDDisk::TPersistentBufferLsnRecordHeader); pos += sizeof(NDDisk::TPersistentBufferSectorInfo); // (Sectors.size()-1) entries + if (recB->Flags & NDDisk::TPersistentBufferLsnRecordHeader::HAS_PAYLOAD_CHECKSUMS) { + pos += sizeof(ui64); // (Sectors.size()-1) checksums + } // Record C: immediately follows. const auto* recC = reinterpret_cast<const NDDisk::TPersistentBufferLsnRecordHeader*>(pos); diff --git a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp index b36d7ebc065..19758b86f74 100644 --- a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp +++ b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_checksum_ut.cpp @@ -11,6 +11,7 @@ #include <ydb/core/blobstorage/ddisk/ddisk.h> #include <ydb/core/blobstorage/ddisk/ddisk_actor.h> +#include <ydb/core/blobstorage/ddisk/persistent_buffer_header.h> #include <ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.h> #include <ydb/core/blobstorage/pdisk/blobstorage_pdisk.h> #include <ydb/core/blobstorage/pdisk/blobstorage_pdisk_data.h> @@ -22,6 +23,8 @@ #include <util/string/printf.h> +#include <cstring> + namespace NKikimr { namespace { @@ -184,6 +187,113 @@ public: auto res = new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0); SendPDiskResponse(disk, *checkSpace, res); } + + // Create a DDisk instance that simulates a restart where the PB chunks from a + // previous instance are passed via StartingPoints. Mirrors the helper of the same + // name in ddisk_actor_batch_write_ut.cpp. + TDiskHandle CreateDDiskWithRestoredChunkData(ui32 pdiskId, ui32 slotId, + const std::vector<ui32>& preExistingChunkIds, ui64 oldUniqueId, + const std::unordered_map<ui32, TString>& chunkData) { + const TActorId pdiskEdge = Runtime.AllocateEdgeActor(NodeId, __FILE__, __LINE__); + const TActorId pdiskServiceId = MakeBlobStoragePDiskID(NodeId, pdiskId); + Runtime.RegisterService(pdiskServiceId, pdiskEdge); + + TVector<TActorId> actorIds = { + MakeBlobStorageDDiskId(NodeId, pdiskId, slotId), + }; + auto groupInfo = MakeIntrusive<TBlobStorageGroupInfo>(TBlobStorageGroupType::ErasureNone, ui32(1), ui32(1), + ui32(1), &actorIds); + + TVDiskConfig::TBaseInfo baseInfo( + TVDiskIdShort(groupInfo->GetVDiskId(0)), + pdiskServiceId, + 0x100000 + pdiskId, + pdiskId, + NPDisk::DEVICE_TYPE_NVME, + slotId, + NKikimrBlobStorage::TVDiskKind::Default, + 1, + "ddisk_pool"); + NDDisk::TPersistentBufferFormat pbFormat{256, 4, BlockSize * 128, 8, 5000, 512 * 1024}; + const TActorId ddiskActor = Runtime.Register(NDDisk::CreateDDiskActor(std::move(baseInfo), groupInfo, + std::move(pbFormat), NDDisk::TDDiskConfig{}, Counters), + NodeId); + const TActorId ddiskServiceId = MakeBlobStorageDDiskId(NodeId, pdiskId, slotId); + const TActorId pbServiceId = MakeBlobStoragePersistentBufferId(NodeId, pdiskId, slotId); + Runtime.RegisterService(ddiskServiceId, ddiskActor); + + TDiskHandle disk{ddiskServiceId, pbServiceId, pdiskEdge, pdiskId, slotId, 100000 + pdiskId * 1000}; + + const NPDisk::TOwner Owner = 1; + const NPDisk::TOwnerRound OwnerRound = 1; + + auto init = WaitPDiskRequest<NPDisk::TEvYardInit>(disk); + TVector<ui32> ownedChunks; + auto initReply = std::make_unique<NPDisk::TEvYardInitResult>( + NKikimrProto::OK, + 0, 0, 0, + BlockSize, BlockSize, BlockSize, + ChunkSize, + BlockSize, + Owner, + OwnerRound, + 1, + 0, + std::move(ownedChunks), + NPDisk::DEVICE_TYPE_NVME, + false, + BlockSize, + ""); + + NPDisk::TDiskFormat format = {}; + format.Clear(false); + initReply->DiskFormat = NPDisk::TDiskFormatPtr(new NPDisk::TDiskFormat(format), +[](NPDisk::TDiskFormat* ptr) { + delete ptr; + }); + + { + NKikimrBlobStorage::NDDisk::NInternal::TPersistentBufferChunkMapLogRecord pbChunkMap; + for (ui32 chunkIdx : preExistingChunkIds) { + pbChunkMap.AddChunkIdxs(chunkIdx); + } + pbChunkMap.SetUniqueId(oldUniqueId + 1); + + TString pbChunkMapData; + const bool serializeOk = pbChunkMap.SerializeToString(&pbChunkMapData); + Y_ABORT_UNLESS(serializeOk); + initReply->StartingPoints[TLogSignature::SignaturePersistentBufferChunkMap] = + NPDisk::TLogRecord(TLogSignature::SignaturePersistentBufferChunkMap, + TRcBuf(pbChunkMapData), 1 /*lsn*/); + } + + SendPDiskResponse(disk, *init, initReply.release()); + + auto readLog = WaitPDiskRequest<NPDisk::TEvReadLog>(disk); + auto readLogReply = std::make_unique<NPDisk::TEvReadLogResult>( + NKikimrProto::OK, + readLog->Get()->Position, + readLog->Get()->Position, + true, + 0, + "", + Owner); + SendPDiskResponse(disk, *readLog, readLogReply.release()); + + for (ui32 i = 0; i < preExistingChunkIds.size(); ++i) { + auto readRaw = WaitPDiskRequest<NPDisk::TEvChunkReadRaw>(disk); + const ui32 chunkIdx = readRaw->Get()->ChunkIdx; + auto it = chunkData.find(chunkIdx); + TString data; + if (it != chunkData.end()) { + data = it->second; + } else { + data = TString(TTestContext::ChunkSize, '\0'); + } + SendPDiskResponse(disk, *readRaw, new NPDisk::TEvChunkReadRawResult(TRope(data))); + } + + return disk; + } }; void SendToDDisk(TTestContext& ctx, const TActorId& serviceId, IEventBase* event, ui64 cookie = 0) { @@ -560,6 +670,765 @@ Y_UNIT_TEST_SUITE(TDDiskChecksumTests) { UNIT_ASSERT_VALUES_EQUAL(GetChecksumCounter(ctx, disk, "WritesWithoutChecksums")->Val(), 1); } + + // 9. Persistence: a PB write carrying sender checksums stores them, and an in-memory read + // (record still cached, no restart) returns exactly those checksums, in order, alongside the + // original payload. + Y_UNIT_TEST(SinglePBWriteWithChecksumsReadBackInMemory) { + TTestContext ctx; + const TDiskHandle disk = ctx.CreateDDisk(60, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk.PBServiceId, 110, 1); + + const ui64 lsn = 1; + const TString payload = MakeData('Z', 2 * BlockSize); + const NDDisk::TBlockSelector selector{5, 0, 2 * BlockSize}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> expectedChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + UNIT_ASSERT_VALUES_EQUAL(expectedChecksums.size(), 2u); + + SendToDDisk(ctx, disk.PBServiceId, write.release()); + + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk); + ctx.SendPDiskResponse(disk, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL(readResult->Get()->GetPayload(0).ConvertToString(), payload); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), expectedChecksums.size()); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(readResult->Get()->Record.GetChecksums(i), expectedChecksums[i]); + } + } + + // 10. Persistence across restart, both the direct (non-batch) write path and the batched write + // path: record A is written directly, B and C are accumulated into a single batch header + // sector while A's disk write is in-flight. All three carry sender checksums. After a + // simulated restart (StartingPoints replay from a fresh DDisk instance), all three records + // must be listed and readable with the same checksums that were originally sent. + Y_UNIT_TEST(WritesWithChecksumsAndBatchSurviveRestart) { + TTestContext ctx; + + const TDiskHandle disk1 = ctx.CreateDDisk(61, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 111, 1); + + const ui64 lsnA = 1; + const ui64 lsnB = 77; + const ui64 lsnC = 88; + const TString payloadA = MakeData('A', BlockSize); + const TString payloadB = MakeData('B', BlockSize); + const TString payloadC = MakeData('C', BlockSize); + const NDDisk::TBlockSelector selectorA{1, 0, BlockSize}; + const NDDisk::TBlockSelector selectorB{2, 0, BlockSize}; + const NDDisk::TBlockSelector selectorC{3, 0, BlockSize}; + + // Accumulate all raw PDisk writes into per-chunk buffers so we can feed them back to + // disk2 during restore. + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + std::vector<ui64> checksumsA, checksumsB, checksumsC; + + // Step 1: send A, keep its PDisk write pending so B and C enter the batch path. + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>( + creds1, selectorA, lsnA, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadA)); + checksumsA.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + auto rawA = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + const TActorId disk1ActorId = rawA->Sender; + captureWrite(rawA); + + // Step 2/3: B and C accumulate into the same batch while A is in-flight. + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>( + creds1, selectorB, lsnB, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadB)); + checksumsB.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>( + creds1, selectorC, lsnC, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadC)); + checksumsC.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + + // Step 4: complete A. + ctx.SendPDiskResponse(disk1, *rawA, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrA = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrA, TReplyStatus::OK); + + // Step 5: wakeup fires -> combined write for B and C (header + B data + C data). + auto rawBC = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawBC->Get()->Data.size(), 3 * BlockSize, + "Combined write must contain header + B data + C data (3 sectors)"); + captureWrite(rawBC); + + const ui64 actualUniqueId = [&]() -> ui64 { + const TString& buf = chunkBufs[rawBC->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawBC->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->BatchSize, 2u, "Header must record BatchSize=2 for B and C"); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->Version, 1u, + "Header Version must be 1 when any record in the batch carries payload checksums"); + return hdr->PersistentBufferUniqueId; + }(); + UNIT_ASSERT_C(actualUniqueId != 0, "PersistentBufferUniqueId must not be zero"); + + // Step 6: complete the combined write -> B and C both reply. + ctx.SendPDiskResponse(disk1, *rawBC, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrB = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrB, TReplyStatus::OK); + auto wrC = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrC, TReplyStatus::OK); + + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + + // Phase 2: restart with instance 2 (same PDiskId, same UniqueId). + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->Sender == disk1ActorId) { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, disk1.PDiskEdge, + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + } + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 61, 1, + pbChunkIds, + /*oldUniqueId=*/actualUniqueId - 1, + chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 111, 1); + + auto listResult = SendToDDiskAndWait<NDDisk::TEvListPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvListPersistentBuffer(creds2)); + AssertStatus(listResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(listResult->Get()->Record.RecordsSize(), 3, + "Records A, B and C must all be restored after restart"); + + auto checkRead = [&](const NDDisk::TBlockSelector& selector, ui64 lsn, const TString& payload, + const std::vector<ui64>& expectedChecksums, const TString& label) { + auto read = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selector, lsn, 1, {true})); + AssertStatus(read, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(read->Get()->GetPayload(0).ConvertToString(), payload, + TStringBuilder() << label << ": restored payload must match"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(read->Get()->Record.ChecksumsSize()), expectedChecksums.size(), + TStringBuilder() << label << ": checksum count mismatch after restore"); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(read->Get()->Record.GetChecksums(i), expectedChecksums[i], + TStringBuilder() << label << ": checksum " << i << " mismatch after restore"); + } + }; + + checkRead(selectorA, lsnA, payloadA, checksumsA, "record A (non-batch)"); + checkRead(selectorB, lsnB, payloadB, checksumsB, "record B (batched)"); + checkRead(selectorC, lsnC, payloadC, checksumsC, "record C (batched)"); + } + + // 11. Signature-correction sector: the sender's checksum is validated over, and must be + // persisted as, the ORIGINAL payload bytes -- even though PB zeroes the colliding byte on disk + // and restores it on read. If the checksum were ever recomputed from the on-disk (corrected) + // bytes it would not match what the sender sent. + Y_UNIT_TEST(SignatureCorrectionChecksumPersistedAsOriginal) { + TTestContext ctx; + const TDiskHandle disk = ctx.CreateDDisk(62, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk.PBServiceId, 112, 1); + + const ui64 lsn = 1; + // The second data sector's first byte deliberately equals the PB header signature byte, + // forcing TDDiskActor to zero it on disk (HasSignatureCorrection) and restore it on read. + TString payload = MakeData('S', 2 * BlockSize); + payload.Detach()[BlockSize] = static_cast<char>(NDDisk::TPersistentBufferHeader::PersistentBufferHeaderSignature[0]); + const NDDisk::TBlockSelector selector{7, 0, 2 * BlockSize}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> expectedChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + + SendToDDisk(ctx, disk.PBServiceId, write.release()); + + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk); + // Sanity: the on-disk sector's colliding byte was actually zeroed (header sector + 2 data + // sectors are written contiguously, so the second data sector starts at offset 2*BlockSize). + const TString writtenData = pbWriteRaw->Get()->Data.ConvertToString(); + UNIT_ASSERT_VALUES_EQUAL_C(writtenData[2 * BlockSize], '\0', + "Signature-correction must zero the colliding byte on disk"); + ctx.SendPDiskResponse(disk, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload, + "Original signature byte must be restored on read"); + UNIT_ASSERT_VALUES_EQUAL(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), expectedChecksums.size()); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(i), expectedChecksums[i], + "Persisted checksum must be the pre-correction (original) value, never recomputed from on-disk bytes"); + } + } + + // 12. Legacy / no-checksum compatibility across restart: a write without sender checksums + // (opt-in, same as SinglePBWriteWithoutChecksums) must restore successfully and its read must + // return an empty Checksums list, both immediately and after a restart. + Y_UNIT_TEST(LegacyWriteWithoutChecksumsSurvivesRestartWithEmptyChecksums) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(63, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 113, 1); + + const ui64 lsn = 1; + const TString payload = MakeData('L', BlockSize); + const NDDisk::TBlockSelector selector{4, 0, BlockSize}; + + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payload)); + // Deliberately not calling AddPayloadThenChecksum(): checksum persistence is opt-in. + UNIT_ASSERT_VALUES_EQUAL(write->Record.ChecksumsSize(), 0u); + + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + + auto rawWrite = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + const TActorId disk1SenderActorId = rawWrite->Sender; + captureWrite(rawWrite); + const ui64 actualUniqueId = [&]() -> ui64 { + const TString& buf = chunkBufs[rawWrite->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawWrite->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->Version, 0u, "Header Version must stay 0 for a checksum-less write"); + return hdr->PersistentBufferUniqueId; + }(); + ctx.SendPDiskResponse(disk1, *rawWrite, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->Sender == disk1SenderActorId) { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, disk1.PDiskEdge, + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + } + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 63, 1, + pbChunkIds, + /*oldUniqueId=*/actualUniqueId - 1, + chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 113, 1); + + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload, + "Legacy record's payload must survive restart"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), 0u, + "A checksum-less record must return an empty Checksums list after restore"); + } + + // 13. Re-replication (TEvReadThenWritePersistentBuffers) must forward the checksums that were + // persisted on the source record, not silently drop them: the coordinator PB reads its own + // record (which carries sender-supplied checksums) and re-writes it to the target PBs via + // TEvWritePersistentBuffers -- the checksums must reach and be validated/persisted on every + // target, exactly as if the original sender had written them directly. Modeled on + // PersistentBufferReadThenWriteTunnel (ddisk_actor_ut.cpp). + Y_UNIT_TEST(ReadThenWriteForwardsPersistedChecksums) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(64, 1); + const TDiskHandle disk2 = ctx.CreateDDisk(65, 1); + const TDiskHandle disk3 = ctx.CreateDDisk(66, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk1.PBServiceId, 120, 1); + + const ui64 lsn = 10; + const TString payload = MakeData('P', 2 * BlockSize); + const NDDisk::TBlockSelector selector{3, 0, 2 * BlockSize}; + + auto pbs = std::vector<std::tuple<ui32, ui32, ui32>>{ + {NodeId, disk2.PDiskId, disk2.SlotId}, + {NodeId, disk3.PDiskId, disk3.SlotId}}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> expectedChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + UNIT_ASSERT_VALUES_EQUAL(expectedChecksums.size(), 2u); + + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + ctx.SendPDiskResponse(disk1, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + auto readThenWrite = std::make_unique<NDDisk::TEvReadThenWritePersistentBuffers>(creds, lsn, 1, pbs, 1000); + SendToDDisk(ctx, disk1.PBServiceId, readThenWrite.release()); + for (int i = 0; i < 2; ++i) { + auto peerWriteRaw = ctx.WaitPDiskRequests<NPDisk::TEvChunkWriteRaw>({disk2.PDiskEdge, disk3.PDiskEdge}); + const TDiskHandle& respondingDisk = (peerWriteRaw->Recipient == disk2.PDiskEdge) ? disk2 : disk3; + ctx.SendPDiskResponse(respondingDisk, *peerWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + } + + auto tunnelResult = ctx.Runtime.WaitForEdgeActorEvent<NDDisk::TEvWritePersistentBuffersResult>(ctx.Edge, false); + UNIT_ASSERT_VALUES_EQUAL(tunnelResult->Get()->Record.ResultSize(), 2u); + for (ui32 i = 0; i < tunnelResult->Get()->Record.ResultSize(); ++i) { + const auto& wr = tunnelResult->Get()->Record.GetResult(i); + UNIT_ASSERT_C(wr.GetResult().GetStatus() == TReplyStatus::OK, TStringBuilder() + << "Re-replicated write " << i << " expected OK, got " + << TReplyStatus::E_Name(static_cast<TReplyStatus::E>(wr.GetResult().GetStatus()))); + } + + for (auto disk : {disk2, disk3}) { + NDDisk::TQueryCredentials peerCreds = Connect(ctx, disk.PBServiceId, 120, 1); + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(peerCreds, selector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload, + "Re-replicated payload must match the original"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), expectedChecksums.size(), + "Re-replicated record must carry the same checksums as the original write, not an empty list"); + for (ui32 i = 0; i < expectedChecksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(i), expectedChecksums[i], + "Checksum " << i << " must survive TEvReadThenWritePersistentBuffers re-replication"); + } + } + } + + // 14. Batch header overflow: checksummed 8-sector records (the largest size still eligible for + // batching) accumulate in an open batch until one more record's metadata (TPersistentBufferLsnRecordHeader + // + its sector-location array + one checksum per data sector, see LsnRecordMetadataSize) no longer + // fits the shared 4 KiB header sector -- even though the batch's outer record-count limit + // (MaxLsnsPerPack, sized for checksum-less records) is not yet reached. That overflowing record + // must fall back to a direct (own-header) write instead of corrupting the header sector, and every + // record -- batched and direct alike -- must restore with correct payload and checksums after a + // restart. + Y_UNIT_TEST(BatchHeaderOverflowFallsBackToDirectWrite) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(67, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 121, 1); + + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + // Write A: keep-alive PDisk write; keeps PersistentBufferDiskOperationInflight non-empty so + // subsequent small writes take the batch path instead of a direct write. + const TString payloadA = MakeData('A', BlockSize); + const NDDisk::TBlockSelector selectorA{1, 0, BlockSize}; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorA, 1, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payloadA)); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + auto rawA = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + captureWrite(rawA); + + // Compute, the same way LsnRecordMetadataSize does, exactly how many checksummed 8-sector + // records fit in the shared 4 KiB header sector alongside TPersistentBufferHeader: that many + // are sent first (all must batch, i.e. produce no PDisk write of their own), then one more is + // sent (must overflow and fall back to a direct write). This is deterministic -- no need to + // race for "did a write appear immediately", since A's own raw write was already dequeued above + // and batched records never produce a raw write, so the next TEvChunkWriteRaw on disk1 can only + // be the overflowing record's direct write. + constexpr ui32 kSectorsPerRecord = 8; + const ui32 recordMetadataSize = sizeof(NDDisk::TPersistentBufferLsnRecordHeader) + + kSectorsPerRecord * sizeof(NDDisk::TPersistentBufferSectorInfo) + + kSectorsPerRecord * sizeof(ui64); + const ui32 headerBudget = BlockSize - sizeof(NDDisk::TPersistentBufferHeader); + const ui32 batchedCount = headerBudget / recordMetadataSize; + UNIT_ASSERT_C(batchedCount >= 1, "Sanity: at least one checksummed 8-sector record must fit the header sector"); + + struct TRecordInfo { + TString Payload; + std::vector<ui64> Checksums; + NDDisk::TBlockSelector Selector; + ui64 Lsn; + }; + auto makeRecord = [&](ui32 i) { + TString payload = MakeData(static_cast<char>('a' + (i % 26)), kSectorsPerRecord * BlockSize); + const NDDisk::TBlockSelector selector{ui64(10 + i), 0, kSectorsPerRecord * BlockSize}; + const ui64 lsn = 100 + i; + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + TRecordInfo info{payload, + std::vector<ui64>(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()), + selector, lsn}; + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + return info; + }; + + std::vector<TRecordInfo> batched; + for (ui32 i = 0; i < batchedCount; ++i) { + batched.push_back(makeRecord(i)); + } + const TRecordInfo overflow = makeRecord(batchedCount); + + auto rawOverflow = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawOverflow->Get()->Data.size(), (kSectorsPerRecord + 1) * BlockSize, + "Overflowing record must fall back to its own direct (header + data) write"); + captureWrite(rawOverflow); + ctx.SendPDiskResponse(disk1, *rawOverflow, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto overflowResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(overflowResult, TReplyStatus::OK); + + // Complete A's write, then the batch's combined write (header + all batched records' data). + ctx.SendPDiskResponse(disk1, *rawA, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrA = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrA, TReplyStatus::OK); + + auto rawBatch = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawBatch->Get()->Data.size(), (1 + batched.size() * kSectorsPerRecord) * BlockSize, + "Batch combined write must contain the header sector plus every batched record's data sectors"); + const ui64 actualUniqueId = [&]() -> ui64 { + captureWrite(rawBatch); + const TString& buf = chunkBufs[rawBatch->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawBatch->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->BatchSize, static_cast<ui32>(batched.size()), + "BatchSize must equal the number of successfully batched records"); + return hdr->PersistentBufferUniqueId; + }(); + ctx.SendPDiskResponse(disk1, *rawBatch, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + for (ui32 i = 0; i < batched.size(); ++i) { + auto wr = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wr, TReplyStatus::OK); + } + + // Restart: restore from the captured chunk data and verify every record -- the overflowing + // direct write and every batched record -- comes back with the correct payload and checksums. + // Reading a restored record fetches its data sectors from PDisk on demand (restore only + // reconstructs in-memory metadata, not payload bytes), so intercept TEvChunkReadRaw to disk2 + // and serve it from the captured chunkBufs; also auto-answer TEvCheckSpace, which the actor + // may still emit periodically. + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, ev->GetRecipientRewrite(), + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 67, 1, pbChunkIds, /*oldUniqueId=*/actualUniqueId - 1, chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 121, 1); + + auto checkRecord = [&](const TRecordInfo& info, const TString& label) { + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, info.Selector, info.Lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), info.Payload, + label << " payload mismatch"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), info.Checksums.size(), + label << " checksum count mismatch"); + for (ui32 i = 0; i < info.Checksums.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(i), info.Checksums[i], + label << " checksum " << i << " mismatch"); + } + }; + + checkRecord(overflow, "overflow (direct) record"); + for (ui32 i = 0; i < batched.size(); ++i) { + checkRecord(batched[i], TStringBuilder() << "batched record " << i); + } + } + + // 15. Partial-range read: a read selector narrower than the written record (and starting at a + // nonzero offset within it) must return both the correct payload sub-range and the matching + // checksum sub-range -- exercising the firstBlock/blockCount slicing in ReplyReadPersistentBuffer. + Y_UNIT_TEST(PartialRangeReadSlicesChecksums) { + TTestContext ctx; + const TDiskHandle disk = ctx.CreateDDisk(68, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk.PBServiceId, 122, 1); + + const ui64 lsn = 1; + const TString payload = MakeData('R', 4 * BlockSize); + const NDDisk::TBlockSelector selector{5, 0, 4 * BlockSize}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + const std::vector<ui64> allChecksums(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + UNIT_ASSERT_VALUES_EQUAL(allChecksums.size(), 4u); + + SendToDDisk(ctx, disk.PBServiceId, write.release()); + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk); + ctx.SendPDiskResponse(disk, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + // Read blocks [1, 3) out of the 4-block record, i.e. a 2-block selector starting one block in. + const NDDisk::TBlockSelector partialSelector{5, BlockSize, 2 * BlockSize}; + auto readResult = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds, partialSelector, lsn, 1, {true})); + AssertStatus(readResult, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->GetPayload(0).ConvertToString(), payload.substr(BlockSize, 2 * BlockSize), + "Partial read must return only the requested payload sub-range"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readResult->Get()->Record.ChecksumsSize()), 2u, + "Partial read must return only the checksums for the requested sub-range, not all 4"); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(0), allChecksums[1], + "First returned checksum must be the record's block 1 (not block 0)"); + UNIT_ASSERT_VALUES_EQUAL_C(readResult->Get()->Record.GetChecksums(1), allChecksums[2], + "Second returned checksum must be the record's block 2"); + } + + // 16. Mixed batch: a checksummed record and a checksum-less record share the same batch header. + // After a restart, the checksummed record must restore with its checksums and the checksum-less + // one must restore with an empty Checksums list -- exercising the per-record HAS_PAYLOAD_CHECKSUMS + // flag check during batch restore parsing, as opposed to a single Version-wide switch. + Y_UNIT_TEST(MixedChecksummedAndLegacyBatchRestoresCorrectly) { + TTestContext ctx; + const TDiskHandle disk1 = ctx.CreateDDisk(69, 1); + NDDisk::TQueryCredentials creds1 = Connect(ctx, disk1.PBServiceId, 123, 1); + + std::unordered_map<ui32, TString> chunkBufs; + auto captureWrite = [&](const std::unique_ptr<TEventHandle<NPDisk::TEvChunkWriteRaw>>& raw) { + const ui32 chunkIdx = raw->Get()->ChunkIdx; + const ui32 offset = raw->Get()->Offset; + TString written = raw->Get()->Data.ConvertToString(); + if (chunkBufs.find(chunkIdx) == chunkBufs.end()) { + chunkBufs[chunkIdx] = TString(TTestContext::ChunkSize, '\0'); + } + UNIT_ASSERT_C(offset + written.size() <= TTestContext::ChunkSize, "Write exceeds chunk size"); + memcpy(chunkBufs[chunkIdx].Detach() + offset, written.data(), written.size()); + }; + + // Write A: keep-alive PDisk write; keeps PersistentBufferDiskOperationInflight non-empty so + // subsequent small writes take the batch path. + const TString payloadA = MakeData('A', BlockSize); + const NDDisk::TBlockSelector selectorA{1, 0, BlockSize}; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorA, 1, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payloadA)); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + auto rawA = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + captureWrite(rawA); + + // Write B: batched, WITHOUT checksums (legacy/opt-out). + const TString payloadB = MakeData('B', BlockSize); + const NDDisk::TBlockSelector selectorB{2, 0, BlockSize}; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorB, 2, NDDisk::TWriteInstruction(0)); + write->AddPayload(TRope(payloadB)); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + + // Write C: batched, WITH checksums -- lands in the same batch header as B. + const TString payloadC = MakeData('C', BlockSize); + const NDDisk::TBlockSelector selectorC{3, 0, BlockSize}; + std::vector<ui64> checksumsC; + { + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds1, selectorC, 3, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payloadC)); + checksumsC.assign(write->Record.GetChecksums().begin(), write->Record.GetChecksums().end()); + SendToDDisk(ctx, disk1.PBServiceId, write.release()); + } + UNIT_ASSERT_VALUES_EQUAL(checksumsC.size(), 1u); + + // Complete A's write, then the batch's combined write (header + B's + C's data sectors). + ctx.SendPDiskResponse(disk1, *rawA, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrA = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrA, TReplyStatus::OK); + + auto rawBatch = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk1); + UNIT_ASSERT_VALUES_EQUAL_C(rawBatch->Get()->Data.size(), 3 * BlockSize, + "Batch combined write must contain the header sector plus B's and C's data sectors"); + const ui64 actualUniqueId = [&]() -> ui64 { + captureWrite(rawBatch); + const TString& buf = chunkBufs[rawBatch->Get()->ChunkIdx]; + const auto* hdr = reinterpret_cast<const NDDisk::TPersistentBufferHeader*>(buf.data() + rawBatch->Get()->Offset); + UNIT_ASSERT_VALUES_EQUAL_C(hdr->BatchSize, 2u, "Both B and C must land in the same batch"); + return hdr->PersistentBufferUniqueId; + }(); + ctx.SendPDiskResponse(disk1, *rawBatch, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto wrB = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrB, TReplyStatus::OK); + auto wrC = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(wrC, TReplyStatus::OK); + + // Restart. Reading a restored record fetches its data sectors from PDisk on demand, so + // intercept TEvChunkReadRaw to disk2 and serve it from the captured chunkBufs; also + // auto-answer TEvCheckSpace, which the actor may still emit periodically. + std::vector<ui32> pbChunkIds; + for (ui32 i = 0; i < PersistentBufferInitChunks; ++i) { + pbChunkIds.push_back(disk1.FirstChunkId + i); + } + std::optional<TActorId> disk2PdiskEdge; + ctx.Runtime.FilterFunction = [&](ui32 /*nodeId*/, std::unique_ptr<IEventHandle>& ev) -> bool { + if (ev->GetTypeRewrite() == NPDisk::TEvCheckSpace::EventType) { + ctx.Runtime.Send(new IEventHandle(ev->Sender, ev->GetRecipientRewrite(), + new NPDisk::TEvCheckSpaceResult(NKikimrProto::OK, 0, 0, 0, 0, 0, 0, 0, "", 0), + 0, ev->Cookie), NodeId); + return false; + } + if (disk2PdiskEdge && + ev->GetTypeRewrite() == NPDisk::TEvChunkReadRaw::EventType && + ev->GetRecipientRewrite() == *disk2PdiskEdge) { + auto* req = ev->Get<NPDisk::TEvChunkReadRaw>(); + const ui32 chunkIdx = req->ChunkIdx; + const ui32 offset = req->Offset; + const ui32 size = req->Size; + auto it = chunkBufs.find(chunkIdx); + TString slice(size, '\0'); + if (it != chunkBufs.end()) { + const TString& buf = it->second; + UNIT_ASSERT_C(offset + size <= buf.size(), "Read request out of chunk bounds"); + memcpy(slice.Detach(), buf.data() + offset, size); + } + ctx.Runtime.Send(new IEventHandle(ev->Sender, *disk2PdiskEdge, + new NPDisk::TEvChunkReadRawResult(TRope(slice)), + 0, ev->Cookie), NodeId); + return false; + } + return true; + }; + + const TDiskHandle disk2 = ctx.CreateDDiskWithRestoredChunkData( + 69, 1, pbChunkIds, /*oldUniqueId=*/actualUniqueId - 1, chunkBufs); + disk2PdiskEdge = disk2.PDiskEdge; + NDDisk::TQueryCredentials creds2 = Connect(ctx, disk2.PBServiceId, 123, 1); + + auto readB = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selectorB, 2, 1, {true})); + AssertStatus(readB, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readB->Get()->GetPayload(0).ConvertToString(), payloadB, + "Checksum-less batched record's payload must survive restart"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readB->Get()->Record.ChecksumsSize()), 0u, + "Checksum-less batched record must restore with an empty Checksums list"); + + auto readC = SendToDDiskAndWait<NDDisk::TEvReadPersistentBufferResult>( + ctx, disk2.PBServiceId, new NDDisk::TEvReadPersistentBuffer(creds2, selectorC, 3, 1, {true})); + AssertStatus(readC, TReplyStatus::OK); + UNIT_ASSERT_VALUES_EQUAL_C(readC->Get()->GetPayload(0).ConvertToString(), payloadC, + "Checksummed batched record's payload must survive restart"); + UNIT_ASSERT_VALUES_EQUAL_C(static_cast<ui32>(readC->Get()->Record.ChecksumsSize()), checksumsC.size(), + "Checksummed batched record must restore with its checksums, not an empty list"); + for (ui32 i = 0; i < checksumsC.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(readC->Get()->Record.GetChecksums(i), checksumsC[i], + "Checksum " << i << " must survive restart for the checksummed batched record"); + } + } } } // anonymous namespace diff --git a/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp b/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp index cec89be4f79..6b1e3b13457 100644 --- a/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp +++ b/ydb/core/blobstorage/ddisk/write_persistent_buffers_request_actor.cpp @@ -147,6 +147,11 @@ namespace NKikimr::NDDisk { auto msg = std::make_unique<TEvWritePersistentBuffers>(creds, selector, inflight.Lsn, NDDisk::TWriteInstruction(0), inflight.PersistentBufferIds, inflight.Timeout); msg->AddPayload(TRope(payload)); + // Forward the checksums persisted on the source record, if any (TEvReadPersistentBufferResult + // carries them opt-in, see TPersistentBuffer::TRecord::PayloadChecksums). Without this, records + // re-replicated via TEvReadThenWritePersistentBuffers would silently lose their checksums on + // every target PB, exactly the recovery scenario where corruption detection matters most. + msg->Record.MutableChecksums()->CopyFrom(record.GetChecksums()); auto h = std::make_unique<IEventHandle>(SelfId(), inflight.Sender, msg.release(), 0, inflight.Cookie); TActivationContext::Send(h.release()); diff --git a/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.cpp b/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.cpp index 601645f3ef9..534ec86a629 100644 --- a/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.cpp +++ b/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.cpp @@ -1,8 +1,10 @@ #include "dsproxy_mock.h" #include "model.h" #include <ydb/core/base/blobstorage.h> +#include <ydb/core/blobstorage/dsproxy/dsproxy.h> #include <ydb/core/blobstorage/vdisk/common/vdisk_events.h> #include <ydb/core/util/stlog.h> +#include <util/random/fast.h> #define YDB_LOG_THIS_FILE_COMPONENT BS_PROXY @@ -148,6 +150,105 @@ namespace NKikimr { , Model(MakeIntrusive<NFake::TProxyDS>(groupId)) {} }; + + //////////////////////////////////////////////////////////////////////////////// + + class TBlobStorageFailureInjectingActor final + : public TActor<TBlobStorageFailureInjectingActor> + { + private: + const TActorId RealProxy; + const TGroupId GroupId; + const double FailureProbability; + const ui64 RandomFailureSeed; + const NKikimrProto::EReplyStatus ErrorReplyStatus; + TFastRng64 Rng; + const TString FailureErrorReason; + + public: + TBlobStorageFailureInjectingActor( + TActorId realProxy, + TGroupId groupId, + TBSFailureInjectionConfig config) + : TActor(&TThis::StateWork) + , RealProxy(realProxy) + , GroupId(groupId) + , FailureProbability(std::clamp(config.GetFailureProbability(), 0.0, 1.0)) + , RandomFailureSeed(config.HasRandomSeed() ? config.GetRandomSeed() : RandomNumber<ui64>()) + , ErrorReplyStatus(config.GetErrorReplyStatus()) + , Rng(RandomFailureSeed) + , FailureErrorReason(TStringBuilder() + << "injected by BSProxyInterceptor" + << " group " << GroupId + << " seed " << RandomFailureSeed) + { + } + + private: + bool ShouldInjectFailure() + { + return Rng.GenRandReal4() < FailureProbability; + } + + template <typename TRequest> + bool MaybeInjectFailure( + TAutoPtr<IEventHandle>& ev, + TRequest& request, + const TString& eventName) + { + if (!ShouldInjectFailure()) { + return false; + } + + auto response = request.MakeErrorResponse(ErrorReplyStatus, FailureErrorReason, GroupId); + response->ExecutionRelay = std::move(request.ExecutionRelay); + + LOG_WARN_S(*TlsActivationContext, NKikimrServices::BS_PROXY, + "[BSProxyInterceptor] group " << GroupId + << " injecting " << eventName + << " failure; not forwarding to " << RealProxy.ToString() + << " sender " << ev->Sender.ToString() + << " cookie " << ev->Cookie + << " probability " << FailureProbability + << " seed " << RandomFailureSeed); + + Send(ev->Sender, response.release(), 0, ev->Cookie); + return true; + } + + STFUNC(StateWork) + { + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::BS_PROXY, + "[BSProxyInterceptor] group " << GroupId << " " << ev->GetTypeName() << " " << ev->ToString()); + + #define HANDLE_EVENT(evType) \ + case evType::EventType: { \ + auto* msg = ev->Get<evType>(); \ + if (MaybeInjectFailure(ev, *msg, ev->GetTypeName())) { \ + return; \ + } \ + break; \ + } + + switch (ev->GetTypeRewrite()) { + DSPROXY_ENUM_EVENTS(HANDLE_EVENT) + case TEvents::TEvPoison::EventType: { + TActor::PassAway(); + [[fallthrough]]; + } + default: { + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::BS_PROXY, + "[BSProxyInterceptor] group " << GroupId + << " skipping event type " << ev->GetTypeName()); + break; + } + } + + TActivationContext::Forward(ev, RealProxy); + #undef HANDLE_EVENT + } + }; + } // anon IActor *CreateBlobStorageGroupProxyMockActor(TIntrusivePtr<NFake::TProxyDS> model) { @@ -158,4 +259,8 @@ namespace NKikimr { return new TBlobStorageGroupProxyMockActor(groupId); } + IActor *CreateBlobStorageGroupFailureInjectingActor(TActorId actorId, TGroupId groupId, const TBSFailureInjectionConfig& config) { + return new TBlobStorageFailureInjectingActor(actorId, groupId, config); + } + } // NKikimr diff --git a/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.h b/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.h index eb0aa2e0e51..ea57c8dd1cc 100644 --- a/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.h +++ b/ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.h @@ -2,6 +2,8 @@ #include "defs.h" #include <ydb/core/base/blobstorage_common.h> +#include <ydb/core/protos/blobstorage.pb.h> + namespace NKikimr { namespace NFake { @@ -11,4 +13,7 @@ namespace NKikimr { IActor *CreateBlobStorageGroupProxyMockActor(TIntrusivePtr<NFake::TProxyDS> model); IActor *CreateBlobStorageGroupProxyMockActor(TGroupId groupId); + using TBSFailureInjectionConfig = NKikimrBlobStorage::TNodeWardenServiceSet::TFailureInjectionConfig; + IActor *CreateBlobStorageGroupFailureInjectingActor(TActorId actorId, TGroupId groupId, const TBSFailureInjectionConfig& config); + } // NKikimr diff --git a/ydb/core/blobstorage/nodewarden/distconf.h b/ydb/core/blobstorage/nodewarden/distconf.h index f22c277079f..a194af1a701 100644 --- a/ydb/core/blobstorage/nodewarden/distconf.h +++ b/ydb/core/blobstorage/nodewarden/distconf.h @@ -476,17 +476,25 @@ namespace NKikimr::NStorage { std::optional<TString> GenerateFirstConfig(NKikimrBlobStorage::TStorageConfig *config, const TString& selfAssemblyUUID); - void AllocateStaticGroup(NKikimrBlobStorage::TStorageConfig *config, TGroupId groupId, ui32 groupGeneration, - TBlobStorageGroupType gtype, const NKikimrBlobStorage::TGroupGeometry& geometry, - const NProtoBuf::RepeatedPtrField<NKikimrBlobStorage::TPDiskFilter>& pdiskFilters, - std::optional<NKikimrBlobStorage::EPDiskType> pdiskType, - THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks, - const NBsController::TGroupMapper::TForbiddenPDisks& forbid, - i64 requiredSpace, NKikimrBlobStorage::TBaseConfig *baseConfig, - bool convertToDonor, bool ignoreVSlotQuotaCheck, bool isSelfHealReasonDecommit, TBridgePileId bridgePileId, - std::optional<TGroupId> bridgeProxyGroupId, - const NProtoBuf::RepeatedField<ui32>& selfHealAllowedNodes = {}, - bool applyNodeAllowList = false); + struct TAllocateStaticGroupParams { + NKikimrBlobStorage::TStorageConfig *Config = nullptr; + TGroupId GroupId; + ui32 GroupGeneration = 0; + TBlobStorageGroupType GroupType; + THashMap<TVDiskIdShort, NBsController::TPDiskId> ReplacedDisks; + NBsController::TGroupMapper::TForbiddenPDisks ForbiddenPDisks; + i64 RequiredSpace = 0; + const NKikimrBlobStorage::TBaseConfig *BaseConfig = nullptr; + bool ConvertToDonor = false; + bool IgnoreVSlotQuotaCheck = false; + bool AllowUnusableDisks = false; + bool IsSelfHealReasonDecommit = false; + TBridgePileId BridgePileId; + std::optional<TGroupId> BridgeProxyGroupId; + bool ApplySelfHealNodeAllowList = false; + }; + + void AllocateStaticGroup(TAllocateStaticGroupParams params); bool UpdateConfig(NKikimrBlobStorage::TStorageConfig *config); @@ -518,7 +526,9 @@ namespace NKikimr::NStorage { bool GenerateStateStorageConfig(NKikimrConfig::TDomainsConfig::TStateStorage *ss , const NKikimrBlobStorage::TStorageConfig& baseConfig , std::unordered_set<ui32>& usedNodes + , const std::unordered_set<ui32>& nodesToUse = {} , const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig = {} + , bool automaticManagement = true , ui32 overrideReplicasInRingCount = 0 , ui32 overrideRingsCount = 0 , ui32 replicasSpecificVolume = 200 diff --git a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp index 4dec85b0917..96e83a946b7 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_generate.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_generate.cpp @@ -37,10 +37,15 @@ namespace NKikimr::NStorage { TGroupId groupId = TGroupId::Zero(); auto allocateGroup = [&](TBridgePileId bridgePileId, std::optional<TGroupId> bridgeProxyGroupId) { - AllocateStaticGroup(config, groupId, /*groupGeneration=*/ 1, TBlobStorageGroupType(species), - smConfig.GetGeometry(), smConfig.GetPDiskFilter(), - smConfig.HasPDiskType() ? std::make_optional(smConfig.GetPDiskType()) : std::nullopt, {}, {}, 0, - nullptr, false, true, false, bridgePileId, bridgeProxyGroupId); + AllocateStaticGroup({ + .Config = config, + .GroupId = groupId, + .GroupGeneration = 1, + .GroupType = TBlobStorageGroupType(species), + .IgnoreVSlotQuotaCheck = true, + .BridgePileId = bridgePileId, + .BridgeProxyGroupId = bridgeProxyGroupId, + }); const auto& groups = config->GetBlobStorageConfig().GetServiceSet().GetGroups(); const auto& allocatedGroup = groups.at(groups.size() - 1); @@ -127,7 +132,12 @@ namespace NKikimr::NStorage { config->Mutable##NAME##Config()->CopyFrom(Cfg->DomainsConfig->GetExplicit##NAME##Config()); \ } \ if (!config->Has##NAME##Config()) { \ - GenerateStateStorageConfig(config->Mutable##NAME##Config(), *config, usedNodes); \ + std::unordered_set<ui32> nodesToUse; \ + if (Cfg->SelfManagementConfig && Cfg->SelfManagementConfig->NAME##SelfHealAllowedNodesSize()) { \ + const auto& allowedNodes = Cfg->SelfManagementConfig->Get##NAME##SelfHealAllowedNodes(); \ + nodesToUse.insert(allowedNodes.begin(), allowedNodes.end()); \ + } \ + GenerateStateStorageConfig(config->Mutable##NAME##Config(), *config, usedNodes, nodesToUse); \ } UPDATE_EXPLICIT_CONFIG(StateStorage) @@ -139,32 +149,28 @@ namespace NKikimr::NStorage { return std::nullopt; } - void TDistributedConfigKeeper::AllocateStaticGroup(NKikimrBlobStorage::TStorageConfig *config, TGroupId groupId, - ui32 groupGeneration, TBlobStorageGroupType gtype, const NKikimrBlobStorage::TGroupGeometry& geometry, - const NProtoBuf::RepeatedPtrField<NKikimrBlobStorage::TPDiskFilter>& pdiskFilters, - std::optional<NKikimrBlobStorage::EPDiskType> pdiskType, - THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks, - const NBsController::TGroupMapper::TForbiddenPDisks& forbid, i64 requiredSpace, - NKikimrBlobStorage::TBaseConfig *baseConfig, bool convertToDonor, bool ignoreVSlotQuotaCheck, - bool isSelfHealReasonDecommit, TBridgePileId bridgePileId, std::optional<TGroupId> bridgeProxyGroupId, - const NProtoBuf::RepeatedField<ui32>& selfHealAllowedNodes, bool applyNodeAllowList) { + void TDistributedConfigKeeper::AllocateStaticGroup(TAllocateStaticGroupParams params) { using TPDiskId = NBsController::TPDiskId; - NKikimrConfig::TBlobStorageConfig *bsConfig = config->MutableBlobStorageConfig(); + auto& config = *params.Config; + auto& replacedDisks = params.ReplacedDisks; + const auto& selfManagementConfig = config.GetSelfManagementConfig(); + const auto& selfHealAllowedNodes = selfManagementConfig.GetStaticGroupSelfHealAllowedNodes(); + NKikimrConfig::TBlobStorageConfig *bsConfig = config.MutableBlobStorageConfig(); // build node location map THashMap<ui32, TNodeLocation> nodeLocations; THashSet<ui32> allowedNodeIds; - for (const auto& node : config->GetAllNodes()) { + for (const auto& node : config.GetAllNodes()) { TNodeLocation location(node.GetLocation()); nodeLocations.try_emplace(node.GetNodeId(), location); - if (bridgePileId == ResolveNodePileId(location)) { + if (params.BridgePileId == ResolveNodePileId(location)) { allowedNodeIds.insert(node.GetNodeId()); } } // when restricting self-heal targets, only these node ids may host relocated vdisks - const bool restrictSelfHealNodes = applyNodeAllowList && !selfHealAllowedNodes.empty(); + const bool restrictSelfHealNodes = params.ApplySelfHealNodeAllowList && !selfHealAllowedNodes.empty(); const THashSet<ui32> selfHealAllowedNodeSet = restrictSelfHealNodes ? THashSet<ui32>(selfHealAllowedNodes.begin(), selfHealAllowedNodes.end()) : THashSet<ui32>{}; @@ -181,10 +187,10 @@ namespace NKikimr::NStorage { THashMap<TPDiskId, TPDiskInfo> pdisks; auto checkMatch = [&](NKikimrBlobStorage::EPDiskType type, bool sharedWithOs, bool readCentric, ui64 kind) { - if (type == pdiskType) { + if (selfManagementConfig.HasPDiskType() && type == selfManagementConfig.GetPDiskType()) { return true; } - for (const auto& pdiskFilter : pdiskFilters) { + for (const auto& pdiskFilter : selfManagementConfig.GetPDiskFilter()) { bool m = true; for (const auto& p : pdiskFilter.GetProperty()) { bool pMatch = false; @@ -218,12 +224,13 @@ namespace NKikimr::NStorage { ui32 defaultMaxSlots = 16; - if (baseConfig) { + if (params.BaseConfig) { + const auto& baseConfig = *params.BaseConfig; std::optional<NKikimrBlobStorage::TPDiskSpaceColor::E> pdiskSpaceColorBorder; ui32 pdiskSpaceMarginPromille = 150; - if (baseConfig->HasSettings()) { - const auto& settings = baseConfig->GetSettings(); + if (baseConfig.HasSettings()) { + const auto& settings = baseConfig.GetSettings(); if (settings.DefaultMaxSlotsSize()) { defaultMaxSlots = settings.GetDefaultMaxSlots(0); } @@ -235,7 +242,7 @@ namespace NKikimr::NStorage { } } - for (const auto& pdisk : baseConfig->GetPDisk()) { + for (const auto& pdisk : baseConfig.GetPDisk()) { if (!checkMatch(pdisk.GetType(), pdisk.GetSharedWithOs(), pdisk.GetReadCentric(), pdisk.GetKind())) { continue; } @@ -261,13 +268,13 @@ namespace NKikimr::NStorage { } const bool usableInTermsOfDecommission = pdisk.GetDecommitStatus() == NKikimrBlobStorage::EDecommitStatus::DECOMMIT_NONE || - pdisk.GetDecommitStatus() == NKikimrBlobStorage::EDecommitStatus::DECOMMIT_REJECTED && !isSelfHealReasonDecommit; + pdisk.GetDecommitStatus() == NKikimrBlobStorage::EDecommitStatus::DECOMMIT_REJECTED && !params.IsSelfHealReasonDecommit; if (!usableInTermsOfDecommission) { pdiskInfo.Usable = false; pdiskInfo.WhyUnusable += 'D'; } - if (!ignoreVSlotQuotaCheck && pdiskInfo.Usable && pdisk.HasPDiskMetrics() && baseConfig->HasSettings()) { + if (!params.IgnoreVSlotQuotaCheck && pdiskInfo.Usable && pdisk.HasPDiskMetrics() && baseConfig.HasSettings()) { const auto& m = pdisk.GetPDiskMetrics(); if (m.HasEnforcedDynamicSlotSize() && pdiskSpaceColorBorder >= NKikimrBlobStorage::TPDiskSpaceColor::YELLOW) { pdiskInfo.SpaceAvailable = m.GetEnforcedDynamicSlotSize() * (1000 - pdiskSpaceMarginPromille) / 1000; @@ -282,7 +289,7 @@ namespace NKikimr::NStorage { } THashMap<ui32, ui64> maxGroupSlotSize; - for (const auto& vslot : baseConfig->GetVSlot()) { + for (const auto& vslot : baseConfig.GetVSlot()) { if (vslot.HasVDiskMetrics()) { if (const auto& m = vslot.GetVDiskMetrics(); m.HasAllocatedSize()) { ui64& size = maxGroupSlotSize[vslot.GetGroupId()]; @@ -291,7 +298,7 @@ namespace NKikimr::NStorage { } } - for (const auto& vslot : baseConfig->GetVSlot()) { + for (const auto& vslot : baseConfig.GetVSlot()) { const auto& vslotId = vslot.GetVSlotId(); const TPDiskId pdiskId(vslotId.GetNodeId(), vslotId.GetPDiskId()); if (const auto it = pdisks.find(pdiskId); it != pdisks.end()) { @@ -321,7 +328,7 @@ namespace NKikimr::NStorage { std::optional<ui32> generation; for (const auto& group : ss.GetGroups()) { - if (TGroupId::FromProto(&group, &NKikimrBlobStorage::TGroupInfo::GetGroupID) == groupId) { + if (TGroupId::FromProto(&group, &NKikimrBlobStorage::TGroupInfo::GetGroupID) == params.GroupId) { generation.emplace(group.GetGroupGeneration()); ui32 failRealmIdx = 0; @@ -362,7 +369,7 @@ namespace NKikimr::NStorage { for (const auto& vdisk : ss.GetVDisks()) { const TVDiskID vdiskId = VDiskIDFromVDiskID(vdisk.GetVDiskID()); - if (vdiskId.GroupID == groupId) { + if (vdiskId.GroupID == params.GroupId) { if (!generation) { throw TExConfigError() << "missing record for group being reconfigured"; } else if (vdiskId.GroupGeneration == *generation && !replacedDisks.contains(vdiskId)) { @@ -375,10 +382,12 @@ namespace NKikimr::NStorage { for (const auto& pdisk : ss.GetPDisks()) { const TPDiskId pdiskId(pdisk.GetNodeID(), pdisk.GetPDiskID()); - if (requiredPDiskIds.contains(pdiskId)) { + if (params.AllowUnusableDisks && requiredPDiskIds.contains(pdiskId)) { if (const auto [it, inserted] = pdisks.try_emplace(pdiskId); inserted) { - auto& r = it->second.Record; - r.CopyFrom(pdisk); + TPDiskInfo& pdiskInfo = it->second; + pdiskInfo.Record.CopyFrom(pdisk); + pdiskInfo.Usable = false; + pdiskInfo.WhyUnusable += 'X'; } } @@ -438,10 +447,10 @@ namespace NKikimr::NStorage { } } }; - EnumerateConfigDrives(*config, 0, processDrive, nullptr, true); + EnumerateConfigDrives(config, 0, processDrive, nullptr, true); // group mapper - NBsController::TGroupGeometryInfo geom(gtype.GetErasure(), geometry); + NBsController::TGroupGeometryInfo geom(params.GroupType.GetErasure(), selfManagementConfig.GetGeometry()); NBsController::TGroupMapper mapper(geom); for (const auto& [pdiskId, item] : pdisks) { @@ -460,7 +469,7 @@ namespace NKikimr::NStorage { slotSizeInUnits = pdiskConfig.GetSlotSizeInUnits(); } - const bool pileFilter = !bridgePileId || allowedNodeIds.contains(pdiskId.NodeId); + const bool pileFilter = !params.BridgePileId || allowedNodeIds.contains(pdiskId.NodeId); const bool nodeAllowFilter = !restrictSelfHealNodes || selfHealAllowedNodeSet.contains(pdiskId.NodeId); TString whyUnusable = item.WhyUnusable; @@ -502,8 +511,8 @@ namespace NKikimr::NStorage { NBsController::TGroupMapperError error; const ui32 groupSizeInUnits = 1; // static groups are always single-unit - if (!mapper.AllocateGroup(groupId.GetRawId(), groupDefinition, replacedDisks, forbid, - groupSizeInUnits, requiredSpace, false, {}, error)) { + if (!mapper.AllocateGroup(params.GroupId.GetRawId(), groupDefinition, replacedDisks, + params.ForbiddenPDisks, groupSizeInUnits, params.RequiredSpace, false, {}, error)) { throw TExConfigError() << "group allocation failed Error# " << error.ErrorMessage << " groupDefinition# " << dumpGroupDefinition(); } @@ -513,38 +522,38 @@ namespace NKikimr::NStorage { NKikimrBlobStorage::TGroupInfo *sGroup = nullptr; for (size_t i = 0; i < sSet->GroupsSize(); ++i) { if (const auto& group = sSet->GetGroups(i); TGroupId::FromProto(&group, - &NKikimrBlobStorage::TGroupInfo::GetGroupID) == groupId) { + &NKikimrBlobStorage::TGroupInfo::GetGroupID) == params.GroupId) { sGroup = sSet->MutableGroups(i); break; } } if (!sGroup) { sGroup = sSet->AddGroups(); - groupId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetGroupID); - sGroup->SetErasureSpecies(gtype.GetErasure()); + params.GroupId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetGroupID); + sGroup->SetErasureSpecies(params.GroupType.GetErasure()); } else { sGroup->ClearRings(); } - sGroup->SetGroupGeneration(groupGeneration); + sGroup->SetGroupGeneration(params.GroupGeneration); - if (bridgeProxyGroupId) { - bridgeProxyGroupId->CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgeProxyGroupId); + if (params.BridgeProxyGroupId) { + params.BridgeProxyGroupId->CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgeProxyGroupId); } - bridgePileId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgePileId); + params.BridgePileId.CopyToProto(sGroup, &NKikimrBlobStorage::TGroupInfo::SetBridgePileId); THashMap<TVDiskIdShort, NProtoBuf::RepeatedPtrField<NKikimrBlobStorage::TNodeWardenServiceSet::TVDisk::TDonor>> donors; for (size_t i = 0; i < sSet->VDisksSize(); ++i) { const auto& vdisk = sSet->GetVDisks(i); const TVDiskID vdiskId = VDiskIDFromVDiskID(vdisk.GetVDiskID()); - if (vdiskId.GroupID != groupId || vdisk.GetEntityStatus() == NKikimrBlobStorage::EEntityStatus::DESTROY) { + if (vdiskId.GroupID != params.GroupId || vdisk.GetEntityStatus() == NKikimrBlobStorage::EEntityStatus::DESTROY) { continue; } auto *m = sSet->MutableVDisks(i); if (replacedDisks.contains(vdiskId)) { if (m->HasDonorMode()) { // this disk is already a donor, nothing to do about it - } else if (convertToDonor) { + } else if (params.ConvertToDonor) { // make this disk a donor auto *donorMode = m->MutableDonorMode(); donorMode->SetNumFailRealms(groupDefinition.size()); @@ -560,7 +569,7 @@ namespace NKikimr::NStorage { donor->MutableVDiskId()->CopyFrom(m->GetVDiskID()); donor->MutableVDiskLocation()->CopyFrom(m->GetVDiskLocation()); } else { - m->MutableVDiskID()->SetGroupGeneration(groupGeneration); + m->MutableVDiskID()->SetGroupGeneration(params.GroupGeneration); } } @@ -596,7 +605,7 @@ namespace NKikimr::NStorage { sLoc->SetPDiskGuid(pdisk.GetPDiskGuid()); auto *sDisk = sSet->AddVDisks(); - VDiskIDFromVDiskID(TVDiskID(groupId, groupGeneration, vdiskId), sDisk->MutableVDiskID()); + VDiskIDFromVDiskID(TVDiskID(params.GroupId, params.GroupGeneration, vdiskId), sDisk->MutableVDiskID()); sDisk->SetVDiskKind(NKikimrBlobStorage::TVDiskKind::Default); sDisk->MutableVDiskLocation()->CopyFrom(*sLoc); if (const auto it = donors.find(vdiskId); it != donors.end()) { @@ -608,14 +617,23 @@ namespace NKikimr::NStorage { bool TDistributedConfigKeeper::GenerateStateStorageConfig(NKikimrConfig::TDomainsConfig::TStateStorage *ss , const NKikimrBlobStorage::TStorageConfig& baseConfig, std::unordered_set<ui32>& usedNodes + , const std::unordered_set<ui32>& nodesToUse , const NKikimrConfig::TDomainsConfig::TStateStorage& oldConfig + , bool automaticManagement , ui32 overrideReplicasInRingCount , ui32 overrideRingsCount , ui32 replicasSpecificVolume ) { + if (!automaticManagement) { + ss->CopyFrom(oldConfig); + return true; + } std::map<TBridgePileId, THashMap<TString, std::vector<std::tuple<ui32, TNodeLocation>>>> nodes; bool goodConfig = true; for (const auto& node : baseConfig.GetAllNodes()) { + if (!nodesToUse.empty() && !nodesToUse.contains(node.GetNodeId())) { + continue; + } TNodeLocation location(node.GetLocation()); TBridgePileId pileId = ResolveNodePileId(location); nodes[pileId][location.GetDataCenterId()].emplace_back(node.GetNodeId(), location); diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp b/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp index 2dc6f9d839d..9b3d68d6c2f 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_invoke_state_storage.cpp @@ -15,15 +15,27 @@ namespace NKikimr::NStorage { const NKikimrBlobStorage::TStorageConfig& config = *Self->StorageConfig; bool result = true; std::unordered_set<ui32> usedNodes; - result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageConfig(), config, usedNodes, config.GetStateStorageConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); - if (pileupReplicas) { - usedNodes.clear(); - } - result &= Self->GenerateStateStorageConfig(currentConfig->MutableStateStorageBoardConfig(), config, usedNodes, config.GetStateStorageBoardConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); - if (pileupReplicas) { - usedNodes.clear(); - } - result &= Self->GenerateStateStorageConfig(currentConfig->MutableSchemeBoardConfig(), config, usedNodes, config.GetSchemeBoardConfig(), overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + +#define GENERATE_CONFIG(NAME) \ + { \ + std::unordered_set<ui32> nodesToUse; \ + if (Self->Cfg->SelfManagementConfig && Self->Cfg->SelfManagementConfig->NAME##SelfHealAllowedNodesSize()) { \ + const auto& allowedNodes = Self->Cfg->SelfManagementConfig->Get##NAME##SelfHealAllowedNodes(); \ + nodesToUse.insert(allowedNodes.begin(), allowedNodes.end()); \ + } \ + const bool automaticManagement = !Self->Cfg->SelfManagementConfig || Self->Cfg->SelfManagementConfig->GetAutomatic##NAME##Management(); \ + result &= Self->GenerateStateStorageConfig(currentConfig->Mutable##NAME##Config(), config, usedNodes, nodesToUse, config.Get##NAME##Config(), automaticManagement, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); \ + if (pileupReplicas) { \ + usedNodes.clear(); \ + } \ + } + + GENERATE_CONFIG(StateStorage) + GENERATE_CONFIG(StateStorageBoard) + GENERATE_CONFIG(SchemeBoard) + +#undef GENERATE_CONFIG + return result; } diff --git a/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp b/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp index 30884b3f0cf..fc3e718fe03 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_invoke_static_group.cpp @@ -176,8 +176,6 @@ namespace NKikimr::NStorage { } const auto& ss = bsConfig.GetServiceSet(); - const auto& smConfig = config.GetSelfManagementConfig(); - THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks; NBsController::TGroupMapper::TForbiddenPDisks forbid; for (const auto& vdisk : ss.GetVDisks()) { @@ -207,14 +205,23 @@ namespace NKikimr::NStorage { std::optional<TGroupId> bridgeProxyGroupId = group.HasBridgeProxyGroupId() ? std::make_optional(TGroupId::FromProto(&group, &NKikimrBlobStorage::TGroupInfo::GetBridgeProxyGroupId)) : std::nullopt; - Self->AllocateStaticGroup(&config, vdiskId.GroupID, vdiskId.GroupGeneration + 1, - TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)group.GetErasureSpecies()), - smConfig.GetGeometry(), smConfig.GetPDiskFilter(), - smConfig.HasPDiskType() ? std::make_optional(smConfig.GetPDiskType()) : std::nullopt, - replacedDisks, forbid, maxSlotSize, - &BaseConfig.value(), cmd.GetConvertToDonor(), cmd.GetIgnoreVSlotQuotaCheck(), - cmd.GetIsSelfHealReasonDecommit(), bridgePileId, bridgeProxyGroupId, - smConfig.GetStaticGroupSelfHealAllowedNodes(), cmd.GetFromSelfHeal()); + Self->AllocateStaticGroup({ + .Config = &config, + .GroupId = vdiskId.GroupID, + .GroupGeneration = vdiskId.GroupGeneration + 1, + .GroupType = TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)group.GetErasureSpecies()), + .ReplacedDisks = std::move(replacedDisks), + .ForbiddenPDisks = std::move(forbid), + .RequiredSpace = static_cast<i64>(maxSlotSize), + .BaseConfig = &BaseConfig.value(), + .ConvertToDonor = cmd.GetConvertToDonor(), + .IgnoreVSlotQuotaCheck = cmd.GetIgnoreVSlotQuotaCheck(), + .AllowUnusableDisks = cmd.GetAllowUnusableDisks(), + .IsSelfHealReasonDecommit = cmd.GetIsSelfHealReasonDecommit(), + .BridgePileId = bridgePileId, + .BridgeProxyGroupId = bridgeProxyGroupId, + .ApplySelfHealNodeAllowList = cmd.GetFromSelfHeal(), + }); } catch (const TExConfigError& ex) { YDB_LOG_NOTICE("ReassignGroupDisk failed to allocate group", {"marker", "NWDC76"}, diff --git a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp index aa8b5bc41fb..052c77e5e6d 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.cpp @@ -70,13 +70,23 @@ namespace NKikimr::NStorage { RingsInGroupCount = minNodesInGroup < 8 ? minNodesInGroup : 8; if (RingsInGroupCount > minNodesInGroup) { RingsInGroupCount = minNodesInGroup; + if (OverrideRingsCount > 0) { + EnoughNodesForOverride = false; + } } NToSelect = RingsInGroupCount < 3 ? 1 : (RingsInGroupCount < 5 ? 3 : 5); ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + minNodesInGroup / ReplicasSpecificVolume); } else { if (OverrideRingsCount == 3 || OverrideRingsCount == 9) { RingsInGroupCount = OverrideRingsCount / 3; + if (RingsInGroupCount > minNodesInGroup) { + RingsInGroupCount = minNodesInGroup < 3 ? 1 : 3; + EnoughNodesForOverride = false; + } } else { + // any other value (including 0, meaning "no override") is not applicable to the + // multi-group (3-DC) topology and is silently ignored - this is not an insufficient + // nodes condition, just a non-applicable override RingsInGroupCount = minNodesInGroup < 3 ? 1 : 3; } NToSelect = RingsInGroupCount < 3 ? 3 : 9; @@ -87,11 +97,17 @@ namespace NKikimr::NStorage { ReplicasInRingCount = OverrideReplicasInRingCount > 0 ? OverrideReplicasInRingCount : (1 + nodesCnt / ReplicasSpecificVolume); } if (ReplicasInRingCount * RingsInGroupCount > minNodesInGroup) { + if (OverrideReplicasInRingCount > 1) { + EnoughNodesForOverride = false; + } ReplicasInRingCount = 1; } } bool TStateStoragePerPileGenerator::IsGoodConfig() const { + if (!EnoughNodesForOverride) { + return false; + } for (auto &nodes : Rings) { for (auto nodeId : nodes) { if (CalcNodeState(nodeId, false) > 1) { @@ -102,6 +118,10 @@ namespace NKikimr::NStorage { return true; } + bool TStateStoragePerPileGenerator::IsEnoughNodesForOverride() const { + return EnoughNodesForOverride; + } + void TStateStoragePerPileGenerator::AddRingGroup(NKikimrConfig::TDomainsConfig::TStateStorage *ss) { auto *rg = ss->AddRingGroups(); PileId.CopyToProto(rg, &NKikimrConfig::TDomainsConfig::TStateStorage::TRing::SetBridgePileId); diff --git a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h index 1a55b2c7bd0..4fb5702b0f9 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h +++ b/ydb/core/blobstorage/nodewarden/distconf_statestorage_config_generator.h @@ -20,6 +20,8 @@ namespace NKikimr::NStorage { bool IsGoodConfig() const; void AddRingGroup(NKikimrConfig::TDomainsConfig::TStateStorage *ss); + bool IsEnoughNodesForOverride() const; + private: static constexpr ui32 NodeStatesSize = (ui32)ENodeState::NODE_STATE_MAX; @@ -48,5 +50,6 @@ namespace NKikimr::NStorage { ui32 OverrideReplicasInRingCount = 0; ui32 OverrideRingsCount = 0; ui32 ReplicasSpecificVolume = 200; + bool EnoughNodesForOverride = true; }; } diff --git a/ydb/core/blobstorage/nodewarden/distconf_ut.cpp b/ydb/core/blobstorage/nodewarden/distconf_ut.cpp index 28d707927f0..cbb9d93aa45 100644 --- a/ydb/core/blobstorage/nodewarden/distconf_ut.cpp +++ b/ydb/core/blobstorage/nodewarden/distconf_ut.cpp @@ -23,7 +23,7 @@ namespace NBlobStorageNodeWardenTest{ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { - NKikimrConfig::TDomainsConfig::TStateStorage GenerateSimpleStateStorage(ui32 nodes, std::unordered_set<ui32> usedNodes = {}, ui32 overrideReplicasInRingCount = 0, ui32 overrideRingsCount = 0, ui32 replicasSpecificVolume = 200) { + NKikimrConfig::TDomainsConfig::TStateStorage GenerateSimpleStateStorage(ui32 nodes, std::unordered_set<ui32> usedNodes = {}, ui32 overrideReplicasInRingCount = 0, ui32 overrideRingsCount = 0, ui32 replicasSpecificVolume = 200, std::unordered_set<ui32> nodesToUse = {}, bool *goodConfigOut = nullptr, const NKikimrConfig::TDomainsConfig::TStateStorage& oldSS = {}, bool automaticManagement = true) { NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true); NKikimrConfig::TDomainsConfig::TStateStorage ss; NKikimrBlobStorage::TStorageConfig config; @@ -31,7 +31,10 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { auto *node = config.AddAllNodes(); node->SetNodeId(i + 1); } - keeper.GenerateStateStorageConfig(&ss, config, usedNodes, {}, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + bool goodConfig = keeper.GenerateStateStorageConfig(&ss, config, usedNodes, nodesToUse, oldSS, automaticManagement, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + if (goodConfigOut) { + *goodConfigOut = goodConfig; + } return ss; } @@ -46,6 +49,9 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { , ui32 overrideReplicasInRingCount = 0 , ui32 overrideRingsCount = 0 , ui32 replicasSpecificVolume = 1000 + , std::unordered_set<ui32> nodesToUse = {} + , bool *goodConfigOut = nullptr + , bool automaticManagement = true ) { NKikimrBlobStorage::TStorageConfig config; ui32 nodeId = 1; @@ -74,7 +80,10 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { for (auto [nodeId, state] : nodesState) { keeper.SelfHealNodesState[nodeId] = state; } - keeper.GenerateStateStorageConfig(&ss, config, usedNodes, oldSS, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + bool goodConfig = keeper.GenerateStateStorageConfig(&ss, config, usedNodes, nodesToUse, oldSS, automaticManagement, overrideReplicasInRingCount, overrideRingsCount, replicasSpecificVolume); + if (goodConfigOut) { + *goodConfigOut = goodConfig; + } return ss; } @@ -209,6 +218,158 @@ Y_UNIT_TEST_SUITE(TDistconfGenerateConfigTest) { " Ring { Node: 4 Node: 5 Node: 6 } Ring { Node: 7 Node: 8 Node: 9 } } }"); CheckStateStorage2(GenerateSimpleStateStorage(16, {}, 0, 2, 5), "{ RingGroups { NToSelect: 1 Ring { Node: 1 Node: 2 Node: 3 Node: 4 } Ring { Node: 5 Node: 6 Node: 7 Node: 8 } } }"); } + + Y_UNIT_TEST(NodesToUseEmptyMeansNoRestriction) { + // Explicit regression check: passing an empty nodesToUse set behaves exactly + // like not passing the parameter at all (default argument in other tests). + CheckStateStorage(GenerateSimpleStateStorage(8, {}, 0, 0, 200, {}), 5, {1, 2, 3, 4, 5, 6, 7, 8}); + CheckStateStorage(GenerateDCStateStorage(3, 1, 1, {}, {}, {}, 9, 0, 0, 1000, {}), 3, {1, 2, 3}); + } + + Y_UNIT_TEST(NodesToUseRestrictsSimplePool) { + // Out of 10 available nodes, only {5, 6, 7} are allowed to be used. + CheckStateStorage(GenerateSimpleStateStorage(10, {}, 0, 0, 200, {5, 6, 7}), 3, {5, 6, 7}); + } + + Y_UNIT_TEST(NodesToUseKeepsOriginalNodeIds) { + // Restrict a big pool of 100 nodes down to 8 specific nodes; the algorithm + // should behave exactly as if only those 8 nodes existed, while preserving + // their original (non-contiguous) node ids. + CheckStateStorage( + GenerateSimpleStateStorage(100, {}, 0, 0, 200, {10, 20, 30, 40, 50, 60, 70, 80}), + 5, {10, 20, 30, 40, 50, 60, 70, 80}); + } + + Y_UNIT_TEST(NodesToUseIgnoresUnknownNodeIds) { + // Node ids in nodesToUse that are not present in baseConfig's AllNodes + // must simply be ignored, not cause a crash or a bad config. + CheckStateStorage(GenerateSimpleStateStorage(3, {}, 0, 0, 200, {1, 2, 3, 9999}), 3, {1, 2, 3}); + } + + Y_UNIT_TEST(NodesToUseForcesAlternateNodeSelectionInDC) { + // Without restriction, the default pick within each 3-node rack is its + // first node: {1, 4, 7, 10, 13, 16, 19, 22, 25}. + CheckStateStorage(GenerateDCStateStorage(3, 3, 3), 9, {1, 4, 7, 10, 13, 16, 19, 22, 25}); + + // Excluding those default nodes from nodesToUse forces the generator to + // fall back to the next available node in each rack. + std::unordered_set<ui32> nodesToUse = { + 2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 17, 18, 20, 21, 23, 24, 26, 27 + }; + CheckStateStorage(GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 0, 0, 1000, nodesToUse), + 9, {2, 5, 8, 11, 14, 17, 20, 23, 26}); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Cases where the requested configuration (via overrideRingsCount/overrideReplicasInRingCount) + // can not be honored because nodesToUse restricts the available nodes too much. + + Y_UNIT_TEST(NodesToUseInsufficientForOverrideRingsCountSingleGroup) { + // Only 3 nodes are usable (all with a known GOOD self-heal state, via GenerateDCStateStorage), + // but the caller explicitly asks for 8 rings: impossible to satisfy. + bool goodConfig = true; + GenerateDCStateStorage(1, 1, 100, {}, {}, {}, 9, /*overrideReplicasInRingCount=*/0, /*overrideRingsCount=*/8, + 1000, /*nodesToUse=*/{1, 2, 3}, &goodConfig); + UNIT_ASSERT(!goodConfig); + + // Sanity check: with exactly as many nodes as requested rings, the override is satisfiable. + bool goodConfig2 = false; + GenerateDCStateStorage(1, 1, 100, {}, {}, {}, 9, 0, /*overrideRingsCount=*/3, 1000, /*nodesToUse=*/{1, 2, 3}, &goodConfig2); + UNIT_ASSERT(goodConfig2); + } + + Y_UNIT_TEST(NodesToUseInsufficientForOverrideReplicasInRingCountSingleGroup) { + // Only 2 nodes are usable, but the caller explicitly asks for 3 replicas per ring: + // impossible to satisfy (would need at least 3 nodes for a single ring). + bool goodConfig = true; + GenerateDCStateStorage(1, 1, 100, {}, {}, {}, 9, /*overrideReplicasInRingCount=*/3, /*overrideRingsCount=*/1, + 1000, /*nodesToUse=*/{1, 2}, &goodConfig); + UNIT_ASSERT(!goodConfig); + } + + Y_UNIT_TEST(NodesToUseInsufficientForOverrideRingsCountMultiGroup) { + // 3-DC topology, but nodesToUse leaves only 2 nodes in one of the DCs while the + // caller explicitly requests 9 rings (3 per group) - the smallest group can't supply 3 rings. + std::unordered_set<ui32> nodesToUse = { + 1, 4, 7, // dc-0: full rack representation (3 nodes) + 10, 13, // dc-1: only 2 nodes available + 19, 22, 25, // dc-2: full rack representation (3 nodes) + }; + bool goodConfig = true; + GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 0, /*overrideRingsCount=*/9, 1000, nodesToUse, &goodConfig); + UNIT_ASSERT(!goodConfig); + } + + Y_UNIT_TEST(NodesToUseSufficientForOverrideKeepsGoodConfig) { + // Same restriction style as above, but this time nodesToUse leaves exactly enough + // nodes in every group to satisfy the override - the config must be reported as good. + std::unordered_set<ui32> nodesToUse = { + 1, 4, 7, + 10, 13, 16, + 19, 22, 25, + }; + bool goodConfig = false; + GenerateDCStateStorage(3, 3, 3, {}, {}, {}, 9, 0, /*overrideRingsCount=*/9, 1000, nodesToUse, &goodConfig); + UNIT_ASSERT(goodConfig); + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // AutomaticManagement (AutomaticStateStorageManagement / AutomaticStateStorageBoardManagement / + // AutomaticSchemeBoardManagement) semantics at the GenerateStateStorageConfig level: when the + // subsystem's automatic management is disabled, self-heal must keep the current (old) config + // untouched and report the result as "good", regardless of node states or overrides. + + Y_UNIT_TEST(AutomaticManagementDisabledKeepsOldConfigUnchanged) { + NKikimrConfig::TDomainsConfig::TStateStorage oldSS; + { + auto* rg = oldSS.AddRingGroups(); + rg->SetNToSelect(1); + auto* ring = rg->AddRing(); + ring->AddNode(42); + } + + bool goodConfig = false; + auto ss = GenerateSimpleStateStorage(8, {}, 0, 0, 200, {}, &goodConfig, oldSS, /*automaticManagement=*/false); + + // The generator must not have touched node selection at all: result equals oldSS verbatim. + UNIT_ASSERT_EQUAL(TStringBuilder() << ss, TStringBuilder() << oldSS); + UNIT_ASSERT(goodConfig); + } + + Y_UNIT_TEST(AutomaticManagementEnabledGeneratesNewConfig) { + NKikimrConfig::TDomainsConfig::TStateStorage oldSS; + { + auto* rg = oldSS.AddRingGroups(); + rg->SetNToSelect(1); + auto* ring = rg->AddRing(); + ring->AddNode(42); + } + + bool goodConfig = false; + auto ss = GenerateSimpleStateStorage(8, {}, 0, 0, 200, {}, &goodConfig, oldSS, /*automaticManagement=*/true); + + // With automatic management enabled (default), a fresh configuration must be generated + // from the current node pool, ignoring the stale oldSS content. + UNIT_ASSERT(TStringBuilder() << ss != TStringBuilder() << oldSS); + CheckStateStorage(ss, 5, {1, 2, 3, 4, 5, 6, 7, 8}); + } + + Y_UNIT_TEST(AutomaticManagementDisabledIgnoresBadNodeStates) { + // Even if nodesToUse/overrides would normally produce a bad config, disabling automatic + // management must short-circuit generation entirely and still report "good", since the + // old config is kept as-is and self-heal is not supposed to touch it. + NKikimrConfig::TDomainsConfig::TStateStorage oldSS; + { + auto* rg = oldSS.AddRingGroups(); + rg->SetNToSelect(1); + auto* ring = rg->AddRing(); + ring->AddNode(7); + } + bool goodConfig = false; + GenerateSimpleStateStorage(100, {}, /*overrideReplicasInRingCount=*/0, /*overrideRingsCount=*/8, + 200, /*nodesToUse=*/{1, 2, 3}, &goodConfig, oldSS, /*automaticManagement=*/false); + UNIT_ASSERT(goodConfig); + } } Y_UNIT_TEST_SUITE(TDeriveStorageConfigCleanupTest) { @@ -438,7 +599,7 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { NKikimrBlobStorage::TStorageConfig Config; NKikimrBlobStorage::TBaseConfig BaseConfig; - void AddNode(ui32 nodeId, const TString& host, ui32 port = 19001, + void AddNode(ui32 nodeId, const TString& host, ui32 port = 19001, const TString& dataCenter = "", const TString& rack = "") { auto *node = Config.AddAllNodes(); node->SetNodeId(nodeId); @@ -552,6 +713,16 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { } return result; } + + void SetPDiskType(ui32 nodeId, NKikimrBlobStorage::EPDiskType type) { + for (auto& pdisk : *BaseConfig.MutablePDisk()) { + if (pdisk.GetNodeId() == nodeId) { + pdisk.SetType(type); + return; + } + } + UNIT_FAIL("PDisk not found"); + } }; NKikimrBlobStorage::TGroupGeometry Geometry(ui32 numFailDomains) { @@ -563,16 +734,28 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { } void Reallocate(TSetup& s, const NProtoBuf::RepeatedField<ui32>& allowedNodeIds, bool applyNodeAllowList, - i32 erasureSpecies = TBlobStorageGroupType::ErasureNone, ui32 numFailDomains = 1) { + i32 erasureSpecies = TBlobStorageGroupType::ErasureNone, ui32 numFailDomains = 1, bool allowUnusableDisks = false) { + auto *selfManagementConfig = s.Config.MutableSelfManagementConfig(); + selfManagementConfig->MutableGeometry()->CopyFrom(Geometry(numFailDomains)); + selfManagementConfig->SetPDiskType(NKikimrBlobStorage::EPDiskType::ROT); + selfManagementConfig->MutableStaticGroupSelfHealAllowedNodes()->CopyFrom(allowedNodeIds); + NKikimr::NStorage::TDistributedConfigKeeper keeper(nullptr, nullptr, true); THashMap<TVDiskIdShort, NBsController::TPDiskId> replacedDisks; replacedDisks.emplace(TVDiskIdShort(0, 0, 0), NBsController::TPDiskId()); NBsController::TGroupMapper::TForbiddenPDisks forbid; - keeper.AllocateStaticGroup(&s.Config, TGroupId::FromValue(0), /*groupGeneration=*/ 2, - TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)erasureSpecies), Geometry(numFailDomains), - /*pdiskFilters=*/ {}, NKikimrBlobStorage::EPDiskType::ROT, replacedDisks, forbid, - /*requiredSpace=*/ 0, &s.BaseConfig, /*convertToDonor=*/ false, /*ignoreVSlotQuotaCheck=*/ true, - /*isSelfHealReasonDecommit=*/ false, TBridgePileId(), std::nullopt, allowedNodeIds, applyNodeAllowList); + keeper.AllocateStaticGroup({ + .Config = &s.Config, + .GroupId = TGroupId::FromValue(0), + .GroupGeneration = 2, + .GroupType = TBlobStorageGroupType((TBlobStorageGroupType::EErasureSpecies)erasureSpecies), + .ReplacedDisks = std::move(replacedDisks), + .ForbiddenPDisks = std::move(forbid), + .BaseConfig = &s.BaseConfig, + .IgnoreVSlotQuotaCheck = true, + .AllowUnusableDisks = allowUnusableDisks, + .ApplySelfHealNodeAllowList = applyNodeAllowList, + }); } NProtoBuf::RepeatedField<ui32> NodeIds(const std::vector<ui32>& nodeIds) { @@ -651,6 +834,23 @@ Y_UNIT_TEST_SUITE(TDistconfStaticGroupSelfHealTest) { Reallocate(s, NodeIds({2}), /*applyNodeAllowList=*/ true, TBlobStorageGroupType::Erasure4Plus2Block, /*numFailDomains=*/ 8), NKikimr::NStorage::TDistributedConfigKeeper::TExConfigError); } + + Y_UNIT_TEST(Block42RejectsUnusablePreservedPDiskByDefault) { + TSetup s = MakeBlock42Setup(); + s.SetPDiskType(2, NKikimrBlobStorage::EPDiskType::SSD); + UNIT_ASSERT_EXCEPTION( + Reallocate(s, NodeIds({}), false, TBlobStorageGroupType::Erasure4Plus2Block, /*numFailDomains=*/ 8), + NKikimr::NStorage::TDistributedConfigKeeper::TExConfigError); + } + + Y_UNIT_TEST(Block42AllowsUnusablePreservedPDiskWhenRequested) { + TSetup s = MakeBlock42Setup(); + s.SetPDiskType(2, NKikimrBlobStorage::EPDiskType::SSD); + UNIT_ASSERT_NO_EXCEPTION( + Reallocate(s, NodeIds({}), false, TBlobStorageGroupType::Erasure4Plus2Block, /*numFailDomains=*/ 8, + /*allowUnusableDisks=*/ true)); + UNIT_ASSERT_VALUES_EQUAL(s.GetGroupDomainNodes()[1], 2u); + } } } diff --git a/ydb/core/blobstorage/nodewarden/node_warden_impl.h b/ydb/core/blobstorage/nodewarden/node_warden_impl.h index 1416551804a..7c5c13b3541 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_impl.h +++ b/ydb/core/blobstorage/nodewarden/node_warden_impl.h @@ -6,6 +6,7 @@ #include <ydb/core/base/tablet_pipe.h> #include <ydb/core/blobstorage/dsproxy/group_sessions.h> #include <ydb/core/blobstorage/dsproxy/dsproxy_nodemon.h> +#include <ydb/core/blobstorage/dsproxy/mock/dsproxy_mock.h> #include <ydb/core/blobstorage/incrhuge/incrhuge.h> #include <ydb/core/cms/console/configs_dispatcher.h> #include <ydb/core/cms/console/console.h> diff --git a/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp b/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp index 21056c02f9a..688b5824fb0 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_proxy.cpp @@ -126,8 +126,23 @@ void TNodeWarden::StartLocalProxy(ui32 groupId) { // subscribe for group information changes through distconf cache Send(SelfId(), new TEvNodeWardenQueryCache(Sprintf("G%08" PRIx32, groupId), true)); - group.ProxyId = as->Register(proxy.release(), TMailboxType::ReadAsFilled, AppData()->SystemPoolId); - as->RegisterLocalService(MakeBlobStorageProxyID(groupId), group.ProxyId); + auto id = as->Register(proxy.release(), TMailboxType::ReadAsFilled, AppData()->SystemPoolId); + + // determine if we want to inject BS errors + if (Cfg->BlobStorageConfig.GetServiceSet().HasFailureInjectionConfig()) { + auto const& fiConfig = Cfg->BlobStorageConfig.GetServiceSet().GetFailureInjectionConfig(); + if (fiConfig.GetFailureProbability() > 0) { + const auto gid = TGroupId::FromValue(groupId); + if (IsDynamicGroup(gid) || fiConfig.GetIncludeStaticGroups()) { + id = as->Register( + CreateBlobStorageGroupFailureInjectingActor(id, gid, fiConfig), + TMailboxType::ReadAsFilled, AppData()->SystemPoolId); + } + } + } + + group.ProxyId = id; + as->RegisterLocalService(MakeBlobStorageProxyID(groupId), id); } void TNodeWarden::StartVirtualGroupAgent(ui32 groupId) { diff --git a/ydb/core/blobstorage/ut_blobstorage/group_mapping.cpp b/ydb/core/blobstorage/ut_blobstorage/group_mapping.cpp index 7f2abb69aa5..febc2353cba 100644 --- a/ydb/core/blobstorage/ut_blobstorage/group_mapping.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/group_mapping.cpp @@ -648,4 +648,286 @@ Y_UNIT_TEST_SUITE(GroupAllocation) { checkGroupStatus(); } } + + // DomainLevelEnd values corresponding to fail_domain_type setting (see ydb/tools/cfg/types.py, DistinctionLevels) + constexpr ui32 DomainLevelEndRack = 40; // fail_domain_type: rack + constexpr ui32 DomainLevelEndUnit = 50; // fail_domain_type: body (unit) + constexpr ui32 DomainLevelEndDiskScope = 60; // fail_domain_type: disk_scope + constexpr ui32 DomainLevelEndDisk = 256; // fail_domain_type: disk + + std::function<TNodeLocation(ui32)> MakeLocationGenerator() { + return [](ui32 nodeId) { + NActorsInterconnect::TNodeLocation proto; + proto.SetDataCenter("A"); + proto.SetRack(std::to_string(nodeId)); + proto.SetUnit("1"); + return TNodeLocation(proto); + }; + } + + NKikimrBlobStorage::TGroupGeometry MakeGeometry(ui32 domainLevelEnd) { + NKikimrBlobStorage::TGroupGeometry geometry; + geometry.SetNumFailRealms(1); + geometry.SetNumFailDomainsPerFailRealm(8); + geometry.SetNumVDisksPerFailDomain(1); + geometry.SetRealmLevelBegin(10); + geometry.SetRealmLevelEnd(20); + geometry.SetDomainLevelBegin(10); + geometry.SetDomainLevelEnd(domainLevelEnd); + return geometry; + } + + // scopes: list of {diskScope, numDrives}; empty diskScope means drives without DiskScope set + void DefineHostConfig(NKikimrBlobStorage::TConfigRequest& request, ui32 hostConfigId, + const std::vector<std::pair<TString, ui32>>& scopes) { + auto *cmd = request.AddCommand()->MutableDefineHostConfig(); + cmd->SetHostConfigId(hostConfigId); + for (const auto& [diskScope, numDrives] : scopes) { + for (ui32 pdiskIdx = 0; pdiskIdx < numDrives; ++pdiskIdx) { + auto *drive = cmd->AddDrive(); + const TString pdiskName = (diskScope ? diskScope : "noscope") + ToString(pdiskIdx); + drive->SetPath("SectorMap:" + pdiskName + ":1000"); + drive->SetType(NKikimrBlobStorage::EPDiskType::ROT); + if (diskScope) { + drive->SetDiskScope(diskScope); + } + } + } + } + + void DefineBox(NKikimrBlobStorage::TConfigRequest& request, TEnvironmentSetup& env, + const std::function<ui32(ui32)>& hostConfigIdForNode) { + auto *cmd = request.AddCommand()->MutableDefineBox(); + cmd->SetBoxId(1); + for (ui32 nodeId : env.Runtime->GetNodes()) { + auto *host = cmd->AddHost(); + host->MutableKey()->SetNodeId(nodeId); + host->SetHostConfigId(hostConfigIdForNode(nodeId)); + } + } + + void DefineStoragePool(NKikimrBlobStorage::TConfigRequest& request, TEnvironmentSetup& env, + ui32 storagePoolId, const TString& name, ui32 numGroups, ui32 domainLevelEnd) { + auto *cmd = request.AddCommand()->MutableDefineStoragePool(); + cmd->SetBoxId(1); + cmd->SetStoragePoolId(storagePoolId); + cmd->SetName(name); + cmd->SetKind(name); + cmd->SetErasureSpecies(TBlobStorageGroupType::ErasureSpeciesName(env.Settings.Erasure.GetErasure())); + cmd->SetVDiskKind("Default"); + cmd->SetNumGroups(numGroups); + cmd->AddPDiskFilter()->AddProperty()->SetType(NKikimrBlobStorage::EPDiskType::ROT); + cmd->MutableGeometry()->CopyFrom(MakeGeometry(domainLevelEnd)); + } + + std::map<std::pair<ui32, ui32>, TString> GetPDiskScopes(const NKikimrBlobStorage::TBaseConfig& cfg) { + std::map<std::pair<ui32, ui32>, TString> scopes; + for (const auto& pdisk : cfg.GetPDisk()) { + if (pdisk.HasDiskScope()) { + scopes.emplace(std::make_pair(pdisk.GetNodeId(), pdisk.GetPDiskId()), pdisk.GetDiskScope()); + } + } + return scopes; + } + + Y_UNIT_TEST(DiskScopeUniformDistribution) { + const ui32 numNodes = 4; + const ui32 disksPerNode = 4; + const ui32 numGroups = 16; + TBlobStorageGroupType groupType = TBlobStorageGroupType::Erasure4Plus2Block; + TEnvironmentSetup env(TEnvironmentSetup::TSettings{ + .NodeCount = numNodes, + .Erasure = groupType, + .ControllerNodeId = numNodes, + .LocationGenerator = MakeLocationGenerator(), + }); + + { + NKikimrBlobStorage::TConfigRequest request; + DefineHostConfig(request, 1, {{"scope-A", 2}, {"scope-B", 2}}); + DefineBox(request, env, [](ui32) { return 1; }); + DefineStoragePool(request, env, 1, env.StoragePoolName, numGroups, DomainLevelEndDiskScope); + auto response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + } + + NKikimrBlobStorage::TBaseConfig base = env.FetchBaseConfig(); + UNIT_ASSERT_VALUES_EQUAL(base.GroupSize(), numGroups); + UNIT_ASSERT_VALUES_EQUAL(base.PDiskSize(), numNodes * disksPerNode); + + TGroupGeometryInfo geom = CreateGroupGeometry(groupType, 1, 8, 1, 10, 20, 10, DomainLevelEndDiskScope); + TString error; + UNIT_ASSERT_C(CheckBaseConfigLayout(geom, base, true, error), error); + + const auto scopes = GetPDiskScopes(base); + + std::map<ui32, std::set<std::pair<ui32, TString>>> groupNodeScopePairs; + std::map<std::pair<ui32, ui32>, ui32> slotsPerPDisk; + for (const auto& vslot : base.GetVSlot()) { + const auto key = std::make_pair(vslot.GetVSlotId().GetNodeId(), vslot.GetVSlotId().GetPDiskId()); + ++slotsPerPDisk[key]; + const auto it = scopes.find(key); + UNIT_ASSERT_C(it != scopes.end(), "group VDisk resides on PDisk without DiskScope, nodeId# " + << key.first << " pdiskId# " << key.second); + const bool inserted = groupNodeScopePairs[vslot.GetGroupId()].emplace(key.first, it->second).second; + UNIT_ASSERT_C(inserted, "two VDisks of the same group share node and DiskScope, groupId# " + << vslot.GetGroupId() << " nodeId# " << key.first << " scope# " << it->second); + } + + // every group occupies 8 distinct (node, scope) fail domains + UNIT_ASSERT_VALUES_EQUAL(groupNodeScopePairs.size(), numGroups); + for (const auto& [groupId, nodeScopePairs] : groupNodeScopePairs) { + UNIT_ASSERT_VALUES_EQUAL_C(nodeScopePairs.size(), 8, "groupId# " << groupId); + } + + // VDisks are distributed uniformly among all PDisks + const ui32 expectedSlotsPerPDisk = numGroups * 8 / (numNodes * disksPerNode); + UNIT_ASSERT_VALUES_EQUAL(slotsPerPDisk.size(), numNodes * disksPerNode); + for (const auto& [pdisk, slots] : slotsPerPDisk) { + UNIT_ASSERT_VALUES_EQUAL_C(slots, expectedSlotsPerPDisk, "uneven VDisk distribution, nodeId# " + << pdisk.first << " pdiskId# " << pdisk.second); + } + } + + Y_UNIT_TEST(AllFailDomainTypes) { + const ui32 numNodes = 8; + TBlobStorageGroupType groupType = TBlobStorageGroupType::Erasure4Plus2Block; + TEnvironmentSetup env(TEnvironmentSetup::TSettings{ + .NodeCount = numNodes, + .Erasure = groupType, + .ControllerNodeId = numNodes, + .LocationGenerator = MakeLocationGenerator(), + }); + + const std::vector<std::pair<TString, ui32>> pools = { + {"disk_scope", DomainLevelEndDiskScope}, + {"unit", DomainLevelEndUnit}, + {"disk", DomainLevelEndDisk}, + {"rack", DomainLevelEndRack}, + }; + + { + NKikimrBlobStorage::TConfigRequest request; + DefineHostConfig(request, 1, {{"scope-A", 2}, {"scope-B", 2}}); + DefineBox(request, env, [](ui32) { return 1; }); + auto response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + } + + for (size_t i = 0; i < pools.size(); ++i) { + NKikimrBlobStorage::TConfigRequest request; + DefineStoragePool(request, env, i + 1, "pool-" + pools[i].first, 1, pools[i].second); + auto response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), "failed to allocate group with fail_domain_type# " + << pools[i].first << " error# " << response.GetErrorDescription()); + } + + NKikimrBlobStorage::TBaseConfig base = env.FetchBaseConfig(); + UNIT_ASSERT_VALUES_EQUAL(base.GroupSize(), pools.size()); + + std::map<ui32, ui32> groupToPool; + for (const auto& group : base.GetGroup()) { + groupToPool[group.GetGroupId()] = group.GetStoragePoolId(); + } + + const auto scopes = GetPDiskScopes(base); + + std::map<ui32, std::vector<std::pair<ui32, ui32>>> pdisksPerPool; // storagePoolId -> pdisks with group VDisks + for (const auto& vslot : base.GetVSlot()) { + const auto it = groupToPool.find(vslot.GetGroupId()); + UNIT_ASSERT(it != groupToPool.end()); + pdisksPerPool[it->second].emplace_back(vslot.GetVSlotId().GetNodeId(), vslot.GetVSlotId().GetPDiskId()); + } + + for (size_t i = 0; i < pools.size(); ++i) { + const auto& [name, domainLevelEnd] = pools[i]; + const auto& pdisks = pdisksPerPool[i + 1]; + UNIT_ASSERT_VALUES_EQUAL_C(pdisks.size(), 8, "fail_domain_type# " << name); + + // verify layout of this pool's group against pool geometry + NKikimrBlobStorage::TBaseConfig poolConfig = base; + poolConfig.ClearVSlot(); + for (const auto& vslot : base.GetVSlot()) { + if (groupToPool[vslot.GetGroupId()] == i + 1) { + poolConfig.AddVSlot()->CopyFrom(vslot); + } + } + TGroupGeometryInfo geom = CreateGroupGeometry(groupType, 1, 8, 1, 10, 20, 10, domainLevelEnd); + TString error; + UNIT_ASSERT_C(CheckBaseConfigLayout(geom, poolConfig, true, error), "fail_domain_type# " << name + << " error# " << error); + + if (domainLevelEnd == DomainLevelEndDiskScope) { + // disk_scope fail domain type must not allow two VDisks to share the same node and DiskScope + std::set<std::pair<ui32, TString>> nodeScopePairs; + for (const auto& pdisk : pdisks) { + const auto it = scopes.find(pdisk); + UNIT_ASSERT_C(it != scopes.end(), "group VDisk resides on PDisk without DiskScope, nodeId# " + << pdisk.first << " pdiskId# " << pdisk.second); + const bool inserted = nodeScopePairs.emplace(pdisk.first, it->second).second; + UNIT_ASSERT_C(inserted, "two VDisks of the same group share node and DiskScope, nodeId# " + << pdisk.first << " scope# " << it->second); + } + } else if (domainLevelEnd == DomainLevelEndRack) { + // rack fail domain type must not allow two VDisks on the same node + std::set<ui32> distinctNodes; + for (const auto& pdisk : pdisks) { + distinctNodes.insert(pdisk.first); + } + UNIT_ASSERT_VALUES_EQUAL_C(distinctNodes.size(), pdisks.size(), + "two VDisks of the same group occupy the same node, fail_domain_type# " << name); + } + } + } + + Y_UNIT_TEST(PartialDiskScopeMarkdown) { + const ui32 numNodes = 4; + TBlobStorageGroupType groupType = TBlobStorageGroupType::Erasure4Plus2Block; + TEnvironmentSetup env(TEnvironmentSetup::TSettings{ + .NodeCount = numNodes, + .Erasure = groupType, + .ControllerNodeId = numNodes, + .LocationGenerator = MakeLocationGenerator(), + }); + + { + NKikimrBlobStorage::TConfigRequest request; + DefineHostConfig(request, 1, {{"scope-A", 2}, {"scope-B", 2}}); // with disk scopes + DefineHostConfig(request, 2, {{"", 4}}); // without disk scopes + DefineBox(request, env, [&](ui32 nodeId) { return nodeId <= numNodes / 2 ? 1 : 2; }); + auto response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + } + + { + // allocation with fail_domain_type: disk_scope is impossible: nodes without + // DiskScope markdown provide only one fail domain each, 6 fail domains total + NKikimrBlobStorage::TConfigRequest request; + DefineStoragePool(request, env, 1, "pool-disk-scope", 1, DomainLevelEndDiskScope); + auto response = env.Invoke(request); + UNIT_ASSERT_C(!response.GetSuccess(), "group allocation with fail_domain_type# disk_scope must fail"); + } + + { + // allocation with fail_domain_type: disk is possible: every PDisk is a separate fail domain + NKikimrBlobStorage::TConfigRequest request; + DefineStoragePool(request, env, 2, "pool-disk", 1, DomainLevelEndDisk); + auto response = env.Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + } + + NKikimrBlobStorage::TBaseConfig base = env.FetchBaseConfig(); + UNIT_ASSERT_VALUES_EQUAL(base.GroupSize(), 1); + + TGroupGeometryInfo geom = CreateGroupGeometry(groupType, 1, 8, 1, 10, 20, 10, DomainLevelEndDisk); + TString error; + UNIT_ASSERT_C(CheckBaseConfigLayout(geom, base, true, error), error); + + std::set<std::pair<ui32, ui32>> usedPDisks; + for (const auto& vslot : base.GetVSlot()) { + const bool inserted = usedPDisks.emplace(vslot.GetVSlotId().GetNodeId(), vslot.GetVSlotId().GetPDiskId()).second; + UNIT_ASSERT_C(inserted, "two VDisks of the same group occupy the same PDisk"); + } + UNIT_ASSERT_VALUES_EQUAL(usedPDisks.size(), 8); + } } diff --git a/ydb/core/config/init/init_impl.h b/ydb/core/config/init/init_impl.h index 4db609adf59..eff49dd92eb 100644 --- a/ydb/core/config/init/init_impl.h +++ b/ydb/core/config/init/init_impl.h @@ -389,6 +389,7 @@ struct TCommonAppOptions { ui32 GRpcPublicPort = 0; ui32 GRpcsPublicPort = 0; ui32 KafkaPort = 0; + TString KafkaListenAddress = ""; TVector<TString> GRpcPublicAddressesV4; TVector<TString> GRpcPublicAddressesV6; TString GRpcPublicTargetNameOverride = ""; @@ -470,6 +471,7 @@ struct TCommonAppOptions { opts.AddLongOption("grpc-public-port", "set public gRPC port for discovery").RequiredArgument("PORT").StoreResult(&GRpcPublicPort); opts.AddLongOption("grpcs-public-port", "set public gRPC SSL port for discovery").RequiredArgument("PORT").StoreResult(&GRpcsPublicPort); opts.AddLongOption("kafka-port", "enable kafka proxy to listen on port").OptionalArgument("PORT").StoreResult(&KafkaPort); + opts.AddLongOption("kafka-address", "set kafka proxy listen address").RequiredArgument("ADDR").StoreResult(&KafkaListenAddress); // Should be provided in yaml config: TGRpcConfig.PublicAddressesV4 opts.AddLongOption("grpc-public-address-v4", "set public ipv4 address for discovery").RequiredArgument("ADDR").Hidden().EmplaceTo(&GRpcPublicAddressesV4); // Should be provided in yaml config: TGRpcConfig.PublicAddressesV6 @@ -732,6 +734,11 @@ struct TCommonAppOptions { conf.SetListeningPort(KafkaPort); ConfigUpdateTracer.AddUpdate(NKikimrConsole::TConfigItem::KafkaProxyConfigItem, TConfigItemInfo::EUpdateKind::UpdateExplicitly); } + if (KafkaListenAddress) { + auto& conf = *appConfig.MutableKafkaProxyConfig(); + conf.SetListeningAddress(KafkaListenAddress); + ConfigUpdateTracer.AddUpdate(NKikimrConsole::TConfigItem::KafkaProxyConfigItem, TConfigItemInfo::EUpdateKind::UpdateExplicitly); + } if (HttpProxyPort) { auto* httpProxyConfig = appConfig.MutableHttpProxyConfig(); httpProxyConfig->SetEnabled(true); diff --git a/ydb/core/driver_lib/run/config_parser.cpp b/ydb/core/driver_lib/run/config_parser.cpp index db06805e410..82980028679 100644 --- a/ydb/core/driver_lib/run/config_parser.cpp +++ b/ydb/core/driver_lib/run/config_parser.cpp @@ -69,6 +69,7 @@ void TRunCommandConfigParser::SetupLastGetOptForConfigFiles(NLastGetopt::TOpts& opts.AddLongOption("grpc-port", "enable gRPC server on port").RequiredArgument("PORT"); opts.AddLongOption("grpcs-port", "enable gRPC SSL server on port").RequiredArgument("PORT"); opts.AddLongOption("kafka-port", "enable kafka proxy server on port").OptionalArgument("PORT"); + opts.AddLongOption("kafka-address", "set kafka proxy listen address").RequiredArgument("ADDR"); opts.AddLongOption("grpc-public-host", "set public gRPC host for discovery").RequiredArgument("HOST"); opts.AddLongOption("grpc-public-port", "set public gRPC port for discovery").RequiredArgument("PORT"); opts.AddLongOption("grpcs-public-port", "set public gRPC SSL port for discovery").RequiredArgument("PORT"); @@ -169,6 +170,11 @@ void TRunCommandConfigParser::ParseConfigFiles(const NLastGetopt::TOptsParseResu conf.SetListeningPort(FromString<ui16>(res.Get("kafka-port"))); } + if (res.Has("kafka-address")) { + auto& conf = *Config.AppConfig.MutableKafkaProxyConfig(); + conf.SetListeningAddress(res.Get("kafka-address")); + } + if (res.Has("grpc-public-host")) { auto& conf = *Config.AppConfig.MutableGRpcConfig(); conf.SetPublicHost(res.Get("grpc-public-host")); diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index fe1e6570650..27e6c618542 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -3227,6 +3227,7 @@ void TKafkaProxyServiceInitializer::InitializeServices(NActors::TActorSystemSetu if (Config.GetKafkaProxyConfig().GetEnableKafkaProxy()) { NKafka::TListenerSettings settings; settings.Port = Config.GetKafkaProxyConfig().GetListeningPort(); + settings.Address = Config.GetKafkaProxyConfig().GetListeningAddress(); settings.SslCertificatePem = Config.GetKafkaProxyConfig().GetSslCertificate(); settings.CertificateFile = Config.GetKafkaProxyConfig().GetCert(); settings.PrivateKeyFile = Config.GetKafkaProxyConfig().GetKey(); diff --git a/ydb/core/formats/arrow/accessor/common/json_value_view.cpp b/ydb/core/formats/arrow/accessor/common/json_value_view.cpp index 73d3b0907d1..8e20e6b7ca0 100644 --- a/ydb/core/formats/arrow/accessor/common/json_value_view.cpp +++ b/ydb/core/formats/arrow/accessor/common/json_value_view.cpp @@ -1,17 +1,36 @@ #include "json_value_view.h" +#include <library/cpp/json/json_reader.h> +#include <library/cpp/json/json_writer.h> + #include <ydb/library/actors/core/log.h> #include <yql/essentials/types/binary_json/read.h> +#include <yql/essentials/types/binary_json/write.h> #include <util/generic/ylimits.h> +#include <util/stream/str.h> #include <util/string/cast.h> +#include <util/string/escape.h> #include <cmath> #include <limits> namespace NKikimr::NArrow::NAccessor { +namespace { + +NBinaryJson::TBinaryJson JsonValueToBinaryJsonVerified(const NJson::TJsonValue& json) { + const auto dumpedJson = WriteJsonRoundTripSafe(json); + auto result = NBinaryJson::SerializeToBinaryJson(dumpedJson); + if (std::holds_alternative<TString>(result)) { + AFL_VERIFY(false)("error", std::get<TString>(result))("type", json.GetType())("input_dump", EscapeC(dumpedJson)); + } + return std::get<NBinaryJson::TBinaryJson>(std::move(result)); +} + +} // namespace + std::optional<TString> TJsonValueView::JsonNumberToString(double jsonNumber) { if (std::isnan(jsonNumber)) { return std::nullopt; @@ -90,6 +109,34 @@ std::optional<TStringBuf> TJsonValueView::GetBinaryJsonBlobOptional() const { return Bytes; } +NJson::TJsonValue TJsonValueView::ToJsonValue() const { + switch (Kind) { + case EKind::BinaryJson: { + const auto text = NBinaryJson::SerializeToJson(Bytes); + NJson::TJsonValue result; + AFL_VERIFY(NJson::ReadJsonTree(text, &result)); + return result; + } + case EKind::String: + return NJson::TJsonValue(Bytes); + case EKind::Number: + return NJson::TJsonValue(Number); + case EKind::Bool: + return NJson::TJsonValue(Bool); + } +} + +NBinaryJson::TBinaryJson TJsonValueView::ToBinaryJson() const { + switch (Kind) { + case EKind::BinaryJson: + return NBinaryJson::TBinaryJson(Bytes.data(), Bytes.size()); + case EKind::String: + case EKind::Number: + case EKind::Bool: + return JsonValueToBinaryJsonVerified(ToJsonValue()); + } +} + std::optional<TStringBuf> TJsonValueView::ScalarFromBinaryJson() const { if (Bytes.empty()) { return std::nullopt; @@ -117,4 +164,13 @@ std::optional<TStringBuf> TJsonValueView::ScalarFromBinaryJson() const { } } +TString WriteJsonRoundTripSafe(const NJson::TJsonValue& json) { + NJson::TJsonWriterConfig config; + // avoid precision loss on double conversion + config.FloatToStringMode = PREC_AUTO; + TStringStream dumpedJson; + NJson::WriteJson(&dumpedJson, &json, config); + return dumpedJson.Str(); +} + } // namespace NKikimr::NArrow::NAccessor diff --git a/ydb/core/formats/arrow/accessor/common/json_value_view.h b/ydb/core/formats/arrow/accessor/common/json_value_view.h index 4a680f68cb9..3d6bbc39fc0 100644 --- a/ydb/core/formats/arrow/accessor/common/json_value_view.h +++ b/ydb/core/formats/arrow/accessor/common/json_value_view.h @@ -1,5 +1,8 @@ #pragma once +#include <library/cpp/json/writer/json_value.h> +#include <yql/essentials/types/binary_json/format.h> + #include <util/generic/string.h> #include <optional> @@ -22,6 +25,9 @@ public: // So result will be different for a native scalar and the same scalar wrapped in BinaryJson. // Expected to be used only for containers, but no checks enforce that. std::optional<TStringBuf> GetBinaryJsonBlobOptional() const; + + NJson::TJsonValue ToJsonValue() const; + NBinaryJson::TBinaryJson ToBinaryJson() const; private: static std::optional<TString> JsonNumberToString(double jsonNumber); @@ -51,4 +57,7 @@ private: mutable TString ScalarHolder; }; +// Serialize a JSON value to text with round-trip-safe number formatting. +TString WriteJsonRoundTripSafe(const NJson::TJsonValue& json); + } // namespace NKikimr::NArrow::NAccessor diff --git a/ydb/core/formats/arrow/accessor/common/ut/ut_json_value_view.cpp b/ydb/core/formats/arrow/accessor/common/ut/ut_json_value_view.cpp index fc3773bd177..b9c83bbeb0a 100644 --- a/ydb/core/formats/arrow/accessor/common/ut/ut_json_value_view.cpp +++ b/ydb/core/formats/arrow/accessor/common/ut/ut_json_value_view.cpp @@ -73,4 +73,13 @@ Y_UNIT_TEST_SUITE(JsonValueView) { UNIT_ASSERT_VALUES_EQUAL(ScalarStr(TJsonValueView::OfNumber(2.5)), BlobScalar("2.5")); UNIT_ASSERT_VALUES_EQUAL(ScalarStr(TJsonValueView::OfBool(true)), BlobScalar("true")); } + + Y_UNIT_TEST(ToJsonValue) { + UNIT_ASSERT_VALUES_EQUAL(TJsonValueView::OfString("hi").ToJsonValue().GetString(), "hi"); + UNIT_ASSERT_VALUES_EQUAL(TJsonValueView::OfNumber(2.5).ToJsonValue().GetDouble(), 2.5); + UNIT_ASSERT(TJsonValueView::OfBool(true).ToJsonValue().GetBoolean()); + const auto blob = ToBinaryJson(R"({"a":1})"); + const auto json = TJsonValueView::OfBinaryJson(TStringBuf(blob.data(), blob.size())).ToJsonValue(); + UNIT_ASSERT_VALUES_EQUAL(json["a"].GetInteger(), 1); + } } diff --git a/ydb/core/formats/arrow/accessor/common/ya.make b/ydb/core/formats/arrow/accessor/common/ya.make index 06cb1580445..9d9cd062ace 100644 --- a/ydb/core/formats/arrow/accessor/common/ya.make +++ b/ydb/core/formats/arrow/accessor/common/ya.make @@ -2,6 +2,7 @@ LIBRARY(library-formats-arrow-accessor-common) PEERDIR( contrib/libs/apache/arrow + library/cpp/json library/cpp/json/writer ydb/library/actors/core ydb/library/formats/arrow/protos diff --git a/ydb/core/formats/arrow/accessor/plain/accessor.h b/ydb/core/formats/arrow/accessor/plain/accessor.h index 5a3e7d36a77..246217052c2 100644 --- a/ydb/core/formats/arrow/accessor/plain/accessor.h +++ b/ydb/core/formats/arrow/accessor/plain/accessor.h @@ -70,41 +70,36 @@ public: { } - template <class TArrowDataType = arrow::StringType> - class TPlainBuilder { + class TPlainBuilderBase { private: std::unique_ptr<arrow::ArrayBuilder> Builder; std::optional<ui32> LastRecordIndex; - public: - TPlainBuilder(const ui32 reserveItems = 0, const ui32 reserveSize = 0) { - Builder = NArrow::MakeBuilder(arrow::TypeTraits<TArrowDataType>::type_singleton(), reserveItems, reserveSize); - } - - void AddRecord(const ui32 recordIndex, const std::string_view value) { - AddValue(recordIndex, arrow::util::string_view(value.data(), value.size())); + protected: + arrow::ArrayBuilder& GetBuilder() { + return *Builder; } - template <class TValue> - void AddValue(const ui32 recordIndex, const TValue& value) { + template <class TAppender> + void AddAt(const ui32 recordIndex, const TAppender& append) { if (LastRecordIndex) { AFL_VERIFY(*LastRecordIndex < recordIndex)("last", LastRecordIndex)("index", recordIndex); TStatusValidator::Validate(Builder->AppendNulls(recordIndex - *LastRecordIndex - 1)); } else { TStatusValidator::Validate(Builder->AppendNulls(recordIndex)); } + append(*Builder); LastRecordIndex = recordIndex; - AFL_VERIFY(NArrow::Append<TArrowDataType>(*Builder, value)); + } + + public: + TPlainBuilderBase(std::unique_ptr<arrow::ArrayBuilder> builder) + : Builder(std::move(builder)) { + AFL_VERIFY(!!Builder); } void AddNull(const ui32 recordIndex) { - if (LastRecordIndex) { - AFL_VERIFY(*LastRecordIndex < recordIndex)("last", LastRecordIndex)("index", recordIndex); - TStatusValidator::Validate(Builder->AppendNulls(recordIndex - *LastRecordIndex)); - } else { - TStatusValidator::Validate(Builder->AppendNulls(recordIndex + 1)); - } - LastRecordIndex = recordIndex; + AddAt(recordIndex, [](arrow::ArrayBuilder& builder) { TStatusValidator::Validate(builder.AppendNull()); }); } std::shared_ptr<IChunkedArray> Finish(const ui32 recordsCount) { @@ -118,6 +113,23 @@ public: } }; + template <class TArrowDataType = arrow::StringType> + class TPlainBuilder: public TPlainBuilderBase { + public: + TPlainBuilder(const ui32 reserveItems = 0, const ui32 reserveSize = 0) + : TPlainBuilderBase(NArrow::MakeBuilder(arrow::TypeTraits<TArrowDataType>::type_singleton(), reserveItems, reserveSize)) { + } + + void AddRecord(const ui32 recordIndex, const std::string_view value) { + AddValue(recordIndex, arrow::util::string_view(value.data(), value.size())); + } + + template <class TValue> + void AddValue(const ui32 recordIndex, const TValue& value) { + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { AFL_VERIFY(NArrow::Append<TArrowDataType>(builder, value)); }); + } + }; + static TPlainBuilder<arrow::BinaryType> MakeBuilderBinary(const ui32 reserveItems = 0, const ui32 reserveData = 0) { return TPlainBuilder<arrow::BinaryType>(reserveItems, reserveData); } diff --git a/ydb/core/formats/arrow/accessor/sparsed/accessor.h b/ydb/core/formats/arrow/accessor/sparsed/accessor.h index 63758d3969a..c72a9e0031b 100644 --- a/ydb/core/formats/arrow/accessor/sparsed/accessor.h +++ b/ydb/core/formats/arrow/accessor/sparsed/accessor.h @@ -253,38 +253,46 @@ public: return Record.GetRecords(); } - template <class TDataType> - class TSparsedBuilder { + class TSparsedBuilderBase { private: std::unique_ptr<arrow::ArrayBuilder> IndexBuilder; std::unique_ptr<arrow::ArrayBuilder> ValueBuilder; - ui32 RecordsCount = 0; + const std::shared_ptr<arrow::DataType> ValueType; const std::shared_ptr<arrow::Scalar> DefaultValue; std::optional<ui32> LastRecordIndex; - public: - TSparsedBuilder(const std::shared_ptr<arrow::Scalar>& defaultValue, const ui32 reserveItems, const ui32 reserveData) - : DefaultValue(defaultValue) - { - IndexBuilder = NArrow::MakeBuilder(arrow::uint32(), reserveItems, 0); - ValueBuilder = NArrow::MakeBuilder(arrow::TypeTraits<TDataType>::type_singleton(), reserveItems, reserveData); - } - - void AddRecord(const ui32 recordIndex, const std::string_view value) { - if (!!LastRecordIndex) { - AFL_VERIFY(*LastRecordIndex < recordIndex); + void AdvanceRecordIndex(const ui32 recordIndex) { + if (LastRecordIndex) { + AFL_VERIFY(*LastRecordIndex < recordIndex)("last", LastRecordIndex)("index", recordIndex); } LastRecordIndex = recordIndex; + } + + protected: + arrow::ArrayBuilder& GetValueBuilder() { + return *ValueBuilder; + } + + template <class TAppender> + void AddAt(const ui32 recordIndex, const TAppender& append) { + AdvanceRecordIndex(recordIndex); AFL_VERIFY(NArrow::Append<arrow::UInt32Type>(*IndexBuilder, recordIndex)); - AFL_VERIFY(NArrow::Append<TDataType>(*ValueBuilder, arrow::util::string_view(value.data(), value.size()))); - ++RecordsCount; + append(*ValueBuilder); + } + + public: + TSparsedBuilderBase(std::unique_ptr<arrow::ArrayBuilder> valueBuilder, const std::shared_ptr<arrow::DataType>& valueType, + const std::shared_ptr<arrow::Scalar>& defaultValue, const ui32 reserveItems) + : IndexBuilder(NArrow::MakeBuilder(arrow::uint32(), reserveItems, 0)) + , ValueBuilder(std::move(valueBuilder)) + , ValueType(valueType) + , DefaultValue(defaultValue) + { + AFL_VERIFY(!!ValueBuilder); } void AddNull(const ui32 recordIndex) { - if (!!LastRecordIndex) { - AFL_VERIFY(*LastRecordIndex < recordIndex); - } - LastRecordIndex = recordIndex; + AdvanceRecordIndex(recordIndex); if (!!DefaultValue && DefaultValue->type->id() != arrow::null()->id()) { AFL_VERIFY(NArrow::Append<arrow::UInt32Type>(*IndexBuilder, recordIndex)); TStatusValidator::Validate(ValueBuilder->AppendNull()); @@ -292,16 +300,27 @@ public: } std::shared_ptr<IChunkedArray> Finish(const ui32 recordsCount) { - TSparsedArray::TBuilder builder(DefaultValue, arrow::TypeTraits<TDataType>::type_singleton()); - std::vector<std::unique_ptr<arrow::ArrayBuilder>> builders; - builders.emplace_back(std::move(IndexBuilder)); - builders.emplace_back(std::move(ValueBuilder)); - builder.AddChunk(recordsCount, arrow::RecordBatch::Make(TSparsedArray::BuildSchema(arrow::TypeTraits<TDataType>::type_singleton()), - RecordsCount, NArrow::Finish(std::move(builders)))); + TBuilder builder(DefaultValue, ValueType); + builder.AddChunk(recordsCount, NArrow::FinishBuilder(std::move(IndexBuilder)), NArrow::FinishBuilder(std::move(ValueBuilder))); return builder.Finish(); } }; + template <class TDataType> + class TSparsedBuilder: public TSparsedBuilderBase { + public: + TSparsedBuilder(const std::shared_ptr<arrow::Scalar>& defaultValue, const ui32 reserveItems, const ui32 reserveData) + : TSparsedBuilderBase(NArrow::MakeBuilder(arrow::TypeTraits<TDataType>::type_singleton(), reserveItems, reserveData), + arrow::TypeTraits<TDataType>::type_singleton(), defaultValue, reserveItems) { + } + + void AddRecord(const ui32 recordIndex, const std::string_view value) { + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { + AFL_VERIFY(NArrow::Append<TDataType>(builder, arrow::util::string_view(value.data(), value.size()))); + }); + } + }; + static TSparsedBuilder<arrow::BinaryType> MakeBuilderBinary(const ui32 reserveItems = 0, const ui32 reserveData = 0) { return TSparsedBuilder<arrow::BinaryType>(nullptr, reserveItems, reserveData); } diff --git a/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp b/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp index 58894ffd10f..2e69ef7d567 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/accessor.cpp @@ -1,4 +1,5 @@ #include "accessor.h" +#include "types.h" #include "direct_builder.h" #include "signals.h" @@ -137,7 +138,7 @@ TConclusion<NBinaryJson::TBinaryJson> ToBinaryJson(const TJsonRestorer& restorer [](NBinaryJson::TBinaryJson&& val) -> TConclusion<NBinaryJson::TBinaryJson> { return std::move(val); }}, - NBinaryJson::SerializeToBinaryJson(restorer.GetResult().GetStringRobust())); + NBinaryJson::SerializeToBinaryJson(WriteJsonRoundTripSafe(restorer.GetResult()))); } std::shared_ptr<arrow::Array> TSubColumnsArray::BuildBJsonArray(const TColumnConstructionContext& context) const { diff --git a/ydb/core/formats/arrow/accessor/sub_columns/columns_storage.h b/ydb/core/formats/arrow/accessor/sub_columns/columns_storage.h index 840694c9f04..3ea238c2f79 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/columns_storage.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/columns_storage.h @@ -80,12 +80,12 @@ public: const arrow::Array& GetArray() const { return *CurrentArrayData; } - i64 GetLocalIndex() const { + ui32 GetLocalIndex() const { return ChunkAddress->GetAddress().GetLocalIndex(CurrentIndex); } NArrow::NAccessor::TJsonValueView GetValue() const { - return ArrayElementToJsonValueView(*CurrentArrayData, GetLocalIndex(), ValueType); + return GetCodecForValueType(ValueType)->ReadValueView(*CurrentArrayData, GetLocalIndex()); } bool HasValue() const { diff --git a/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.cpp b/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.cpp index a67b4ef3867..eea3998e997 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.cpp @@ -1,78 +1,40 @@ #include "accessor.h" #include "columns_storage.h" #include "direct_builder.h" +#include "encoding_builders.h" #include <util/string/escape.h> #include <ydb/core/formats/arrow/accessor/common/chunk_data.h> #include <ydb/core/formats/arrow/accessor/dictionary/constructor.h> #include <ydb/core/formats/arrow/accessor/plain/accessor.h> -#include <ydb/core/formats/arrow/accessor/sparsed/accessor.h> #include <ydb/core/formats/arrow/serializer/abstract.h> #include <contrib/libs/simdjson/include/simdjson.h> namespace NKikimr::NArrow::NAccessor::NSubColumns { -namespace { - -std::string_view MakeStoredBytesView(const NBinaryJson::TBinaryJson& rec, const EValueType valueType) { - if (valueType == EValueType::BinaryJson) { - return std::string_view(rec.Data(), rec.Size()); - } - AFL_VERIFY(valueType == EValueType::String)("value_type", (ui32)valueType); - const auto scalar = ExtractStringScalar(rec); - return std::string_view(scalar.data(), scalar.size()); -} - -template <class TArrow, class TExtractor> -std::shared_ptr<IChunkedArray> BuildTypedPlain(const std::deque<NBinaryJson::TBinaryJson>& values, - const std::vector<ui32>& recordIndexes, const ui32 recordsCount, const ui32 reserveData, const TExtractor& extract) { - TTrivialArray::TPlainBuilder<TArrow> builder(recordsCount, reserveData); - for (ui32 i = 0; i < recordIndexes.size(); ++i) { - builder.AddValue(recordIndexes[i], extract(values[i])); - } - return builder.Finish(recordsCount); -} -} // namespace - void TColumnElements::BuildSparsedAccessor(const ui32 recordsCount, const EValueType valueType) { AFL_VERIFY(!Accessor); - // Columns with non-binary encoding are not supported. - AFL_VERIFY(valueType == EValueType::BinaryJson || valueType == EValueType::String)("value_type", (ui32)valueType); - auto recordsBuilder = TSparsedArray::MakeBuilderBinary(RecordIndexes.size(), DataSize); - for (ui32 idx = 0; idx < RecordIndexes.size(); ++idx) { - recordsBuilder.AddRecord(RecordIndexes[idx], MakeStoredBytesView(Values[idx], valueType)); + TEncodingSparsedBuilder builder(GetCodecForValueType(valueType), RecordIndexes.size(), DataSize); + for (ui32 i = 0; i < RecordIndexes.size(); ++i) { + builder.AddFromBinaryJson(RecordIndexes[i], Values[i]); } - Accessor = recordsBuilder.Finish(recordsCount); + Accessor = builder.Finish(recordsCount); } void TColumnElements::BuildPlainAccessor(const ui32 recordsCount, const EValueType valueType) { AFL_VERIFY(!Accessor); - switch (valueType) { - case EValueType::BinaryJson: - case EValueType::String: - Accessor = BuildTypedPlain<arrow::BinaryType>(Values, RecordIndexes, recordsCount, DataSize, - [valueType](const NBinaryJson::TBinaryJson& rec) { - const auto sv = MakeStoredBytesView(rec, valueType); - return arrow::util::string_view(sv.data(), sv.size()); - }); - break; - case EValueType::Double: - // No need to reserve by size for fixed-size arrays - Accessor = BuildTypedPlain<arrow::DoubleType>(Values, RecordIndexes, recordsCount, 0, - &ExtractDoubleScalar); - break; - case EValueType::Bool: - Accessor = BuildTypedPlain<arrow::BooleanType>(Values, RecordIndexes, recordsCount, 0, - &ExtractBoolScalar); - break; + TEncodingPlainBuilder builder(GetCodecForValueType(valueType), recordsCount, DataSize); + for (ui32 i = 0; i < RecordIndexes.size(); ++i) { + builder.AddFromBinaryJson(RecordIndexes[i], Values[i]); } + Accessor = builder.Finish(recordsCount); } void TColumnElements::BuildDictionaryAccessor(const ui32 recordsCount, const EValueType valueType) { BuildPlainAccessor(recordsCount, valueType); - const TChunkConstructionData cData( - recordsCount, nullptr, GetArrowTypeForValueType(valueType), NSerialization::TSerializerContainer::GetDefaultSerializer()); + const TChunkConstructionData cData(recordsCount, nullptr, GetCodecForValueType(valueType)->GetArrowType(), + NSerialization::TSerializerContainer::GetDefaultSerializer()); Accessor = NDictionary::TConstructor().Construct(Accessor, cData).DetachResult(); } diff --git a/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.h b/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.h index 5c2148566b0..59e979499c1 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/direct_builder.h @@ -192,10 +192,8 @@ public: if (separateColumns && settings.GetEnableNativeColumnsResolved()) { valueType = DetectValueTypeForArray(i->GetValues()); } - IChunkedArray::EType accessorType = IChunkedArray::EType::Array; - if (settings.IsSparsed(presentCount, recordsCount)) { - accessorType = IChunkedArray::EType::SparsedArray; - } else if (separateColumns && DictionaryApplicableForValueType(valueType) && + auto accessorType = settings.IsSparsed(presentCount, recordsCount) ? IChunkedArray::EType::SparsedArray : IChunkedArray::EType::Array; + if (separateColumns && CanBeDictionaryEncoded(valueType) && settings.IsDictionary(presentCount, enumerateValues)) { accessorType = IChunkedArray::EType::Dictionary; } diff --git a/ydb/core/formats/arrow/accessor/sub_columns/encoding_builders.h b/ydb/core/formats/arrow/accessor/sub_columns/encoding_builders.h new file mode 100644 index 00000000000..e1441e91421 --- /dev/null +++ b/ydb/core/formats/arrow/accessor/sub_columns/encoding_builders.h @@ -0,0 +1,57 @@ +#pragma once + +#include "types.h" + +#include <ydb/core/formats/arrow/accessor/plain/accessor.h> +#include <ydb/core/formats/arrow/accessor/sparsed/accessor.h> + +#include <ydb/library/actors/core/log.h> + +#include <yql/essentials/types/binary_json/format.h> + +// Subcolumn builders that either copy an arrow-native value or decode one from BinaryJson. +namespace NKikimr::NArrow::NAccessor::NSubColumns { + +class TEncodingPlainBuilder: public TTrivialArray::TPlainBuilderBase { +private: + std::shared_ptr<const IValueArrowCodec> Codec; + +public: + TEncodingPlainBuilder(const std::shared_ptr<const IValueArrowCodec>& codec, const ui32 reserveItems, const ui32 reserveData) + : TPlainBuilderBase(codec->MakeBuilder(reserveItems, reserveData)) + , Codec(codec) + { + } + + void AddFromBinaryJson(const ui32 recordIndex, const NBinaryJson::TBinaryJson& blob) { + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { Codec->AppendFromBinaryJson(builder, blob); }); + } + + void AddArrayElement(const ui32 recordIndex, const arrow::Array& array, const ui32 position) { + AFL_VERIFY(GetBuilder().type()->id() == array.type_id())("builder", GetBuilder().type()->ToString())("array", array.type()->ToString()); + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { AFL_VERIFY(NArrow::Append(builder, array, position)); }); + } +}; + +class TEncodingSparsedBuilder: public TSparsedArray::TSparsedBuilderBase { +private: + std::shared_ptr<const IValueArrowCodec> Codec; + +public: + TEncodingSparsedBuilder(const std::shared_ptr<const IValueArrowCodec>& codec, const ui32 reserveItems, const ui32 reserveData) + : TSparsedBuilderBase(codec->MakeBuilder(reserveItems, reserveData), codec->GetArrowType(), nullptr, reserveItems) + , Codec(codec) + { + } + + void AddFromBinaryJson(const ui32 recordIndex, const NBinaryJson::TBinaryJson& blob) { + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { Codec->AppendFromBinaryJson(builder, blob); }); + } + + void AddArrayElement(const ui32 recordIndex, const arrow::Array& array, const ui32 position) { + AFL_VERIFY(GetValueBuilder().type()->id() == array.type_id())("builder", GetValueBuilder().type()->ToString())("array", array.type()->ToString()); + AddAt(recordIndex, [&](arrow::ArrayBuilder& builder) { AFL_VERIFY(NArrow::Append(builder, array, position)); }); + } +}; + +} // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/formats/arrow/accessor/sub_columns/iterators.cpp b/ydb/core/formats/arrow/accessor/sub_columns/iterators.cpp index 27b59bb364c..6d981ce4dc9 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/iterators.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/iterators.cpp @@ -5,7 +5,7 @@ namespace NKikimr::NArrow::NAccessor::NSubColumns { NJson::TJsonValue TGeneralIterator::GetValue() const { AFL_VERIFY(IsValidFlag); - return ArrayElementToJsonValue(*CurrentArray, LocalIndex, ValueType); + return Codec->ReadValueView(*CurrentArray, LocalIndex).ToJsonValue(); } } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/formats/arrow/accessor/sub_columns/iterators.h b/ydb/core/formats/arrow/accessor/sub_columns/iterators.h index de5b129de4e..f9f4d516051 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/iterators.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/iterators.h @@ -15,10 +15,10 @@ private: bool IsValidFlag = false; bool HasValueFlag = false; bool IsColumnKeyFlag = false; - EValueType ValueType = EValueType::BinaryJson; - // Current value as (array, local index); the reader interprets it per ValueType. + std::shared_ptr<const IValueArrowCodec> Codec; + // Current physical position as (array, local index) const arrow::Array* CurrentArray = nullptr; - i64 LocalIndex = 0; + ui32 LocalIndex = 0; void InitFromIterator(const TColumnsData::TIterator& iterator) { RecordIndex = iterator.GetCurrentRecordIndex(); @@ -72,12 +72,14 @@ public: TGeneralIterator(TColumnsData::TIterator&& iterator, const EValueType valueType, const std::optional<ui32> remappedKey = {}) : Iterator(iterator) , RemappedKey(remappedKey) - , ValueType(valueType) { + , Codec(GetCodecForValueType(valueType)) { Initialize(); } + TGeneralIterator(TOthersData::TIterator&& iterator, const std::vector<ui32>& remapKeys = {}) : Iterator(iterator) - , RemapKeys(remapKeys) { + , RemapKeys(remapKeys) + , Codec(GetCodecForValueType(EValueType::BinaryJson)) { Initialize(); } bool IsColumnKey() const { @@ -159,28 +161,23 @@ public: // Re-encode the current value to BinaryJson. NBinaryJson::TBinaryJson GetValueAsBinaryJson() const { AFL_VERIFY(IsValidFlag); - return ArrayElementToBinaryJson(*CurrentArray, LocalIndex, ValueType); + return Codec->ReadValueView(*CurrentArray, LocalIndex).ToBinaryJson(); } EValueType GetValueType() const { - return ValueType; + return Codec->GetValueType(); } const arrow::Array& GetArray() const { AFL_VERIFY(IsValidFlag); return *CurrentArray; } - i64 GetLocalIndex() const { + ui32 GetLocalIndex() const { + AFL_VERIFY(IsValidFlag); return LocalIndex; } ui32 GetValueSize() const { AFL_VERIFY(IsValidFlag); - return ArrayElementSize(*CurrentArray, LocalIndex, ValueType); - } - TStringBuf GetStorageView() const { - AFL_VERIFY(IsValidFlag); - AFL_VERIFY(ValueType == EValueType::String || ValueType == EValueType::BinaryJson)("value_type", (ui32)ValueType); - const auto view = static_cast<const arrow::BinaryArray&>(*CurrentArray).GetView(LocalIndex); - return TStringBuf(view.data(), view.size()); + return Codec->GetElementSize(*CurrentArray, LocalIndex); } NJson::TJsonValue GetValue() const; diff --git a/ydb/core/formats/arrow/accessor/sub_columns/json_value_path.cpp b/ydb/core/formats/arrow/accessor/sub_columns/json_value_path.cpp index 166f9b8773f..067f60d14e6 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/json_value_path.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/json_value_path.cpp @@ -139,6 +139,7 @@ void TJsonPathAccessor::VisitValues(const TValuesVisitor& visitor) const { return; } + const auto codec = GetCodecForValueType(ValueType); ChunkedArrayAccessor->VisitValues([&](std::shared_ptr<arrow::Array> arr) { AFL_VERIFY(arr); for (int64_t i = 0; i < arr->length(); ++i) { @@ -147,7 +148,7 @@ void TJsonPathAccessor::VisitValues(const TValuesVisitor& visitor) const { continue; } - const auto value = ArrayElementToJsonValueView(*arr, i, ValueType); + const auto value = codec->ReadValueView(*arr, i); if (auto scalar = value.GetScalarOptional()) { // A scalar has no sub-structure, so a remaining path cannot resolve against it. visitor(RemainingPathPtr ? std::nullopt : scalar); diff --git a/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp b/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp index 7c01560f1fc..f6f8fd975e0 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/others_storage.cpp @@ -121,7 +121,7 @@ public: TDictStats::TBuilder statBuilder; for (auto&& i : UsedKeys) { statBuilder.Add(i.second.GetKeyName(), i.second.GetRecordsCount(), i.second.GetDataSize(), - i.second.GetAccessorType(settings, recordsCount), i.second.GetValueType()); + i.second.GetAccessorType(settings, recordsCount), OthersExplicitBinaryJson); } return statBuilder.Finish(); } @@ -250,7 +250,7 @@ TOthersData TOthersData::BuildEmpty() { TConclusion<std::shared_ptr<TJsonPathAccessor>> TOthersData::GetPathAccessor(const std::string_view path, const ui32 recordsCount) const { auto jsonPathAccessorTrie = std::make_shared<NKikimr::NArrow::NAccessor::NSubColumns::TJsonPathAccessorTrie>(); for (ui32 i = 0; i < Stats.GetColumnsCount(); ++i) { - auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(Stats.GetColumnName(i)), nullptr, EValueType::BinaryJson, i); + auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(Stats.GetColumnName(i)), nullptr, OthersExplicitBinaryJson, i); AFL_VERIFY(insertResult.IsSuccess())("error", insertResult.GetErrorMessage()); } auto accessorResult = jsonPathAccessorTrie->GetAccessor(path); @@ -266,7 +266,7 @@ TConclusion<std::shared_ptr<TJsonPathAccessor>> TOthersData::GetPathAccessor(con auto idx = accessor->GetCookie(); if (!idx) { return std::make_shared<TJsonPathAccessor>( - std::make_shared<TSparsedArray>(nullptr, arrow::binary(), recordsCount), TString{}, EValueType::BinaryJson); + std::make_shared<TSparsedArray>(nullptr, arrow::binary(), recordsCount), TString{}, OthersExplicitBinaryJson); } TColumnFilter filter = TColumnFilter::BuildAllowFilter(); for (TIterator it(Records); it.IsValid(); it.Next()) { @@ -278,7 +278,7 @@ TConclusion<std::shared_ptr<TJsonPathAccessor>> TOthersData::GetPathAccessor(con TSparsedArray::TBuilder builder(nullptr, arrow::binary()); auto batch = ToBatch(table); builder.AddChunk(recordsCount, batch->GetColumnByName("record_idx"), batch->GetColumnByName("value")); - return std::make_shared<TJsonPathAccessor>(builder.Finish(), accessor->GetRemainingPath(), EValueType::BinaryJson); + return std::make_shared<TJsonPathAccessor>(builder.Finish(), accessor->GetRemainingPath(), OthersExplicitBinaryJson); } NArrow::NAccessor::TJsonValueView TOthersData::TIterator::GetValue() const { diff --git a/ydb/core/formats/arrow/accessor/sub_columns/others_storage.h b/ydb/core/formats/arrow/accessor/sub_columns/others_storage.h index 00233ff7555..799ef5f2396 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/others_storage.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/others_storage.h @@ -123,7 +123,7 @@ public: AFL_VERIFY(IsValid()); return *Values; } - i64 GetLocalIndex() const { + ui32 GetLocalIndex() const { AFL_VERIFY(IsValid()); return CurrentIndex; } diff --git a/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp b/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp index b08040273ae..02e05754011 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp @@ -12,7 +12,7 @@ namespace NKikimr::NArrow::NAccessor::NSubColumns { -TSplittedColumns TDictStats::SplitByVolume(const TSettings& settings, const ui32 recordsCount) const { +TDictStats TDictStats::SelectSeparatedColumns(const TSettings& settings, const ui32 recordsCount) const { std::map<ui64, std::vector<TRTStats>> bySize; ui64 sumSize = 0; for (ui32 i = 0; i < GetColumnsCount(); ++i) { @@ -20,31 +20,21 @@ TSplittedColumns TDictStats::SplitByVolume(const TSettings& settings, const ui32 sumSize += GetColumnSize(i); } std::vector<TRTStats> columnStats; - std::vector<TRTStats> otherStats; TSettings::TColumnsDistributor distributor = settings.BuildDistributor(sumSize, recordsCount); for (auto it = bySize.rbegin(); it != bySize.rend(); ++it) { for (auto&& i : it->second) { - switch (distributor.TakeAndDetect(it->first, i.GetRecordsCount())) { - case TSettings::TColumnsDistributor::EColumnType::Separated: - columnStats.emplace_back(std::move(i)); - break; - case TSettings::TColumnsDistributor::EColumnType::Other: - otherStats.emplace_back(std::move(i)); - break; + // Keys not taken as separated fall into the Others store, whose stats are built elsewhere. + if (distributor.TakeAndDetect(it->first, i.GetRecordsCount()) == TSettings::TColumnsDistributor::EColumnType::Separated) { + columnStats.emplace_back(std::move(i)); } } } std::sort(columnStats.begin(), columnStats.end()); - std::sort(otherStats.begin(), otherStats.end()); auto columnsBuilder = MakeBuilder(); - auto othersBuilder = MakeBuilder(); for (auto&& i : columnStats) { columnsBuilder.Add(i.GetKeyName(), i.GetRecordsCount(), i.GetDataSize(), i.GetAccessorType(settings, recordsCount), i.GetValueType()); } - for (auto&& i : otherStats) { - othersBuilder.Add(i.GetKeyName(), i.GetRecordsCount(), i.GetDataSize(), i.GetAccessorType(settings, recordsCount), i.GetValueType()); - } - return TSplittedColumns(columnsBuilder.Finish(), othersBuilder.Finish()); + return columnsBuilder.Finish(); } TDictStats TDictStats::Merge(const std::vector<const TDictStats*>& stats, const TSettings& settings, const ui32 recordsCount) { diff --git a/ydb/core/formats/arrow/accessor/sub_columns/stats.h b/ydb/core/formats/arrow/accessor/sub_columns/stats.h index 8f6c7388495..b065ce7290f 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/stats.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/stats.h @@ -15,8 +15,6 @@ namespace NKikimr::NArrow::NAccessor::NSubColumns { -class TSplittedColumns; - class TDictStats { private: using TJsonPathAccessorTriePtr = std::shared_ptr<NKikimr::NArrow::NAccessor::NSubColumns::TJsonPathAccessorTrie>; @@ -33,7 +31,7 @@ private: auto jsonPathAccessorTrie = std::make_shared<NKikimr::NArrow::NAccessor::NSubColumns::TJsonPathAccessorTrie>(); for (ui32 i = 0; i < DataNames->length(); ++i) { const auto arrView = DataNames->GetView(i); - auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(TStringBuf(arrView.data(), arrView.size())), nullptr, EValueType::BinaryJson, i); + auto insertResult = jsonPathAccessorTrie->Insert(ToJsonPath(TStringBuf(arrView.data(), arrView.size())), nullptr, OthersExplicitBinaryJson, i); AFL_VERIFY(insertResult.IsSuccess())("key_name", TStringBuf(arrView.data(), arrView.size())) ("json_path", ToJsonPath(TStringBuf(arrView.data(), arrView.size()))) ("error", insertResult.GetErrorMessage()); @@ -156,7 +154,9 @@ public: static TDictStats Merge(const std::vector<const TDictStats*>& stats, const TSettings& settings, const ui32 recordsCount); - TSplittedColumns SplitByVolume(const TSettings& settings, const ui32 recordsCount) const; + // Selects which keys become separated columns; + // the rest fall into the Others store, whose stats are built by the caller. + TDictStats SelectSeparatedColumns(const TSettings& settings, const ui32 recordsCount) const; class TBuilder: TNonCopyable { private: @@ -194,7 +194,7 @@ public: std::shared_ptr<arrow::Field> GetField(const ui32 index) const { AFL_VERIFY(index < DataNames->length()); auto name = DataNames->GetView(index); - return std::make_shared<arrow::Field>(std::string(name.data(), name.size()), GetArrowTypeForValueType(GetValueType(index))); + return std::make_shared<arrow::Field>(std::string(name.data(), name.size()), GetCodecForValueType(GetValueType(index))->GetArrowType()); } TRTStats GetRTStats(const ui32 index) const { @@ -240,25 +240,6 @@ public: return result; } - bool IsSparsed(const ui32 columnIndex, const ui32 recordsCount, const TSettings& settings) const; TDictStats(const std::shared_ptr<arrow::RecordBatch>& original); }; - -class TSplittedColumns { -private: - TDictStats Columns; - TDictStats Others; - -public: - TDictStats ExtractColumns() { - return std::move(Columns); - } - TDictStats ExtractOthers() { - return std::move(Others); - } - TSplittedColumns(TDictStats&& columns, TDictStats&& others) - : Columns(std::move(columns)) - , Others(std::move(others)) { - } -}; } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/formats/arrow/accessor/sub_columns/types.cpp b/ydb/core/formats/arrow/accessor/sub_columns/types.cpp index 26f517749ab..6a2e18197b8 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/types.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/types.cpp @@ -3,22 +3,29 @@ #include <contrib/libs/apache/arrow/cpp/src/arrow/array/array_binary.h> #include <contrib/libs/apache/arrow/cpp/src/arrow/array/array_primitive.h> -#include <library/cpp/json/json_reader.h> -#include <library/cpp/json/json_writer.h> - #include <ydb/library/actors/core/log.h> +#include <ydb/library/formats/arrow/arrow_helpers.h> +#include <ydb/library/formats/arrow/switch/switch_type.h> #include <yql/essentials/types/binary_json/read.h> -#include <yql/essentials/types/binary_json/write.h> namespace NKikimr::NArrow::NAccessor::NSubColumns { namespace { -NBinaryJson::TBinaryJson ToBinaryJson(const NJson::TJsonValue& json) { - auto result = NBinaryJson::SerializeToBinaryJson(NJson::WriteJson(&json, false)); - AFL_VERIFY(std::holds_alternative<NBinaryJson::TBinaryJson>(result)); - return std::get<NBinaryJson::TBinaryJson>(std::move(result)); +TStringBuf ExtractStringScalar(const NBinaryJson::TBinaryJson& blob) { + auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); + return reader->GetRootCursor().GetElement(0).GetString(); +} + +double ExtractDoubleScalar(const NBinaryJson::TBinaryJson& blob) { + auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); + return reader->GetRootCursor().GetElement(0).GetNumber(); +} + +bool ExtractBoolScalar(const NBinaryJson::TBinaryJson& blob) { + auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); + return reader->GetRootCursor().GetElement(0).GetType() == NBinaryJson::EEntryType::BoolTrue; } EValueType ValueTypeForItem(const NBinaryJson::TBinaryJson& blob) { @@ -41,27 +48,104 @@ EValueType ValueTypeForItem(const NBinaryJson::TBinaryJson& blob) { } } -} // namespace +TStringBuf BinaryView(const arrow::Array& array, const i64 index) { + AFL_VERIFY(array.type()->id() == arrow::Type::BINARY); + const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); + return TStringBuf(view.data(), view.size()); +} -std::shared_ptr<arrow::DataType> GetArrowTypeForValueType(const EValueType valueType) { - switch (valueType) { - case EValueType::BinaryJson: - case EValueType::String: - return arrow::binary(); - case EValueType::Double: - return arrow::float64(); - case EValueType::Bool: - return arrow::boolean(); +// BinaryJson and String share arrow::binary() storage; they differ only in what the bytes mean. +class TBinaryBackedCodec: public IValueArrowCodec { +public: + std::shared_ptr<arrow::DataType> GetArrowType() const override { + return arrow::binary(); + } + + ui32 GetElementSize(const arrow::Array& array, const i64 index) const override { + return BinaryView(array, index).size(); + } + std::unique_ptr<arrow::ArrayBuilder> MakeBuilder(const ui32 reserveItems, const ui32 reserveData) const override { + return NArrow::MakeBuilder(arrow::binary(), reserveItems, reserveData); } +}; + +class TBinaryJsonCodec: public TBinaryBackedCodec { +public: + EValueType GetValueType() const override { + return EValueType::BinaryJson; + } + TJsonValueView ReadValueView(const arrow::Array& array, const i64 index) const override { + return TJsonValueView::OfBinaryJson(BinaryView(array, index)); + } + void AppendFromBinaryJson(arrow::ArrayBuilder& builder, const NBinaryJson::TBinaryJson& blob) const override { + AFL_VERIFY(NArrow::Append<arrow::BinaryType>(builder, arrow::util::string_view(blob.data(), blob.size()))); + } +}; + +class TStringCodec: public TBinaryBackedCodec { +public: + EValueType GetValueType() const override { + return EValueType::String; + } + TJsonValueView ReadValueView(const arrow::Array& array, const i64 index) const override { + return TJsonValueView::OfString(BinaryView(array, index)); + } + void AppendFromBinaryJson(arrow::ArrayBuilder& builder, const NBinaryJson::TBinaryJson& blob) const override { + const auto scalar = ExtractStringScalar(blob); + AFL_VERIFY(NArrow::Append<arrow::BinaryType>(builder, arrow::util::string_view(scalar.data(), scalar.size()))); + } +}; + +template <class TArrow, EValueType ValueType, auto ExtractScalar, auto MakeView> +class TNativeScalarCodec: public IValueArrowCodec { +private: + using TArray = typename arrow::TypeTraits<TArrow>::ArrayType; + +public: + EValueType GetValueType() const override { + return ValueType; + } + std::shared_ptr<arrow::DataType> GetArrowType() const override { + return arrow::TypeTraits<TArrow>::type_singleton(); + } + ui32 GetElementSize(const arrow::Array& /*array*/, const i64 /*index*/) const override { + return sizeof(typename TArrow::c_type); + } + TJsonValueView ReadValueView(const arrow::Array& array, const i64 index) const override { + AFL_VERIFY(array.type()->id() == TArrow::type_id); + return MakeView(static_cast<const TArray&>(array).Value(index)); + } + std::unique_ptr<arrow::ArrayBuilder> MakeBuilder(const ui32 reserveItems, const ui32 /*reserveData*/) const override { + return NArrow::MakeBuilder(arrow::TypeTraits<TArrow>::type_singleton(), reserveItems, 0); + } + void AppendFromBinaryJson(arrow::ArrayBuilder& builder, const NBinaryJson::TBinaryJson& blob) const override { + AFL_VERIFY(NArrow::Append<TArrow>(builder, ExtractScalar(blob))); + } +}; + +using TDoubleCodec = TNativeScalarCodec<arrow::DoubleType, EValueType::Double, ExtractDoubleScalar, TJsonValueView::OfNumber>; +using TBoolCodec = TNativeScalarCodec<arrow::BooleanType, EValueType::Bool, ExtractBoolScalar, TJsonValueView::OfBool>; + +} // namespace + +bool CanBeDictionaryEncoded(EValueType valueType) { + return valueType == EValueType::BinaryJson || valueType == EValueType::String; } -bool DictionaryApplicableForValueType(const EValueType valueType) { +std::shared_ptr<const IValueArrowCodec> GetCodecForValueType(const EValueType valueType) { + static const std::shared_ptr<const IValueArrowCodec> binaryJson = std::make_shared<TBinaryJsonCodec>(); + static const std::shared_ptr<const IValueArrowCodec> string = std::make_shared<TStringCodec>(); + static const std::shared_ptr<const IValueArrowCodec> doubleValue = std::make_shared<TDoubleCodec>(); + static const std::shared_ptr<const IValueArrowCodec> boolValue = std::make_shared<TBoolCodec>(); switch (valueType) { case EValueType::BinaryJson: + return binaryJson; case EValueType::String: - return true; - default: - return false; + return string; + case EValueType::Double: + return doubleValue; + case EValueType::Bool: + return boolValue; } } @@ -83,82 +167,4 @@ EValueType DetectValueTypeForArray(const std::deque<NBinaryJson::TBinaryJson>& v return common.value_or(EValueType::BinaryJson); } -TStringBuf ExtractStringScalar(const NBinaryJson::TBinaryJson& blob) { - auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); - return reader->GetRootCursor().GetElement(0).GetString(); -} - -double ExtractDoubleScalar(const NBinaryJson::TBinaryJson& blob) { - auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); - return reader->GetRootCursor().GetElement(0).GetNumber(); -} - -bool ExtractBoolScalar(const NBinaryJson::TBinaryJson& blob) { - auto reader = NBinaryJson::TBinaryJsonReader::Make(blob); - return reader->GetRootCursor().GetElement(0).GetType() == NBinaryJson::EEntryType::BoolTrue; -} - -NJson::TJsonValue ArrayElementToJsonValue(const arrow::Array& array, const i64 index, const EValueType valueType) { - switch (valueType) { - case EValueType::String: { - const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); - return NJson::TJsonValue(TStringBuf(view.data(), view.size())); - } - case EValueType::Double: - return NJson::TJsonValue(static_cast<const arrow::DoubleArray&>(array).Value(index)); - case EValueType::Bool: - return NJson::TJsonValue(static_cast<const arrow::BooleanArray&>(array).Value(index)); - case EValueType::BinaryJson: { - const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); - const auto text = NBinaryJson::SerializeToJson(TStringBuf(view.data(), view.size())); - NJson::TJsonValue result; - AFL_VERIFY(NJson::ReadJsonTree(text, &result)); - return result; - } - } -} - -NBinaryJson::TBinaryJson ArrayElementToBinaryJson(const arrow::Array& array, const i64 index, const EValueType valueType) { - switch (valueType) { - case EValueType::BinaryJson: { - const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); - return NBinaryJson::TBinaryJson(view.data(), view.size()); - } - case EValueType::String: - case EValueType::Double: - case EValueType::Bool: - return ToBinaryJson(ArrayElementToJsonValue(array, index, valueType)); - } -} - -TJsonValueView ArrayElementToJsonValueView(const arrow::Array& array, const i64 index, const EValueType valueType) { - switch (valueType) { - case EValueType::String: { - const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); - return TJsonValueView::OfString(TStringBuf(view.data(), view.size())); - } - case EValueType::Double: - return TJsonValueView::OfNumber(static_cast<const arrow::DoubleArray&>(array).Value(index)); - case EValueType::Bool: - return TJsonValueView::OfBool(static_cast<const arrow::BooleanArray&>(array).Value(index)); - case EValueType::BinaryJson: { - const auto view = static_cast<const arrow::BinaryArray&>(array).GetView(index); - return TJsonValueView::OfBinaryJson(TStringBuf(view.data(), view.size())); - } - } -} - -ui32 ArrayElementSize(const arrow::Array& array, const i64 index, const EValueType valueType) { - switch (valueType) { - case EValueType::BinaryJson: - case EValueType::String: - return static_cast<const arrow::BinaryArray&>(array).GetView(index).size(); - case EValueType::Double: - return sizeof(double); - case EValueType::Bool: - // actually only 1 bit in arrow representation, not 1 byte, but let's not overcomplicate things - return 1; - } -} - } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/formats/arrow/accessor/sub_columns/types.h b/ydb/core/formats/arrow/accessor/sub_columns/types.h index 90be2c2a8c8..2ed2ab6d78b 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/types.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/types.h @@ -5,43 +5,43 @@ #include <deque> #include <contrib/libs/apache/arrow/cpp/src/arrow/array/array_base.h> +#include <contrib/libs/apache/arrow/cpp/src/arrow/array/builder_base.h> #include <ydb/core/formats/arrow/accessor/common/json_value_view.h> #include <library/cpp/json/writer/json_value.h> #include <yql/essentials/types/binary_json/format.h> -// Type conversions between BinaryJson, dedicated scalar types and arrow storage types. namespace NKikimr::NArrow::NAccessor::NSubColumns { -std::shared_ptr<arrow::DataType> GetArrowTypeForValueType(const EValueType valueType); +// Conversion between physical arrow storage and logical JsonValue/BinaryJson in-program representation +class IValueArrowCodec { +public: + virtual ~IValueArrowCodec() = default; + virtual EValueType GetValueType() const = 0; + virtual std::shared_ptr<arrow::DataType> GetArrowType() const = 0; -// Dictionary encoding only enabled for the binary-backed types (BinaryJson blobs and raw strings). -// Integral types would trade fixed-size position in array for fixed-size dictionary ref. -// May still be good for compression, but requires further experiments. -bool DictionaryApplicableForValueType(const EValueType valueType); + // Read path - wrap the physical element `index` as a logical value view (a native scalar for + // Double/Bool/String, or a BinaryJson blob for BinaryJson). + // The view aliases `array`, which must outlive it. + virtual TJsonValueView ReadValueView(const arrow::Array& array, const i64 index) const = 0; + // Approximate size of element `index` for accounting: variable for binary values, fixed for scalar types. + virtual ui32 GetElementSize(const arrow::Array& array, const i64 index) const = 0; -// Element type to represent result of merging arrays with arg types -EValueType MergeValueTypes(const std::optional<EValueType>& acc, const EValueType next); - -EValueType DetectValueTypeForArray(const std::deque<NBinaryJson::TBinaryJson>& values); + // Write path - initialize builder and write values to it. + // reserveData makes sense only for variable-length types (BinaryJson and string). + virtual std::unique_ptr<arrow::ArrayBuilder> MakeBuilder(const ui32 reserveItems, const ui32 reserveData) const = 0; + virtual void AppendFromBinaryJson(arrow::ArrayBuilder& builder, const NBinaryJson::TBinaryJson& blob) const = 0; +}; -// Convert json blob to its contained scalar. -// Fail on verify if blob is not of specified type. -TStringBuf ExtractStringScalar(const NBinaryJson::TBinaryJson& blob); -double ExtractDoubleScalar(const NBinaryJson::TBinaryJson& blob); -bool ExtractBoolScalar(const NBinaryJson::TBinaryJson& blob); +bool CanBeDictionaryEncoded(EValueType valueType); -// Read element `index` of a materialized native column array (interpreted per valueType) as a JSON -// value (document reconstruction) or a BinaryJson blob. The array's physical arrow type must match valueType. -NJson::TJsonValue ArrayElementToJsonValue(const arrow::Array& array, const i64 index, const EValueType valueType); -NBinaryJson::TBinaryJson ArrayElementToBinaryJson(const arrow::Array& array, const i64 index, const EValueType valueType); +std::shared_ptr<const IValueArrowCodec> GetCodecForValueType(const EValueType valueType); -// Wrap element `index` as a logical value view: a native scalar for Double/Bool/String, or a BinaryJson -// blob for BinaryJson. The returned view aliases `array`, which must outlive it. -TJsonValueView ArrayElementToJsonValueView(const arrow::Array& array, const i64 index, const EValueType valueType); +// Element type to represent result of merging arrays with arg types +EValueType MergeValueTypes(const std::optional<EValueType>& acc, const EValueType next); -ui32 ArrayElementSize(const arrow::Array& array, const i64 index, const EValueType valueType); +EValueType DetectValueTypeForArray(const std::deque<NBinaryJson::TBinaryJson>& values); } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp b/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp index 8335760024a..5f83768df24 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_native_scalars.cpp @@ -11,6 +11,7 @@ #include <yql/essentials/types/binary_json/write.h> #include <algorithm> +#include <limits> using NKikimr::NArrow::NAccessor::NSubColumns::NTesting::PrintBinaryJsons; @@ -220,4 +221,25 @@ Y_UNIT_TEST_SUITE(SubColumnsNativeScalars) { UNIT_ASSERT_VALUES_EQUAL(CollectPushdown(native, "$.n"), "1;2.5;<null>;-3;"); UNIT_ASSERT_VALUES_EQUAL(CollectPushdown(native, "$.b"), "true;false;<null>;true;"); } + + Y_UNIT_TEST(ReencodeNearMaxDouble) { + const std::vector<std::pair<TString, double>> cases = { + {"1.7976931348623157e308", 1.7976931348623157e308}, + {"-1.7976931348623157e308", -1.7976931348623157e308}, + }; + for (const auto& [lit, expected] : cases) { + const TString doc = TString(TStringBuilder() << "{\"n\":" << lit << "}"); + auto native = BuildSubColumns({doc}, NativeSettings(0)); + UNIT_ASSERT_VALUES_EQUAL_C(CountValueType(native, EValueType::Double), 1, native->DebugJson().GetStringRobust()); + UNIT_ASSERT(native->GetChunkedArray()); // read-back document reconstruction must not abort + const auto assertExact = [&](const std::shared_ptr<TSubColumnsArray>& arr) { + auto it = arr->GetColumnsData().BuildIterator(0); + auto bj = GetCodecForValueType(EValueType::Double)->ReadValueView(it.GetArray(), it.GetLocalIndex()).ToBinaryJson(); + auto reader = NBinaryJson::TBinaryJsonReader::Make(bj); + UNIT_ASSERT_VALUES_EQUAL(reader->GetRootCursor().GetElement(0).GetNumber(), expected); + }; + assertExact(native); + assertExact(SerializeRoundTrip(native, NativeSettings(0))); + } + } }; diff --git a/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_sparsed.cpp b/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_sparsed.cpp new file mode 100644 index 00000000000..eb49570fc68 --- /dev/null +++ b/ydb/core/formats/arrow/accessor/sub_columns/ut/ut_sparsed.cpp @@ -0,0 +1,64 @@ +#include <ydb/core/formats/arrow/accessor/sub_columns/direct_builder.h> +#include <ydb/core/formats/arrow/accessor/sub_columns/types.h> + +#include <contrib/libs/apache/arrow/cpp/src/arrow/chunked_array.h> + +#include <library/cpp/testing/unittest/registar.h> +#include <yql/essentials/types/binary_json/write.h> + +Y_UNIT_TEST_SUITE(SubColumnsSparsedBuilder) { + using namespace NKikimr; + using namespace NKikimr::NArrow; + using namespace NKikimr::NArrow::NAccessor; + using namespace NKikimr::NArrow::NAccessor::NSubColumns; + + NBinaryJson::TBinaryJson ToBinaryJson(const TString& json) { + auto v = NBinaryJson::SerializeToBinaryJson(json); + auto* bj = std::get_if<NBinaryJson::TBinaryJson>(&v); + UNIT_ASSERT(bj); + return *bj; + } + + std::shared_ptr<IChunkedArray> BuildColumn( + const std::vector<std::pair<ui32, TString>>& data, const ui32 recordsCount, const EValueType vt, const bool sparsed) { + TColumnElements col("k"); + for (auto&& [idx, json] : data) { + col.AddData(ToBinaryJson(json), idx); + } + if (sparsed) { + col.BuildSparsedAccessor(recordsCount, vt); + } else { + col.BuildPlainAccessor(recordsCount, vt); + } + return col.GetAccessorVerified(); + } + + void CheckSparsedMatchesPlain( + const std::vector<std::pair<ui32, TString>>& data, const ui32 recordsCount, const EValueType vt) { + auto sparsed = BuildColumn(data, recordsCount, vt, true); + auto plain = BuildColumn(data, recordsCount, vt, false); + UNIT_ASSERT_VALUES_EQUAL((ui32)sparsed->GetType(), (ui32)IChunkedArray::EType::SparsedArray); + UNIT_ASSERT_VALUES_EQUAL((ui32)plain->GetType(), (ui32)IChunkedArray::EType::Array); + UNIT_ASSERT_VALUES_EQUAL(sparsed->GetRecordsCount(), recordsCount); + UNIT_ASSERT_VALUES_EQUAL(plain->GetRecordsCount(), recordsCount); + auto sc = sparsed->GetChunkedArray(); + auto pc = plain->GetChunkedArray(); + UNIT_ASSERT_C(sc->Equals(pc), TStringBuilder() << "sparsed=" << sc->ToString() << " plain=" << pc->ToString()); + } + + Y_UNIT_TEST(StringMatchesPlain) { + CheckSparsedMatchesPlain({ { 1, R"("x")" }, { 4, R"("yy")" }, { 7, R"("zzz")" } }, 10, EValueType::String); + } + + Y_UNIT_TEST(DoubleMatchesPlain) { + CheckSparsedMatchesPlain({ { 0, "3.5" }, { 2, "-2" }, { 9, "1000000" } }, 10, EValueType::Double); + } + + Y_UNIT_TEST(BoolMatchesPlain) { + CheckSparsedMatchesPlain({ { 3, "true" }, { 5, "false" } }, 8, EValueType::Bool); + } + + Y_UNIT_TEST(BinaryJsonMatchesPlain) { + CheckSparsedMatchesPlain({ { 1, R"([1,2])" }, { 2, R"({"a":1})" } }, 5, EValueType::BinaryJson); + } +} diff --git a/ydb/core/formats/arrow/accessor/sub_columns/ut/ya.make b/ydb/core/formats/arrow/accessor/sub_columns/ut/ya.make index 45aa769c406..e32403a1a1b 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/ut/ya.make +++ b/ydb/core/formats/arrow/accessor/sub_columns/ut/ya.make @@ -13,6 +13,7 @@ SRCS( ut_sub_columns.cpp ut_native_scalars.cpp ut_dictionary.cpp + ut_sparsed.cpp ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/formats/arrow/accessor/sub_columns/value_type.h b/ydb/core/formats/arrow/accessor/sub_columns/value_type.h index a6174a04bf2..a42012d4807 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/value_type.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/value_type.h @@ -13,4 +13,7 @@ enum class EValueType : ui8 { String = 3, }; +// The Others store holds and reads every value as BinaryJson, whatever its logical type. +inline constexpr EValueType OthersExplicitBinaryJson = EValueType::BinaryJson; + } // namespace NKikimr::NArrow::NAccessor::NSubColumns diff --git a/ydb/core/fq/libs/actors/logging/log.h b/ydb/core/fq/libs/actors/logging/log.h deleted file mode 100644 index a5e79b85706..00000000000 --- a/ydb/core/fq/libs/actors/logging/log.h +++ /dev/null @@ -1,197 +0,0 @@ -#pragma once -#include <ydb/library/services/services.pb.h> - -#include <ydb/library/actors/core/log.h> - -#define LOG_STREAMS_IMPL(level, component, logRecordStream) \ - LOG_LOG_S(::NActors::TActivationContext::AsActorContext(), ::NActors::NLog:: Y_CAT(PRI_, level), ::NKikimrServices::component, logRecordStream); - -#define LOG_STREAMS_IMPL_AS(actorSystem, level, component, logRecordStream) \ - LOG_LOG_S(actorSystem, ::NActors::NLog:: Y_CAT(PRI_, level), ::NKikimrServices::component, logRecordStream); - -// Component: STREAMS. -#define LOG_STREAMS_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS, logRecordStream) -#define LOG_STREAMS_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS, logRecordStream) -#define LOG_STREAMS_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS, logRecordStream) -#define LOG_STREAMS_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS, logRecordStream) -#define LOG_STREAMS_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS, logRecordStream) -#define LOG_STREAMS_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS, logRecordStream) -#define LOG_STREAMS_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS, logRecordStream) -#define LOG_STREAMS_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS, logRecordStream) -#define LOG_STREAMS_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS, logRecordStream) - -// Component: STREAMS_SERVICE. -#define LOG_STREAMS_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_SERVICE, logRecordStream) -#define LOG_STREAMS_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_SERVICE, logRecordStream) - -// Component: STREAMS_STORAGE_SERVICE. -#define LOG_STREAMS_STORAGE_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_STORAGE_SERVICE, logRecordStream) - -#define LOG_STREAMS_STORAGE_SERVICE_AS_DEBUG(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, DEBUG, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_AS_INFO(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, INFO, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_AS_WARN(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, WARN, STREAMS_STORAGE_SERVICE, logRecordStream) -#define LOG_STREAMS_STORAGE_SERVICE_AS_ERROR(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, ERROR, STREAMS_STORAGE_SERVICE, logRecordStream) - -// Component: ROW_DISPATCHER. -#define LOG_ROW_DISPATCHER_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, FQ_ROW_DISPATCHER, LogPrefix << logRecordStream) -#define LOG_ROW_DISPATCHER_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, FQ_ROW_DISPATCHER, LogPrefix << logRecordStream) -#define LOG_ROW_DISPATCHER_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, FQ_ROW_DISPATCHER, LogPrefix << logRecordStream) -#define LOG_ROW_DISPATCHER_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, FQ_ROW_DISPATCHER, LogPrefix << logRecordStream) -#define LOG_ROW_DISPATCHER_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, FQ_ROW_DISPATCHER, LogPrefix << logRecordStream) - -// Component: STREAMS_SCHEDULER_SERVICE. -#define LOG_STREAMS_SCHEDULER_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_SCHEDULER_SERVICE, logRecordStream) -#define LOG_STREAMS_SCHEDULER_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_SCHEDULER_SERVICE, logRecordStream) - -// Component: STREAMS_RESOURCE_SERVICE. -#define LOG_STREAMS_RESOURCE_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_RESOURCE_SERVICE, logRecordStream) -#define LOG_STREAMS_RESOURCE_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_RESOURCE_SERVICE, logRecordStream) - -// Component: STREAMS_CHECKPOINT_COORDINATOR. -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) -#define LOG_STREAMS_CHECKPOINT_COORDINATOR_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_CHECKPOINT_COORDINATOR, logRecordStream) - -// Component: STREAMS_CONTROL_PLANE_SERVICE. -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) -#define LOG_STREAMS_CONTROL_PLANE_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_CONTROL_PLANE_SERVICE, logRecordStream) - -// Component: STREAMS_GRAPH_LEADER. -#define LOG_STREAMS_GRAPH_LEADER_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, STREAMS_GRAPH_LEADER, logRecordStream) -#define LOG_STREAMS_GRAPH_LEADER_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, STREAMS_GRAPH_LEADER, logRecordStream) - -// Component: YQ_CONTROL_PLANE_STORAGE. -#define LOG_YQ_CONTROL_PLANE_STORAGE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, YQ_CONTROL_PLANE_STORAGE, logRecordStream) - - -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_EMERG(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, EMERG, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_ALERT(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, ALERT, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_CRIT(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, CRIT, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_ERROR(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, ERROR, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_WARN(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, WARN, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_NOTICE(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, NOTICE, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_INFO(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, INFO, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_DEBUG(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, DEBUG, YQ_CONTROL_PLANE_STORAGE, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_STORAGE_AS_TRACE(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, TRACE, YQ_CONTROL_PLANE_STORAGE, logRecordStream) - -// Component: YQ_CONTROL_PLANE_PROXY. -#define LOG_YQ_CONTROL_PLANE_PROXY_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, YQ_CONTROL_PLANE_PROXY, logRecordStream) -#define LOG_YQ_CONTROL_PLANE_PROXY_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, YQ_CONTROL_PLANE_PROXY, logRecordStream) - -// Component: YQ_TEST_CONNECTION. -#define LOG_YQ_TEST_CONNECTION_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, YQ_TEST_CONNECTION, logRecordStream) - -#define LOG_YQ_TEST_CONNECTION_AS_EMERG(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, EMERG, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_ALERT(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, ALERT, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_CRIT(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, CRIT, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_ERROR(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, ERROR, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_WARN(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, WARN, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_NOTICE(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, NOTICE, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_INFO(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, INFO, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_DEBUG(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, DEBUG, YQ_TEST_CONNECTION, logRecordStream) -#define LOG_YQ_TEST_CONNECTION_AS_TRACE(actorSystem, logRecordStream) LOG_STREAMS_IMPL_AS(actorSystem, TRACE, YQ_TEST_CONNECTION, logRecordStream) - -// Component: YQ_AUDIT. -#define LOG_YQ_AUDIT_SERVICE_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, YQ_AUDIT, logRecordStream) -#define LOG_YQ_AUDIT_SERVICE_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, YQ_AUDIT, logRecordStream) - -// Component: YQ_HEALTH. -#define LOG_YQ_HEALTH_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, YQ_HEALTH, logRecordStream) -#define LOG_YQ_HEALTH_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, YQ_HEALTH, logRecordStream) - -// Component: FQ_CONTROL_PLANE_CONFIG. -#define LOG_FQ_CONTROL_PLANE_CONFIG_EMERG(logRecordStream) LOG_STREAMS_IMPL(EMERG, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_ALERT(logRecordStream) LOG_STREAMS_IMPL(ALERT, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_CRIT(logRecordStream) LOG_STREAMS_IMPL(CRIT, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_ERROR(logRecordStream) LOG_STREAMS_IMPL(ERROR, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_WARN(logRecordStream) LOG_STREAMS_IMPL(WARN, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_NOTICE(logRecordStream) LOG_STREAMS_IMPL(NOTICE, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_INFO(logRecordStream) LOG_STREAMS_IMPL(INFO, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_DEBUG(logRecordStream) LOG_STREAMS_IMPL(DEBUG, FQ_CONTROL_PLANE_CONFIG, logRecordStream) -#define LOG_FQ_CONTROL_PLANE_CONFIG_TRACE(logRecordStream) LOG_STREAMS_IMPL(TRACE, FQ_CONTROL_PLANE_CONFIG, logRecordStream) diff --git a/ydb/core/fq/libs/actors/logging/ya.make b/ydb/core/fq/libs/actors/logging/ya.make deleted file mode 100644 index 92b7640dc0a..00000000000 --- a/ydb/core/fq/libs/actors/logging/ya.make +++ /dev/null @@ -1,14 +0,0 @@ -LIBRARY() - -SRCS( - log.h -) - -PEERDIR( - ydb/library/actors/core - ydb/core/protos -) - -YQL_LAST_ABI_VERSION() - -END() diff --git a/ydb/core/fq/libs/actors/nodes_health_check.cpp b/ydb/core/fq/libs/actors/nodes_health_check.cpp index 850ec710369..4f2f0c78734 100644 --- a/ydb/core/fq/libs/actors/nodes_health_check.cpp +++ b/ydb/core/fq/libs/actors/nodes_health_check.cpp @@ -13,10 +13,7 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <google/protobuf/util/time_util.h> -#define LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) -#define LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_NODES_MANAGER namespace NFq { @@ -44,7 +41,7 @@ public: static constexpr char ActorName[] = "YQ_PRIVATE_NODES_HEALTH_CHECK"; void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { - LOG_E("TNodesHealthCheckActor::OnUndelivered"); + YDB_LOG_ERROR("TNodesHealthCheckActor::OnUndelivered"); auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>(); res->Status = Ydb::StatusIds::GENERIC_ERROR; res->Issues.AddIssue("UNDELIVERED"); @@ -60,9 +57,9 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("Failed with", + {"code", codeStr}, + {"details", Issues}); auto res = MakeHolder<TEvents::TEvNodesHealthCheckResponse>(); res->Status = reqStatus; res->Issues.AddIssues(Issues); diff --git a/ydb/core/fq/libs/actors/nodes_manager.cpp b/ydb/core/fq/libs/actors/nodes_manager.cpp index 60dc041f93f..87968e38686 100644 --- a/ydb/core/fq/libs/actors/nodes_manager.cpp +++ b/ydb/core/fq/libs/actors/nodes_manager.cpp @@ -22,15 +22,8 @@ #include <random> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_NODES_MANAGER -#define LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) -#define LOG_I(stream) \ - LOG_INFO_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) -#define LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) -#define LOG_T(stream) \ - LOG_TRACE_S(*TlsActivationContext, NKikimrServices::YQL_NODES_MANAGER, stream) namespace NFq { @@ -78,14 +71,16 @@ public: static constexpr char ActorName[] = "YQ_NODES_MANAGER"; void PassAway() final { - LOG_I("PassAway, InstanceId: " << InstanceId); + YDB_LOG_INFO("PassAway", + {"instanceId", InstanceId}); NActors::IActor::PassAway(); } void Bootstrap() { Become(&TNodesManagerActor::StateFunc); ServiceCounters.Counters->GetCounter("EvBootstrap", true)->Inc(); - LOG_I("Bootstrap, InstanceId: " << InstanceId); + YDB_LOG_INFO("Bootstrap", + {"instanceId", InstanceId}); ResolveSelfAddress(); } @@ -122,7 +117,8 @@ private: error.SetMessage(TStringBuilder{} << "Error choosing scheduler. Invalid settings: " << scheduler << ", error: " << CurrentExceptionMessage()); } } - LOG_D("TEvAllocateWorkersResponse " << response->Record.DebugString()); + YDB_LOG_DEBUG("TEvAllocateWorkersResponse", + {"response", response->Record.DebugString()}); Send(ev->Sender, response.Release()); } @@ -229,7 +225,7 @@ private: nodeFilter.insert(nodeIdProto.begin(), nodeIdProto.end()); } TPeer node = {SelfId().NodeId(), InstanceId + "," + HostName(), 0, 0, 0, DataCenter}; - + if (!Peers.empty()) { for (ui32 i = 0; i < Peers.size(); ++i) { auto nextNode = Peers[SingleNodeScheduler.NodeOrder[NextPeer]]; @@ -290,18 +286,20 @@ private: Address = NAddr::PrintHost(*ev->Get()->Address); NodesHealthCheck(); } else { - LOG_E("TNodesManagerActor::TEvAddressInfo: empty Address"); + YDB_LOG_ERROR("TNodesManagerActor::TEvAddressInfo: empty Address"); ResolveSelfAddress(); } } void Handle(NActors::TEvResolveError::TPtr& ev) { - LOG_E("TNodesManagerActor::TEvResolveError: " << ev->Get()->Explain << ", Host: " << ev->Get()->Host); + YDB_LOG_ERROR("TNodesManagerActor::TEvResolveError", + {"ev", ev->Get()->Explain}, + {"host", ev->Get()->Host}); ResolveSelfAddress(); } void Handle(NFq::TEvNodesManager::TEvGetNodesRequest::TPtr& ev) { - LOG_T("Received TNodesManagerActor::TEvGetNodesRequest"); + YDB_LOG_TRACE("Received TNodesManagerActor::TEvGetNodesRequest"); auto response = MakeHolder<NFq::TEvNodesManager::TEvGetNodesResponse>(); response->NodeIds.reserve(Peers.size()); for (const auto& info : Peers) { @@ -311,7 +309,7 @@ private: } void ResolveSelfAddress() { - LOG_D("TNodesManagerActor::ResolveSelfAddress"); + YDB_LOG_DEBUG("TNodesManagerActor::ResolveSelfAddress"); auto resolve = MakeHolder<NActors::TEvResolveAddress>(); resolve->Address = HostName(); resolve->Port = IcPort; @@ -319,7 +317,7 @@ private: } void NodesHealthCheck() { - LOG_T("TNodesManagerActor::NodesHealthCheck"); + YDB_LOG_TRACE("TNodesManagerActor::NodesHealthCheck"); const TDuration ttl = TDuration::Seconds(5); Schedule(ttl, new NActors::TEvents::TEvWakeup(WU_NodesHealthCheck)); @@ -346,7 +344,7 @@ private: } void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&) { - LOG_E("TNodesManagerActor::OnUndelivered"); + YDB_LOG_ERROR("TNodesManagerActor::OnUndelivered"); ServiceCounters.Counters->GetCounter("OnUndelivered", true)->Inc(); } @@ -390,13 +388,15 @@ private: ServiceCounters.Counters->GetCounter("PeerCount", false)->Set(Peers.size()); ServiceCounters.Counters->GetCounter("NodesHealthCheckOk", true)->Inc(); - LOG_T("Send NodeInfo with size: " << nodesInfo->size() << " to DynamicNameserver"); + YDB_LOG_TRACE("Send NodeInfo with to DynamicNameserver", + {"size", nodesInfo->size()}); if (!nodesInfo->empty()) { THolder<TEvInterconnect::TEvNodesInfo> nameServiceUpdateReq(new TEvInterconnect::TEvNodesInfo(nodesInfo)); Send(GetNameserviceActorId(), nameServiceUpdateReq.Release()); } } catch (yexception &e) { - LOG_E(e.what()); + YDB_LOG_ERROR("Exception", + {"exception", e.what()}); ServiceCounters.Counters->GetCounter("NodesHealthCheckFail", true)->Inc(); } } diff --git a/ydb/core/fq/libs/actors/pending_fetcher.cpp b/ydb/core/fq/libs/actors/pending_fetcher.cpp index c665e61b52c..f2a7960ae60 100644 --- a/ydb/core/fq/libs/actors/pending_fetcher.cpp +++ b/ydb/core/fq/libs/actors/pending_fetcher.cpp @@ -64,11 +64,7 @@ #include <util/generic/guid.h> #include <util/system/hostname.h> -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_PENDING_FETCHER, stream) -#define LOG_W(stream) LOG_WARN_S (*TlsActivationContext, NKikimrServices::FQ_PENDING_FETCHER, stream) -#define LOG_I(stream) LOG_INFO_S (*TlsActivationContext, NKikimrServices::FQ_PENDING_FETCHER, stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_PENDING_FETCHER, stream) -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_PENDING_FETCHER, stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::FQ_PENDING_FETCHER namespace NFq { @@ -197,7 +193,7 @@ public: static constexpr char ActorName[] = "YQ_PENDING_FETCHER"; void PassAway() final { - LOG_D("Stop Fetcher"); + YDB_LOG_DEBUG("Stop Fetcher"); Send(DatabaseResolver, new NActors::TEvents::TEvPoison()); NActors::IActor::PassAway(); } @@ -219,7 +215,7 @@ public: private: void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&) { - LOG_E("TYqlPendingFetcher::OnUndelivered"); + YDB_LOG_ERROR("TYqlPendingFetcher::OnUndelivered"); HasRunningRequest = false; } @@ -251,19 +247,21 @@ private: void Handle(TEvInternalService::TEvGetTaskResponse::TPtr& ev) { HasRunningRequest = false; - LOG_T("Got GetTask response from PrivateApi"); + YDB_LOG_TRACE("Got GetTask response from PrivateApi"); GetTaskCounters.LatencyMs->Collect((TInstant::Now() - StartGetTaskTime).MilliSeconds()); GetTaskCounters.InFly->Dec(); if (!ev->Get()->Status.IsSuccess()) { GetTaskCounters.Error->Inc(); - LOG_E("Error with GetTask: "<< ev->Get()->Status.GetIssues().ToString()); + YDB_LOG_ERROR("Error with", + {"getTask", ev->Get()->Status.GetIssues()}); return; } GetTaskCounters.Ok->Inc(); const auto& res = ev->Get()->Result; - LOG_T("Tasks count: " << res.tasks().size()); + YDB_LOG_TRACE("Tasks", + {"count", res.tasks().size()}); if (!res.tasks().empty()) { ProcessTask(res); HasRunningRequest = true; @@ -309,7 +307,8 @@ private: auto itA = RunActorMap.find(runActorId); if (itA == RunActorMap.end()) { - LOG_W("Unknown RunActor " << runActorId << " destroyed"); + YDB_LOG_WARN("Unknown RunActor destroyed", + {"runActorId", runActorId}); return; } auto queryId = itA->second.QueryId; @@ -326,7 +325,10 @@ private: void GetPendingTask() { FetcherGeneration++; - LOG_T("Request Private::GetTask" << ", Owner: " << GetOwnerId() << ", Host: " << HostName() << ", Tenant: " << TenantName); + YDB_LOG_TRACE("Request Private::GetTask", + {"owner", GetOwnerId()}, + {"host", HostName()}, + {"tenant", TenantName}); Fq::Private::GetTaskRequest request; request.set_owner_id(GetOwnerId()); request.set_host(HostName()); @@ -354,11 +356,11 @@ private: } void RunTask(const Fq::Private::GetTaskResult::Task& task) { - LOG_D("NewTask:" - << " Scope: " << task.scope() - << " Id: " << task.query_id().value() - << " UserId: " << task.user_id() - << " AuthToken: " << NKikimr::MaskTicket(task.user_token())); + YDB_LOG_DEBUG("NewTask", + {"scope", task.scope()}, + {"id", task.query_id().value()}, + {"userId", task.user_id()}, + {"authToken", NKikimr::MaskTicket(task.user_token())}); THashMap<TString, TString> serviceAccounts; for (const auto& identity : task.service_accounts()) { diff --git a/ydb/core/fq/libs/actors/proxy_private.cpp b/ydb/core/fq/libs/actors/proxy_private.cpp index e335244ead9..114c04c6892 100644 --- a/ydb/core/fq/libs/actors/proxy_private.cpp +++ b/ydb/core/fq/libs/actors/proxy_private.cpp @@ -15,9 +15,7 @@ #include <util/generic/guid.h> #include <util/system/hostname.h> -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream) -#define LOG_I(stream) LOG_INFO_S (*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_PRIVATE_PROXY namespace NFq { @@ -182,7 +180,7 @@ private: ) void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr&) { - LOG_E("TYqlAnalyticsPrivateProxy::OnUndelivered"); + YDB_LOG_ERROR("TYqlAnalyticsPrivateProxy::OnUndelivered"); Counters->GetCounter("OnUndelivered", true)->Inc(); } diff --git a/ydb/core/fq/libs/actors/rate_limiter_resources.cpp b/ydb/core/fq/libs/actors/rate_limiter_resources.cpp index a02a4773146..54b4d127111 100644 --- a/ydb/core/fq/libs/actors/rate_limiter_resources.cpp +++ b/ydb/core/fq/libs/actors/rate_limiter_resources.cpp @@ -15,15 +15,7 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <google/protobuf/util/time_util.h> -#define C_LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateCreateRateLimiterResource - QueryId: " << OperationId << ", Owner: " << OwnerId << ", " << stream) -#define C_LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateCreateRateLimiterResource - QueryId: " << OperationId << ", Owner: " << OwnerId << ", "<< stream) - -#define D_LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateDeleteRateLimiterResource - QueryId: " << OperationId << ", Owner: " << OwnerId << ", " << stream) -#define D_LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateDeleteRateLimiterResource - QueryId: " << OperationId << ", Owner: " << OwnerId << ", "<< stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_PRIVATE_PROXY namespace NFq { @@ -60,9 +52,11 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - C_LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("PrivateCreateRateLimiterResource - Failed with", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"code", codeStr}, + {"details", Issues}); auto res = MakeHolder<TEvents::TEvCreateRateLimiterResourceResponse>(); res->Status = reqStatus; res->Issues.AddIssues(Issues); @@ -75,7 +69,10 @@ public: const auto& req = Ev->Record; OperationId = req.query_id().value(); OwnerId = req.owner_id(); - C_LOG_D("Request CP::CreateRateLimiterResource with size: " << req.ByteSize() << " bytes"); + YDB_LOG_DEBUG("PrivateCreateRateLimiterResource - Request CP::CreateRateLimiterResource with bytes", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"size", req.ByteSize()}); RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); try { auto event = CreateControlPlaneEvent(); @@ -99,7 +96,9 @@ private: } void HandleResponse(NFq::TEvControlPlaneStorage::TEvCreateRateLimiterResourceResponse::TPtr& ev) { - C_LOG_D("Got CP::CreateRateLimiterResourceResponse"); + YDB_LOG_DEBUG("PrivateCreateRateLimiterResource - Got CP::CreateRateLimiterResourceResponse", + {"queryId", OperationId}, + {"owner", OwnerId}); const auto& issues = ev->Get()->Issues; if (issues) { Issues.AddIssues(issues); @@ -157,9 +156,11 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - D_LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("PrivateDeleteRateLimiterResource - Failed with", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"code", codeStr}, + {"details", Issues}); auto res = MakeHolder<TEvents::TEvDeleteRateLimiterResourceResponse>(); res->Status = reqStatus; res->Issues.AddIssues(Issues); @@ -172,7 +173,10 @@ public: const auto& req = Ev->Record; OperationId = req.query_id().value(); OwnerId = req.owner_id(); - D_LOG_D("Request CP::DeleteRateLimiterResource with size: " << req.ByteSize() << " bytes"); + YDB_LOG_DEBUG("PrivateDeleteRateLimiterResource - Request CP::DeleteRateLimiterResource with bytes", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"size", req.ByteSize()}); RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); try { auto event = CreateControlPlaneEvent(); @@ -196,7 +200,9 @@ private: } void HandleResponse(NFq::TEvControlPlaneStorage::TEvDeleteRateLimiterResourceResponse::TPtr& ev) { - D_LOG_D("Got CP::DeleteRateLimiterResourceResponse"); + YDB_LOG_DEBUG("PrivateDeleteRateLimiterResource - Got CP::DeleteRateLimiterResourceResponse", + {"queryId", OperationId}, + {"owner", OwnerId}); const auto& issues = ev->Get()->Issues; if (issues) { Issues.AddIssues(issues); diff --git a/ydb/core/fq/libs/actors/result_writer.cpp b/ydb/core/fq/libs/actors/result_writer.cpp index 5b239463742..a253b211301 100644 --- a/ydb/core/fq/libs/actors/result_writer.cpp +++ b/ydb/core/fq/libs/actors/result_writer.cpp @@ -22,10 +22,7 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <ydb/core/fq/libs/private_client/internal_service.h> -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RESULT_WRITER, "TraceId: " << TraceId << " " << stream) -#define LOG_I(stream) LOG_INFO_S (*TlsActivationContext, NKikimrServices::FQ_RESULT_WRITER, "TraceId: " << TraceId << " " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RESULT_WRITER, "TraceId: " << TraceId << " " << stream) -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RESULT_WRITER, "TraceId: " << TraceId << " " << stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::FQ_RESULT_WRITER namespace NFq { @@ -59,7 +56,8 @@ public: static constexpr char ActorName[] = "YQ_RESULT_WRITER"; void Bootstrap() { - LOG_I("Bootstrap"); + YDB_LOG_INFO("Bootstrap", + {"traceId", TraceId}); Become(&TResultWriter::StateFunc); } @@ -77,14 +75,16 @@ private: void PassAway() { auto duration = (TInstant::Now()-StartTime); - LOG_I("FinishWrite, Records: " << RowIndex - << " HasError: " << HasError - << " Size: " << Size - << " Rows: " << Rows - << " FreeSpace: " << FreeSpace - << " Duration: " << duration - << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024 - << " ResultChunks.size(): " << ResultChunks.size()); + YDB_LOG_INFO("FinishWrite", + {"traceId", TraceId}, + {"records", RowIndex}, + {"hasError", HasError}, + {"size", Size}, + {"rows", Rows}, + {"freeSpace", FreeSpace}, + {"duration", duration}, + {"avgSpeed", Size/(duration.Seconds()+1)/1024/1024}, + {"resultChunks", ResultChunks.size()}); NActors::IActor::PassAway(); } @@ -95,7 +95,9 @@ private: } void SendIssuesAndSetErrorFlag(const TIssues& issues, NYql::NDqProto::StatusIds::StatusCode statusCode = NYql::NDqProto::StatusIds::INTERNAL_ERROR) { - LOG_E("ControlPlane WriteResult Issues: " << issues.ToString()); + YDB_LOG_ERROR("ControlPlane WriteResult", + {"traceId", TraceId}, + {"issues", issues}); Issues.AddIssues(issues); HasError = true; auto req = MakeHolder<TEvDqFailure>(statusCode, Issues); @@ -123,7 +125,8 @@ private: queryResult.SetTruncated(Truncated); queryResult.SetRowsCount(Rows); - LOG_D("Send response to executer"); + YDB_LOG_DEBUG("Send response to executer", + {"traceId", TraceId}); Send(ExecuterId, new TEvQueryResponse(std::move(queryResult))); } @@ -166,13 +169,15 @@ private: auto duration = (TInstant::Now()-StartTime); - LOG_T("ChannelData, Records: " << RowIndex - << " HasError: " << HasError - << " Size: " << Size - << " Rows: " << Rows - << " FreeSpace: " << FreeSpace - << " Duration: " << duration - << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024); + YDB_LOG_TRACE("ChannelData", + {"traceId", TraceId}, + {"records", RowIndex}, + {"hasError", HasError}, + {"size", Size}, + {"rows", Rows}, + {"freeSpace", FreeSpace}, + {"duration", duration}, + {"avgSpeed", Size/(duration.Seconds()+1)/1024/1024}); } Requests.erase(it); @@ -222,13 +227,15 @@ private: auto duration = (TInstant::Now()-StartTime); - LOG_T("ChannelData Shift, Records: " << RowIndex - << " HasError: " << HasError - << " Size: " << Size - << " Rows: " << Rows - << " FreeSpace: " << FreeSpace - << " Duration: " << duration - << " AvgSpeed: " << Size/(duration.Seconds()+1)/1024/1024); + YDB_LOG_TRACE("ChannelData Shift", + {"traceId", TraceId}, + {"records", RowIndex}, + {"hasError", HasError}, + {"size", Size}, + {"rows", Rows}, + {"freeSpace", FreeSpace}, + {"duration", duration}, + {"avgSpeed", Size/(duration.Seconds()+1)/1024/1024}); ResultChunks.pop_front(); } @@ -328,7 +335,9 @@ private: try { ProcessData(ev); } catch (...) { - LOG_E(CurrentExceptionMessage()); + YDB_LOG_ERROR("Exception", + {"traceId", TraceId}, + {"currentExceptionMessage", CurrentExceptionMessage()}); auto req = MakeHolder<TEvDqFailure>(NYql::NDqProto::StatusIds::INTERNAL_ERROR, TIssue("Internal error on data write").SetCode(NYql::DEFAULT_ERROR, TSeverityIds::S_ERROR)); Send(ExecuterId, req.Release()); HasError = true; diff --git a/ydb/core/fq/libs/actors/run_actor.cpp b/ydb/core/fq/libs/actors/run_actor.cpp index db33230a4f2..52c67f50fc1 100644 --- a/ydb/core/fq/libs/actors/run_actor.cpp +++ b/ydb/core/fq/libs/actors/run_actor.cpp @@ -82,12 +82,7 @@ #include <util/string/split.h> #include <util/system/hostname.h> -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << Params.QueryId << " " << stream) -#define LOG_W(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << Params.QueryId << " " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << Params.QueryId << " " << stream) -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << Params.QueryId << " " << stream) -#define LOG_QE(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << QueryId << " " << stream) -#define LOG_QW(stream) LOG_WARN_S( *TlsActivationContext, NKikimrServices::FQ_RUN_ACTOR, "QueryId: " << QueryId << " " << stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::FQ_RUN_ACTOR namespace NFq { @@ -290,13 +285,16 @@ public: TString Plan2Json(const TString& ysonPlan) { if (!ysonPlan) { - LOG_QW("Can't convert plan from yson to json: plan is empty"); + YDB_LOG_WARN("Can't convert plan from yson to json: plan is empty", + {"queryId", QueryId}); return {}; } try { return NJson2Yson::ConvertYson2Json(ysonPlan); } catch (...) { - LOG_QE("Can't convert plan from yson to json: " << CurrentExceptionMessage()); + YDB_LOG_ERROR("Can't convert plan from yson", + {"queryId", QueryId}, + {"json", CurrentExceptionMessage()}); } return {}; } @@ -373,7 +371,9 @@ public: static constexpr char ActorName[] = "YQ_RUN_ACTOR"; void Bootstrap() { - LOG_D("Start run actor. Compute state: " << FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status)); + YDB_LOG_DEBUG("Start run actor. Compute", + {"queryId", Params.QueryId}, + {"state", FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status)}); FillConnections(); @@ -467,11 +467,14 @@ private: ) void FillConnections() { - LOG_D("FillConnections"); + YDB_LOG_DEBUG("FillConnections", + {"queryId", Params.QueryId}); for (const auto& connection : Params.Connections) { if (!connection.content().name()) { - LOG_D("Connection with empty name " << connection.meta().id()); + YDB_LOG_DEBUG("Connection with empty name", + {"queryId", Params.QueryId}, + {"connectionId", connection.meta().id()}); continue; } YqConnections.emplace(connection.meta().id(), connection); @@ -510,7 +513,8 @@ private: void CancelRunningQuery() { if (ReadRulesCreatorId) { - LOG_D("Cancel read rules creation"); + YDB_LOG_DEBUG("Cancel read rules creation", + {"queryId", Params.QueryId}); Send(ReadRulesCreatorId, new NActors::TEvents::TEvPoison()); } @@ -526,7 +530,8 @@ private: } if (ControlId) { - LOG_D("Cancel running query"); + YDB_LOG_DEBUG("Cancel running query", + {"queryId", Params.QueryId}); Send(ControlId, new NDq::TEvDq::TEvAbortExecution(NYql::NDqProto::StatusIds::ABORTED, FederatedQuery::QueryMeta::ComputeStatus_Name(FinalQueryStatus))); } else { QueryResponseArrived = true; @@ -534,7 +539,8 @@ private: } void PassAway() override { - LOG_D("PassAway"); + YDB_LOG_DEBUG("PassAway", + {"queryId", Params.QueryId}); Send(FetcherId, new NActors::TEvents::TEvPoisonTaken()); KillChildrenActors(); NActors::TActorBootstrapped<TRunActor>::PassAway(); @@ -569,7 +575,8 @@ private: if (QueryStateUpdateRequest.resources().rate_limiter() == Fq::Private::TaskResources::PREPARE) { if (!RateLimiterResourceCreatorId) { - LOG_D("Start rate limiter resource creator"); + YDB_LOG_DEBUG("Start rate limiter resource creator", + {"queryId", Params.QueryId}); RateLimiterResourceCreatorId = Register(CreateRateLimiterResourceCreator(SelfId(), Params.Owner, Params.QueryId, Params.Scope, Params.TenantName)); } return; @@ -656,7 +663,10 @@ private: } void Fail(const TString& errorMessage) { - LOG_E("Fail for query " << Params.QueryId << ", finishing: " << Finishing << ", details: " << errorMessage); + YDB_LOG_ERROR("Fail for query", + {"queryId", Params.QueryId}, + {"finishing", Finishing}, + {"details", errorMessage}); if (YqConnections.empty()) { Issues.AddIssue("YqConnections array is empty"); @@ -688,7 +698,9 @@ private: void Handle(TEvents::TEvQueryActionResult::TPtr& ev) { Action = ev->Get()->Action; - LOG_D("New query action received: " << FederatedQuery::QueryAction_Name(Action)); + YDB_LOG_DEBUG("New query action", + {"queryId", Params.QueryId}, + {"received", FederatedQuery::QueryAction_Name(Action)}); switch (Action) { case FederatedQuery::ABORT: case FederatedQuery::ABORT_GRACEFULLY: // not fully implemented @@ -756,7 +768,9 @@ private: THashMap<TString, TTopicIndependentConsumers> topicToIndependentConsumers; ui32 graphIndex = 0; for (auto& graphParams : DqGraphParams) { - LOG_D("Graph " << graphIndex); + YDB_LOG_DEBUG("Graph", + {"queryId", Params.QueryId}, + {"graphIndex", graphIndex}); graphIndex++; const TString consumerNamePrefix = graphIndex == 1 ? Params.QueryId : TStringBuilder() << Params.QueryId << '-' << graphIndex; // Simple name in simple case for (NYql::NDqProto::TDqTask& task : *graphParams.MutableTasks()) { @@ -774,7 +788,10 @@ private: srcDesc.SetConsumerName(consumerName); settingsAny.PackFrom(srcDesc); if (isNewConsumer) { - LOG_D("Create consumer \"" << srcDesc.GetConsumerName() << "\" for topic \"" << srcDesc.GetTopicPath() << "\""); + YDB_LOG_DEBUG("Create consumer for topic", + {"queryId", Params.QueryId}, + {"consumerName", srcDesc.GetConsumerName()}, + {"topicPath", srcDesc.GetTopicPath()}); auto& consumer = *QueryStateUpdateRequest.mutable_resources()->add_topic_consumers(); consumer.set_database_id(srcDesc.GetDatabaseId()); consumer.set_database(srcDesc.GetDatabase()); @@ -859,7 +876,10 @@ private: } void Handle(TEvents::TEvForwardPingResponse::TPtr& ev) { - LOG_T("Forward ping response. Success: " << ev->Get()->Success << ". Cookie: " << ev->Cookie); + YDB_LOG_TRACE("Forward ping response", + {"queryId", Params.QueryId}, + {"success", ev->Get()->Success}, + {"cookie", ev->Cookie}); if (!ev->Get()->Success) { // Failed setting new status or lease was lost ResignQuery(NYql::NDqProto::StatusIds::UNAVAILABLE); return; @@ -878,14 +898,18 @@ private: } void HandleFinish(TEvents::TEvForwardPingResponse::TPtr& ev) { - LOG_T("Forward ping response. Success: " << ev->Get()->Success << ". Cookie: " << ev->Cookie); + YDB_LOG_TRACE("Forward ping response", + {"queryId", Params.QueryId}, + {"success", ev->Get()->Success}, + {"cookie", ev->Cookie}); if (!ev->Get()->Success) { // Failed setting new status or lease was lost Fail("Failed to write finalizing status"); return; } if (ev->Cookie == SaveFinalizingStatusCookie) { - LOG_D("Finalizing status is saved"); + YDB_LOG_DEBUG("Finalizing status is saved", + {"queryId", Params.QueryId}); FinalizingStatusIsWritten = true; ContinueFinish(); } @@ -897,7 +921,9 @@ private: return; } if (ev->Get()->Issues) { - LOG_W("Effect Issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_WARN("Effect", + {"queryId", Params.QueryId}, + {"issues", ev->Get()->Issues.ToOneLineString()}); Issues.AddIssues(ev->Get()->Issues); } ++EffectApplicatorFinished; @@ -909,7 +935,9 @@ private: for (const auto& dqGraph : DqGraphParams) { dqTasks += dqGraph.TasksSize(); } - LOG_D("Overall dq tasks: " << dqTasks); + YDB_LOG_DEBUG("Overall dq", + {"queryId", Params.QueryId}, + {"tasks", dqTasks}); if (dqTasks > MaxTasksPerOperation) { return TStringBuilder() << "Too many tasks per operation: " << dqTasks << ". Allowed: less than " << MaxTasksPerOperation; } @@ -926,7 +954,10 @@ private: } void Handle(TEvents::TEvGraphParams::TPtr& ev) { - LOG_D("Graph (" << (ev->Get()->IsEvaluation ? "evaluation" : "execution") << ") with tasks: " << ev->Get()->GraphParams.TasksSize()); + YDB_LOG_DEBUG("Graph with", + {"queryId", Params.QueryId}, + {"kind", (ev->Get()->IsEvaluation ? "evaluation" : "execution")}, + {"tasks", ev->Get()->GraphParams.TasksSize()}); if (Params.Resources.rate_limiter_path()) { const TString rateLimiterResource = GetRateLimiterResourcePath(Params.CloudId, Params.Scope.ParseFolder(), Params.QueryId); @@ -947,7 +978,8 @@ private: } void Handle(TEvCheckpointCoordinator::TEvZeroCheckpointDone::TPtr&) { - LOG_D("Coordinator saved zero checkpoint"); + YDB_LOG_DEBUG("Coordinator saved zero checkpoint", + {"queryId", Params.QueryId}); Y_ABORT_UNLESS(ControlId); SetLoadFromCheckpointMode(); } @@ -970,7 +1002,9 @@ private: GraphKey = "Precompute=" + ToString(it->second.Index); } else { if (ev->Sender != ExecuterId) { - LOG_E("TEvDqStats received from UNKNOWN Actor (TDqExecuter?) " << ev->Sender); + YDB_LOG_ERROR("TEvDqStats received from UNKNOWN Actor (TDqExecuter?)", + {"queryId", Params.QueryId}, + {"sender", ev->Sender}); return; } GraphKey = "Graph=" + ToString(DqGraphIndex); @@ -1209,7 +1243,9 @@ private: statistics = NJson2Yson::ConvertYson2Json(out.Str()); return true; } catch (NYson::TYsonException& ex) { - LOG_E(ex.what()); + YDB_LOG_ERROR("Exception", + {"queryId", Params.QueryId}, + {"exception", ex.what()}); return false; } } @@ -1223,16 +1259,21 @@ private: TIssue WrapInternalIssues(const TIssues& issues) { NYql::IssuesToMessage(TruncateIssues(issues), QueryStateUpdateRequest.mutable_internal_issues()); TString referenceId = GetEntityIdAsString(Params.Config.GetCommon().GetIdsPrefix(), EEntityType::UNDEFINED); - LOG_E(referenceId << ": " << issues.ToOneLineString()); + YDB_LOG_ERROR("", + {"queryId", Params.QueryId}, + {"referenceId", referenceId}, + {"issues", issues.ToOneLineString()}); return TIssue("Contact technical support and provide query information and this id: " + referenceId + "_" + Now().ToStringUpToSeconds()); } void SaveQueryResponse(NYql::NDqs::TEvQueryResponse::TPtr& ev) { auto& result = ev->Get()->Record; - LOG_D("Query response " << NYql::NDqProto::StatusIds_StatusCode_Name(ev->Get()->Record.GetStatusCode()) - << ". Result set index: " << DqGraphIndex - << ". Issues count: " << result.IssuesSize() - << ". Rows count: " << result.GetRowsCount()); + YDB_LOG_DEBUG("Query response result set", + {"queryId", Params.QueryId}, + {"statusCode", NYql::NDqProto::StatusIds_StatusCode_Name(ev->Get()->Record.GetStatusCode())}, + {"index", DqGraphIndex}, + {"issuesCount", result.IssuesSize()}, + {"rowsCount", result.GetRowsCount()}); if (ev->Get()->Record.GetStatusCode() == NYql::NDqProto::StatusIds::INTERNAL_ERROR && !Params.Config.GetCommon().GetKeepInternalErrors()) { TIssues issues; @@ -1287,9 +1328,14 @@ private: QueryEvalStatusCode = result.GetStatusCode(); - LOG_D("Query evaluation " << NYql::NDqProto::StatusIds_StatusCode_Name(QueryEvalStatusCode) - << ". " << it->second.Index << " response. Issues count: " << result.IssuesSize() - << ". Rows count: " << result.GetRowsCount() << ", Sample count: " << result.SampleSize() << ", Truncated: " << result.GetTruncated()); + YDB_LOG_DEBUG("Query evaluation response", + {"queryId", Params.QueryId}, + {"queryEvalStatusCode", NYql::NDqProto::StatusIds_StatusCode_Name(QueryEvalStatusCode)}, + {"index", it->second.Index}, + {"issuesCount", result.IssuesSize()}, + {"rowsCount", result.GetRowsCount()}, + {"sampleCount", result.SampleSize()}, + {"truncated", result.GetTruncated()}); TVector<NDq::TDqSerializedBatch> rows; for (const auto& s : result.GetSample()) { @@ -1348,7 +1394,9 @@ private: } if (ev->Sender != ExecuterId) { - LOG_E("TEvQueryResponse received from UNKNOWN Actor (TDqExecuter?) " << ev->Sender); + YDB_LOG_ERROR("TEvQueryResponse received from UNKNOWN Actor (TDqExecuter?)", + {"queryId", Params.QueryId}, + {"sender", ev->Sender}); return; } @@ -1356,7 +1404,8 @@ private: auto statusCode = ev->Get()->Record.GetStatusCode(); if (statusCode == NYql::NDqProto::StatusIds::UNSPECIFIED) { - LOG_E("StatusCode == NYql::NDqProto::StatusIds::UNSPECIFIED, it is not expected, the query will be failed."); + YDB_LOG_ERROR("StatusCode == NYql::NDqProto::StatusIds::UNSPECIFIED, it is not expected, the query will be failed", + {"queryId", Params.QueryId}); } if (statusCode != NYql::NDqProto::StatusIds::SUCCESS) { @@ -1375,7 +1424,8 @@ private: // Continue with the next graph QueryStateUpdateRequest.set_dq_graph_index(++DqGraphIndex); RunNextDqGraph(); - LOG_D("Send save query response request to pinger"); + YDB_LOG_DEBUG("Send save query response request to pinger", + {"queryId", Params.QueryId}); Send(Pinger, new TEvents::TEvForwardPingRequest(QueryStateUpdateRequest)); } @@ -1392,11 +1442,14 @@ private: } if (ev->Sender != ExecuterId) { - LOG_E("TEvQueryResponse received from UNKNOWN Actor (TDqExecuter?) when FINISHED " << ev->Sender); + YDB_LOG_ERROR("TEvQueryResponse received from UNKNOWN Actor (TDqExecuter?) when FINISHED", + {"queryId", Params.QueryId}, + {"sender", ev->Sender}); return; } - LOG_D("TEvQueryResponse received on finish"); + YDB_LOG_DEBUG("TEvQueryResponse received on finish", + {"queryId", Params.QueryId}); QueryResponseArrived = true; SaveQueryResponse(ev); @@ -1404,11 +1457,15 @@ private: } void Handle(TEvents::TEvDataStreamsReadRulesCreationResult::TPtr& ev) { - LOG_D("Read rules creation finished. Issues: " << ev->Get()->Issues.Size()); + YDB_LOG_DEBUG("Read rules creation finished", + {"queryId", Params.QueryId}, + {"issues", ev->Get()->Issues.Size()}); ReadRulesCreatorId = {}; if (ev->Get()->Issues) { AddIssueWithSubIssues("Problems with read rules creation", ev->Get()->Issues); - LOG_D(Issues.ToOneLineString()); + YDB_LOG_DEBUG("Dump queryId, #_Issues.ToOneLineString", + {"queryId", Params.QueryId}, + {"issues", Issues.ToOneLineString()}); Finish(FederatedQuery::QueryMeta::FAILED); } else { QueryStateUpdateRequest.mutable_resources()->set_topic_consumers_state(Fq::Private::TaskResources::READY); @@ -1420,7 +1477,9 @@ private: ReadRulesCreatorId = {}; if (ev->Get()->Issues) { TransientIssues.AddIssues(ev->Get()->Issues); - LOG_D(TransientIssues.ToOneLineString()); + YDB_LOG_DEBUG("Dump queryId, #_TransientIssues.ToOneLineString", + {"queryId", Params.QueryId}, + {"transientIssues", TransientIssues.ToOneLineString()}); } if (CanRunReadRulesDeletionActor()) { RunReadRulesDeletionActor(); @@ -1476,11 +1535,15 @@ private: } void Handle(NFq::TEvInternalService::TEvCreateRateLimiterResourceResponse::TPtr& ev) { - LOG_D("Rate limiter resource creation finished. Success: " << ev->Get()->Status.IsSuccess()); + YDB_LOG_DEBUG("Rate limiter resource creation finished", + {"queryId", Params.QueryId}, + {"success", ev->Get()->Status.IsSuccess()}); RateLimiterResourceCreatorId = {}; if (!ev->Get()->Status.IsSuccess()) { AddIssueWithSubIssues("Problems with rate limiter resource creation", NYdb::NAdapters::ToYqlIssues(ev->Get()->Status.GetIssues())); - LOG_D(Issues.ToOneLineString()); + YDB_LOG_DEBUG("Dump queryId, #_Issues.ToOneLineString", + {"queryId", Params.QueryId}, + {"issues", Issues.ToOneLineString()}); Finish(FederatedQuery::QueryMeta::FAILED); } else { Params.Resources.set_rate_limiter_path(ev->Get()->Result.rate_limiter()); @@ -1491,14 +1554,20 @@ private: } void HandleFinish(NFq::TEvInternalService::TEvCreateRateLimiterResourceResponse::TPtr& ev) { - LOG_D("Rate limiter resource creation finished. Success: " << ev->Get()->Status.IsSuccess() << ". Issues: " << ev->Get()->Status.GetIssues().ToOneLineString()); + YDB_LOG_DEBUG("Rate limiter resource creation finished", + {"queryId", Params.QueryId}, + {"success", ev->Get()->Status.IsSuccess()}, + {"issues", ev->Get()->Status.GetIssues().ToOneLineString()}); RateLimiterResourceCreatorId = {}; StartRateLimiterResourceDeleterIfCan(); } void HandleFinish(NFq::TEvInternalService::TEvDeleteRateLimiterResourceResponse::TPtr& ev) { - LOG_D("Rate limiter resource deletion finished. Success: " << ev->Get()->Status.IsSuccess() << ". Issues: " << ev->Get()->Status.GetIssues().ToOneLineString()); + YDB_LOG_DEBUG("Rate limiter resource deletion finished", + {"queryId", Params.QueryId}, + {"success", ev->Get()->Status.IsSuccess()}, + {"issues", ev->Get()->Status.GetIssues().ToOneLineString()}); RateLimiterResourceDeleterId = {}; RateLimiterResourceWasDeleted = true; @@ -1507,7 +1576,8 @@ private: bool StartRateLimiterResourceDeleterIfCan() { if (!RateLimiterResourceDeleterId && !RateLimiterResourceCreatorId && FinalizingStatusIsWritten && QueryResponseArrived && Params.Config.GetRateLimiter().GetEnabled()) { - LOG_D("Start rate limiter resource deleter"); + YDB_LOG_DEBUG("Start rate limiter resource deleter", + {"queryId", Params.QueryId}); RateLimiterResourceDeleterId = Register(CreateRateLimiterResourceDeleter(SelfId(), Params.Owner, Params.QueryId, Params.Scope, Params.TenantName)); return true; } @@ -1516,7 +1586,8 @@ private: TEvaluationGraphInfo RunEvalDqGraph(NFq::NProto::TGraphParams& dqGraphParams) { - LOG_D("RunEvalDqGraph"); + YDB_LOG_DEBUG("RunEvalDqGraph", + {"queryId", Params.QueryId}); FillGraphMemoryInfo(dqGraphParams); @@ -1544,7 +1615,8 @@ private: dqGraphParams.GetResultType(), empty, false).Release()); } else { - LOG_D("ResultReceiver was NOT CREATED since ResultType is empty"); + YDB_LOG_DEBUG("ResultReceiver was NOT CREATED since ResultType is empty", + {"queryId", Params.QueryId}); info.ResultId = info.ExecuterId; } @@ -1560,7 +1632,11 @@ private: PrepareResultFormatSettings(info.ResultFormatSettings, dqGraphParams, *dqConfiguration); NTasksPacker::UnPack(*request.MutableTask(), dqGraphParams.GetTasks(), dqGraphParams.GetStageProgram()); Send(info.ExecuterId, new NYql::NDqs::TEvGraphRequest(request, info.ControlId, info.ResultId)); - LOG_D("Evaluation Executer: " << info.ExecuterId << ", Controller: " << info.ControlId << ", ResultActor: " << info.ResultId); + YDB_LOG_DEBUG("Evaluation", + {"queryId", Params.QueryId}, + {"executer", info.ExecuterId}, + {"controller", info.ControlId}, + {"resultActor", info.ResultId}); return info; } @@ -1598,7 +1674,8 @@ private: PrepareResultFormatSettings(ResultFormatSettings, dqGraphParams, *dqConfiguration); } else { - LOG_D("ResultWriter was NOT CREATED since ResultType is empty"); + YDB_LOG_DEBUG("ResultWriter was NOT CREATED since ResultType is empty", + {"queryId", Params.QueryId}); resultId = ExecuterId; ClearResultFormatSettings(); } @@ -1652,7 +1729,11 @@ private: NTasksPacker::UnPack(*request.MutableTask(), dqGraphParams.GetTasks(), dqGraphParams.GetStageProgram()); Send(ExecuterId, new NYql::NDqs::TEvGraphRequest(request, ControlId, resultId)); - LOG_D("Executer: " << ExecuterId << ", Controller: " << ControlId << ", ResultIdActor: " << resultId); + YDB_LOG_DEBUG("Dump queryId, executer, controller, resultIdActor", + {"queryId", Params.QueryId}, + {"executer", ExecuterId}, + {"controller", ControlId}, + {"resultIdActor", resultId}); } void PrepareResultFormatSettings(TMaybe<NCommon::TResultFormatSettings>& resultFormatSettings, NFq::NProto::TGraphParams& dqGraphParams, const TDqConfiguration& dqConfiguration) { @@ -1819,14 +1900,18 @@ private: void WriteFinalizingStatus() { const FederatedQuery::QueryMeta::ComputeStatus finalizingStatus = GetFinalizingStatus(); Params.Status = finalizingStatus; - LOG_D("Write finalizing status: " << FederatedQuery::QueryMeta::ComputeStatus_Name(finalizingStatus)); + YDB_LOG_DEBUG("Write finalizing", + {"queryId", Params.QueryId}, + {"status", FederatedQuery::QueryMeta::ComputeStatus_Name(finalizingStatus)}); Fq::Private::PingTaskRequest request; request.set_status(finalizingStatus); Send(Pinger, new TEvents::TEvForwardPingRequest(request), 0, SaveFinalizingStatusCookie); } void Finish(FederatedQuery::QueryMeta::ComputeStatus status) { - LOG_D("Is about to finish query with status " << FederatedQuery::QueryMeta::ComputeStatus_Name(status)); + YDB_LOG_DEBUG("Is about to finish query with status", + {"queryId", Params.QueryId}, + {"status", FederatedQuery::QueryMeta::ComputeStatus_Name(status)}); Finishing = true; FinalQueryStatus = status; @@ -1846,7 +1931,9 @@ private: } void StartEffectApplicators() { - LOG_D("Apply effects with status: " << FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status)); + YDB_LOG_DEBUG("Apply effects with", + {"queryId", Params.QueryId}, + {"status", FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status)}); for (const auto& externalEffect : QueryStateUpdateRequest.resources().external_effects()) { auto providerName = externalEffect.GetProviderName(); if (providerName == "S3Sink") { @@ -1872,12 +1959,16 @@ private: if (Params.Status == FederatedQuery::QueryMeta::ABORTING_BY_USER) { for (auto& prefix : S3Prefixes) { - LOG_W("Partial results are possible with prefix: " << prefix); + YDB_LOG_WARN("Partial results are possible with", + {"queryId", Params.QueryId}, + {"prefix", prefix}); Issues.AddIssue(TIssue(TStringBuilder() << "Partial results are possible with prefix: " << prefix)); } } } else { - LOG_E("Unknown effect applicator: " << providerName); + YDB_LOG_ERROR("Unknown effect", + {"queryId", Params.QueryId}, + {"applicator", providerName}); } } } @@ -1913,12 +2004,16 @@ private: void ResignQuery(NYql::NDqProto::StatusIds::StatusCode statusCode) { QueryStateUpdateRequest.set_resign_query(true); QueryStateUpdateRequest.set_status_code(statusCode); - LOG_W("ResignQuery, status " << NYql::NDqProto::StatusIds::StatusCode_Name(statusCode)); + YDB_LOG_WARN("ResignQuery, status", + {"queryId", Params.QueryId}, + {"statusCode", NYql::NDqProto::StatusIds::StatusCode_Name(statusCode)}); SendPingAndPassAway(); } void SendPingAndPassAway() { - LOG_D("SendPingAndPassAway, FinalQueryStatus " << FederatedQuery::QueryMeta::ComputeStatus_Name(FinalQueryStatus)); + YDB_LOG_DEBUG("SendPingAndPassAway, FinalQueryStatus", + {"queryId", Params.QueryId}, + {"finalQueryStatus", FederatedQuery::QueryMeta::ComputeStatus_Name(FinalQueryStatus)}); // Run ping. if (QueryStateUpdateRequest.resign_query()) { // Retry state => all issues are not fatal. TransientIssues.AddIssues(Issues); @@ -1965,7 +2060,8 @@ private: //NYql::NLog::YqlLogger().SetComponentLevel(NYql::NLog::EComponent::CoreEval, NYql::NLog::ELevel::TRACE); //NYql::NLog::YqlLogger().SetComponentLevel(NYql::NLog::EComponent::CorePeepHole, NYql::NLog::ELevel::TRACE); - LOG_D("Compiling query ..."); + YDB_LOG_DEBUG("Compiling query", + {"queryId", Params.QueryId}); NYql::TGatewaysConfig gatewaysConfig; SetupYqlCore(*gatewaysConfig.MutableYqlCore()); @@ -2119,7 +2215,8 @@ private: } void Handle(TEvents::TEvAsyncContinue::TPtr& ev) { - LOG_D("Compiling finished"); + YDB_LOG_DEBUG("Compiling finished", + {"queryId", Params.QueryId}); NYql::TProgram::TStatus status = TProgram::TStatus::Error; const auto& f = ev->Get()->Future; @@ -2320,24 +2417,24 @@ private: } void LogReceivedParams() { - LOG_D("Run actors params: { QueryId: " << Params.QueryId - << " CloudId: " << Params.CloudId - << " UserId: " << Params.UserId - << " Owner: " << Params.Owner - << " PreviousQueryRevision: " << Params.PreviousQueryRevision - << " Connections: " << Params.Connections.size() - << " Bindings: " << Params.Bindings.size() - << " AccountIdSignatures: " << Params.AccountIdSignatures.size() - << " QueryType: " << FederatedQuery::QueryContent::QueryType_Name(Params.QueryType) - << " ExecuteMode: " << FederatedQuery::ExecuteMode_Name(Params.ExecuteMode) - << " ResultId: " << Params.ResultId - << " StateLoadMode: " << FederatedQuery::StateLoadMode_Name(Params.StateLoadMode) - << " StreamingDisposition: " << Params.StreamingDisposition - << " Status: " << FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status) - << " DqGraphs: " << Params.DqGraphs.size() - << " DqGraphIndex: " << Params.DqGraphIndex - << " Resource.TopicConsumers: " << Params.Resources.topic_consumers().size() - << " }"); + YDB_LOG_DEBUG("Run actors params", + {"queryId", Params.QueryId}, + {"cloudId", Params.CloudId}, + {"userId", Params.UserId}, + {"owner", Params.Owner}, + {"previousQueryRevision", Params.PreviousQueryRevision}, + {"connections", Params.Connections.size()}, + {"bindings", Params.Bindings.size()}, + {"accountIdSignatures", Params.AccountIdSignatures.size()}, + {"queryType", FederatedQuery::QueryContent::QueryType_Name(Params.QueryType)}, + {"executeMode", FederatedQuery::ExecuteMode_Name(Params.ExecuteMode)}, + {"resultId", Params.ResultId}, + {"stateLoadMode", FederatedQuery::StateLoadMode_Name(Params.StateLoadMode)}, + {"streamingDisposition", Params.StreamingDisposition}, + {"status", FederatedQuery::QueryMeta::ComputeStatus_Name(Params.Status)}, + {"dqGraphs", Params.DqGraphs.size()}, + {"dqGraphIndex", Params.DqGraphIndex}, + {"topicConsumers", Params.Resources.topic_consumers().size()}); } bool CollectBasic() { diff --git a/ydb/core/fq/libs/actors/task_get.cpp b/ydb/core/fq/libs/actors/task_get.cpp index 6ffbd5066fd..d5393e70778 100644 --- a/ydb/core/fq/libs/actors/task_get.cpp +++ b/ydb/core/fq/libs/actors/task_get.cpp @@ -18,10 +18,7 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <ydb/library/security/util.h> -#define LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateGetTask - Owner: " << OwnerId << ", " << "Host: " << Host << ", Tenant: " << Tenant << ", " << stream) -#define LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateGetTask - Owner: " << OwnerId << ", " << "Host: " << Host << ", Tenant: " << Tenant << ", " << stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_PRIVATE_PROXY namespace NFq { @@ -51,7 +48,10 @@ public: static constexpr char ActorName[] = "YQ_PRIVATE_GET_TASK"; void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { - LOG_E("TGetTaskRequestActor::OnUndelivered"); + YDB_LOG_ERROR("PrivateGetTask - TGetTaskRequestActor::OnUndelivered", + {"owner", OwnerId}, + {"host", Host}, + {"tenant", Tenant}); auto response = MakeHolder<TEvents::TEvGetTaskResponse>(); response->Status = Ydb::StatusIds::GENERIC_ERROR; response->Issues.AddIssue("UNDELIVERED"); @@ -67,9 +67,12 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("PrivateGetTask - Failed with", + {"owner", OwnerId}, + {"host", Host}, + {"tenant", Tenant}, + {"code", codeStr}, + {"details", Issues}); auto response = MakeHolder<TEvents::TEvGetTaskResponse>(); response->Status = reqStatus; response->Issues.AddIssues(Issues); @@ -83,14 +86,21 @@ public: OwnerId = request.owner_id(); Host = request.host(); Tenant = request.tenant(); - LOG_D("Request CP::GetTask with size: " << request.ByteSize() << " bytes"); + YDB_LOG_DEBUG("PrivateGetTask - Request CP::GetTask with bytes", + {"owner", OwnerId}, + {"host", Host}, + {"tenant", Tenant}, + {"size", request.ByteSize()}); RequestedMBytes->Collect(request.ByteSize() / 1024 / 1024); Send(ControlPlaneConfigActorId(), new TEvControlPlaneConfig::TEvGetTenantInfoRequest()); } private: void HandleResponse(NFq::TEvControlPlaneStorage::TEvGetTaskResponse::TPtr& ev) { // YQ - LOG_D("Got CP::GetTask Response"); + YDB_LOG_DEBUG("PrivateGetTask - Got CP::GetTask Response", + {"owner", OwnerId}, + {"host", Host}, + {"tenant", Tenant}); const auto& issues = ev->Get()->Issues; if (issues) { diff --git a/ydb/core/fq/libs/actors/task_ping.cpp b/ydb/core/fq/libs/actors/task_ping.cpp index 0e187a072ab..ea9bd6a433c 100644 --- a/ydb/core/fq/libs/actors/task_ping.cpp +++ b/ydb/core/fq/libs/actors/task_ping.cpp @@ -17,10 +17,7 @@ #include <google/protobuf/util/time_util.h> -#define LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivatePingTask - QueryId: " << OperationId << ", Owner: " << OwnerId << ", " << stream) -#define LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivatePingTask - QueryId: " << OperationId << ", Owner: " << OwnerId << ", "<< stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_PRIVATE_PROXY namespace NFq { @@ -48,7 +45,9 @@ public: static constexpr char ActorName[] = "YQ_PRIVATE_PING_TASK"; void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { - LOG_E("TTaskPingRequestActor::OnUndelivered"); + YDB_LOG_ERROR("PrivatePingTask - TTaskPingRequestActor::OnUndelivered", + {"queryId", OperationId}, + {"owner", OwnerId}); auto res = MakeHolder<TEvents::TEvPingTaskResponse>(); res->Status = Ydb::StatusIds::GENERIC_ERROR; res->Issues.AddIssue("UNDELIVERED"); @@ -64,9 +63,11 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("PrivatePingTask - Failed with", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"code", codeStr}, + {"details", Issues}); auto res = MakeHolder<TEvents::TEvPingTaskResponse>(); res->Status = reqStatus; res->Issues.AddIssues(Issues); @@ -82,7 +83,10 @@ public: TenantName = req.tenant(); Scope = req.scope(); Deadline = NProtoInterop::CastFromProto(req.deadline()); - LOG_D("Request CP::PingTask with size: " << req.ByteSize() << " bytes"); + YDB_LOG_DEBUG("PrivatePingTask - Request CP::PingTask with bytes", + {"queryId", OperationId}, + {"owner", OwnerId}, + {"size", req.ByteSize()}); RequestedMBytes->Collect(req.ByteSize() / 1024 / 1024); Send(ControlPlaneConfigActorId(), new TEvControlPlaneConfig::TEvGetTenantInfoRequest()); } @@ -102,7 +106,9 @@ private: } void HandleResponse(NFq::TEvControlPlaneStorage::TEvPingTaskResponse::TPtr& ev) { - LOG_D("Got CP::PingTaskResponse"); + YDB_LOG_DEBUG("PrivatePingTask - Got CP::PingTaskResponse", + {"queryId", OperationId}, + {"owner", OwnerId}); const auto& issues = ev->Get()->Issues; if (issues) { Issues.AddIssues(issues); diff --git a/ydb/core/fq/libs/actors/task_result_write.cpp b/ydb/core/fq/libs/actors/task_result_write.cpp index cf2a2efb70e..78a538048a3 100644 --- a/ydb/core/fq/libs/actors/task_result_write.cpp +++ b/ydb/core/fq/libs/actors/task_result_write.cpp @@ -14,10 +14,7 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <ydb/core/fq/libs/control_plane_storage/control_plane_storage.h> -#define LOG_E(stream) \ - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateWriteTask - ResultId: " << ResultId << ", RequestId: " << RequestId << ", " << stream) -#define LOG_D(stream) \ - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::YQL_PRIVATE_PROXY, "PrivateWriteTask - ResultId: " << ResultId << ", RequestId: " << RequestId << ", " << stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::YQL_PRIVATE_PROXY namespace NFq { @@ -47,7 +44,9 @@ public: static constexpr char ActorName[] = "YQ_PRIVATE_WRITE_RESULT_TASK"; void OnUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { - LOG_E("TWriteTaskRequestActor::OnUndelivered"); + YDB_LOG_ERROR("PrivateWriteTask - TWriteTaskRequestActor::OnUndelivered", + {"resultId", ResultId}, + {"requestId", RequestId}); Res->Status = Ydb::StatusIds::GENERIC_ERROR; Res->Issues.AddIssue("UNDELIVERED"); Send(ev->Sender, Res.Release()); @@ -62,9 +61,11 @@ public: void Fail(const TString& message, Ydb::StatusIds::StatusCode reqStatus = Ydb::StatusIds::INTERNAL_ERROR) { Issues.AddIssue(message); const auto codeStr = Ydb::StatusIds_StatusCode_Name(reqStatus); - LOG_E(TStringBuilder() - << "Failed with code: " << codeStr - << " Details: " << Issues.ToString()); + YDB_LOG_ERROR("PrivateWriteTask - Failed with", + {"resultId", ResultId}, + {"requestId", RequestId}, + {"code", codeStr}, + {"details", Issues}); Res->Status = reqStatus; Res->Issues.AddIssues(Issues); Send(Sender, Res.Release()); @@ -76,7 +77,10 @@ public: auto request = Ev->Record; ResultId = request.result_id().value(); RequestId = request.request_id(); - LOG_D("Request CP::WriteTaskResult with size: " << request.ByteSize() << " bytes"); + YDB_LOG_DEBUG("PrivateWriteTask - Request CP::WriteTaskResult with bytes", + {"resultId", ResultId}, + {"requestId", RequestId}, + {"size", request.ByteSize()}); RequestedMBytes->Collect(request.ByteSize() / 1024 / 1024); Send(NFq::ControlPlaneStorageServiceActorId(), new NFq::TEvControlPlaneStorage::TEvWriteResultDataRequest(std::move(request)), 0, RequestId); @@ -91,7 +95,9 @@ private: ) void HandleResponse(NFq::TEvControlPlaneStorage::TEvWriteResultDataResponse::TPtr& ev) { - LOG_D("Got CP::WriteTaskResult Response"); + YDB_LOG_DEBUG("PrivateWriteTask - Got CP::WriteTaskResult Response", + {"resultId", ResultId}, + {"requestId", RequestId}); const auto& issues = ev->Get()->Issues; if (issues) { Issues.AddIssues(issues); diff --git a/ydb/core/fq/libs/actors/ya.make b/ydb/core/fq/libs/actors/ya.make index 644b6907375..a94d516b906 100644 --- a/ydb/core/fq/libs/actors/ya.make +++ b/ydb/core/fq/libs/actors/ya.make @@ -27,7 +27,6 @@ PEERDIR( library/cpp/yson library/cpp/yson/node ydb/core/base - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/checkpointing ydb/core/fq/libs/checkpointing_common ydb/core/fq/libs/common @@ -96,10 +95,6 @@ YQL_LAST_ABI_VERSION() END() -RECURSE( - logging -) - RECURSE_FOR_TESTS( ut ) diff --git a/ydb/core/fq/libs/checkpoint_storage/ut/ya.make b/ydb/core/fq/libs/checkpoint_storage/ut/ya.make index 6f2c93b83f7..45c744c0035 100644 --- a/ydb/core/fq/libs/checkpoint_storage/ut/ya.make +++ b/ydb/core/fq/libs/checkpoint_storage/ut/ya.make @@ -12,7 +12,6 @@ FORK_SUBTESTS() PEERDIR( library/cpp/retry library/cpp/testing/unittest - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/checkpoint_storage/events ydb/core/testlib ydb/core/testlib/default diff --git a/ydb/core/fq/libs/checkpoint_storage/ya.make b/ydb/core/fq/libs/checkpoint_storage/ya.make index c3357f3237f..e086b27b01d 100644 --- a/ydb/core/fq/libs/checkpoint_storage/ya.make +++ b/ydb/core/fq/libs/checkpoint_storage/ya.make @@ -12,7 +12,6 @@ SRCS( PEERDIR( contrib/libs/fmt ydb/library/actors/core - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/config/protos ydb/core/fq/libs/control_plane_storage ydb/core/fq/libs/ydb diff --git a/ydb/core/fq/libs/checkpointing/ya.make b/ydb/core/fq/libs/checkpointing/ya.make index e1553e35bab..cb436b3244b 100644 --- a/ydb/core/fq/libs/checkpointing/ya.make +++ b/ydb/core/fq/libs/checkpointing/ya.make @@ -11,7 +11,7 @@ SRCS( PEERDIR( ydb/library/actors/core - ydb/core/fq/libs/actors/logging + ydb/core/fq/libs/config/protos ydb/core/fq/libs/checkpointing_common ydb/core/fq/libs/checkpoint_storage/events ydb/library/yql/dq/actors/compute diff --git a/ydb/core/fq/libs/control_plane_proxy/ut/ya.make b/ydb/core/fq/libs/control_plane_proxy/ut/ya.make index 11f42c49525..6c1870d6486 100644 --- a/ydb/core/fq/libs/control_plane_proxy/ut/ya.make +++ b/ydb/core/fq/libs/control_plane_proxy/ut/ya.make @@ -5,7 +5,6 @@ SIZE(MEDIUM) PEERDIR( library/cpp/testing/unittest ydb/core/base - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/control_plane_storage ydb/core/fq/libs/test_connection ydb/core/fq/libs/quota_manager/ut_helpers diff --git a/ydb/core/fq/libs/control_plane_proxy/utils/logs.h b/ydb/core/fq/libs/control_plane_proxy/utils/logs.h deleted file mode 100644 index e2d2f8432a6..00000000000 --- a/ydb/core/fq/libs/control_plane_proxy/utils/logs.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include <ydb/core/fq/libs/actors/logging/log.h> - -#define CPP_LOG_D(s) \ - LOG_YQ_CONTROL_PLANE_PROXY_DEBUG(s) -#define CPP_LOG_I(s) \ - LOG_YQ_CONTROL_PLANE_PROXY_INFO(s) -#define CPP_LOG_W(s) \ - LOG_YQ_CONTROL_PLANE_PROXY_WARN(s) -#define CPP_LOG_E(s) \ - LOG_YQ_CONTROL_PLANE_PROXY_ERROR(s) -#define CPP_LOG_T(s) \ - LOG_YQ_CONTROL_PLANE_PROXY_TRACE(s) diff --git a/ydb/core/fq/libs/control_plane_proxy/utils/ya.make b/ydb/core/fq/libs/control_plane_proxy/utils/ya.make index c6adf22c468..e95cdc4e053 100644 --- a/ydb/core/fq/libs/control_plane_proxy/utils/ya.make +++ b/ydb/core/fq/libs/control_plane_proxy/utils/ya.make @@ -5,7 +5,6 @@ SRCS( ) PEERDIR( - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/compute/common ydb/core/fq/libs/config/protos ydb/core/fq/libs/control_plane_storage diff --git a/ydb/core/fq/libs/control_plane_proxy/ya.make b/ydb/core/fq/libs/control_plane_proxy/ya.make index 47713ff69eb..45e1ffb27be 100644 --- a/ydb/core/fq/libs/control_plane_proxy/ya.make +++ b/ydb/core/fq/libs/control_plane_proxy/ya.make @@ -9,7 +9,6 @@ PEERDIR( ydb/library/actors/core ydb/core/base ydb/core/fq/libs/actors - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/compute/ydb ydb/core/fq/libs/compute/ydb/control_plane ydb/core/fq/libs/control_plane_config diff --git a/ydb/core/fq/libs/control_plane_storage/ya.make b/ydb/core/fq/libs/control_plane_storage/ya.make index a2af1b7af86..c74ea6448b0 100644 --- a/ydb/core/fq/libs/control_plane_storage/ya.make +++ b/ydb/core/fq/libs/control_plane_storage/ya.make @@ -22,7 +22,6 @@ PEERDIR( library/cpp/protobuf/interop ydb/core/base ydb/core/external_sources - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/common ydb/core/fq/libs/config ydb/core/fq/libs/config/protos diff --git a/ydb/core/fq/libs/row_dispatcher/coordinator.cpp b/ydb/core/fq/libs/row_dispatcher/coordinator.cpp index 42037c096cf..20414235b63 100644 --- a/ydb/core/fq/libs/row_dispatcher/coordinator.cpp +++ b/ydb/core/fq/libs/row_dispatcher/coordinator.cpp @@ -1,6 +1,5 @@ #include "coordinator.h" -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/actors/nodes_manager_events.h> #include <ydb/core/fq/libs/ydb/ydb.h> #include <ydb/core/fq/libs/ydb/schema.h> @@ -12,11 +11,14 @@ #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> #include <ydb/library/actors/core/interconnect.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/actors/protos/actors.pb.h> #include <ydb/public/sdk/cpp/adapters/issue/issue.h> #include <yql/essentials/public/issue/yql_issue_message.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq { using namespace NActors; @@ -25,7 +27,7 @@ using namespace NThreading; using NYql::TIssues; namespace { - + //////////////////////////////////////////////////////////////////////////////// struct TCoordinatorMetrics { @@ -116,12 +118,12 @@ class TActorCoordinator : public TActorBootstrapped<TActorCoordinator> { }; struct TRowDispatcherInfo { - TRowDispatcherInfo(bool connected, ENodeState state, bool isLocal) + TRowDispatcherInfo(bool connected, ENodeState state, bool isLocal) : Connected(connected) , State(state) , IsLocal(isLocal) {} bool Connected = false; - ENodeState State; + ENodeState State; bool IsLocal = false; THashSet<TPartitionKey, TPartitionKeyHash> Locations; }; @@ -289,15 +291,19 @@ void TActorCoordinator::Bootstrap() { if (PrintStateEnabled) { Schedule(TDuration::Seconds(PrintStatePeriodSec), new TEvPrivate::TEvPrintState()); } - LOG_ROW_DISPATCHER_DEBUG("Successfully bootstrapped coordinator, id " << SelfId() - << ", NodesManagerId " << NodesManagerId - << ", rebalancing timeout " << RebalancingTimeout); + YDB_LOG_DEBUG("Successfully bootstrapped coordinator, id NodesManagerId rebalancing timeout", + {"logPrefix", LogPrefix}, + {"selfId", SelfId()}, + {"nodesManagerId", NodesManagerId}, + {"rebalancingTimeout", RebalancingTimeout}); auto nodeGroup = Metrics.Counters->GetSubgroup("node", ToString(SelfId().NodeId())); Metrics.IsActive = nodeGroup->GetCounter("IsActive"); } void TActorCoordinator::UpdateKnownRowDispatchers(NActors::TActorId actorId, bool isLocal) { - LOG_ROW_DISPATCHER_TRACE("UpdateKnownRowDispatchers " << actorId.ToString()); + YDB_LOG_TRACE("UpdateKnownRowDispatchers", + {"logPrefix", LogPrefix}, + {"actorId", actorId}); auto it = RowDispatchers.find(actorId); if (it != RowDispatchers.end()) { @@ -311,7 +317,10 @@ void TActorCoordinator::UpdateKnownRowDispatchers(NActors::TActorId actorId, boo continue; } - LOG_ROW_DISPATCHER_TRACE("Move all Locations from old actor " << oldActorId.ToString() << " to new " << actorId.ToString()); + YDB_LOG_TRACE("Move all Locations from old actor to new", + {"logPrefix", LogPrefix}, + {"oldActorId", oldActorId}, + {"actorId", actorId}); for (auto& key : info.Locations) { PartitionLocations[key] = actorId; } @@ -327,12 +336,15 @@ void TActorCoordinator::UpdateKnownRowDispatchers(NActors::TActorId actorId, boo nodeState = ENodeState::Started; } - LOG_ROW_DISPATCHER_TRACE("Add new row dispatcher to map (state " << static_cast<int>(nodeState) << ")"); + YDB_LOG_TRACE("Add new row dispatcher to map (state", + {"logPrefix", LogPrefix}, + {"nodeState", static_cast<int>(nodeState)}); RowDispatchers.emplace(actorId, TRowDispatcherInfo{true, nodeState, isLocal}); UpdateGlobalState(); if (nodeState == ENodeState::Initializing && !RebalancingScheduled) { - LOG_ROW_DISPATCHER_TRACE("Schedule TEvRebalancing"); + YDB_LOG_TRACE("Schedule TEvRebalancing", + {"logPrefix", LogPrefix}); Schedule(RebalancingTimeout, new TEvPrivate::TEvRebalancing()); RebalancingScheduled = true; } @@ -354,10 +366,14 @@ void TActorCoordinator::UpdateInterconnectSessions(const NActors::TActorId& inte } void TActorCoordinator::Handle(NActors::TEvents::TEvPing::TPtr& ev) { - LOG_ROW_DISPATCHER_TRACE("TEvPing received, " << ev->Sender); + YDB_LOG_TRACE("TEvPing received", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); UpdateInterconnectSessions(ev->InterconnectSession); UpdateKnownRowDispatchers(ev->Sender, false); - LOG_ROW_DISPATCHER_TRACE("Send TEvPong to " << ev->Sender); + YDB_LOG_TRACE("Send TEvPong", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); Send(ev->Sender, new NActors::TEvents::TEvPong(), IEventHandle::FlagTrackDelivery); } @@ -388,19 +404,25 @@ void TActorCoordinator::PrintInternalState() { auto str = GetInternalState(); auto buf = TStringBuf(str); for (ui64 offset = 0; offset < buf.size(); offset += PrintStateToLogSplitSize) { - LOG_ROW_DISPATCHER_DEBUG(buf.SubString(offset, PrintStateToLogSplitSize)); + YDB_LOG_DEBUG("Dump logPrefix, state", + {"logPrefix", LogPrefix}, + {"state", buf.SubString(offset, PrintStateToLogSplitSize)}); } } void TActorCoordinator::HandleConnected(TEvInterconnect::TEvNodeConnected::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("EvNodeConnected " << ev->Get()->NodeId); + YDB_LOG_DEBUG("EvNodeConnected", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); // Dont set Connected = true. // Wait TEvPing from row dispatchers. } void TActorCoordinator::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("TEvNodeDisconnected, node id " << ev->Get()->NodeId); - + YDB_LOG_DEBUG("TEvNodeDisconnected, node id", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); + for (auto& [actorId, info] : RowDispatchers) { if (ev->Get()->NodeId != actorId.NodeId()) { continue; @@ -409,7 +431,8 @@ void TActorCoordinator::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected: info.Connected = false; if (!RebalancingScheduled) { - LOG_ROW_DISPATCHER_TRACE("Schedule TEvRebalancing"); + YDB_LOG_TRACE("Schedule TEvRebalancing", + {"logPrefix", LogPrefix}); Schedule(RebalancingTimeout, new TEvPrivate::TEvRebalancing()); RebalancingScheduled = true; } @@ -417,10 +440,14 @@ void TActorCoordinator::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected: } void TActorCoordinator::Handle(NActors::TEvents::TEvUndelivered::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("TEvUndelivered, ev: " << ev->Get()->ToString()); + YDB_LOG_DEBUG("TEvUndelivered", + {"logPrefix", LogPrefix}, + {"ev", ev->Get()->ToString()}); if (ev->Sender == NodesManagerId) { - LOG_ROW_DISPATCHER_INFO("TEvUndelivered, from nodes manager, reason: " << ev->Get()->Reason); + YDB_LOG_INFO("TEvUndelivered, from nodes manager", + {"logPrefix", LogPrefix}, + {"reason", ev->Get()->Reason}); NActors::TActivationContext::Schedule(NodesManagerRetryPeriod, new IEventHandle(NodesManagerId, SelfId(), new NFq::TEvNodesManager::TEvGetNodesRequest(), IEventHandle::FlagTrackDelivery)); return; } @@ -432,7 +459,8 @@ void TActorCoordinator::Handle(NActors::TEvents::TEvUndelivered::TPtr& ev) { info.Connected = false; if (!RebalancingScheduled) { - LOG_ROW_DISPATCHER_TRACE("Schedule TEvRebalancing"); + YDB_LOG_TRACE("Schedule TEvRebalancing", + {"logPrefix", LogPrefix}); Schedule(RebalancingTimeout, new TEvPrivate::TEvRebalancing()); RebalancingScheduled = true; } @@ -441,7 +469,10 @@ void TActorCoordinator::Handle(NActors::TEvents::TEvUndelivered::TPtr& ev) { } void TActorCoordinator::Handle(NFq::TEvRowDispatcher::TEvCoordinatorChanged::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("New leader " << ev->Get()->CoordinatorActorId << ", SelfId " << SelfId()); + YDB_LOG_DEBUG("New leader SelfId", + {"logPrefix", LogPrefix}, + {"coordinatorActorId", ev->Get()->CoordinatorActorId}, + {"selfId", SelfId()}); Metrics.LeaderChanged->Inc(); bool isActive = (ev->Get()->CoordinatorActorId == SelfId()); @@ -508,14 +539,21 @@ void TActorCoordinator::Handle(NFq::TEvRowDispatcher::TEvCoordinatorRequest::TPt UpdateInterconnectSessions(ev->InterconnectSession); TStringStream str; - LOG_ROW_DISPATCHER_INFO("TEvCoordinatorRequest from " << ev->Sender.ToString() << ", " << source.GetTopicPath() << ", partIds: " << JoinSeq(", ", ev->Get()->Record.GetPartitionIds())); + YDB_LOG_INFO("TEvCoordinatorRequest", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"topicPath", source.GetTopicPath()}, + {"partIds", JoinSeq(", ", ev->Get()->Record.GetPartitionIds())}); Metrics.IncomingRequests->Inc(); TCoordinatorRequest request = {.Cookie = ev->Cookie, .Record = ev->Get()->Record}; if (ComputeCoordinatorRequest(ev->Sender, request)) { PendingReadActors.erase(ev->Sender); } else { - LOG_ROW_DISPATCHER_INFO("Not all nodes connected, nodes count: " << NodesCount << ", known rd count: " << RowDispatchers.size() << ", add request into pending queue"); + YDB_LOG_INFO("Not all nodes connected, known add request into pending queue", + {"logPrefix", LogPrefix}, + {"nodesCount", NodesCount}, + {"rdCount", RowDispatchers.size()}); // We save only last request from each read actor PendingReadActors[ev->Sender] = request; } @@ -563,7 +601,9 @@ bool TActorCoordinator::ComputeCoordinatorRequest(TActorId readActorId, const TC } } - LOG_ROW_DISPATCHER_DEBUG("Send TEvCoordinatorResult to " << readActorId); + YDB_LOG_DEBUG("Send TEvCoordinatorResult", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}); Send(readActorId, response.release(), IEventHandle::FlagTrackDelivery, request.Cookie); return true; } @@ -588,11 +628,15 @@ void TActorCoordinator::Handle(TEvPrivate::TEvPrintState::TPtr&) { void TActorCoordinator::Handle(NKikimr::TEvTenantNodeEnumerator::TEvLookupResult::TPtr& ev) { if (!ev->Get()->Success) { - LOG_ROW_DISPATCHER_ERROR("Failed to get TEvLookupResult, try later..."); + YDB_LOG_ERROR("Failed to get TEvLookupResult, try later", + {"logPrefix", LogPrefix}); ScheduleNodeInfoRequest(); return; } - LOG_ROW_DISPATCHER_INFO("Updated node info, node count: " << ev->Get()->AssignedNodes.size() << ", AssignedNodes: " << JoinSeq(", ", ev->Get()->AssignedNodes)); + YDB_LOG_INFO("Updated node info, node", + {"logPrefix", LogPrefix}, + {"count", ev->Get()->AssignedNodes.size()}, + {"assignedNodes", JoinSeq(", ", ev->Get()->AssignedNodes)}); NodesCount = ev->Get()->AssignedNodes.size(); UpdateGlobalState(); UpdatePendingReadActors(); @@ -600,30 +644,41 @@ void TActorCoordinator::Handle(NKikimr::TEvTenantNodeEnumerator::TEvLookupResult void TActorCoordinator::Handle(TEvPrivate::TEvListNodes::TPtr&) { if (NodesManagerId) { - LOG_ROW_DISPATCHER_DEBUG("Send TEvGetNodesRequest to NodesManager"); + YDB_LOG_DEBUG("Send TEvGetNodesRequest to NodesManager", + {"logPrefix", LogPrefix}); Send(NodesManagerId, new NFq::TEvNodesManager::TEvGetNodesRequest(), IEventHandle::FlagTrackDelivery); } else { - LOG_ROW_DISPATCHER_DEBUG("Send NodeEnumerationLookup request"); + YDB_LOG_DEBUG("Send NodeEnumerationLookup request", + {"logPrefix", LogPrefix}); Register(NKikimr::CreateTenantNodeEnumerationLookup(SelfId(), Tenant)); } } void TActorCoordinator::Handle(TEvPrivate::TEvRebalancing::TPtr&) { - LOG_ROW_DISPATCHER_DEBUG("Rebalancing..."); + YDB_LOG_DEBUG("Rebalancing", + {"logPrefix", LogPrefix}); RebalancingScheduled = false; bool needRebalance = false; TSet<TActorId> toDelete; auto printState = [&](const TString& str){ - LOG_ROW_DISPATCHER_DEBUG(str); + YDB_LOG_DEBUG("Dump logPrefix, str", + {"logPrefix", LogPrefix}, + {"str", str}); for (auto& [actorId, info] : RowDispatchers) { - LOG_ROW_DISPATCHER_DEBUG(" node " << actorId.NodeId() << " (" << actorId << ") state " << (info.State == ENodeState::Initializing ? "Initializing" : "Started") << " connected " << info.Connected << " partitions count " << info.Locations.size()); + YDB_LOG_DEBUG("Node state connected partitions count", + {"logPrefix", LogPrefix}, + {"nodeId", actorId.NodeId()}, + {"actorId", actorId}, + {"state", (info.State == ENodeState::Initializing ? "Initializing" : "Started")}, + {"connected", info.Connected}, + {"locations", info.Locations.size()}); } }; printState("Current state (rebalancing):"); - + for (auto& [actorId, info] : RowDispatchers) { if (info.State == ENodeState::Initializing) { if (info.Connected) { @@ -649,7 +704,9 @@ void TActorCoordinator::Handle(TEvPrivate::TEvRebalancing::TPtr&) { } for (const auto& readActorId : KnownReadActors) { - LOG_ROW_DISPATCHER_TRACE("Send TEvCoordinatorDistributionReset to " << readActorId); + YDB_LOG_TRACE("Send TEvCoordinatorDistributionReset", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}); Send(readActorId, new TEvRowDispatcher::TEvCoordinatorDistributionReset(), IEventHandle::FlagTrackDelivery); } @@ -666,7 +723,8 @@ void TActorCoordinator::Handle(TEvPrivate::TEvRebalancing::TPtr&) { void TActorCoordinator::Handle(TEvPrivate::TEvStartingTimeout::TPtr&) { if (State != ENodeState::Started) { - LOG_ROW_DISPATCHER_TRACE("Change global state to Started (by timeout)"); + YDB_LOG_TRACE("Change global state to Started (by timeout)", + {"logPrefix", LogPrefix}); State = ENodeState::Started; PrintInternalState(); } @@ -680,7 +738,10 @@ bool TActorCoordinator::IsReadyPartitionDistribution() const { } void TActorCoordinator::SendError(TActorId readActorId, const TCoordinatorRequest& request, const TString& message) { - LOG_ROW_DISPATCHER_WARN("Send TEvCoordinatorResult to " << readActorId << ", issues: " << message); + YDB_LOG_WARN("Send TEvCoordinatorResult", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}, + {"issues", message}); auto response = std::make_unique<TEvRowDispatcher::TEvCoordinatorResult>(); NYql::IssuesToMessage(NYql::TIssues{NYql::TIssue{message}}, response->Record.MutableIssues()); Send(readActorId, response.release(), IEventHandle::FlagTrackDelivery, request.Cookie); @@ -692,7 +753,9 @@ void TActorCoordinator::ScheduleNodeInfoRequest() const { void TActorCoordinator::Handle(NFq::TEvNodesManager::TEvGetNodesResponse::TPtr& ev) { NodesCount = ev->Get()->NodeIds.size(); - LOG_ROW_DISPATCHER_INFO("Updated node info, node count: " << NodesCount); + YDB_LOG_INFO("Updated node info, node", + {"logPrefix", LogPrefix}, + {"count", NodesCount}); UpdateGlobalState(); if (!NodesCount) { ScheduleNodeInfoRequest(); @@ -702,7 +765,8 @@ void TActorCoordinator::Handle(NFq::TEvNodesManager::TEvGetNodesResponse::TPtr& void TActorCoordinator::UpdateGlobalState() { if (State != ENodeState::Started && NodesCount && RowDispatchers.size() >= NodesCount) { - LOG_ROW_DISPATCHER_TRACE("Change global state to Started (by nodes count)"); + YDB_LOG_TRACE("Change global state to Started (by nodes count)", + {"logPrefix", LogPrefix}); State = ENodeState::Started; PrintInternalState(); } diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.cpp index 17dba9be709..2cc58e6e887 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/filters_set.cpp @@ -2,7 +2,9 @@ #include "purecalc_filter.h" -#include <ydb/core/fq/libs/actors/logging/log.h> +#include <ydb/library/actors/core/log.h> + +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER namespace NFq::NRowDispatcher { @@ -31,7 +33,10 @@ public: {} void ProcessData(const TVector<ui64>& columnIndex, const TVector<ui64>& offsets, const TVector<std::span<NYql::NUdf::TUnboxedValue>>& values, ui64 numberRows) override { - LOG_ROW_DISPATCHER_TRACE("ProcessData for " << RunHandlers_.size() << " clients, number rows: " << numberRows); + YDB_LOG_TRACE("ProcessData for clients, number", + {"logPrefix", LogPrefix}, + {"runHandlers", RunHandlers_.size()}, + {"rows", numberRows}); if (!numberRows) { return; @@ -44,7 +49,9 @@ public: continue; } if (const auto nextOffset = consumer->GetNextMessageOffset(); nextOffset && offsets[numberRows - 1] < *nextOffset) { - LOG_ROW_DISPATCHER_TRACE("Ignore processing for " << consumer->GetClientId() << ", historical offset"); + YDB_LOG_TRACE("Ignore processing for historical offset", + {"logPrefix", LogPrefix}, + {"clientId", consumer->GetClientId()}); continue; } @@ -54,11 +61,15 @@ public: } void OnCompileResponse(TEvRowDispatcher::TEvPurecalcCompileResponse::TPtr& ev) override { - LOG_ROW_DISPATCHER_TRACE("Got compile response for request with id " << ev->Cookie); + YDB_LOG_TRACE("Got compile response for request with id", + {"logPrefix", LogPrefix}, + {"cookie", ev->Cookie}); auto compileHandlerStatus = RemoveCompileProgram(ev->Cookie); if (compileHandlerStatus.IsFail()) { - LOG_ROW_DISPATCHER_ERROR(compileHandlerStatus.GetError().GetErrorMessage()); + YDB_LOG_ERROR("", + {"logPrefix", LogPrefix}, + {"error", compileHandlerStatus.GetError().GetErrorMessage()}); return; } const auto compileHandler = compileHandlerStatus.DetachResult(); @@ -71,7 +82,9 @@ public: auto runHandlerStatus = AddRunProgram(compileHandler->GetConsumer(), compileHandler->GetProgram()); if (runHandlerStatus.IsFail()) { - LOG_ROW_DISPATCHER_ERROR(runHandlerStatus.GetError().GetErrorMessage()); + YDB_LOG_ERROR("", + {"logPrefix", LogPrefix}, + {"error", runHandlerStatus.GetError().GetErrorMessage()}); return; } @@ -91,7 +104,9 @@ public: } void RemoveProgram(NActors::TActorId clientId) override { - LOG_ROW_DISPATCHER_TRACE("Remove program with client id " << clientId); + YDB_LOG_TRACE("Remove program with client id", + {"logPrefix", LogPrefix}, + {"clientId", clientId}); AbortCompileProgram(clientId); RemoveRunProgram(clientId); @@ -107,20 +122,27 @@ private: return; } - LOG_ROW_DISPATCHER_TRACE("Start program with client id " << consumer->GetClientId()); + YDB_LOG_TRACE("Start program with client id", + {"logPrefix", LogPrefix}, + {"clientId", consumer->GetClientId()}); consumer->OnStart(); } TStatus AddProgram(IProcessedDataConsumer::TPtr consumer, IProgramHolder::TPtr programHolder) { - LOG_ROW_DISPATCHER_TRACE("Create program with client id " << consumer->GetClientId()); + YDB_LOG_TRACE("Create program with client id", + {"logPrefix", LogPrefix}, + {"clientId", consumer->GetClientId()}); if (!programHolder) { auto runHandlerStatus = AddRunProgram(std::move(consumer), std::move(programHolder)); return runHandlerStatus.IsSuccess() ? TStatus::Success() : runHandlerStatus.GetError(); } - LOG_ROW_DISPATCHER_TRACE("Create purecalc program for query '" << programHolder->GetQuery() << "' (client id: " << consumer->GetClientId() << ")"); + YDB_LOG_TRACE("Create purecalc program for query (client", + {"logPrefix", LogPrefix}, + {"query", programHolder->GetQuery()}, + {"id", consumer->GetClientId()}); const auto cookie = NextCookie_++; auto compileHandlerStatus = AddCompileProgram(std::move(consumer), std::move(programHolder), cookie); @@ -215,12 +237,18 @@ private: if (const auto value = values[index]; !value.empty()) { result.emplace_back(value); } else { - LOG_ROW_DISPATCHER_TRACE("Ignore processing for " << consumer->GetClientId() << ", client got parsing error for column " << columnId); + YDB_LOG_TRACE("Ignore processing for client got parsing error for column", + {"logPrefix", LogPrefix}, + {"clientId", consumer->GetClientId()}, + {"columnId", columnId}); return; } } - LOG_ROW_DISPATCHER_TRACE("Pass " << numberRows << " rows to purecalc filter (client id: " << consumer->GetClientId() << ")"); + YDB_LOG_TRACE("Pass rows to purecalc filter (client", + {"logPrefix", LogPrefix}, + {"numberRows", numberRows}, + {"id", consumer->GetClientId()}); programRunHandler->ProcessData(result, numberRows); } diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/purecalc_filter.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/purecalc_filter.cpp index d6567f06cca..9f908654100 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/purecalc_filter.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/purecalc_filter.cpp @@ -2,12 +2,14 @@ #include <fmt/format.h> -#include <ydb/core/fq/libs/actors/logging/log.h> +#include <ydb/library/actors/core/log.h> #include <yql/essentials/minikql/computation/mkql_computation_node_holders.h> #include <yql/essentials/providers/common/schema/parser/yql_type_parser.h> #include <yql/essentials/public/purecalc/common/interface.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq::NRowDispatcher { namespace { @@ -338,7 +340,9 @@ public: } void Compile() override { - LOG_ROW_DISPATCHER_TRACE("Send compile request with id " << Cookie_); + YDB_LOG_TRACE("Send compile request with id", + {"logPrefix", LogPrefix}, + {"cookie", Cookie_}); auto compileRequest = std::make_unique<TEvRowDispatcher::TEvPurecalcCompileRequest>(std::exchange(ProgramHolder_, nullptr), Consumer_->GetPurecalcSettings()); NActors::TActivationContext::ActorSystem()->Send( @@ -353,7 +357,9 @@ public: } void AbortCompilation() override { - LOG_ROW_DISPATCHER_TRACE("Send abort compile request with id " << Cookie_); + YDB_LOG_TRACE("Send abort compile request with id", + {"logPrefix", LogPrefix}, + {"cookie", Cookie_}); NActors::TActivationContext::ActorSystem()->Send( new NActors::IEventHandle( CompileServiceId_, @@ -367,12 +373,15 @@ public: void OnCompileResponse(TEvRowDispatcher::TEvPurecalcCompileResponse::TPtr& ev) override { ProgramHolder_ = ev->Get()->ProgramHolder.Release(); - LOG_ROW_DISPATCHER_TRACE("Program compilation finished"); + YDB_LOG_TRACE("Program compilation finished", + {"logPrefix", LogPrefix}); } void OnCompileError(TEvRowDispatcher::TEvPurecalcCompileResponse::TPtr& ev) override { auto status = TStatus::Fail(ev->Get()->Status, std::move(ev->Get()->Issues)); - LOG_ROW_DISPATCHER_ERROR("Program compilation error: " << status.GetErrorMessage()); + YDB_LOG_ERROR("Program compilation", + {"logPrefix", LogPrefix}, + {"error", status.GetErrorMessage()}); CompileErrors_->Inc(); Consumer_->OnError(status.AddParentIssue("Failed to compile client program")); } @@ -405,10 +414,15 @@ public: } void ProcessData(const TVector<std::span<NYql::NUdf::TUnboxedValue>>& values, ui64 numberRows) const override { - LOG_ROW_DISPATCHER_TRACE("ProcessData for " << numberRows << " rows"); + YDB_LOG_TRACE("ProcessData for rows", + {"logPrefix", LogPrefix}, + {"numberRows", numberRows}); if (!ProgramHolder_) { - LOG_ROW_DISPATCHER_TRACE("Add " << numberRows << " rows to client " << Consumer_->GetClientId() << " without processing"); + YDB_LOG_TRACE("Add rows to client without processing", + {"logPrefix", LogPrefix}, + {"numberRows", numberRows}, + {"clientId", Consumer_->GetClientId()}); for (ui64 rowId = 0; rowId < numberRows; ++rowId) { NYql::NUdf::TUnboxedValue value = NYql::NUdf::TUnboxedValuePod{rowId}; Consumer_->OnData(&value); @@ -436,7 +450,8 @@ private: const auto& watermarkExpr = consumer->GetWatermarkExpr(); if (!filterExpr && !watermarkExpr) { - LOG_ROW_DISPATCHER_TRACE("No sql was generated"); + YDB_LOG_TRACE("No sql was generated", + {"logPrefix", LogPrefix}); return {}; } @@ -458,7 +473,9 @@ private: "watermark_expr"_a = watermarkExpr ? static_cast<TString>(TStringBuilder() << ", (" << watermarkExpr << ") AS " << WATERMARK_FIELD_NAME) : "" ); - LOG_ROW_DISPATCHER_DEBUG("Generated sql:\n" << result); + YDB_LOG_DEBUG("Generated sql:\n", + {"logPrefix", LogPrefix}, + {"result", result}); return result; } diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/ya.make b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/ya.make index ac9610d29a5..4e4f8687753 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/filters/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/filters/ya.make @@ -8,7 +8,6 @@ SRCS( PEERDIR( contrib/libs/fmt - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/row_dispatcher/events ydb/core/fq/libs/row_dispatcher/format_handler/common ydb/core/fq/libs/row_dispatcher/purecalc_no_pg_wrapper diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp index c43453b08e2..9a92ca79244 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.cpp @@ -5,14 +5,16 @@ #include <util/generic/queue.h> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/row_dispatcher/format_handler/parsers/parser_base.h> #include <ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/yql/dq/common/rope_over_buffer.h> #include <yql/essentials/minikql/computation/mkql_computation_node_pack.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq::NRowDispatcher { namespace { @@ -69,12 +71,16 @@ private: } void OnParsingError(TStatus status) override { - LOG_ROW_DISPATCHER_ERROR("Got parsing error: " << status.GetErrorMessage()); + YDB_LOG_ERROR("Got parsing", + {"logPrefix", LogPrefix}, + {"error", status.GetErrorMessage()}); Self.FatalError(status); } void OnParsedData(ui64 numberRows) override { - LOG_ROW_DISPATCHER_TRACE("Got parsed data, number rows: " << numberRows); + YDB_LOG_TRACE("Got parsed data, number", + {"logPrefix", LogPrefix}, + {"rows", numberRows}); Self.ParsedData.assign(ParerSchema.size(), std::span<NYql::NUdf::TUnboxedValue>()); for (size_t i = 0; i < ParerSchema.size(); ++i) { @@ -93,7 +99,10 @@ private: private: void OnColumnError(ui64 columnIndex, TStatus status) { const auto& column = ParerSchema[columnIndex]; - LOG_ROW_DISPATCHER_WARN("Failed to parse column " << column.ToString() << ", " << status.GetErrorMessage()); + YDB_LOG_WARN("Failed to parse column", + {"logPrefix", LogPrefix}, + {"column", column}, + {"error", status.GetErrorMessage()}); const auto columnIt = Self.ColumnsDesc.find(column.Name); if (columnIt == Self.ColumnsDesc.end()) { @@ -171,12 +180,16 @@ private: FinishPacking(); TQueue<TDataBatch> result; result.swap(ClientData); - LOG_ROW_DISPATCHER_TRACE("ExtractClientData, number batches: " << result.size()); + YDB_LOG_TRACE("ExtractClientData, number", + {"logPrefix", LogPrefix}, + {"batches", result.size()}); return result; } void OnClientError(TStatus status) { - LOG_ROW_DISPATCHER_WARN("OnClientError, " << status.GetErrorMessage()); + YDB_LOG_WARN("OnClientError", + {"logPrefix", LogPrefix}, + {"error", status.GetErrorMessage()}); Client->OnClientError(std::move(status)); } @@ -227,7 +240,10 @@ private: if (Watermark < watermark) { Watermark = watermark; } - LOG_ROW_DISPATCHER_TRACE("OnWatermark, offset: " << offset << ", watermark: " << watermark); + YDB_LOG_TRACE("OnWatermark", + {"logPrefix", LogPrefix}, + {"offset", offset}, + {"watermark", watermark}); } public: @@ -253,7 +269,10 @@ private: Offset = Self.Offsets->at(rowId); if (const auto nextOffset = Client->GetNextMessageOffset(); nextOffset && Offset < *nextOffset) { - LOG_ROW_DISPATCHER_TRACE("OnData, skip historical offset: " << Offset << ", next message offset: " << *nextOffset); + YDB_LOG_TRACE("OnData, skip next", + {"logPrefix", LogPrefix}, + {"historicalOffset", Offset}, + {"messageOffset", *nextOffset}); return; } @@ -289,7 +308,12 @@ private: return; } - LOG_ROW_DISPATCHER_TRACE("OnBatchFinish, offset: " << Offset << ", number rows: " << numberRows << ", row size: " << rowSize << ", watermark: " << Watermark); + YDB_LOG_TRACE("OnBatchFinish, number row", + {"logPrefix", LogPrefix}, + {"offset", Offset}, + {"rows", numberRows}, + {"size", rowSize}, + {"watermark", Watermark}); Client->AddDataToClient(Offset, numberRows, rowSize, Watermark); @@ -321,7 +345,10 @@ private: void FinishPacking() { if (!DataPacker->IsEmpty() || !Watermark.Empty()) { - LOG_ROW_DISPATCHER_TRACE("FinishPacking, batch size: " << DataPackerSize << ", number rows: " << FilteredOffsets.size()); + YDB_LOG_TRACE("FinishPacking, batch number", + {"logPrefix", LogPrefix}, + {"size", DataPackerSize}, + {"rows", FilteredOffsets.size()}); if (FilteredOffsets.empty()) { FilteredOffsets.push_back(Offset); } @@ -393,7 +420,8 @@ public: RefreshScheduled = false; if (Parser) { - LOG_ROW_DISPATCHER_TRACE("Refresh parser"); + YDB_LOG_TRACE("Refresh parser", + {"logPrefix", LogPrefix}); Parser->Refresh(); ScheduleRefresh(); } @@ -410,13 +438,17 @@ public: } void HandleException(const std::exception& error) { - LOG_ROW_DISPATCHER_ERROR("Got unexpected exception: " << error.what()); + YDB_LOG_ERROR("Got unexpected", + {"logPrefix", LogPrefix}, + {"exception", error.what()}); FatalError(TStatus::Fail(EStatusId::INTERNAL_ERROR, TStringBuilder() << "Format handler error, got unexpected exception: " << error.what())); } public: void ParseMessages(const std::vector<NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage>& messages) override { - LOG_ROW_DISPATCHER_TRACE("Send " << messages.size() << " messages to parser"); + YDB_LOG_TRACE("Send messages to parser", + {"logPrefix", LogPrefix}, + {"messages", messages.size()}); if (!messages.empty()) { CurrentOffset = messages.back().GetOffset(); @@ -439,11 +471,16 @@ public: } TStatus AddClient(IClientDataConsumer::TPtr client) override { - LOG_ROW_DISPATCHER_DEBUG("Add client with id " << client->GetClientId()); + YDB_LOG_DEBUG("Add client with id", + {"logPrefix", LogPrefix}, + {"clientId", client->GetClientId()}); if (const auto clientOffset = client->GetNextMessageOffset()) { if (Parser && CurrentOffset && *CurrentOffset > *clientOffset) { - LOG_ROW_DISPATCHER_DEBUG("Parser was flushed due to new historical offset " << *clientOffset << " (previous parser offset: " << *CurrentOffset << ")"); + YDB_LOG_DEBUG("Parser was flushed due to new historical offset (previous parser", + {"logPrefix", LogPrefix}, + {"clientOffset", *clientOffset}, + {"offset", *CurrentOffset}); Parser->Refresh(true); } } @@ -476,7 +513,9 @@ public: } void RemoveClient(NActors::TActorId clientId) override { - LOG_ROW_DISPATCHER_DEBUG("Remove client with id " << clientId); + YDB_LOG_DEBUG("Remove client with id", + {"logPrefix", LogPrefix}, + {"clientId", clientId}); if (Filters) { Filters->RemoveProgram(clientId); @@ -558,7 +597,9 @@ private: Parser->Refresh(true); } - LOG_ROW_DISPATCHER_DEBUG("UpdateParser to new schema with size " << parerSchema.size()); + YDB_LOG_DEBUG("UpdateParser to new schema with size", + {"logPrefix", LogPrefix}, + {"schemaSize", parerSchema.size()}); ParserHandler = MakeIntrusive<TParserHandler>(*this, std::move(parerSchema)); if (const ui64 schemaSize = ParserHandler->GetColumns().size()) { @@ -569,13 +610,17 @@ private: } Parser = newParser.DetachResult(); - LOG_ROW_DISPATCHER_DEBUG("Parser was created on schema with " << schemaSize << " columns"); + YDB_LOG_DEBUG("Parser was created on schema with columns", + {"logPrefix", LogPrefix}, + {"schemaSize", schemaSize}); } else { if (auto status = Parser->ChangeConsumer(ParserHandler); status.IsFail()) { return status; } - LOG_ROW_DISPATCHER_DEBUG("Parser was updated on new schema with " << schemaSize << " columns"); + YDB_LOG_DEBUG("Parser was updated on new schema with columns", + {"logPrefix", LogPrefix}, + {"schemaSize", schemaSize}); } ParserSchemaIndex.resize(MaxColumnId, std::numeric_limits<ui64>::max()); @@ -583,7 +628,8 @@ private: ParserSchemaIndex[columnDesc.ColumnId] = i++; } } else { - LOG_ROW_DISPATCHER_INFO("No columns to parse, reset parser"); + YDB_LOG_INFO("No columns to parse, reset parser", + {"logPrefix", LogPrefix}); Parser.Reset(); } @@ -613,7 +659,11 @@ private: } const ui64 lastOffset = Offsets->at(numberRows - 1); - LOG_ROW_DISPATCHER_TRACE("Send " << numberRows << " messages to programs, first offset: " << Offsets->front() << ", last offset: " << lastOffset); + YDB_LOG_TRACE("Send messages to programs", + {"logPrefix", LogPrefix}, + {"numberRows", numberRows}, + {"firstOffset", Offsets->front()}, + {"lastOffset", lastOffset}); if (Filters) { Filters->ProcessData(ParserSchemaIndex, *Offsets, ParsedData, numberRows); @@ -621,14 +671,19 @@ private: for (const auto& [_, client] : Clients) { if (client->IsStarted()) { - LOG_ROW_DISPATCHER_TRACE("Commit client " << client->GetClient()->GetClientId() << " offset " << lastOffset); + YDB_LOG_TRACE("Commit client offset", + {"logPrefix", LogPrefix}, + {"clientId", client->GetClient()->GetClientId()}, + {"lastOffset", lastOffset}); client->GetClient()->UpdateClientOffset(lastOffset); } } } void FatalError(TStatus status) const { - LOG_ROW_DISPATCHER_ERROR("Got fatal error: " << status.GetErrorMessage()); + YDB_LOG_ERROR("Got fatal", + {"logPrefix", LogPrefix}, + {"error", status.GetErrorMessage()}); for (const auto& [_, client] : Clients) { client->OnClientError(status); } diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp index 888bd4da50b..e6c050ae1ce 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/json_parser.cpp @@ -7,7 +7,7 @@ #include <library/cpp/containers/absl/flat_hash_map.h> #include <util/string/join.h> -#include <ydb/core/fq/libs/actors/logging/log.h> +#include <ydb/library/actors/core/log.h> #include <yql/essentials/minikql/dom/json.h> #include <yql/essentials/minikql/mkql_node_cast.h> @@ -17,6 +17,8 @@ #include <sstream> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq::NRowDispatcher { namespace { @@ -66,7 +68,10 @@ struct TJsonParserBuffer { const auto offset = message.GetOffset(); if (Y_UNLIKELY(Offsets && Offsets.back() > offset)) { - LOG_ROW_DISPATCHER_WARN("Got message with offset " << offset << " which is less than previous offset " << Offsets.back()); + YDB_LOG_WARN("Got message with offset which is less than previous offset", + {"logPrefix", LogPrefix}, + {"offset", offset}, + {"offsetsBack", Offsets.back()}); } NumberValues++; @@ -827,11 +832,15 @@ public: FillColumnsBuffers(); Buffer.Reserve(Config.BatchSize, MaxNumberRows); - LOG_ROW_DISPATCHER_INFO("JsonParser was created, simdjson active implementation " << simdjson::get_active_implementation()->name() - << " (" << simdjson::get_active_implementation()->description() << ")" - << ", config: error skip mode: " << Config.SkipErrors << ", batch size: " << Config.BatchSize << ", latency limit " << Config.LatencyLimit - << ", buffer cell count: " << Config.BufferCellCount - << ", max number rows: " << MaxNumberRows ); + YDB_LOG_INFO("JsonParser was created, simdjson active implementation config: error skip batch latency limit buffer cell max number", + {"logPrefix", LogPrefix}, + {"name", simdjson::get_active_implementation()->name()}, + {"description", simdjson::get_active_implementation()->description()}, + {"mode", Config.SkipErrors}, + {"size", Config.BatchSize}, + {"latencyLimit", Config.LatencyLimit}, + {"count", Config.BufferCellCount}, + {"rows", MaxNumberRows}); Parser.threaded = false; } @@ -864,7 +873,9 @@ public: public: void ParseMessages(const std::vector<NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage>& messages) override { - LOG_ROW_DISPATCHER_TRACE("Add " << messages.size() << " messages to parse"); + YDB_LOG_TRACE("Add messages to parse", + {"logPrefix", LogPrefix}, + {"messages", messages.size()}); Y_ENSURE(!Buffer.Finished, "Cannot parse messages with finished buffer"); for (const auto& message : messages) { @@ -878,7 +889,9 @@ public: if (!Config.LatencyLimit) { ParseBuffer(); } else { - LOG_ROW_DISPATCHER_TRACE("Collecting data to parse, skip parsing, current buffer size: " << Buffer.GetSize()); + YDB_LOG_TRACE("Collecting data to parse, skip parsing, current buffer", + {"logPrefix", LogPrefix}, + {"size", Buffer.GetSize()}); } } } @@ -894,7 +907,9 @@ public: if (force || creationDuration > Config.LatencyLimit) { ParseBuffer(); } else { - LOG_ROW_DISPATCHER_TRACE("Refresh, skip parsing, buffer creation duration: " << creationDuration); + YDB_LOG_TRACE("Refresh, skip parsing, buffer creation", + {"logPrefix", LogPrefix}, + {"duration", creationDuration}); } } @@ -907,7 +922,10 @@ public: MaxNumberRows = CalculateMaxNumberRows(); FillColumnsBuffers(); - LOG_ROW_DISPATCHER_DEBUG("Parser columns count changed from " << Columns.size() << " to " << Consumer->GetColumns().size()); + YDB_LOG_DEBUG("Parser columns count changed", + {"logPrefix", LogPrefix}, + {"columns", Columns.size()}, + {"consumerColumns", Consumer->GetColumns().size()}); return InitColumnsParsers(); } @@ -930,7 +948,10 @@ protected: auto [values, size] = Buffer.Finish(); OutputOffsets.resize(Buffer.Offsets.size()); - LOG_ROW_DISPATCHER_TRACE("Do parsing, first offset: " << Buffer.Offsets.front() << ", values:\n" << values); + YDB_LOG_TRACE("Do parsing, first values:\n", + {"logPrefix", LogPrefix}, + {"offset", Buffer.Offsets.front()}, + {"values", values}); if (Config.SkipErrors) { OutputOffsets = Buffer.Offsets; @@ -942,7 +963,7 @@ protected: if (status == EParsingStatus::Finish) { break; } - size_t inputRowId = state.OutputRowId + state.ErrorsCount; + size_t inputRowId = state.OutputRowId + state.ErrorsCount; if (inputRowId < Buffer.MessageOffsets.size()) { auto nextJsonOffset = Buffer.MessageOffsets[inputRowId]; state.CurrentBufferPtr = values + nextJsonOffset; @@ -990,7 +1011,7 @@ protected: ParsingFailedRowCount = 0; } -private: +private: struct TParsingState { const char* InitialBufferPtr; const char* CurrentBufferPtr; @@ -1016,7 +1037,7 @@ private: } auto currentJsonOffset = Buffer.MessageOffsets[inputRowId]; - auto nextJsonOffset = Buffer.MessageOffsets[inputRowId + 1]; + auto nextJsonOffset = Buffer.MessageOffsets[inputRowId + 1]; ui16 parsedNonOptional = 0; auto len = nextJsonOffset - currentJsonOffset; @@ -1059,7 +1080,11 @@ private: state.Status = status; return EParsingStatus::Finish; } - LOG_ROW_DISPATCHER_DEBUG("Unbatched parser, skipped " << state.ErrorsCount << ", outputRowId " << state.OutputRowId << ", recovering from " << status.GetErrorMessage()); + YDB_LOG_DEBUG("Unbatched parser, skipped outputRowId recovering", + {"logPrefix", LogPrefix}, + {"errorsCount", state.ErrorsCount}, + {"outputRowId", state.OutputRowId}, + {"error", status.GetErrorMessage()}); ClearRowBuffer(state.OutputRowId); if (TryParseOneJson(state)) { state.OutputRowId++; @@ -1071,7 +1096,11 @@ private: }; EParsingStatus ParseRows(TParsingState& state) { - LOG_ROW_DISPATCHER_TRACE("Init parser, skipped " << state.ErrorsCount << ", outputRowId " << state.OutputRowId << " size " << state.Size); + YDB_LOG_TRACE("Init parser, skipped outputRowId size", + {"logPrefix", LogPrefix}, + {"errorsCount", state.ErrorsCount}, + {"outputRowId", state.OutputRowId}, + {"stateSize", state.Size}); /* Batch size must be at least maximum of document size. diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp index 076f947df22..1d1ff36092e 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/raw_parser.cpp @@ -2,11 +2,13 @@ #include "parser_base.h" -#include <ydb/core/fq/libs/actors/logging/log.h> +#include <ydb/library/actors/core/log.h> #include <yql/essentials/minikql/mkql_node_cast.h> #include <yql/essentials/minikql/mkql_type_ops.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq::NRowDispatcher { namespace { @@ -53,7 +55,9 @@ public: public: void ParseMessages(const std::vector<NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage>& messages) override { - LOG_ROW_DISPATCHER_TRACE("Add " << messages.size() << " messages to parse"); + YDB_LOG_TRACE("Add messages to parse", + {"logPrefix", LogPrefix}, + {"messages", messages.size()}); for (const auto& message : messages) { CurrentMessage = message.GetData(); @@ -90,7 +94,10 @@ public: protected: TStatus DoParsing() override { - LOG_ROW_DISPATCHER_TRACE("Do parsing, first offset: " << Offsets.front() << ", value: " << CurrentMessage); + YDB_LOG_TRACE("Do parsing, first", + {"logPrefix", LogPrefix}, + {"offset", Offsets.front()}, + {"value", CurrentMessage}); NYql::NUdf::TUnboxedValue value = LockObject(NKikimr::NMiniKQL::ValueFromString(DataSlot, CurrentMessage)); if (value) { diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make index 0c36d850dfa..0a479fba44b 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/parsers/ya.make @@ -12,7 +12,6 @@ PEERDIR( library/cpp/containers/absl - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/row_dispatcher/common ydb/core/fq/libs/row_dispatcher/events ydb/core/fq/libs/row_dispatcher/format_handler/common diff --git a/ydb/core/fq/libs/row_dispatcher/format_handler/ya.make b/ydb/core/fq/libs/row_dispatcher/format_handler/ya.make index c2647897cde..4066bf2c0ec 100644 --- a/ydb/core/fq/libs/row_dispatcher/format_handler/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/format_handler/ya.make @@ -5,7 +5,6 @@ SRCS( ) PEERDIR( - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/row_dispatcher/common ydb/core/fq/libs/row_dispatcher/events ydb/core/fq/libs/row_dispatcher/format_handler/common diff --git a/ydb/core/fq/libs/row_dispatcher/leader_election.cpp b/ydb/core/fq/libs/row_dispatcher/leader_election.cpp index dea204c31d8..a0d87d1e369 100644 --- a/ydb/core/fq/libs/row_dispatcher/leader_election.cpp +++ b/ydb/core/fq/libs/row_dispatcher/leader_election.cpp @@ -1,6 +1,5 @@ #include "leader_election.h" -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/events/events.h> #include <ydb/core/fq/libs/row_dispatcher/events/data_plane.h> #include <ydb/core/fq/libs/ydb/schema.h> @@ -8,6 +7,7 @@ #include <ydb/core/fq/libs/ydb/ydb.h> #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/actors/protos/actors.pb.h> #include <ydb/library/logger/actor.h> @@ -15,6 +15,8 @@ #include <memory> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq { using namespace NActors; @@ -245,9 +247,13 @@ void TLeaderElection::Bootstrap() { ActorSystemPtr->store(TActivationContext::ActorSystem(), std::memory_order_release); LogPrefix = "TLeaderElection " + SelfId().ToString() + " "; - LOG_ROW_DISPATCHER_DEBUG("Successfully bootstrapped, local coordinator id " << CoordinatorId.ToString() - << ", tenant id " << TenantId << ", local mode " << Config.GetLocalMode() << ", coordination node path " << CoordinationNodePath - << ", endpoint " << Config.GetDatabase().GetEndpoint()); + YDB_LOG_DEBUG("Successfully bootstrapped, local coordinator id tenant id local mode coordination node path endpoint", + {"logPrefix", LogPrefix}, + {"coordinatorId", CoordinatorId}, + {"tenantId", TenantId}, + {"localMode", Config.GetLocalMode()}, + {"coordinationNodePath", CoordinationNodePath}, + {"endpoint", Config.GetDatabase().GetEndpoint()}); if (Config.GetLocalMode()) { TActivationContext::ActorSystem()->Send(ParentId, new NFq::TEvRowDispatcher::TEvCoordinatorChanged(CoordinatorId, 0)); return; @@ -310,16 +316,18 @@ void TLeaderElection::ResetState() { void TLeaderElection::CreateSemaphore() { Session->CreateSemaphore(SemaphoreName, 1 /* limit */) .Subscribe( + [actorId = this->SelfId(), actorSystemPtr = ActorSystemPtr](const NYdb::NCoordination::TAsyncResult<void>& future) { SendToActorSystem(actorSystemPtr, actorId, new TEvPrivate::TEvCreateSemaphoreResult(future)); - }); + }); } void TLeaderElection::AcquireSemaphore() { if (PendingAcquire) { return; } - LOG_ROW_DISPATCHER_DEBUG("Try to acquire semaphore"); + YDB_LOG_DEBUG("Try to acquire semaphore", + {"logPrefix", LogPrefix}); NActorsProto::TActorId protoId; ActorIdToProto(CoordinatorId, &protoId); @@ -338,11 +346,12 @@ void TLeaderElection::AcquireSemaphore() { } void TLeaderElection::StartSession() { - LOG_ROW_DISPATCHER_DEBUG("Start session"); + YDB_LOG_DEBUG("Start session", + {"logPrefix", LogPrefix}); YdbConnection->CoordinationClient .StartSession( - CoordinationNodePath, + CoordinationNodePath, NYdb::NCoordination::TSessionSettings() .Timeout(CoordinationSessionTimeout) .OnStopped([actorId = this->SelfId(), actorSystemPtr = ActorSystemPtr]() { @@ -355,12 +364,15 @@ void TLeaderElection::StartSession() { void TLeaderElection::Handle(NFq::TEvents::TEvSchemaCreated::TPtr& ev) { if (!IsTableCreated(ev->Get()->Result)) { - LOG_ROW_DISPATCHER_ERROR("Schema creation error " << ev->Get()->Result.GetIssues()); + YDB_LOG_ERROR("Schema creation error", + {"logPrefix", LogPrefix}, + {"issues", ev->Get()->Result.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; } - LOG_ROW_DISPATCHER_DEBUG("Coordination node successfully created"); + YDB_LOG_DEBUG("Coordination node successfully created", + {"logPrefix", LogPrefix}); CoordinationNodeCreated = true; ProcessState(); } @@ -368,26 +380,32 @@ void TLeaderElection::Handle(NFq::TEvents::TEvSchemaCreated::TPtr& ev) { void TLeaderElection::Handle(TEvPrivate::TEvCreateSessionResult::TPtr& ev) { auto result = ev->Get()->Result.GetValue(); if (!result.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("CreateSession failed, " << result.GetIssues()); + YDB_LOG_ERROR("CreateSession failed", + {"logPrefix", LogPrefix}, + {"issues", result.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; } Session = result.GetResult(); - LOG_ROW_DISPATCHER_DEBUG("Session successfully created"); + YDB_LOG_DEBUG("Session successfully created", + {"logPrefix", LogPrefix}); ProcessState(); } void TLeaderElection::Handle(TEvPrivate::TEvCreateSemaphoreResult::TPtr& ev) { auto result = ev->Get()->Result.GetValue(); if (!IsTableCreated(result)) { - LOG_ROW_DISPATCHER_ERROR("Semaphore creating error " << result.GetIssues()); + YDB_LOG_ERROR("Semaphore creating error", + {"logPrefix", LogPrefix}, + {"issues", result.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; } SemaphoreCreated = true; - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully created"); + YDB_LOG_DEBUG("Semaphore successfully created", + {"logPrefix", LogPrefix}); ProcessState(); } @@ -396,16 +414,20 @@ void TLeaderElection::Handle(TEvPrivate::TEvAcquireSemaphoreResult::TPtr& ev) { PendingAcquire = false; if (!result.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("Failed to acquire semaphore, " << result.GetIssues()); + YDB_LOG_ERROR("Failed to acquire semaphore", + {"logPrefix", LogPrefix}, + {"issues", result.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; } - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully acquired"); + YDB_LOG_DEBUG("Semaphore successfully acquired", + {"logPrefix", LogPrefix}); } void TLeaderElection::PassAway() { - LOG_ROW_DISPATCHER_DEBUG("PassAway"); + YDB_LOG_DEBUG("PassAway", + {"logPrefix", LogPrefix}); StopYdb(); TActorBootstrapped::PassAway(); } @@ -423,7 +445,8 @@ void TLeaderElection::StopYdb() { } void TLeaderElection::Handle(TEvPrivate::TEvSessionStopped::TPtr&) { - LOG_ROW_DISPATCHER_DEBUG("TEvSessionStopped"); + YDB_LOG_DEBUG("TEvSessionStopped", + {"logPrefix", LogPrefix}); Session.Clear(); PendingAcquire = false; PendingDescribe = false; @@ -440,15 +463,17 @@ void TLeaderElection::SetTimeout() { void TLeaderElection::Handle(TEvPrivate::TEvRestart::TPtr&) { RestartScheduled = false; - LOG_ROW_DISPATCHER_DEBUG("TEvRestart"); - ProcessState(); + YDB_LOG_DEBUG("TEvRestart", + {"logPrefix", LogPrefix}); + ProcessState(); } void TLeaderElection::DescribeSemaphore() { if (PendingDescribe) { return; } - LOG_ROW_DISPATCHER_DEBUG("Describe semaphore"); + YDB_LOG_DEBUG("Describe semaphore", + {"logPrefix", LogPrefix}); PendingDescribe = true; Session->DescribeSemaphore( SemaphoreName, @@ -466,7 +491,8 @@ void TLeaderElection::DescribeSemaphore() { } void TLeaderElection::Handle(TEvPrivate::TEvOnChangedResult::TPtr& /*ev*/) { - LOG_ROW_DISPATCHER_DEBUG("Semaphore changed"); + YDB_LOG_DEBUG("Semaphore changed", + {"logPrefix", LogPrefix}); PendingDescribe = false; ProcessState(); } @@ -475,7 +501,9 @@ void TLeaderElection::Handle(TEvPrivate::TEvDescribeSemaphoreResult::TPtr& ev) { PendingDescribe = false; auto result = ev->Get()->Result.GetValue(); if (!result.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("Semaphore describe fail, " << result.GetIssues()); + YDB_LOG_ERROR("Semaphore describe fail", + {"logPrefix", LogPrefix}, + {"issues", result.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; @@ -484,7 +512,8 @@ void TLeaderElection::Handle(TEvPrivate::TEvDescribeSemaphoreResult::TPtr& ev) { const NYdb::NCoordination::TSemaphoreDescription& description = result.GetResult(); Y_ABORT_UNLESS(description.GetOwners().size() <= 1, "To many owners"); if (description.GetOwners().empty()) { - LOG_ROW_DISPATCHER_DEBUG("Empty owners"); + YDB_LOG_DEBUG("Empty owners", + {"logPrefix", LogPrefix}); // Wait OnChanged. return; } @@ -497,9 +526,16 @@ void TLeaderElection::Handle(TEvPrivate::TEvDescribeSemaphoreResult::TPtr& ev) { } NActors::TActorId id = ActorIdFromProto(protoId); - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully described: coordinator id " << id << " generation " << generation); + YDB_LOG_DEBUG("Semaphore successfully described: coordinator id generation", + {"logPrefix", LogPrefix}, + {"id", id}, + {"generation", generation}); if (!LeaderActorId || (*LeaderActorId != id)) { - LOG_ROW_DISPATCHER_INFO("Send TEvCoordinatorChanged to " << ParentId << ", new coordinator id " << id << ", previous coordinator id " << LeaderActorId.GetOrElse(TActorId())); + YDB_LOG_INFO("Send TEvCoordinatorChanged to new coordinator id previous coordinator id", + {"logPrefix", LogPrefix}, + {"parentId", ParentId}, + {"id", id}, + {"leaderActorId", LeaderActorId.GetOrElse(TActorId())}); TActivationContext::ActorSystem()->Send(ParentId, new NFq::TEvRowDispatcher::TEvCoordinatorChanged(id, generation)); Metrics.LeaderChanged->Inc(); } @@ -507,7 +543,9 @@ void TLeaderElection::Handle(TEvPrivate::TEvDescribeSemaphoreResult::TPtr& ev) { } void TLeaderElection::HandleException(const std::exception& e) { - LOG_ROW_DISPATCHER_ERROR("Internal error: exception:" << e.what()); + YDB_LOG_ERROR("Internal error", + {"logPrefix", LogPrefix}, + {"exception", e.what()}); Metrics.Errors->Inc(); ResetState(); } diff --git a/ydb/core/fq/libs/row_dispatcher/local_leader_election.cpp b/ydb/core/fq/libs/row_dispatcher/local_leader_election.cpp index 432b2b5508c..08ce60a9473 100644 --- a/ydb/core/fq/libs/row_dispatcher/local_leader_election.cpp +++ b/ydb/core/fq/libs/row_dispatcher/local_leader_election.cpp @@ -1,7 +1,6 @@ #include "local_leader_election.h" #include <memory> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/row_dispatcher/events/data_plane.h> #include <ydb/core/fq/libs/ydb/util.h> @@ -11,9 +10,12 @@ #include <ydb/core/grpc_services/service_coordination.h> #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/actors/protos/actors.pb.h> #include <ydb/library/logger/actor.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq { using namespace NActors; @@ -225,8 +227,10 @@ void TLocalLeaderElection::Bootstrap() { CoordinationNodePath = JoinPath(NKikimr::AppData()->TenantName, DefaultCoordinationNodePath); LogPrefix = "TLeaderElection " + SelfId().ToString() + " "; - LOG_ROW_DISPATCHER_DEBUG("Successfully bootstrapped, local coordinator id " << CoordinatorId.ToString() - << ", coordination node path " << CoordinationNodePath); + YDB_LOG_DEBUG("Successfully bootstrapped, local coordinator id coordination node path", + {"logPrefix", LogPrefix}, + {"coordinatorId", CoordinatorId}, + {"coordinationNodePath", CoordinationNodePath}); ProcessState(); } @@ -274,7 +278,7 @@ void TLocalLeaderElection::ResetState() { State = EState::Init; SetTimeout(); SessionId = 0; - while (PendingRpcResponses) { + while (PendingRpcResponses) { SendSessionEventFail(); } PendingRpcResponses = 0; @@ -288,7 +292,8 @@ void TLocalLeaderElection::ResetState() { } void TLocalLeaderElection::CreateSemaphore() { - LOG_ROW_DISPATCHER_DEBUG("Try to create semaphore"); + YDB_LOG_DEBUG("Try to create semaphore", + {"logPrefix", LogPrefix}); TRpcIn message; auto& inner = *message.mutable_create_semaphore(); uint64_t reqId = NextReqId++; @@ -308,7 +313,8 @@ void TLocalLeaderElection::AcquireSemaphore() { return; } LastAcquireSemaphore = now; - LOG_ROW_DISPATCHER_DEBUG("Try to acquire semaphore"); + YDB_LOG_DEBUG("Try to acquire semaphore", + {"logPrefix", LogPrefix}); NActorsProto::TActorId protoId; ActorIdToProto(CoordinatorId, &protoId); @@ -330,7 +336,8 @@ void TLocalLeaderElection::AcquireSemaphore() { } void TLocalLeaderElection::StartSession() { - LOG_ROW_DISPATCHER_DEBUG("Start session"); + YDB_LOG_DEBUG("Start session", + {"logPrefix", LogPrefix}); auto token = NACLib::TUserToken(BUILTIN_ACL_METADATA, {}).SerializeAsString(); auto ctx = MakeIntrusive<TLocalRpcCtx>(ActorContext().ActorSystem(), SelfId(), TLocalRpcCtx::TSettings{ .Database = NKikimr::AppData()->TenantName, @@ -339,7 +346,7 @@ void TLocalLeaderElection::StartSession() { .RequestType = std::nullopt, .RpcMethodName = "CoordinationService.Session", }); - + auto ev = std::make_unique<NKikimr::NGRpcService::TEvCoordinationSessionRequest>(std::move(ctx), NKikimr::NGRpcService::TRequestAuxSettings{}); if (token) { ev->SetInternalToken(MakeIntrusive<NACLib::TUserToken>(token)); @@ -348,7 +355,8 @@ void TLocalLeaderElection::StartSession() { } void TLocalLeaderElection::PassAway() { - LOG_ROW_DISPATCHER_DEBUG("PassAway"); + YDB_LOG_DEBUG("PassAway", + {"logPrefix", LogPrefix}); if (RpcActor) { CloseSession(NYdb::EStatus::ABORTED, {NYql::TIssue("Actor is shutting down")}); } @@ -356,7 +364,8 @@ void TLocalLeaderElection::PassAway() { } void TLocalLeaderElection::Handle(TEvPrivate::TEvSelfPing::TPtr&) { - LOG_ROW_DISPATCHER_DEBUG("TEvSelfPing"); + YDB_LOG_DEBUG("TEvSelfPing", + {"logPrefix", LogPrefix}); if (SessionClosed) { return; } @@ -391,15 +400,17 @@ void TLocalLeaderElection::SetTimeout() { void TLocalLeaderElection::Handle(TEvPrivate::TEvRestart::TPtr&) { RestartScheduled = false; - LOG_ROW_DISPATCHER_DEBUG("TEvRestart"); - ProcessState(); + YDB_LOG_DEBUG("TEvRestart", + {"logPrefix", LogPrefix}); + ProcessState(); } void TLocalLeaderElection::DescribeSemaphore() { if (PendingDescribe || PendingDescribeChanged) { return; } - LOG_ROW_DISPATCHER_DEBUG("Describe semaphore"); + YDB_LOG_DEBUG("Describe semaphore", + {"logPrefix", LogPrefix}); PendingDescribe = true; PendingDescribeChanged = true; @@ -416,7 +427,9 @@ void TLocalLeaderElection::DescribeSemaphore() { } bool TLocalLeaderElection::OnUnhandledException(const std::exception& e) { - LOG_ROW_DISPATCHER_ERROR("Internal error: exception:" << e.what()); + YDB_LOG_ERROR("Internal error", + {"logPrefix", LogPrefix}, + {"exception", e.what()}); Metrics.Errors->Inc(); ResetState(); return true; @@ -440,12 +453,15 @@ void TLocalLeaderElection::CreateNode(const std::string& path, const NYdb::NCoo void TLocalLeaderElection::Handle(TEvPrivate::TEvCreateNodeResult::TPtr& ev) { const auto& status = ev->Get()->Status; if (!status.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("Coordination node creation error: " << status.GetIssues()); + YDB_LOG_ERROR("Coordination node creation", + {"logPrefix", LogPrefix}, + {"error", status.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; } - LOG_ROW_DISPATCHER_DEBUG("Coordination node successfully created"); + YDB_LOG_DEBUG("Coordination node successfully created", + {"logPrefix", LogPrefix}); CoordinationNodeCreated = true; ProcessState(); } @@ -455,7 +471,9 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvActorAttached::TP RpcActor = ev->Get()->RpcActor; SessionClosed = false; - LOG_ROW_DISPATCHER_DEBUG("RpcActor attached: " << RpcActor); + YDB_LOG_DEBUG("RpcActor", + {"logPrefix", LogPrefix}, + {"attached", RpcActor}); RpcResponses = {}; SendStartSession(); } @@ -464,12 +482,14 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvReadRequest::TPtr PendingRpcResponses++; if (SessionClosed) { - LOG_ROW_DISPATCHER_DEBUG("Rpc read request skipped, session is closed"); + YDB_LOG_DEBUG("Rpc read request skipped, session is closed", + {"logPrefix", LogPrefix}); SendSessionEventFail(); return; } - LOG_ROW_DISPATCHER_DEBUG("Rpc read request"); + YDB_LOG_DEBUG("Rpc read request", + {"logPrefix", LogPrefix}); SendSessionEvents(); } @@ -478,7 +498,8 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvWriteRequest::TPt auto response = std::make_unique<TLocalRpcCtx::TEvWriteFinished>(); if (SessionClosed) { - LOG_ROW_DISPATCHER_DEBUG("Rpc write request skipped, session is closed"); + YDB_LOG_DEBUG("Rpc write request skipped, session is closed", + {"logPrefix", LogPrefix}); response->Success = false; Send(RpcActor, response.release()); return; @@ -489,7 +510,9 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvWriteRequest::TPt auto& message = ev->Get()->Message; const auto messageCase = message.response_case(); - LOG_ROW_DISPATCHER_DEBUG("Rpc write request: " << static_cast<i64>(messageCase)); + YDB_LOG_DEBUG("Rpc write", + {"logPrefix", LogPrefix}, + {"request", static_cast<i64>(messageCase)}); switch (messageCase) { case TRpcOut::kPing: @@ -532,7 +555,8 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvWriteRequest::TPt ProcessDeleteSemaphoreResult(message); break; default: - LOG_ROW_DISPATCHER_DEBUG("Unknown message case"); + YDB_LOG_DEBUG("Unknown message case", + {"logPrefix", LogPrefix}); } ProcessState(); } @@ -540,9 +564,13 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvWriteRequest::TPt void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvFinishRequest::TPtr& ev) { const auto& status = ev->Get()->Status; if (!status.ok()) { - LOG_ROW_DISPATCHER_ERROR("Rpc session finished with error status code: " << static_cast<ui64>(status.error_code()) << ", message: " << status.error_message()); + YDB_LOG_ERROR("Rpc session finished with error status", + {"logPrefix", LogPrefix}, + {"code", static_cast<ui64>(status.error_code())}, + {"message", status.error_message()}); } else { - LOG_ROW_DISPATCHER_INFO("Rpc session successfully finished"); + YDB_LOG_INFO("Rpc session successfully finished", + {"logPrefix", LogPrefix}); } CloseSession(status, "Read session closed"); @@ -551,7 +579,8 @@ void TLocalLeaderElection::Handle(TLocalRpcCtx::TRpcEvents::TEvFinishRequest::TP void TLocalLeaderElection::SendStartSession() { Y_VALIDATE(RpcActor, "RpcActor is not set before read request"); - LOG_ROW_DISPATCHER_DEBUG("Sending start message"); + YDB_LOG_DEBUG("Sending start message", + {"logPrefix", LogPrefix}); TRpcIn message; auto& start = *message.mutable_session_start(); @@ -563,12 +592,15 @@ void TLocalLeaderElection::SendStartSession() { void TLocalLeaderElection::AddSessionEvent(TRpcIn&& message) { if (SessionClosed) { - LOG_ROW_DISPATCHER_DEBUG("Session already closed, skip session event"); + YDB_LOG_DEBUG("Session already closed, skip session event", + {"logPrefix", LogPrefix}); return; } RpcResponses.push(message); - LOG_ROW_DISPATCHER_DEBUG("Added session event: " << static_cast<i64>(message.request_case())); + YDB_LOG_DEBUG("Added session", + {"logPrefix", LogPrefix}, + {"event", static_cast<i64>(message.request_case())}); if (RpcActor) { SendSessionEvents(); @@ -577,7 +609,10 @@ void TLocalLeaderElection::AddSessionEvent(TRpcIn&& message) { void TLocalLeaderElection::SendSessionEvents() { Y_VALIDATE(RpcActor, "RpcActor is not set before read request"); - LOG_ROW_DISPATCHER_DEBUG("Going to send session events, PendingRpcResponses: " << PendingRpcResponses << ", RpcResponses #" << RpcResponses.size()); + YDB_LOG_DEBUG("Going to send session events, RpcResponses", + {"logPrefix", LogPrefix}, + {"pendingRpcResponses", PendingRpcResponses}, + {"rpcResponses", RpcResponses.size()}); while (PendingRpcResponses > 0 && !RpcResponses.empty()) { SendSessionEvent(std::move(RpcResponses.front())); @@ -586,7 +621,10 @@ void TLocalLeaderElection::SendSessionEvents() { } void TLocalLeaderElection::SendSessionEvent(TRpcIn&& message, bool success) { - LOG_ROW_DISPATCHER_DEBUG("Sending session event: " << static_cast<i64>(message.request_case()) << ", success: " << success); + YDB_LOG_DEBUG("Sending session", + {"logPrefix", LogPrefix}, + {"event", static_cast<i64>(message.request_case())}, + {"success", success}); Y_VALIDATE(PendingRpcResponses > 0, "Rpc read is not expected"); PendingRpcResponses--; @@ -606,13 +644,20 @@ void TLocalLeaderElection::CloseSession(NYdb::EStatus status, const NYql::TIssue const bool success = status == NYdb::EStatus::SUCCESS; if (!success) { Metrics.Errors->Inc(); - LOG_ROW_DISPATCHER_ERROR("Closing session with status " << status << " and issues: " << issues.ToOneLineString()); + YDB_LOG_ERROR("Closing session with status and", + {"logPrefix", LogPrefix}, + {"status", status}, + {"issues", issues.ToOneLineString()}); } else { - LOG_ROW_DISPATCHER_INFO("Closing session with success status"); + YDB_LOG_INFO("Closing session with success status", + {"logPrefix", LogPrefix}); } if (SessionClosed) { - LOG_ROW_DISPATCHER_WARN("Session already closed, but got status " << status << " and issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("Session already closed, but got status and", + {"logPrefix", LogPrefix}, + {"status", status}, + {"issues", issues.ToOneLineString()}); return; } SessionClosed = true; @@ -652,7 +697,8 @@ void TLocalLeaderElection::CloseSession(const grpc::Status& status, const TStrin } void TLocalLeaderElection::ProcessPing(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received Ping, send Pong"); + YDB_LOG_DEBUG("Received Ping, send Pong", + {"logPrefix", LogPrefix}); const auto& source = message.ping(); TRpcIn in; in.mutable_pong()->set_opaque(source.opaque()); @@ -660,7 +706,8 @@ void TLocalLeaderElection::ProcessPing(const TRpcOut& message) { } void TLocalLeaderElection::ProcessPong(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received Pong"); + YDB_LOG_DEBUG("Received Pong", + {"logPrefix", LogPrefix}); const auto& source = message.pong(); const uint64_t reqId = source.opaque(); auto* op = FindSentRequest(reqId); @@ -676,14 +723,18 @@ void TLocalLeaderElection::ProcessFailure(const TRpcOut& message) { NYql::TIssues issues; IssuesFromMessage(failure.issues(), issues); - LOG_ROW_DISPATCHER_DEBUG("Received Failure, status: " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_DEBUG("Received Failure", + {"logPrefix", LogPrefix}, + {"status", status}, + {"issues", issues.ToOneLineString()}); CloseSession(static_cast<NYdb::EStatus>(status), issues); ResetState(); } void TLocalLeaderElection::ProcessSessionStarted(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received SessionStarted, session successfully created"); + YDB_LOG_DEBUG("Received SessionStarted, session successfully created", + {"logPrefix", LogPrefix}); const auto& source = message.session_started(); SessionId = source.session_id(); @@ -692,20 +743,26 @@ void TLocalLeaderElection::ProcessSessionStarted(const TRpcOut& message) { } void TLocalLeaderElection::ProcessSessionStopped(const TRpcOut& /*message*/) { - LOG_ROW_DISPATCHER_DEBUG("Received SessionStopped"); + YDB_LOG_DEBUG("Received SessionStopped", + {"logPrefix", LogPrefix}); ResetState(); } void TLocalLeaderElection::ProcessCreateSemaphoreResult(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received CreateSemaphoreResult"); + YDB_LOG_DEBUG("Received CreateSemaphoreResult", + {"logPrefix", LogPrefix}); const auto& source = message.create_semaphore_result(); NYdb::NIssue::TIssues issues; NYdb::NIssue::IssuesFromMessage(source.issues(), issues); - + auto status = NYdb::TStatus(static_cast<NYdb::EStatus>(source.status()), std::move(issues)); - LOG_ROW_DISPATCHER_INFO("Semaphore creating status: " << status); + YDB_LOG_INFO("Semaphore creating", + {"logPrefix", LogPrefix}, + {"status", status}); if (!IsTableCreated(status)) { - LOG_ROW_DISPATCHER_ERROR("Semaphore creating error " << status.GetIssues().ToOneLineString()); + YDB_LOG_ERROR("Semaphore creating error", + {"logPrefix", LogPrefix}, + {"issues", status.GetIssues().ToOneLineString()}); Metrics.Errors->Inc(); ResetState(); return; @@ -717,12 +774,14 @@ void TLocalLeaderElection::ProcessCreateSemaphoreResult(const TRpcOut& message) } SentRequests.erase(reqId); SemaphoreCreated = true; - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully created"); + YDB_LOG_DEBUG("Semaphore successfully created", + {"logPrefix", LogPrefix}); ProcessState(); } void TLocalLeaderElection::ProcessAcquireSemaphoreResult(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received AcquireSemaphoreResult"); + YDB_LOG_DEBUG("Received AcquireSemaphoreResult", + {"logPrefix", LogPrefix}); const auto& source = message.acquire_semaphore_result(); NYdb::NIssue::TIssues issues; NYdb::NIssue::IssuesFromMessage(source.issues(), issues); @@ -731,7 +790,9 @@ void TLocalLeaderElection::ProcessAcquireSemaphoreResult(const TRpcOut& message) PendingAcquire = false; if (!status.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("Failed to acquire semaphore, " << status.GetIssues().ToOneLineString()); + YDB_LOG_ERROR("Failed to acquire semaphore", + {"logPrefix", LogPrefix}, + {"issues", status.GetIssues().ToOneLineString()}); Metrics.Errors->Inc(); ResetState(); return; @@ -743,15 +804,18 @@ void TLocalLeaderElection::ProcessAcquireSemaphoreResult(const TRpcOut& message) } SentRequests.erase(reqId); if (source.acquired()) { - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully acquired"); + YDB_LOG_DEBUG("Semaphore successfully acquired", + {"logPrefix", LogPrefix}); LastAcquireSemaphore = TInstant::Now(); // delay } else { - LOG_ROW_DISPATCHER_DEBUG("Semaphore acquire timed out"); + YDB_LOG_DEBUG("Semaphore acquire timed out", + {"logPrefix", LogPrefix}); } } void TLocalLeaderElection::ProcessDescribeSemaphoreResult(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received DescribeSemaphoreResult"); + YDB_LOG_DEBUG("Received DescribeSemaphoreResult", + {"logPrefix", LogPrefix}); const auto& source = message.describe_semaphore_result(); NYdb::NIssue::TIssues issues; NYdb::NIssue::IssuesFromMessage(source.issues(), issues); @@ -759,7 +823,9 @@ void TLocalLeaderElection::ProcessDescribeSemaphoreResult(const TRpcOut& message PendingDescribe = false; if (!status.IsSuccess()) { - LOG_ROW_DISPATCHER_ERROR("Semaphore describe fail, " << status.GetIssues()); + YDB_LOG_ERROR("Semaphore describe fail", + {"logPrefix", LogPrefix}, + {"issues", status.GetIssues()}); Metrics.Errors->Inc(); ResetState(); return; @@ -776,7 +842,8 @@ void TLocalLeaderElection::ProcessDescribeSemaphoreResult(const TRpcOut& message const NYdb::NCoordination::TSemaphoreDescription& description = source.semaphore_description(); Y_ABORT_UNLESS(description.GetOwners().size() <= 1, "To many owners"); if (description.GetOwners().empty()) { - LOG_ROW_DISPATCHER_DEBUG("Empty owners"); + YDB_LOG_DEBUG("Empty owners", + {"logPrefix", LogPrefix}); // Wait OnChanged. return; } @@ -789,9 +856,16 @@ void TLocalLeaderElection::ProcessDescribeSemaphoreResult(const TRpcOut& message } NActors::TActorId id = ActorIdFromProto(protoId); - LOG_ROW_DISPATCHER_DEBUG("Semaphore successfully described: coordinator id " << id << " generation " << generation); + YDB_LOG_DEBUG("Semaphore successfully described: coordinator id generation", + {"logPrefix", LogPrefix}, + {"id", id}, + {"generation", generation}); if (!LeaderActorId || (*LeaderActorId != id)) { - LOG_ROW_DISPATCHER_INFO("Send TEvCoordinatorChanged to " << ParentId << ", new coordinator id " << id << ", previous coordinator id " << LeaderActorId.GetOrElse(TActorId())); + YDB_LOG_INFO("Send TEvCoordinatorChanged to new coordinator id previous coordinator id", + {"logPrefix", LogPrefix}, + {"parentId", ParentId}, + {"id", id}, + {"leaderActorId", LeaderActorId.GetOrElse(TActorId())}); TActivationContext::ActorSystem()->Send(ParentId, new NFq::TEvRowDispatcher::TEvCoordinatorChanged(id, generation)); Metrics.LeaderChanged->Inc(); } @@ -799,7 +873,8 @@ void TLocalLeaderElection::ProcessDescribeSemaphoreResult(const TRpcOut& message } void TLocalLeaderElection::ProcessDescribeSemaphoreChanged(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received DescribeSemaphoreChanged"); + YDB_LOG_DEBUG("Received DescribeSemaphoreChanged", + {"logPrefix", LogPrefix}); PendingDescribeChanged = false; const auto& source = message.describe_semaphore_changed(); @@ -811,16 +886,19 @@ void TLocalLeaderElection::ProcessDescribeSemaphoreChanged(const TRpcOut& messag SentRequests.erase(reqId); } -void TLocalLeaderElection::ProcessReleaseSemaphoreResult(const TRpcOut& /*message*/) { - LOG_ROW_DISPATCHER_DEBUG("Received ReleaseSemaphoreResult"); +void TLocalLeaderElection::ProcessReleaseSemaphoreResult(const TRpcOut& /*message*/) { + YDB_LOG_DEBUG("Received ReleaseSemaphoreResult", + {"logPrefix", LogPrefix}); } void TLocalLeaderElection::ProcessAcquireSemaphorePending(const TRpcOut& /*message*/) { - LOG_ROW_DISPATCHER_DEBUG("Received AcquireSemaphorePending"); + YDB_LOG_DEBUG("Received AcquireSemaphorePending", + {"logPrefix", LogPrefix}); } void TLocalLeaderElection::ProcessUpdateSemaphoreResult(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received UpdateSemaphoreResult"); + YDB_LOG_DEBUG("Received UpdateSemaphoreResult", + {"logPrefix", LogPrefix}); const auto& source = message.update_semaphore_result(); const uint64_t reqId = source.req_id(); @@ -832,7 +910,8 @@ void TLocalLeaderElection::ProcessUpdateSemaphoreResult(const TRpcOut& message) } void TLocalLeaderElection::ProcessDeleteSemaphoreResult(const TRpcOut& message) { - LOG_ROW_DISPATCHER_DEBUG("Received DeleteSemaphoreResult"); + YDB_LOG_DEBUG("Received DeleteSemaphoreResult", + {"logPrefix", LogPrefix}); const auto& source = message.delete_semaphore_result(); const uint64_t reqId = source.req_id(); auto* op = FindSentRequest(reqId); @@ -844,7 +923,10 @@ void TLocalLeaderElection::ProcessDeleteSemaphoreResult(const TRpcOut& message) void TLocalLeaderElection::UpdateLastKnownGoodTimestampLocked(TInstant timestamp) { SessionLastKnownGoodTimestamp = Max(SessionLastKnownGoodTimestamp, timestamp); - LOG_ROW_DISPATCHER_TRACE("UpdateLastKnownGoodTimestampLocked timestamp " << timestamp << " new " << SessionLastKnownGoodTimestamp); + YDB_LOG_TRACE("UpdateLastKnownGoodTimestampLocked timestamp new", + {"logPrefix", LogPrefix}, + {"timestamp", timestamp}, + {"sessionLastKnownGoodTimestamp", SessionLastKnownGoodTimestamp}); } TLocalLeaderElection::TOperation* TLocalLeaderElection::FindSentRequest(uint64_t reqId) const { @@ -860,7 +942,9 @@ void TLocalLeaderElection::Handle(NActors::TEvents::TEvUndelivered::TPtr& ev) { const auto reason = ev->Get()->Reason; Y_VALIDATE(sourceType == NKikimr::NGRpcService::TEvCoordinationSessionRequest::EventType, "Unexpected undelivered event: " << sourceType << ", reason: " << reason); - LOG_ROW_DISPATCHER_ERROR("Coordination service is unavailable, reason: " << reason); + YDB_LOG_ERROR("Coordination service is unavailable", + {"logPrefix", LogPrefix}, + {"reason", reason}); CloseSession(NYdb::EStatus::INTERNAL_ERROR, {NYql::TIssue("Coordination service is unavailable, please contact internal support")}); ResetState(); } diff --git a/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/compile_service.cpp b/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/compile_service.cpp index a013676774a..ccda4d49c53 100644 --- a/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/compile_service.cpp +++ b/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/compile_service.cpp @@ -1,13 +1,15 @@ #include "compile_service.h" -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/row_dispatcher/events/data_plane.h> #include <ydb/core/fq/libs/row_dispatcher/format_handler/common/common.h> #include <ydb/library/actors/core/actor_bootstrapped.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/actors/core/hfunc.h> #include <yql/essentials/public/purecalc/common/interface.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq::NRowDispatcher { namespace { @@ -49,7 +51,8 @@ public: Finish(); }; - LOG_ROW_DISPATCHER_TRACE("Started compile request"); + YDB_LOG_TRACE("Started compile request", + {"logPrefix", LogPrefix}); IProgramHolder::TPtr programHolder = std::move(Request->Get()->ProgramHolder); TStatus status = TStatus::Success(); @@ -64,10 +67,12 @@ public: } if (status.IsFail()) { - LOG_ROW_DISPATCHER_ERROR("Compilation failed for request"); + YDB_LOG_ERROR("Compilation failed for request", + {"logPrefix", LogPrefix}); Send(Request->Sender, new TEvRowDispatcher::TEvPurecalcCompileResponse(status.GetStatus(), status.GetErrorDescription()), 0, Request->Cookie); } else { - LOG_ROW_DISPATCHER_TRACE("Compilation completed for request"); + YDB_LOG_TRACE("Compilation completed for request", + {"logPrefix", LogPrefix}); Send(Request->Sender, new TEvRowDispatcher::TEvPurecalcCompileResponse(std::move(programHolder)), 0, Request->Cookie); } } @@ -128,7 +133,10 @@ public: void Handle(TEvRowDispatcher::TEvPurecalcCompileRequest::TPtr& ev) { const auto requestActor = ev->Sender; const ui64 requestId = ev->Cookie; - LOG_ROW_DISPATCHER_TRACE("Add to compile queue request with id " << requestId << " from " << requestActor); + YDB_LOG_TRACE("Add to compile queue request with id", + {"logPrefix", LogPrefix}, + {"requestId", requestId}, + {"requestActor", requestActor}); // Remove old compile request RemoveRequest(requestActor, requestId); @@ -142,13 +150,19 @@ public: } void Handle(TEvRowDispatcher::TEvPurecalcCompileAbort::TPtr& ev) { - LOG_ROW_DISPATCHER_TRACE("Abort compile request with id " << ev->Cookie << " from " << ev->Sender); + YDB_LOG_TRACE("Abort compile request with id", + {"logPrefix", LogPrefix}, + {"cookie", ev->Cookie}, + {"sender", ev->Sender}); RemoveRequest(ev->Sender, ev->Cookie); } void Handle(TEvPrivate::TEvCompileFinished::TPtr& ev) { - LOG_ROW_DISPATCHER_TRACE("Compile finished for request with id " << ev->Get()->RequestId << " from " << ev->Get()->RequestActor); + YDB_LOG_TRACE("Compile finished for request with id", + {"logPrefix", LogPrefix}, + {"requestId", ev->Get()->RequestId}, + {"requestActor", ev->Get()->RequestActor}); InFlightCompilations.erase(ev->Sender); Counters.ActiveCompileActors->Dec(); diff --git a/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/ya.make b/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/ya.make index 39657da6767..90611406e68 100644 --- a/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/purecalc_compilation/ya.make @@ -5,7 +5,6 @@ SRCS( ) PEERDIR( - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/config/protos ydb/core/fq/libs/row_dispatcher/common ydb/core/fq/libs/row_dispatcher/events diff --git a/ydb/core/fq/libs/row_dispatcher/row_dispatcher.cpp b/ydb/core/fq/libs/row_dispatcher/row_dispatcher.cpp index 11fc56c6d26..0495a4acbd9 100644 --- a/ydb/core/fq/libs/row_dispatcher/row_dispatcher.cpp +++ b/ydb/core/fq/libs/row_dispatcher/row_dispatcher.cpp @@ -9,7 +9,6 @@ #include <ydb/core/base/appdata_fwd.h> #include <ydb/core/base/feature_flags.h> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/events/events.h> #include <ydb/core/fq/libs/metrics/sanitize_label.h> #include <ydb/core/fq/libs/row_dispatcher/events/data_plane.h> @@ -20,6 +19,7 @@ #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> #include <ydb/library/actors/core/interconnect.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/yql/dq/actors/common/retry_queue.h> #include <ydb/library/yql/providers/dq/counters/counters.h> @@ -30,6 +30,8 @@ #include <util/generic/queue.h> #include <util/stream/format.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq { using namespace NActors; @@ -92,7 +94,7 @@ struct TEvPrivate { struct TEvUpdateMetrics : public NActors::TEventLocal<TEvUpdateMetrics, EvUpdateMetrics> {}; struct TEvPrintStateToLog : public NActors::TEventLocal<TEvPrintStateToLog, EvPrintStateToLog> {}; struct TEvTryConnect : public NActors::TEventLocal<TEvTryConnect, EvTryConnect> { - TEvTryConnect(ui32 nodeId = 0) + TEvTryConnect(ui32 nodeId = 0) : NodeId(nodeId) {} ui32 NodeId = 0; }; @@ -333,7 +335,7 @@ class TRowDispatcher : public TActorBootstrapped<TRowDispatcher> { } void PrintInternalState(TStringStream& stream) const { - stream << "Nodes states: \n"; + stream << "Nodes states: \n"; for (const auto& [nodeId, state] : Nodes) { stream << " id " << nodeId << " connected " << state.Connected << " retry scheduled " << state.RetryScheduled << " connected count " << state.Counters.Connected << " disconnected count " << state.Counters.Disconnected << "\n"; @@ -373,7 +375,7 @@ class TRowDispatcher : public TActorBootstrapped<TRowDispatcher> { NActors::TMon* Monitoring; TNodesTracker NodesTracker; NYql::TCounters::TEntry AllSessionsDateRate; - TAggregatedStats AggrStats; + TAggregatedStats AggrStats; ui64 LastCpuTime = 0; NActors::TActorId NodesManagerId; TInstant LastUpdateMetricsTime = TInstant::Now(); @@ -493,7 +495,7 @@ public: void Handle(NFq::TEvPrivate::TEvPrintStateToLog::TPtr&); void Handle(NFq::TEvPrivate::TEvSendStatistic::TPtr&); void Handle(const NMon::TEvHttpInfo::TPtr&); - + void DeleteConsumer(NActors::TActorId readActorId); void UpdateMetrics(); TString GetInternalState(); @@ -569,7 +571,10 @@ TRowDispatcher::TRowDispatcher( void TRowDispatcher::Bootstrap() { Become(&TRowDispatcher::StateFunc); - LOG_ROW_DISPATCHER_DEBUG("Successfully bootstrapped row dispatcher, id " << SelfId() << ", tenant " << Tenant); + YDB_LOG_DEBUG("Successfully bootstrapped row dispatcher, id tenant", + {"logPrefix", LogPrefix}, + {"selfId", SelfId()}, + {"tenant", Tenant}); const auto& config = Config.GetCoordinator(); auto coordinatorId = Register(NewCoordinator(SelfId(), config, Tenant, Counters, NodesManagerId).release()); @@ -597,9 +602,16 @@ void TRowDispatcher::Bootstrap() { void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvCoordinatorChanged::TPtr& ev) { LWPROBE(CoordinatorChanged, ev->Sender.ToString(), ev->Get()->Generation, ev->Get()->CoordinatorActorId.ToString(), CoordinatorGeneration, CoordinatorActorId->ToString()); - LOG_ROW_DISPATCHER_DEBUG("Coordinator changed, old leader " << CoordinatorActorId << ", new " << ev->Get()->CoordinatorActorId << " generation " << ev->Get()->Generation); + YDB_LOG_DEBUG("Coordinator changed, old leader new generation", + {"logPrefix", LogPrefix}, + {"coordinatorActorId", CoordinatorActorId}, + {"evCoordinatorActorId", ev->Get()->CoordinatorActorId}, + {"generation", ev->Get()->Generation}); if (ev->Get()->Generation < CoordinatorGeneration) { - LOG_ROW_DISPATCHER_ERROR("New generation (" << ev->Get()->Generation << ") is less previous (" << CoordinatorGeneration << "), ignore updates"); + YDB_LOG_ERROR("New generation is less previous ignore updates", + {"logPrefix", LogPrefix}, + {"generation", ev->Get()->Generation}, + {"coordinatorGeneration", CoordinatorGeneration}); return; } CoordinatorActorId = ev->Get()->CoordinatorActorId; @@ -615,7 +627,9 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvCoordinatorChanged::TPtr& void TRowDispatcher::HandleConnected(TEvInterconnect::TEvNodeConnected::TPtr& ev) { LWPROBE(NodeConnected, ev->Sender.ToString(), ev->Get()->NodeId); - LOG_ROW_DISPATCHER_DEBUG("EvNodeConnected, node id " << ev->Get()->NodeId); + YDB_LOG_DEBUG("EvNodeConnected, node id", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); Metrics.NodesReconnect->Inc(); NodesTracker.HandleNodeConnected(ev->Get()->NodeId); for (auto& [actorId, consumer] : Consumers) { @@ -625,7 +639,9 @@ void TRowDispatcher::HandleConnected(TEvInterconnect::TEvNodeConnected::TPtr& ev void TRowDispatcher::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { LWPROBE(NodeDisconnected, ev->Sender.ToString(), ev->Get()->NodeId); - LOG_ROW_DISPATCHER_DEBUG("TEvNodeDisconnected, node id " << ev->Get()->NodeId); + YDB_LOG_DEBUG("TEvNodeDisconnected, node id", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); Metrics.NodesReconnect->Inc(); bool isTimeout = NodesTracker.HandleNodeDisconnected(ev->Get()->NodeId); for (auto& [actorId, consumer] : Consumers) { @@ -633,13 +649,15 @@ void TRowDispatcher::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected::TP } TVector<TActorId> toDelete; if (isTimeout) { - LOG_ROW_DISPATCHER_DEBUG("Node disconnected, node id " << ev->Get()->NodeId << " is timeout"); + YDB_LOG_DEBUG("Node disconnected, node id is timeout", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); for (auto& [actorId, consumer] : Consumers) { if (actorId.NodeId() != ev->Get()->NodeId) { continue; } toDelete.push_back(actorId); - } + } } for (auto& actorId : toDelete) { DeleteConsumer(actorId); @@ -648,7 +666,10 @@ void TRowDispatcher::HandleDisconnected(TEvInterconnect::TEvNodeDisconnected::TP void TRowDispatcher::Handle(NActors::TEvents::TEvUndelivered::TPtr& ev) { LWPROBE(UndeliveredStart, ev->Sender.ToString(), ev->Get()->Reason, ev->Cookie); - LOG_ROW_DISPATCHER_TRACE("TEvUndelivered, from " << ev->Sender << ", reason " << ev->Get()->Reason); + YDB_LOG_TRACE("TEvUndelivered, from reason", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"reason", ev->Get()->Reason}); for (auto& [key, consumer] : Consumers) { if (ev->Cookie != consumer->Generation) { // Several partitions in one read_actor have different Generation. LWPROBE(UndeliveredSkipGeneration, ev->Sender.ToString(), ev->Get()->Reason, ev->Cookie, consumer->Generation); @@ -668,17 +689,22 @@ void TRowDispatcher::Handle(TEvPrivate::TEvCoordinatorPing::TPtr&) { return; } LWPROBE(CoordinatorPing, CoordinatorActorId->ToString()); - LOG_ROW_DISPATCHER_TRACE("Send ping to " << *CoordinatorActorId); + YDB_LOG_TRACE("Send ping", + {"logPrefix", LogPrefix}, + {"coordinatorActorId", *CoordinatorActorId}); Send(*CoordinatorActorId, new NActors::TEvents::TEvPing()); } void TRowDispatcher::Handle(NActors::TEvents::TEvPong::TPtr&) { LWPROBE(Pong); - LOG_ROW_DISPATCHER_TRACE("NActors::TEvents::TEvPong"); + YDB_LOG_TRACE("NActors::TEvents::TEvPong", + {"logPrefix", LogPrefix}); } void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvCoordinatorChangesSubscribe::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("TEvCoordinatorChangesSubscribe from " << ev->Sender); + YDB_LOG_DEBUG("TEvCoordinatorChangesSubscribe", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); NodesTracker.AddNode(ev->Sender.NodeId()); CoordinatorChangedSubscribers.insert(ev->Sender); if (!CoordinatorActorId) { @@ -819,7 +845,7 @@ TString TRowDispatcher::GetInternalState() { ui64 minInitialOffset = std::numeric_limits<ui64>::max(); for (const auto& [formatName, formatStats] : sessionInfo.Stat.FormatHandlers) { - str << " " << formatName + str << " " << formatName << " parse and filter lantecy " << formatStats.ParseAndFilterLatency << " (parse " << formatStats.ParserStats.ParserLatency << ", filter " << formatStats.FilterStats.FilterLatency << ")\n"; } @@ -834,9 +860,9 @@ TString TRowDispatcher::GetInternalState() { << toHuman(stat.QueuedBytes) << " (" << leftPad(stat.QueuedRows) << " rows) " << " offset " << leftPad(stat.Offset) << " init offset " << leftPad(stat.InitialOffset) << " get " << leftPad(consumer->Counters.GetNextBatch) - << " arr " << leftPad(consumer->Counters.NewDataArrived) << " btc " << leftPad(consumer->Counters.MessageBatch) + << " arr " << leftPad(consumer->Counters.NewDataArrived) << " btc " << leftPad(consumer->Counters.MessageBatch) << " pend get " << leftPad(partition.PendingGetNextBatch) << " pend new " << leftPad(partition.PendingNewDataArrived) - << " waiting " << stat.IsWaiting << " read lag " << leftPad(stat.ReadLagMessages) + << " waiting " << stat.IsWaiting << " read lag " << leftPad(stat.ReadLagMessages) << " conn id " << consumer->Generation << "\n"; maxInitialOffset = std::max(maxInitialOffset, stat.InitialOffset); minInitialOffset = std::min(minInitialOffset, stat.InitialOffset); @@ -848,9 +874,9 @@ TString TRowDispatcher::GetInternalState() { str << "Consumers:\n"; for (auto& [readActorId, consumer] : Consumers) { str << " " << consumer->QueryId << " " << LeftPad(readActorId, 32) << " Generation " << consumer->Generation << "\n"; - str << " partitions: "; + str << " partitions: "; for (const auto& [partitionId, info] : consumer->Partitions) { - str << partitionId << ","; + str << partitionId << ","; } str << "\n retry queue: "; consumer->EventsQueue.PrintInternalState(str); @@ -893,8 +919,14 @@ void TRowDispatcher::UpdateReadActorsInternalState() { } void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("Received TEvStartSession from " << ev->Sender << ", read group " << ev->Get()->Record.GetSource().GetReadGroup() << ", topicPath " << ev->Get()->Record.GetSource().GetTopicPath() << - " part id " << JoinSeq(',', ev->Get()->Record.GetPartitionIds()) << " query id " << ev->Get()->Record.GetQueryId() << " cookie " << ev->Cookie); + YDB_LOG_DEBUG("Received TEvStartSession from read group topicPath part id query id cookie", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"Record.GetSource().GetReadGroup", ev->Get()->Record.GetSource().GetReadGroup()}, + {"Record.GetSource().GetTopicPath", ev->Get()->Record.GetSource().GetTopicPath()}, + {"partitionIds", JoinSeq(',', ev->Get()->Record.GetPartitionIds())}, + {"Record.GetQueryId", ev->Get()->Record.GetQueryId()}, + {"cookie", ev->Cookie}); if (EnableStreamingQueriesCounters) { auto queryGroup = Metrics.Counters->GetSubgroup("query_id", ev->Get()->Record.GetQueryId()); auto topicGroup = queryGroup->GetSubgroup("read_group", SanitizeLabel(ev->Get()->Record.GetSource().GetReadGroup())); @@ -907,12 +939,16 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { auto it = Consumers.find(ev->Sender); if (it != Consumers.end()) { if (ev->Cookie <= it->second->Generation) { - LOG_ROW_DISPATCHER_WARN("Consumer already exists, ignore StartSession"); + YDB_LOG_WARN("Consumer already exists, ignore StartSession", + {"logPrefix", LogPrefix}); return; } - LOG_ROW_DISPATCHER_WARN("Consumer already exists, new consumer with new generation (" << ev->Cookie << ", current " - << it->second->Generation << "), remove old consumer, sender " << ev->Sender << ", topicPath " - << ev->Get()->Record.GetSource().GetTopicPath() << " cookie " << ev->Cookie); + YDB_LOG_WARN("Consumer already exists, new consumer with new generation current remove old consumer, sender topicPath cookie", + {"logPrefix", LogPrefix}, + {"cookie", ev->Cookie}, + {"generation", it->second->Generation}, + {"sender", ev->Sender}, + {"topicPath", ev->Get()->Record.GetSource().GetTopicPath()}); DeleteConsumer(ev->Sender); } const auto& source = ev->Get()->Record.GetSource(); @@ -933,8 +969,11 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { Y_ENSURE(topicSessionInfo.Sessions.size() <= 1); if (topicSessionInfo.Sessions.empty()) { - LOG_ROW_DISPATCHER_DEBUG("Create new session: read group " << source.GetReadGroup() << " topic " << source.GetTopicPath() - << " part id " << partitionId); + YDB_LOG_DEBUG("Create new session: read group topic part id", + {"logPrefix", LogPrefix}, + {"readGroup", source.GetReadGroup()}, + {"topicPath", source.GetTopicPath()}, + {"partitionId", partitionId}); sessionActorId = ActorFactory->RegisterTopicSession( source.GetReadGroup(), source.GetTopicPath(), @@ -974,18 +1013,28 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvGetNextBatch::TPtr& ev) { auto it = Consumers.find(ev->Sender); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore (no consumer) TEvGetNextBatch from " << ev->Sender << " part id " << ev->Get()->Record.GetPartitionId()); + YDB_LOG_WARN("Ignore (no consumer) TEvGetNextBatch from part id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}); return; } auto& session = it->second; LWPROBE(GetNextBatch, ev->Sender.ToString(), ev->Get()->Record.GetPartitionId(), session->QueryId, ev->Get()->Record.ByteSizeLong()); - LOG_ROW_DISPATCHER_TRACE("Received TEvGetNextBatch from " << ev->Sender << " part id " << ev->Get()->Record.GetPartitionId() << " query id " << it->second->QueryId); + YDB_LOG_TRACE("Received TEvGetNextBatch from part id query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}, + {"queryId", it->second->QueryId}); if (!CheckSession(session, ev)) { return; } auto partitionIt = session->Partitions.find(ev->Get()->Record.GetPartitionId()); if (partitionIt == session->Partitions.end()) { - LOG_ROW_DISPATCHER_ERROR("Ignore TEvGetNextBatch from " << ev->Sender << ", wrong partition id " << ev->Get()->Record.GetPartitionId()); + YDB_LOG_ERROR("Ignore TEvGetNextBatch from wrong partition id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}); return; } partitionIt->second.PendingNewDataArrived = false; @@ -997,17 +1046,27 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvGetNextBatch::TPtr& ev) { void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvHeartbeat::TPtr& ev) { auto it = Consumers.find(ev->Sender); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Consumer not found, sender " << ev->Sender << ", part id " << ev->Get()->Record.GetPartitionId() << ", sending TEvNoSession"); + YDB_LOG_WARN("Consumer not found, sender part id sending TEvNoSession", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}); Send(ev->Sender, new NFq::TEvRowDispatcher::TEvNoSession(), 0, ev->Cookie); return; } LWPROBE(Heartbeat, ev->Sender.ToString(), ev->Get()->Record.GetPartitionId(), it->second->QueryId, ev->Get()->Record.ByteSizeLong()); - LOG_ROW_DISPATCHER_TRACE("Received TEvHeartbeat from " << ev->Sender << ", part id " << ev->Get()->Record.GetPartitionId() << " query id " << it->second->QueryId); + YDB_LOG_TRACE("Received TEvHeartbeat from part id query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}, + {"queryId", it->second->QueryId}); CheckSession(it->second, ev); } void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvNoSession::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("Received TEvNoSession from " << ev->Sender << ", generation " << ev->Cookie); + YDB_LOG_DEBUG("Received TEvNoSession from generation", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); auto consumerIt = Consumers.find(ev->Sender); if (consumerIt == Consumers.end()) { return; @@ -1022,12 +1081,23 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvNoSession::TPtr& ev) { template <class TEventPtr> bool TRowDispatcher::CheckSession(TAtomicSharedPtr<TConsumerInfo>& consumer, const TEventPtr& ev) { if (ev->Cookie != consumer->Generation) { - LOG_ROW_DISPATCHER_WARN("Wrong message generation (" << typeid(TEventPtr).name() << "), sender " << ev->Sender << " cookie " << ev->Cookie << ", session generation " << consumer->Generation << ", query id " << consumer->QueryId); + YDB_LOG_WARN("Wrong message generation sender cookie session generation query id", + {"logPrefix", LogPrefix}, + {"typeName", typeid(TEventPtr).name()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"Generation", consumer->Generation}, + {"QueryId", consumer->QueryId}); return false; } if (!consumer->EventsQueue.OnEventReceived(ev)) { const NYql::NDqProto::TMessageTransportMeta& meta = ev->Get()->Record.GetTransportMeta(); - LOG_ROW_DISPATCHER_WARN("Wrong seq num, ignore message (" << typeid(TEventPtr).name() << ") seqNo " << meta.GetSeqNo() << " from " << ev->Sender.ToString() << ", query id " << consumer->QueryId); + YDB_LOG_WARN("Wrong seq num, ignore message seqNo from query id", + {"logPrefix", LogPrefix}, + {"typeName", typeid(TEventPtr).name()}, + {"seqNo", meta.GetSeqNo()}, + {"sender", ev->Sender}, + {"QueryId", consumer->QueryId}); return false; } return true; @@ -1036,12 +1106,18 @@ bool TRowDispatcher::CheckSession(TAtomicSharedPtr<TConsumerInfo>& consumer, con void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStopSession::TPtr& ev) { auto it = Consumers.find(ev->Sender); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore TEvStopSession from " << ev->Sender); + YDB_LOG_WARN("Ignore TEvStopSession", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); return; } LWPROBE(StopSession, ev->Sender.ToString(), it->second->QueryId, ev->Get()->Record.ByteSizeLong()); - LOG_ROW_DISPATCHER_DEBUG("Received TEvStopSession from " << ev->Sender << " topic " << ev->Get()->Record.GetSource().GetTopicPath() << " query id " << it->second->QueryId); + YDB_LOG_DEBUG("Received TEvStopSession from topic query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"topicPath", ev->Get()->Record.GetSource().GetTopicPath()}, + {"queryId", it->second->QueryId}); if (!CheckSession(it->second, ev)) { return; } @@ -1051,12 +1127,18 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvStopSession::TPtr& ev) { void TRowDispatcher::DeleteConsumer(NActors::TActorId readActorId) { auto consumerIt = Consumers.find(readActorId); if (consumerIt == Consumers.end()) { - LOG_ROW_DISPATCHER_ERROR("Ignore (no consumer) DeleteConsumer, " << " read actor id " << readActorId); + YDB_LOG_ERROR("Ignore (no consumer) DeleteConsumer, read actor id", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}); return; } const auto& consumer = consumerIt->second; - LOG_ROW_DISPATCHER_DEBUG("DeleteConsumer, readActorId " << readActorId << " query id " << consumer->QueryId << ", partitions size " << consumer->Partitions.size()); + YDB_LOG_DEBUG("DeleteConsumer, readActorId query id partitions size", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}, + {"QueryId", consumer->QueryId}, + {"Partitions.size", consumer->Partitions.size()}); for (auto& [partitionId, partition] : consumer->Partitions) { auto event = std::make_unique<NFq::TEvRowDispatcher::TEvStopSession>(); *event->Record.MutableSource() = consumer->SourceParams; @@ -1073,10 +1155,14 @@ void TRowDispatcher::DeleteConsumer(NActors::TActorId readActorId) { TTopicSessionInfo& topicSessionInfo = sessionIt->second; TSessionInfo& sessionInfo = topicSessionInfo.Sessions[partition.TopicSessionId]; if (!sessionInfo.Consumers.erase(consumer->ReadActorId)) { - LOG_ROW_DISPATCHER_ERROR("Wrong readActorId " << consumer->ReadActorId << ", no such consumer"); + YDB_LOG_ERROR("Wrong readActorId no such consumer", + {"logPrefix", LogPrefix}, + {"ReadActorId", consumer->ReadActorId}); } if (sessionInfo.Consumers.empty()) { - LOG_ROW_DISPATCHER_DEBUG("Session is not used, sent TEvPoisonPill to " << partition.TopicSessionId); + YDB_LOG_DEBUG("Session is not used, sent TEvPoisonPill", + {"logPrefix", LogPrefix}, + {"topicSessionId", partition.TopicSessionId}); topicSessionInfo.Sessions.erase(partition.TopicSessionId); Send(partition.TopicSessionId, new NActors::TEvents::TEvPoisonPill()); if (topicSessionInfo.Sessions.empty()) { @@ -1088,7 +1174,7 @@ void TRowDispatcher::DeleteConsumer(NActors::TActorId readActorId) { const TString queryId = consumerIt->second->QueryId; ConsumersByEventQueueId.erase(consumerIt->second->EventQueueId); Consumers.erase(consumerIt); - + auto queryIt = ConsumersByQueryId.find(queryId); if (queryIt != ConsumersByQueryId.end()) { queryIt->second.erase(readActorId); @@ -1104,14 +1190,18 @@ void TRowDispatcher::DeleteConsumer(NActors::TActorId readActorId) { void TRowDispatcher::Handle(const TEvPrivate::TEvTryConnect::TPtr& ev) { LWPROBE(TryConnect, ev->Sender.ToString(), ev->Get()->NodeId); - LOG_ROW_DISPATCHER_TRACE("TEvTryConnect to node id " << ev->Get()->NodeId); + YDB_LOG_TRACE("TEvTryConnect to node id", + {"logPrefix", LogPrefix}, + {"nodeId", ev->Get()->NodeId}); NodesTracker.TryConnect(ev->Get()->NodeId); } void TRowDispatcher::Handle(const NYql::NDq::TEvRetryQueuePrivate::TEvEvHeartbeat::TPtr& ev) { auto it = ConsumersByEventQueueId.find(ev->Get()->EventQueueId); if (it == ConsumersByEventQueueId.end()) { - LOG_ROW_DISPATCHER_TRACE("No consumer with EventQueueId = " << ev->Get()->EventQueueId); + YDB_LOG_TRACE("No consumer with EventQueueId", + {"logPrefix", LogPrefix}, + {"eventQueueId", ev->Get()->EventQueueId}); return; } auto& sessionInfo = it->second; @@ -1119,21 +1209,31 @@ void TRowDispatcher::Handle(const NYql::NDq::TEvRetryQueuePrivate::TEvEvHeartbea bool needSend = sessionInfo->EventsQueue.Heartbeat(); if (needSend) { - LOG_ROW_DISPATCHER_TRACE("Send TEvHeartbeat to " << sessionInfo->ReadActorId << " query id " << sessionInfo->QueryId); + YDB_LOG_TRACE("Send TEvHeartbeat to query id", + {"logPrefix", LogPrefix}, + {"readActorId", sessionInfo->ReadActorId}, + {"queryId", sessionInfo->QueryId}); auto event = std::make_unique<NFq::TEvRowDispatcher::TEvHeartbeat>(); sessionInfo->EventsQueue.Send(new NFq::TEvRowDispatcher::TEvHeartbeat(), sessionInfo->Generation); } } -void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvNewDataArrived::TPtr& ev) { +void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvNewDataArrived::TPtr& ev) { auto it = Consumers.find(ev->Get()->ReadActorId); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore (no consumer) TEvNewDataArrived from " << ev->Sender << " part id " << ev->Get()->Record.GetPartitionId()); + YDB_LOG_WARN("Ignore (no consumer) TEvNewDataArrived from part id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"partitionId", ev->Get()->Record.GetPartitionId()}); return; } - auto consumerInfoPtr = it->second; + auto consumerInfoPtr = it->second; LWPROBE(NewDataArrived, ev->Sender.ToString(), ev->Get()->ReadActorId.ToString(), consumerInfoPtr->QueryId, consumerInfoPtr->Generation, ev->Get()->Record.ByteSizeLong()); - LOG_ROW_DISPATCHER_TRACE("Forward TEvNewDataArrived from " << ev->Sender << " to " << ev->Get()->ReadActorId << " query id " << consumerInfoPtr->QueryId); + YDB_LOG_TRACE("Forward TEvNewDataArrived from to query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"readActorId", ev->Get()->ReadActorId}, + {"queryId", consumerInfoPtr->QueryId}); auto partitionIt = consumerInfoPtr->Partitions.find(ev->Get()->Record.GetPartitionId()); if (partitionIt == consumerInfoPtr->Partitions.end()) { // Ignore TEvNewDataArrived because read actor now read others partitions. @@ -1147,12 +1247,19 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvNewDataArrived::TPtr& ev) void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvMessageBatch::TPtr& ev) { auto it = Consumers.find(ev->Get()->ReadActorId); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore (no consumer) TEvMessageBatch from " << ev->Sender << " to " << ev->Get()->ReadActorId); + YDB_LOG_WARN("Ignore (no consumer) TEvMessageBatch", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"readActorId", ev->Get()->ReadActorId}); return; } - auto consumerInfoPtr = it->second; + auto consumerInfoPtr = it->second; LWPROBE(MessageBatch, ev->Sender.ToString(), ev->Get()->ReadActorId.ToString(), consumerInfoPtr->QueryId, consumerInfoPtr->Generation, ev->Get()->Record.ByteSizeLong()); - LOG_ROW_DISPATCHER_TRACE("Forward TEvMessageBatch from " << ev->Sender << " to " << ev->Get()->ReadActorId << " query id " << consumerInfoPtr->QueryId); + YDB_LOG_TRACE("Forward TEvMessageBatch from to query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"readActorId", ev->Get()->ReadActorId}, + {"queryId", consumerInfoPtr->QueryId}); Metrics.RowsSent->Add(ev->Get()->Record.MessagesSize()); auto partitionIt = consumerInfoPtr->Partitions.find(ev->Get()->Record.GetPartitionId()); if (partitionIt == consumerInfoPtr->Partitions.end()) { @@ -1167,17 +1274,26 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvMessageBatch::TPtr& ev) { void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvSessionError::TPtr& ev) { auto it = Consumers.find(ev->Get()->ReadActorId); if (it == Consumers.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore (no consumer) TEvSessionError from " << ev->Sender << " to " << ev->Get()->ReadActorId); + YDB_LOG_WARN("Ignore (no consumer) TEvSessionError", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"readActorId", ev->Get()->ReadActorId}); return; } LWPROBE(SessionError, ev->Sender.ToString(), ev->Get()->ReadActorId.ToString(), it->second->QueryId, it->second->Generation, ev->Get()->Record.ByteSizeLong()); ++*Metrics.ErrorsCount; - LOG_ROW_DISPATCHER_TRACE("Forward TEvSessionError from " << ev->Sender << " to " << ev->Get()->ReadActorId << " query id " << it->second->QueryId); + YDB_LOG_TRACE("Forward TEvSessionError from to query id", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"readActorId", ev->Get()->ReadActorId}, + {"queryId", it->second->QueryId}); if (ev->Get()->IsFatalError) { auto consumerIt = Consumers.find(ev->Get()->ReadActorId); if (consumerIt == Consumers.end()) { - LOG_ROW_DISPATCHER_ERROR("Ignore (no consumer) DeleteConsumer, " << " read actor id " << ev->Get()->ReadActorId); + YDB_LOG_ERROR("Ignore (no consumer) DeleteConsumer, read actor id", + {"logPrefix", LogPrefix}, + {"readActorId", ev->Get()->ReadActorId}); return; } const auto& consumer = consumerIt->second; @@ -1192,7 +1308,9 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvSessionError::TPtr& ev) { if (sessionIt != TopicSessions.end()) { TTopicSessionInfo& topicSessionInfo = sessionIt->second; if (topicSessionInfo.Sessions.erase(ev->Sender)) { - LOG_ROW_DISPATCHER_WARN("Fatal session error, remove session " << ev->Sender); + YDB_LOG_WARN("Fatal session error, remove session", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); Send(ev->Sender, new NActors::TEvents::TEvPoisonPill()); if (topicSessionInfo.Sessions.empty()) { TopicSessions.erase(sessionIt); @@ -1221,12 +1339,15 @@ void TRowDispatcher::PrintStateToLog() { auto str = GetInternalState(); auto buf = TStringBuf(str); for (ui64 offset = 0; offset < buf.size(); offset += PrintStateToLogSplitSize) { - LOG_ROW_DISPATCHER_DEBUG(buf.SubString(offset, PrintStateToLogSplitSize)); + YDB_LOG_DEBUG("Dump logPrefix, #_buf.SubString(offset, PrintStateToLogSplitSize)", + {"logPrefix", LogPrefix}, + {"state", buf.SubString(offset, PrintStateToLogSplitSize)}); } } void TRowDispatcher::Handle(NFq::TEvPrivate::TEvSendStatistic::TPtr&) { - LOG_ROW_DISPATCHER_TRACE("TEvPrivate::TEvSendStatistic"); + YDB_LOG_TRACE("TEvPrivate::TEvSendStatistic", + {"logPrefix", LogPrefix}); UpdateCpuTime(); Schedule(Config.GetSendStatusPeriod(), new NFq::TEvPrivate::TEvSendStatistic()); @@ -1286,11 +1407,13 @@ void TRowDispatcher::Handle(const NMon::TEvHttpInfo::TPtr& ev) { } void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvSessionStatistic::TPtr& ev) { - LOG_ROW_DISPATCHER_TRACE("TEvSessionStatistic from " << ev->Sender); + YDB_LOG_TRACE("TEvSessionStatistic", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); const auto& stat = ev->Get()->Stat; const auto& key = stat.SessionKey; - LWPROBE(SessionStatistic, + LWPROBE(SessionStatistic, ev->Sender.ToString(), key.ReadGroup, key.Endpoint, @@ -1321,7 +1444,7 @@ void TRowDispatcher::Handle(NFq::TEvRowDispatcher::TEvSessionStatistic::TPtr& ev if (it == sessionInfo.Consumers.end()) { continue; } - auto consumerInfoPtr = it->second; + auto consumerInfoPtr = it->second; auto partitionIt = consumerInfoPtr->Partitions.find(key.PartitionId); if (partitionIt == consumerInfoPtr->Partitions.end()) { continue; @@ -1352,7 +1475,10 @@ void TRowDispatcher::UpdateCpuTime() { for (auto& [actorId, consumer] : Consumers) { consumer->CpuMicrosec += diff; } - LOG_ROW_DISPATCHER_TRACE("UpdateCpuTime, currentCpuTime " << currentCpuTime << ", diff " << diff); + YDB_LOG_TRACE("UpdateCpuTime, currentCpuTime diff", + {"logPrefix", LogPrefix}, + {"currentCpuTime", currentCpuTime}, + {"diff", diff}); LastCpuTime = currentCpuTime; } diff --git a/ydb/core/fq/libs/row_dispatcher/topic_session.cpp b/ydb/core/fq/libs/row_dispatcher/topic_session.cpp index db1003f8362..03d587ce519 100644 --- a/ydb/core/fq/libs/row_dispatcher/topic_session.cpp +++ b/ydb/core/fq/libs/row_dispatcher/topic_session.cpp @@ -3,12 +3,12 @@ #include <ydb/core/base/appdata_fwd.h> #include <ydb/core/base/feature_flags.h> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/metrics/sanitize_label.h> #include <ydb/core/fq/libs/row_dispatcher/events/data_plane.h> #include <ydb/core/fq/libs/row_dispatcher/format_handler/format_handler.h> #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/yql/dq/actors/dq.h> #include <ydb/library/yql/providers/pq/common/pq_events_processor.h> #include <ydb/public/sdk/cpp/adapters/issue/issue.h> @@ -16,6 +16,8 @@ #include <util/generic/queue.h> +#define YDB_LOG_THIS_FILE_COMPONENT ::NKikimrServices::FQ_ROW_DISPATCHER + namespace NFq { using namespace NActors; @@ -35,7 +37,7 @@ struct TTopicSessionMetrics { const auto topicGroup = counters->GetSubgroup("topic", SanitizeLabel(topicPath)); ReadGroup = topicGroup->GetSubgroup("read_group", SanitizeLabel(readGroupName)); PartitionGroup = ReadGroup->GetSubgroup("partition", ToString(partitionId)); - PartitionGroup = PartitionGroup->GetSubgroup("host", ""); // This partition report metrics only on one host. + PartitionGroup = PartitionGroup->GetSubgroup("host", ""); // This partition report metrics only on one host. } AllSessionsDataRate = ReadGroup->GetCounter("AllSessionsDataRate", true); InFlyAsyncInputData = PartitionGroup->GetCounter("InFlyAsyncInputData"); @@ -208,7 +210,13 @@ private: void AddDataToClient(ui64 offset, ui64 numberRows, ui64 rowSize, TMaybe<TInstant> watermark) override { Y_ENSURE(!NextMessageOffset || offset >= *NextMessageOffset, "Unexpected historical offset"); - LOG_ROW_DISPATCHER_TRACE("AddDataToClient to " << ReadActorId << ", offset: " << offset << ", number rows: " << numberRows << ", row size: " << rowSize << ", watermark: " << watermark); + YDB_LOG_TRACE("AddDataToClient to number row", + {"logPrefix", LogPrefix}, + {"readActorId", ReadActorId}, + {"offset", offset}, + {"rows", numberRows}, + {"size", rowSize}, + {"watermark", watermark}); NextMessageOffset = offset + 1; QueuedRows += numberRows; @@ -220,7 +228,10 @@ private: } void UpdateClientOffset(ui64 offset) override { - LOG_ROW_DISPATCHER_TRACE("UpdateClientOffset for " << ReadActorId << ", new offset: " << offset); + YDB_LOG_TRACE("UpdateClientOffset for new", + {"logPrefix", LogPrefix}, + {"readActorId", ReadActorId}, + {"offset", offset}); if (!NextMessageOffset || *NextMessageOffset < offset + 1) { NextMessageOffset = offset + 1; } @@ -453,13 +464,16 @@ void TTopicSession::Bootstrap() { Become(&TTopicSession::StateFunc); Metrics.Init(Counters, TopicPath, ReadGroup, PartitionId, EnableStreamingQueriesCounters); LogPrefix = LogPrefix + " " + SelfId().ToString() + " [" + TopicPathPartition + "] "; - LOG_ROW_DISPATCHER_INFO("Bootstrap, Timeout " << Config.GetTimeoutBeforeStartSession() << " sec"); + YDB_LOG_INFO("Bootstrap, Timeout sec", + {"logPrefix", LogPrefix}, + {"timeout", Config.GetTimeoutBeforeStartSession()}); Schedule(TDuration::Seconds(SendStatisticPeriodSec), new NFq::TEvPrivate::TEvSendStatistic()); Schedule(TDuration::Seconds(GetEventByTimerPeriodSec), new NFq::TEvPrivate::TEvGetEventByTimerEvent()); } void TTopicSession::PassAway() { - LOG_ROW_DISPATCHER_INFO("PassAway"); + YDB_LOG_INFO("PassAway", + {"logPrefix", LogPrefix}); StopReadSession(); for (const auto& [actorId, clientInfo] : Clients) { if (const auto formatIt = FormatHandlers.find(clientInfo->HandlerSettings); formatIt != FormatHandlers.end()) { @@ -472,16 +486,22 @@ void TTopicSession::PassAway() { void TTopicSession::SubscribeOnNextEvent(bool checkIsWaitingEvents) { if (!ReadSession || (checkIsWaitingEvents && IsWaitingEvents)) { - LOG_ROW_DISPATCHER_TRACE("Skip SubscribeOnNextEvent, has ReadSession: " << (ReadSession ? "true" : "false") << ", IsWaitingEvents: " << IsWaitingEvents); + YDB_LOG_TRACE("Skip SubscribeOnNextEvent, has", + {"logPrefix", LogPrefix}, + {"readSession", (ReadSession ? "true" : "false")}, + {"isWaitingEvents", IsWaitingEvents}); return; } if (Config.GetMaxSessionUsedMemory() && QueuedBytes > Config.GetMaxSessionUsedMemory()) { - LOG_ROW_DISPATCHER_TRACE("Too much used memory (" << QueuedBytes << " bytes), skip subscribing to WaitEvent()"); + YDB_LOG_TRACE("Too much used memory bytes), skip subscribing to WaitEvent()", + {"logPrefix", LogPrefix}, + {"queuedBytes", QueuedBytes}); return; } - LOG_ROW_DISPATCHER_TRACE("SubscribeOnNextEvent"); + YDB_LOG_TRACE("SubscribeOnNextEvent", + {"logPrefix", LogPrefix}); IsWaitingEvents = true; Metrics.InFlySubscribe->Set(1); TActorSystem* actorSystem = TActivationContext::ActorSystem(); @@ -529,9 +549,12 @@ NYdb::NTopic::TReadSessionSettings TTopicSession::GetReadSessionSettings(const T topicReadSettings.AppendPartitionIds(PartitionId); TInstant minTime = GetMinStartingMessageTimestamp(); - LOG_ROW_DISPATCHER_INFO("Create topic session, Path " << TopicPathPartition - << ", StartingMessageTimestamp " << minTime - << ", BufferSize " << BufferSize << ", GetConsumerMode " << Config.GetConsumerMode()); + YDB_LOG_INFO("Create topic session, Path StartingMessageTimestamp BufferSize GetConsumerMode", + {"logPrefix", LogPrefix}, + {"topicPathPartition", TopicPathPartition}, + {"minTime", minTime}, + {"bufferSize", BufferSize}, + {"consumerMode", Config.GetConsumerMode()}); auto settings = NYdb::NTopic::TReadSessionSettings() .TraceId(LogPrefix) @@ -567,7 +590,9 @@ void TTopicSession::CreateTopicSession() { // Use any sourceParams. ReconnectPeriod = Clients.begin()->second->ReconnectPeriod; if (ReconnectPeriod != TDuration::Zero()) { - LOG_ROW_DISPATCHER_INFO("ReconnectPeriod " << ReconnectPeriod.ToString()); + YDB_LOG_INFO("ReconnectPeriod", + {"logPrefix", LogPrefix}, + {"reconnectPeriod", ReconnectPeriod}); Metrics.ReconnectRate->Inc(); Schedule(ReconnectPeriod, new NFq::TEvPrivate::TEvReconnectSession()); InflightReconnect = true; @@ -576,7 +601,8 @@ void TTopicSession::CreateTopicSession() { } void TTopicSession::Handle(NFq::TEvPrivate::TEvPqEventsReady::TPtr&) { - LOG_ROW_DISPATCHER_TRACE("TEvPqEventsReady"); + YDB_LOG_TRACE("TEvPqEventsReady", + {"logPrefix", LogPrefix}); Metrics.InFlySubscribe->Set(0); IsWaitingEvents = false; auto waitEventDurationMs = (TInstant::Now() - WaitEventStartedAt).MilliSeconds(); @@ -597,20 +623,27 @@ void TTopicSession::Handle(NFq::TEvPrivate::TEvReconnectSession::TPtr&) { return; } StartingMessageTimestamp = GetMinStartingMessageTimestamp(); - LOG_ROW_DISPATCHER_DEBUG("Reconnect topic session, " << TopicPathPartition - << ", StartingMessageTimestamp " << StartingMessageTimestamp - << ", BufferSize " << BufferSize << ", ConsumerMode " << Config.GetConsumerMode()); + YDB_LOG_DEBUG("Reconnect topic session, StartingMessageTimestamp BufferSize ConsumerMode", + {"logPrefix", LogPrefix}, + {"topicPathPartition", TopicPathPartition}, + {"startingMessageTimestamp", StartingMessageTimestamp}, + {"bufferSize", BufferSize}, + {"consumerMode", Config.GetConsumerMode()}); RefreshParsers(); StopReadSession(); CreateTopicSession(); } void TTopicSession::Handle(TEvRowDispatcher::TEvGetNextBatch::TPtr& ev) { - LOG_ROW_DISPATCHER_TRACE("TEvGetNextBatch from " << ev->Sender.ToString()); + YDB_LOG_TRACE("TEvGetNextBatch", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); Metrics.InFlyAsyncInputData->Set(0); auto it = Clients.find(ev->Sender); if (it == Clients.end()) { - LOG_ROW_DISPATCHER_ERROR("Wrong client, sender " << ev->Sender); + YDB_LOG_ERROR("Wrong client, sender", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); return; } SendData(*it->second); @@ -626,7 +659,9 @@ bool TTopicSession::HandleNewEvents() { return false; } if (Config.GetMaxSessionUsedMemory() && QueuedBytes > Config.GetMaxSessionUsedMemory()) { - LOG_ROW_DISPATCHER_TRACE("Too much used memory (" << QueuedBytes << " bytes), stop reading from yds"); + YDB_LOG_TRACE("Too much used memory bytes), stop reading from yds", + {"logPrefix", LogPrefix}, + {"queuedBytes", QueuedBytes}); break; } std::optional<NYdb::NTopic::TReadSessionEvent::TEvent> event = ReadSession->GetEvent(false); @@ -651,7 +686,8 @@ void TTopicSession::CloseTopicSession() { if (!ReadSession) { return; } - LOG_ROW_DISPATCHER_DEBUG("Close session"); + YDB_LOG_DEBUG("Close session", + {"logPrefix", LogPrefix}); ReadSession->Close(TDuration::Zero()); ReadSession.reset(); } @@ -670,13 +706,22 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionE } if (hasOldMessages) { - LOG_ROW_DISPATCHER_TRACE("Skip data. StartingMessageTimestamp: " << Self.StartingMessageTimestamp << ". Write time: " << messages.begin()->GetWriteTime()); + YDB_LOG_TRACE("Skip data. Write", + {"logPrefix", LogPrefix}, + {"startingMessageTimestamp", Self.StartingMessageTimestamp}, + {"time", messages.begin()->GetWriteTime()}); Self.LastMessageOffset = std::prev(it)->GetOffset(); messages.erase(messages.begin(), it); } for (const auto& message : messages) { - LOG_ROW_DISPATCHER_TRACE("Data received: " << message.DebugString(true)); + YDB_LOG_TRACE("Data received", + {"logPrefix", LogPrefix}, + {"data", message.DebugString(true)}); + YDB_LOG_TRACE("Data received", + {"logPrefix", LogPrefix}, + {"data", message.DebugString(true)}); + try { dataSize += message.GetData().size(); } catch (...) { @@ -694,7 +739,10 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionE void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TSessionClosedEvent& ev) { const TString message = TStringBuilder() << "Read session to topic \"" << Self.TopicPathPartition << "\" was closed"; - LOG_ROW_DISPATCHER_DEBUG(message << ": " << ev.DebugString()); + YDB_LOG_DEBUG("Dump logPrefix, message, #_ev.DebugString", + {"logPrefix", LogPrefix}, + {"message", message}, + {"ev", ev.DebugString()}); Self.ThrowFatalError(TStatus::Fail( NYql::NDq::YdbStatusToDqStatus(static_cast<Ydb::StatusIds::StatusCode>(ev.GetStatus())), @@ -703,7 +751,8 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TSessionClose } void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionEvent::TStartPartitionSessionEvent& event) { - LOG_ROW_DISPATCHER_DEBUG("StartPartitionSessionEvent received"); + YDB_LOG_DEBUG("StartPartitionSessionEvent received", + {"logPrefix", LogPrefix}); std::optional<ui64> minOffset; for (const auto& [actorId, info] : Self.Clients) { @@ -711,7 +760,9 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionE minOffset = info->NextMessageOffset; } } - LOG_ROW_DISPATCHER_DEBUG("Confirm StartPartitionSession with offset " << minOffset); + YDB_LOG_DEBUG("Confirm StartPartitionSession with offset", + {"logPrefix", LogPrefix}, + {"minOffset", minOffset}); event.Confirm(minOffset); if (minOffset) { // ensure we restart session if new client wants earlier offset @@ -722,12 +773,15 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionE } void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionEvent::TStopPartitionSessionEvent& event) { - LOG_ROW_DISPATCHER_DEBUG("SessionId: " << Self.GetSessionId() << " StopPartitionSessionEvent received"); + YDB_LOG_DEBUG("StopPartitionSessionEvent received", + {"logPrefix", LogPrefix}, + {"sessionId", Self.GetSessionId()}); event.Confirm(); } void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionEvent::TEndPartitionSessionEvent& /*event*/) { - LOG_ROW_DISPATCHER_WARN("TEndPartitionSessionEvent"); + YDB_LOG_WARN("TEndPartitionSessionEvent", + {"logPrefix", LogPrefix}); Self.ThrowFatalError(TStatus::Fail( EStatusId::SCHEME_ERROR, @@ -735,7 +789,8 @@ void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionE } void TTopicSession::TTopicEventProcessor::operator()(NYdb::NTopic::TReadSessionEvent::TPartitionSessionClosedEvent& /*event*/) { - LOG_ROW_DISPATCHER_WARN("TPartitionSessionClosedEvent"); + YDB_LOG_WARN("TPartitionSessionClosedEvent", + {"logPrefix", LogPrefix}); } TString TTopicSession::GetSessionId() const { @@ -743,7 +798,9 @@ TString TTopicSession::GetSessionId() const { } void TTopicSession::SendToParsing(const std::vector<NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage>& messages) { - LOG_ROW_DISPATCHER_TRACE("SendToParsing, messages: " << messages.size()); + YDB_LOG_TRACE("SendToParsing", + {"logPrefix", LogPrefix}, + {"messages", messages.size()}); for (const auto& [_, formatHandler] : FormatHandlers) { if (formatHandler->HasClients()) { formatHandler->ParseMessages(messages); @@ -759,14 +816,18 @@ void TTopicSession::SendData(TClientsInfo& info) { info.DataArrivedSent = false; if (buffer.empty()) { - LOG_ROW_DISPATCHER_TRACE("Buffer empty"); + YDB_LOG_TRACE("Buffer empty", + {"logPrefix", LogPrefix}); } ui64 dataSize = 0; ui64 eventsSize = info.QueuedRows; if (!info.NextMessageOffset) { - LOG_ROW_DISPATCHER_ERROR("Try SendData() without NextMessageOffset, " << info.ReadActorId - << " unread " << info.QueuedBytes << " DataArrivedSent " << info.DataArrivedSent); + YDB_LOG_ERROR("Try SendData() without NextMessageOffset, unread DataArrivedSent", + {"logPrefix", LogPrefix}, + {"readActorId", info.ReadActorId}, + {"queuedBytes", info.QueuedBytes}, + {"dataArrivedSent", info.DataArrivedSent}); return; } @@ -800,7 +861,10 @@ void TTopicSession::SendData(TClientsInfo& info) { if (buffer.empty()) { event->Record.SetNextMessageOffset(*info.NextMessageOffset); } - LOG_ROW_DISPATCHER_TRACE("SendData to " << info.ReadActorId << ", batch size " << event->Record.MessagesSize()); + YDB_LOG_TRACE("SendData to batch size", + {"logPrefix", LogPrefix}, + {"readActorId", info.ReadActorId}, + {"messagesSize", event->Record.MessagesSize()}); Send(RowDispatcherActorId, event.release()); } while(!buffer.empty()); @@ -816,11 +880,17 @@ void TTopicSession::SendData(TClientsInfo& info) { } void TTopicSession::StartClientSession(TClientsInfo& info) { - LOG_ROW_DISPATCHER_TRACE("StartClientSession, read actor id " << info.ReadActorId << ", offset " << info.GetNextMessageOffset()); + YDB_LOG_TRACE("StartClientSession, read actor id offset", + {"logPrefix", LogPrefix}, + {"readActorId", info.ReadActorId}, + {"getNextMessageOffset", info.GetNextMessageOffset()}); if (ReadSession) { auto offset = info.GetNextMessageOffset(); if (offset && offset <= LastMessageOffset) { - LOG_ROW_DISPATCHER_INFO("New client has less offset (" << offset << ") than the last message (" << LastMessageOffset << "), stop (restart) topic session"); + YDB_LOG_INFO("New client has less offset than the last message stop (restart) topic session", + {"logPrefix", LogPrefix}, + {"offset", offset}, + {"lastMessageOffset", LastMessageOffset}); Metrics.RestartSessionByOffsets->Inc(); ++RestartSessionByOffsets; info.RestartSessionByOffsetsByQuery->Inc(); @@ -840,7 +910,12 @@ void TTopicSession::StartClientSession(TClientsInfo& info) { void TTopicSession::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { auto offset = GetOffset(ev->Get()->Record); const auto& source = ev->Get()->Record.GetSource(); - LOG_ROW_DISPATCHER_INFO("New client: read actor id " << ev->Sender.ToString() << ", predicate: " << source.GetPredicate() << ", watermark expr: " << source.GetWatermarkExpr() << ", offset: " << offset); + YDB_LOG_INFO("New client: read actor id watermark", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"predicate", source.GetPredicate()}, + {"expr", source.GetWatermarkExpr()}, + {"offset", offset}); if (!CheckNewClient(ev)) { return; @@ -880,11 +955,17 @@ void TTopicSession::Handle(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { } void TTopicSession::Handle(NFq::TEvRowDispatcher::TEvStopSession::TPtr& ev) { - LOG_ROW_DISPATCHER_DEBUG("TEvStopSession from " << ev->Sender << " topicPath " << ev->Get()->Record.GetSource().GetTopicPath() << " clients count " << Clients.size()); + YDB_LOG_DEBUG("TEvStopSession from topicPath clients count", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}, + {"topicPath", ev->Get()->Record.GetSource().GetTopicPath()}, + {"clientsSize", Clients.size()}); auto it = Clients.find(ev->Sender); if (it == Clients.end()) { - LOG_ROW_DISPATCHER_WARN("Ignore TEvStopSession from " << ev->Sender << ", no client"); + YDB_LOG_WARN("Ignore TEvStopSession from no client", + {"logPrefix", LogPrefix}, + {"sender", ev->Sender}); return; } auto& info = *it->second; @@ -929,7 +1010,10 @@ void TTopicSession::RestartSessionIfOldestClient(const TClientsInfo& info) { if (info.NextMessageOffset >= minMessageOffset) { return; } - LOG_ROW_DISPATCHER_INFO("Client (on StopSession) has less offset (" << info.NextMessageOffset << ") than others clients (" << minMessageOffset << "), stop (restart) topic session"); + YDB_LOG_INFO("Client (on StopSession) has less offset than others clients stop (restart) topic session", + {"logPrefix", LogPrefix}, + {"nextMessageOffset", info.NextMessageOffset}, + {"minMessageOffset", minMessageOffset}); Metrics.RestartSessionByOffsets->Inc(); ++RestartSessionByOffsets; info.RestartSessionByOffsetsByQuery->Inc(); @@ -942,10 +1026,14 @@ void TTopicSession::RestartSessionIfOldestClient(const TClientsInfo& info) { } void TTopicSession::FatalError(const TStatus& status) { - LOG_ROW_DISPATCHER_ERROR("FatalError: " << status.GetErrorMessage()); + YDB_LOG_ERROR("", + {"logPrefix", LogPrefix}, + {"fatalError", status.GetErrorMessage()}); for (auto& [readActorId, info] : Clients) { - LOG_ROW_DISPATCHER_DEBUG("Send TEvSessionError to " << readActorId); + YDB_LOG_DEBUG("Send TEvSessionError", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}); SendSessionError(readActorId, status, true); } StopReadSession(); @@ -958,7 +1046,10 @@ void TTopicSession::ThrowFatalError(const TStatus& status) { } void TTopicSession::SendSessionError(TActorId readActorId, TStatus status, bool isFatalError) { - LOG_ROW_DISPATCHER_WARN("SendSessionError to " << readActorId << ", status: " << status.GetErrorMessage()); + YDB_LOG_WARN("SendSessionError", + {"logPrefix", LogPrefix}, + {"readActorId", readActorId}, + {"status", status.GetErrorMessage()}); auto event = std::make_unique<TEvRowDispatcher::TEvSessionError>(); event->Record.SetStatusCode(status.GetStatus()); event->Record.SetPartitionId(PartitionId); @@ -973,7 +1064,8 @@ void TTopicSession::SendSessionError(TActorId readActorId, TStatus status, bool void TTopicSession::StopReadSession() { if (ReadSession) { - LOG_ROW_DISPATCHER_DEBUG("Close read session"); + YDB_LOG_DEBUG("Close read session", + {"logPrefix", LogPrefix}); ReadSession->Close(TDuration::Zero()); ReadSession.reset(); } @@ -985,7 +1077,9 @@ void TTopicSession::SendDataArrived(TClientsInfo& info) { return; } info.DataArrivedSent = true; - LOG_ROW_DISPATCHER_TRACE("Send TEvNewDataArrived to " << info.ReadActorId); + YDB_LOG_TRACE("Send TEvNewDataArrived", + {"logPrefix", LogPrefix}, + {"readActorId", info.ReadActorId}); Metrics.InFlyAsyncInputData->Set(1); auto event = std::make_unique<TEvRowDispatcher::TEvNewDataArrived>(); event->Record.SetPartitionId(PartitionId); @@ -1001,7 +1095,8 @@ void TTopicSession::HandleException(const std::exception& e) { } void TTopicSession::SendStatistics() { - LOG_ROW_DISPATCHER_TRACE("SendStatistics"); + YDB_LOG_TRACE("SendStatistics", + {"logPrefix", LogPrefix}); TTopicSessionStatistic sessionStatistic; auto& commonStatistic = sessionStatistic.Common; commonStatistic.QueuedBytes = QueuedBytes; @@ -1046,7 +1141,8 @@ void TTopicSession::Handle(NFq::TEvPrivate::TEvSendStatistic::TPtr&) { } void TTopicSession::Handle(NFq::TEvPrivate::TEvGetEventByTimerEvent::TPtr&) { - LOG_ROW_DISPATCHER_TRACE("TEvGetEventByTimerEvent"); + YDB_LOG_TRACE("TEvGetEventByTimerEvent", + {"logPrefix", LogPrefix}); // Workaround for a partition reading bug: // In some cases, the partition may stop delivering new events due to missed notifications or lost subscriptions, // causing the session to stall and not receive further data. To address this, we periodically schedule a timer event @@ -1063,20 +1159,27 @@ void TTopicSession::Handle(NFq::TEvPrivate::TEvGetEventByTimerEvent::TPtr&) { bool TTopicSession::CheckNewClient(NFq::TEvRowDispatcher::TEvStartSession::TPtr& ev) { auto it = Clients.find(ev->Sender); if (it != Clients.end()) { - LOG_ROW_DISPATCHER_ERROR("Such a client already exists"); + YDB_LOG_ERROR("Such a client already exists", + {"logPrefix", LogPrefix}); SendSessionError(ev->Sender, TStatus::Fail(EStatusId::INTERNAL_ERROR, TStringBuilder() << "Client with id " << ev->Sender << " already exists"), false); return false; } const auto& source = ev->Get()->Record.GetSource(); if (Config.GetConsumerMode() != TRowDispatcherSettings::EConsumerMode::Without && ConsumerName && ConsumerName != source.GetConsumerName()) { - LOG_ROW_DISPATCHER_INFO("Different consumer, expected " << ConsumerName << ", actual " << source.GetConsumerName() << ", send error"); + YDB_LOG_INFO("Different consumer, expected actual send error", + {"logPrefix", LogPrefix}, + {"consumerName", ConsumerName}, + {"sourceConsumerName", source.GetConsumerName()}); SendSessionError(ev->Sender, TStatus::Fail(EStatusId::PRECONDITION_FAILED, TStringBuilder() << "Use the same consumer in all queries via RD (current consumer " << ConsumerName << ")"), false); return false; } if (SkipJsonErrors && SkipJsonErrors != source.GetSkipJsonErrors()) { - LOG_ROW_DISPATCHER_INFO("Different skip json errors mode, expected " << SkipJsonErrors << ", actual " << source.GetSkipJsonErrors() << ", send error"); + YDB_LOG_INFO("Different skip json errors mode, expected actual send error", + {"logPrefix", LogPrefix}, + {"skipJsonErrors", SkipJsonErrors}, + {"sourceSkipJsonErrors", source.GetSkipJsonErrors()}); SendSessionError(ev->Sender, TStatus::Fail(EStatusId::PRECONDITION_FAILED, TStringBuilder() << "Use the same skip json errors settings in all queries via RD (current mode " << SkipJsonErrors << ")"), false); return false; } diff --git a/ydb/core/fq/libs/row_dispatcher/ya.make b/ydb/core/fq/libs/row_dispatcher/ya.make index 6e36f7d67aa..5a4171635f9 100644 --- a/ydb/core/fq/libs/row_dispatcher/ya.make +++ b/ydb/core/fq/libs/row_dispatcher/ya.make @@ -12,7 +12,6 @@ SRCS( ) PEERDIR( - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/config/protos ydb/core/fq/libs/metrics ydb/core/fq/libs/row_dispatcher/common diff --git a/ydb/core/fq/libs/test_connection/ya.make b/ydb/core/fq/libs/test_connection/ya.make index ccc86c1ebbb..cfa347a59bd 100644 --- a/ydb/core/fq/libs/test_connection/ya.make +++ b/ydb/core/fq/libs/test_connection/ya.make @@ -13,7 +13,6 @@ PEERDIR( library/cpp/lwtrace library/cpp/xml/document ydb/core/fq/libs/actors - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/config/protos ydb/core/fq/libs/control_plane_storage ydb/core/fq/libs/test_connection/events diff --git a/ydb/core/grpc_services/rpc_topic_deferred_publish.cpp b/ydb/core/grpc_services/rpc_topic_deferred_publish.cpp index 9f7af45356a..7af2113867d 100644 --- a/ydb/core/grpc_services/rpc_topic_deferred_publish.cpp +++ b/ydb/core/grpc_services/rpc_topic_deferred_publish.cpp @@ -23,13 +23,6 @@ constexpr TStringBuf NotImplementedMessage = "Topic deferred publish is not impl constexpr TStringBuf DisabledMessage = "Topic deferred publish is not enabled"; constexpr TStringBuf AuthenticationRequiredMessage = "Authentication is required"; -TString GetSerializedUserToken(const IRequestOpCtx* request) { - if (request == nullptr) { - return {}; - } - return request->GetSerializedToken(); -} - TString GetUserSID(const IRequestOpCtx* request) { if (request == nullptr) { return BUILTIN_ACL_NO_USER_SID; @@ -423,7 +416,6 @@ public: *database, protoRequest->int_publication_id(), NPQ::NDeferredPublish::EFinalizePublicationOp::Publish, - GetSerializedUserToken(Request.get()), callerSid)); Become(&TPublishRequestActor::StateFunc); } @@ -492,7 +484,6 @@ public: *database, protoRequest->int_publication_id(), NPQ::NDeferredPublish::EFinalizePublicationOp::Cancel, - GetSerializedUserToken(Request.get()), callerSid)); Become(&TCancelPublicationRequestActor::StateFunc); } diff --git a/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp index 8b2dcd59d27..696cd100a5f 100644 --- a/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_alter_configs_actor.cpp @@ -8,6 +8,8 @@ #include <ydb/services/lib/actors/pq_schema_actor.h> #include <ydb/core/persqueue/public/schema/common.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -53,7 +55,10 @@ public: , CleanupPolicy(cleanupPolicy) , TimestampType(timestampType) { - KAFKA_LOG_D("Alter configs actor. DatabaseName: " << databaseName << ". TopicPath: " << TopicPath); + YDB_LOG_DEBUG("Alter configs actor", + {LogPrefix()}, + {"databaseName", databaseName}, + {"topicPath", TopicPath}); }; ~TAlterConfigsActor() = default; @@ -117,7 +122,9 @@ NActors::IActor* CreateKafkaAlterConfigsActor( void TKafkaAlterConfigsActor::Bootstrap(const NActors::TActorContext& ctx) { - KAFKA_LOG_D(InputLogMessage()); + YDB_LOG_DEBUG("Dump logPrefix, inputLogMessage", + {LogPrefix()}, + {"inputLogMessage", InputLogMessage()}); if (Message->ValidateOnly) { ProcessValidateOnly(ctx); @@ -267,7 +274,9 @@ void TKafkaAlterConfigsActor::ProcessValidateOnly(const NActors::TActorContext& response->Responses.push_back(responseResource); } - KAFKA_LOG_D("KLACK TKafkaAlterConfigsActor::ProcessValidateOnly: CorrelationId == " << CorrelationId); + YDB_LOG_DEBUG("KLACK TKafkaAlterConfigsActor::ProcessValidateOnly: CorrelationId", + {LogPrefix()}, + {"correlationId", CorrelationId}); Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, response, NONE_ERROR)); Die(ctx); diff --git a/ydb/core/kafka_proxy/actors/kafka_balancer_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_balancer_actor.cpp index d4b62e0a92b..1ebf566e588 100644 --- a/ydb/core/kafka_proxy/actors/kafka_balancer_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_balancer_actor.cpp @@ -1,5 +1,7 @@ #include "kafka_balancer_actor.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { using namespace NKikimr; @@ -38,7 +40,8 @@ void TKafkaBalancerActor::Handle(NMetadata::NProvider::TEvManagerPrepared::TPtr& } void TKafkaBalancerActor::Die(const TActorContext& ctx) { - KAFKA_LOG_D("Pass away."); + YDB_LOG_DEBUG("Pass away", + {LogPrefix()}); if (Kqp) { Kqp->CloseKqpSession(ctx); } @@ -96,7 +99,8 @@ void TKafkaBalancerActor::RequestFullRetry() { void TKafkaBalancerActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { if (ev->Cookie != KqpReqCookie) { - KAFKA_LOG_CRIT("Unexpected cookie in TEvQueryResponse."); + YDB_LOG_CRIT("Unexpected cookie in TEvQueryResponse", + {LogPrefix()}); return; } @@ -111,7 +115,9 @@ void TKafkaBalancerActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const } if (CurrentTxAbortRetryNumber < TX_ABORT_RETRY_MAX_COUNT) { - KAFKA_LOG_ERROR(TStringBuilder() << "Retry after tx aborted. CurrentTxAbortRetryNumber# " << static_cast<int>(CurrentTxAbortRetryNumber)); + YDB_LOG_ERROR("Retry after tx aborted", + {LogPrefix()}, + {"currentTxAbortRetryNumber", static_cast<int>(CurrentTxAbortRetryNumber)}); RequestFullRetry(); Die(ctx); return; @@ -135,7 +141,8 @@ void TKafkaBalancerActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const } void TKafkaBalancerActor::HandleResponse(NKqp::TEvKqp::TEvQueryResponse::TPtr ev, const TActorContext& ctx) { - KAFKA_LOG_I(TStringBuilder() << "Handle kqp response"); + YDB_LOG_INFO("Handle kqp response", + {LogPrefix()}); switch (RequestType) { case JOIN_GROUP: @@ -151,7 +158,8 @@ void TKafkaBalancerActor::HandleResponse(NKqp::TEvKqp::TEvQueryResponse::TPtr ev HeartbeatNextStep(ev, ctx); break; default: - KAFKA_LOG_ERROR("Unknown RequestType in TEvCreateSessionResponse"); + YDB_LOG_ERROR("Unknown RequestType in TEvCreateSessionResponse", + {LogPrefix()}); Die(ctx); break; } @@ -221,7 +229,8 @@ void TKafkaBalancerActor::JoinGroupNextStep( break; } default: - KAFKA_LOG_CRIT("JOIN_GROUP: Unexpected step" ); + YDB_LOG_CRIT("JOIN_GROUP: Unexpected step", + {LogPrefix()}); SendJoinGroupResponseFail(ctx, CorrelationId, EKafkaErrors::UNKNOWN_SERVER_ERROR, "Unexpected step"); break; } @@ -269,7 +278,8 @@ void TKafkaBalancerActor::SyncGroupNextStep( } default: { - KAFKA_LOG_CRIT("SYNC_GROUP: Unexpected step"); + YDB_LOG_CRIT("SYNC_GROUP: Unexpected step", + {LogPrefix()}); SendSyncGroupResponseFail(ctx, CorrelationId, EKafkaErrors::UNKNOWN_SERVER_ERROR, "Failed to get assignments from master"); break; } @@ -312,7 +322,8 @@ void TKafkaBalancerActor::HeartbeatNextStep( } default: { - KAFKA_LOG_CRIT("HEARTBEAT: Unexpected step"); + YDB_LOG_CRIT("HEARTBEAT: Unexpected step", + {LogPrefix()}); SendHeartbeatResponseFail(ctx, CorrelationId, EKafkaErrors::UNKNOWN_SERVER_ERROR, "Unexpected step"); break; } @@ -340,7 +351,8 @@ void TKafkaBalancerActor::LeaveGroupStep( } default: { - KAFKA_LOG_CRIT("LEAVE_GROUP: Unexpected step"); + YDB_LOG_CRIT("LEAVE_GROUP: Unexpected step", + {LogPrefix()}); SendLeaveGroupResponseFail(ctx, CorrelationId, EKafkaErrors::UNKNOWN_SERVER_ERROR, "Unexpected step"); break; } @@ -382,17 +394,18 @@ void TKafkaBalancerActor::JoinStepCreateNewOrJoinGroup(NKqp::TEvKqp::TEvQueryRes } if (groupStatus->Exists) { - KAFKA_LOG_I(TStringBuilder() << "Check group before join status." - "\n memberId: " << MemberId << - "\n instanceId: " << InstanceId << - "\n group: " << GroupId << - "\n exists: " << groupStatus->Exists << - "\n protocolType: " << groupStatus->ProtocolType << - "\n protocolName: " << groupStatus->ProtocolName << - "\n master: " << groupStatus->MasterId << - "\n generation: " << groupStatus->Generation << - "\n lastSuccessGeneration: " << groupStatus->LastSuccessGeneration << - "\n state: " << groupStatus->State); + YDB_LOG_INFO("Check group before join status", + {LogPrefix()}, + {"memberId", MemberId}, + {"instanceId", InstanceId}, + {"group", GroupId}, + {"exists", groupStatus->Exists}, + {"protocolType", groupStatus->ProtocolType}, + {"protocolName", groupStatus->ProtocolName}, + {"master", groupStatus->MasterId}, + {"generation", groupStatus->Generation}, + {"lastSuccessGeneration", groupStatus->LastSuccessGeneration}, + {"state", groupStatus->State}); } if (!groupStatus->Exists) { @@ -631,13 +644,19 @@ void TKafkaBalancerActor::JoinStepWaitMembersAndChooseProtocol(NKqp::TEvKqp::TEv ui32 memberRebalanceTimeoutMs = prevGenerationMembersAndTimeoutsIt->second.RebalanceTimeoutMs; const TInstant& memberHeartbeatDeadline = prevGenerationMembersAndTimeoutsIt->second.HeartbeatDeadline; if (AllWorkerStates.count(prevGenerationMembersAndTimeoutsIt->first) == 1) { - KAFKA_LOG_D(TStringBuilder() << "Waited member connected: " << prevGenerationMembersAndTimeoutsIt->first); + YDB_LOG_DEBUG("Waited member", + {LogPrefix()}, + {"connected", prevGenerationMembersAndTimeoutsIt->first}); prevGenerationMembersAndTimeoutsIt = WaitedMemberIdsAndTimeouts.erase(prevGenerationMembersAndTimeoutsIt); } else if ((RebalanceStartTime + TDuration::MilliSeconds(memberRebalanceTimeoutMs)) < now) { - KAFKA_LOG_D(TStringBuilder() << "Rebalance deadline: " << prevGenerationMembersAndTimeoutsIt->first); + YDB_LOG_DEBUG("Rebalance", + {LogPrefix()}, + {"deadline", prevGenerationMembersAndTimeoutsIt->first}); prevGenerationMembersAndTimeoutsIt = WaitedMemberIdsAndTimeouts.erase(prevGenerationMembersAndTimeoutsIt); } else if (memberHeartbeatDeadline < now) { - KAFKA_LOG_D(TStringBuilder() << "Waited member connect session deadline: " << prevGenerationMembersAndTimeoutsIt->first); + YDB_LOG_DEBUG("Waited member connect session", + {LogPrefix()}, + {"deadline", prevGenerationMembersAndTimeoutsIt->first}); prevGenerationMembersAndTimeoutsIt = WaitedMemberIdsAndTimeouts.erase(prevGenerationMembersAndTimeoutsIt); } else { ++prevGenerationMembersAndTimeoutsIt; @@ -645,7 +664,9 @@ void TKafkaBalancerActor::JoinStepWaitMembersAndChooseProtocol(NKqp::TEvKqp::TEv } if (WaitedMemberIdsAndTimeouts.size() != 0) { - KAFKA_LOG_D(TStringBuilder() << "Members waited count# : " << WaitedMemberIdsAndTimeouts.size()); + YDB_LOG_DEBUG("Members waited", + {LogPrefix()}, + {"count", WaitedMemberIdsAndTimeouts.size()}); WaitedMemberIdsAndTimeouts.clear(); AllWorkerStates.clear(); WorkerStatesPaginationMemberId = ""; @@ -1321,7 +1342,9 @@ NYdb::TParamsBuilder TKafkaBalancerActor::BuildAssignmentsParams() { auto& assignmentList = params.AddParam("$Assignments").BeginList(); - KAFKA_LOG_D(TStringBuilder() << "Assignments count: " << SyncGroupRequestData->Assignments.size()); + YDB_LOG_DEBUG("Assignments", + {LogPrefix()}, + {"count", SyncGroupRequestData->Assignments.size()}); for (auto& assignment: SyncGroupRequestData->Assignments) { assignmentList.AddListItem() @@ -1446,7 +1469,8 @@ void TKafkaBalancerActor::SendResponseFail(const TActorContext& ctx, EKafkaError } void TKafkaBalancerActor::SendJoinGroupResponseOk(const TActorContext& ctx, ui64 correlationId) { - KAFKA_LOG_I(TStringBuilder() << "JOIN_GROUP success."); + YDB_LOG_INFO("JOIN_GROUP success", + {LogPrefix()}); auto response = std::make_shared<TJoinGroupResponseData>(); response->ProtocolType = ProtocolType; @@ -1474,7 +1498,8 @@ void TKafkaBalancerActor::SendJoinGroupResponseOk(const TActorContext& ctx, ui64 } void TKafkaBalancerActor::SendSyncGroupResponseOk(const TActorContext& ctx, ui64 correlationId) { - KAFKA_LOG_I(TStringBuilder() << "SYNC_GROUP success."); + YDB_LOG_INFO("SYNC_GROUP success", + {LogPrefix()}); auto response = std::make_shared<TSyncGroupResponseData>(); response->ProtocolType = ProtocolType; response->ProtocolName = Protocol; @@ -1486,7 +1511,8 @@ void TKafkaBalancerActor::SendSyncGroupResponseOk(const TActorContext& ctx, ui64 } void TKafkaBalancerActor::SendLeaveGroupResponseOk(const TActorContext& ctx, ui64 corellationId) { - KAFKA_LOG_I(TStringBuilder() << "LEAVE_GROUP success."); + YDB_LOG_INFO("LEAVE_GROUP success", + {LogPrefix()}); auto response = std::make_shared<TLeaveGroupResponseData>(); response->ErrorCode = EKafkaErrors::NONE_ERROR; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, EKafkaErrors::NONE_ERROR)); @@ -1496,7 +1522,8 @@ void TKafkaBalancerActor::SendLeaveGroupResponseOk(const TActorContext& ctx, ui6 void TKafkaBalancerActor::SendHeartbeatResponseOk(const TActorContext& ctx, ui64 corellationId, EKafkaErrors error) { - KAFKA_LOG_I(TStringBuilder() << "HEARTBEAT success."); + YDB_LOG_INFO("HEARTBEAT success", + {LogPrefix()}); auto response = std::make_shared<THeartbeatResponseData>(); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, error)); @@ -1508,7 +1535,9 @@ void TKafkaBalancerActor::SendJoinGroupResponseFail(const TActorContext& ctx, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_ERROR(TStringBuilder() << "JOIN_GROUP failed. reason# " << message); + YDB_LOG_ERROR("JOIN_GROUP failed", + {LogPrefix()}, + {"reason", message}); auto response = std::make_shared<TJoinGroupResponseData>(); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, error)); @@ -1519,7 +1548,9 @@ void TKafkaBalancerActor::SendSyncGroupResponseFail(const TActorContext& ctx, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_ERROR(TStringBuilder() << "SYNC_GROUP failed. reason# " << message); + YDB_LOG_ERROR("SYNC_GROUP failed", + {LogPrefix()}, + {"reason", message}); auto response = std::make_shared<TSyncGroupResponseData>(); response->ErrorCode = error; @@ -1534,7 +1565,9 @@ void TKafkaBalancerActor::SendLeaveGroupResponseFail(const TActorContext& ctx, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_ERROR(TStringBuilder() << "LEAVE_GROUP failed. reason# " << message); + YDB_LOG_ERROR("LEAVE_GROUP failed", + {LogPrefix()}, + {"reason", message}); auto response = std::make_shared<TLeaveGroupResponseData>(); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, error)); @@ -1545,17 +1578,22 @@ void TKafkaBalancerActor::SendHeartbeatResponseFail(const TActorContext& ctx, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_ERROR(TStringBuilder() << "HEARTBEAT failed. reason# " << message); + YDB_LOG_ERROR("HEARTBEAT failed", + {LogPrefix()}, + {"reason", message}); auto response = std::make_shared<THeartbeatResponseData>(); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, error)); Die(ctx); } -TString TKafkaBalancerActor::LogPrefix() { - TStringBuilder sb; - sb << "TKafkaBalancerActor: GroupId# " << GroupId << ", MemberId# " << MemberId << ", CurrentStep# " << static_cast<int>(CurrentStep) << ". "; - return sb; +NActors::NStructuredLog::TStructuredMessage TKafkaBalancerActor::LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaBalancerActor"}, + {"selfId", SelfId()}, + {"groupId", GroupId}, + {"memberId", MemberId}, + {"currentStep", static_cast<int>(CurrentStep)}); } } // namespace NKafka diff --git a/ydb/core/kafka_proxy/actors/kafka_balancer_actor.h b/ydb/core/kafka_proxy/actors/kafka_balancer_actor.h index 6e8a4b6c11b..f4a43e9e688 100644 --- a/ydb/core/kafka_proxy/actors/kafka_balancer_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_balancer_actor.h @@ -143,7 +143,9 @@ public: InstanceId = JoinGroupRequestData->GroupInstanceId.value_or(""); MemberId = JoinGroupRequestData->MemberId.value_or(""); - KAFKA_LOG_D(TStringBuilder() << "JOIN_GROUP request. MemberId# " << MemberId); + YDB_LOG_DEBUG_COMP(NKikimrServices::KAFKA_PROXY, "JOIN_GROUP request", + {LogPrefix()}, + {"memberId", MemberId}); if (JoinGroupRequestData->SessionTimeoutMs) { SessionTimeoutMs = JoinGroupRequestData->SessionTimeoutMs; @@ -308,7 +310,7 @@ private: void SendLeaveGroupResponseFail(const TActorContext&, ui64 corellationId, EKafkaErrors error, TString message = ""); - TString LogPrefix(); + NActors::NStructuredLog::TStructuredMessage LogPrefix(); void SendResponseFail(const TActorContext& ctx, EKafkaErrors error, const TString& message); std::optional<TGroupStatus> ParseGroupState(NKqp::TEvKqp::TEvQueryResponse::TPtr ev); diff --git a/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp index cfe1360802e..ec90091f854 100644 --- a/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_create_partitions_actor.cpp @@ -8,6 +8,8 @@ #include <ydb/core/kafka_proxy/kafka_constants.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -223,10 +225,11 @@ public: databaseName) , PartionsNumber(partitionsNumber) { - KAFKA_LOG_D( - "Create partitions actor. DatabaseName: " << databaseName << - ". TopicPath: " << TopicPath << - ". PartitionsNumber: " << PartionsNumber); + YDB_LOG_DEBUG("Create partitions actor", + {LogPrefix()}, + {"databaseName", databaseName}, + {"topicPath", TopicPath}, + {"partitionsNumber", PartionsNumber}); }; void ModifyPersqueueConfig( @@ -257,7 +260,9 @@ NActors::IActor* CreateKafkaCreatePartitionsActor( } void TKafkaCreatePartitionsActor::Bootstrap(const NActors::TActorContext& ctx) { - KAFKA_LOG_D(InputLogMessage()); + YDB_LOG_DEBUG("Dump logPrefix, inputLogMessage", + {LogPrefix()}, + {"inputLogMessage", InputLogMessage()}); if (Message->ValidateOnly) { ProcessValidateOnly(ctx); diff --git a/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp index 62d858ef481..b33af01aa80 100644 --- a/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp @@ -8,6 +8,8 @@ #include <ydb/core/kafka_proxy/kafka_constants.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -58,7 +60,9 @@ NActors::IActor* CreateKafkaCreateTopicsActor( } void TKafkaCreateTopicsActor::Bootstrap(const NActors::TActorContext& ctx) { - KAFKA_LOG_D(InputLogMessage()); + YDB_LOG_DEBUG("Dump logPrefix, inputLogMessage", + {LogPrefix()}, + {"inputLogMessage", InputLogMessage()}); if (Message->ValidateOnly) { ProcessValidateOnly(ctx); @@ -168,7 +172,10 @@ void TKafkaCreateTopicsActor::Bootstrap(const NActors::TActorContext& ctx) { void TKafkaCreateTopicsActor::Handle(const NKikimr::NPQ::NSchema::TEvSchemaResponse::TPtr& ev) { auto eventPtr = ev->Release(); - KAFKA_LOG_D(TStringBuilder() << "Create topics actor. Topic's " << eventPtr->Path << " response received." << std::to_string(eventPtr->Status)); + YDB_LOG_DEBUG("Create topics actor. Topic's response received", + {LogPrefix()}, + {"path", eventPtr->Path}, + {"status", std::to_string(eventPtr->Status)}); EKafkaErrors status; switch(eventPtr->Status) { diff --git a/ydb/core/kafka_proxy/actors/kafka_describe_configs_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_describe_configs_actor.cpp index 82263c37abf..73456c8455b 100644 --- a/ydb/core/kafka_proxy/actors/kafka_describe_configs_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_describe_configs_actor.cpp @@ -5,6 +5,8 @@ #include <ydb/core/ydb_convert/topic_description.h> #include <ydb/core/ydb_convert/ydb_convert.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -105,7 +107,9 @@ void TKafkaDescribeTopicActor::HandleCacheNavigateResponse(NKikimr::TEvTxProxySc void TKafkaDescribeConfigsActor::Bootstrap(const NActors::TActorContext& ctx) { - KAFKA_LOG_D(InputLogMessage()); + YDB_LOG_DEBUG("Dump logPrefix, inputLogMessage", + {LogPrefix()}, + {"inputLogMessage", InputLogMessage()}); THashSet<TString> requestedTopics{}; diff --git a/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.cpp index 51a98a62a81..16c879adcc2 100644 --- a/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.cpp @@ -17,6 +17,8 @@ #include "kafka_describe_groups_actor.h" #include "kafka_state_name_to_int.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -40,7 +42,8 @@ void TKafkaDescribeGroupsActor::Bootstrap(const NActors::TActorContext& ctx) { new IEventHandle(SelfId(), SelfId(), wakeup.release()) ); } else { - KAFKA_LOG_ERROR("No EnableKafkaNativeBalancing FeatureFlag set."); + YDB_LOG_ERROR("No EnableKafkaNativeBalancing FeatureFlag set", + {LogPrefix()}); TDescribeGroupsResponseData groupsDescriptionResponseWithError; Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, @@ -50,7 +53,8 @@ void TKafkaDescribeGroupsActor::Bootstrap(const NActors::TActorContext& ctx) { } void TKafkaDescribeGroupsActor::Handle(NMetadata::NProvider::TEvManagerPrepared::TPtr&, const TActorContext& ctx) { - KAFKA_LOG_D("Received TEvManagerPrepared. Sending create session request to KQP."); + YDB_LOG_DEBUG("Received TEvManagerPrepared. Sending create session request to KQP", + {LogPrefix()}); InitedTablesCount++; if (InitedTablesCount == TABLES_TO_INIT_COUNT) { StartKqpSession(ctx); @@ -59,7 +63,8 @@ void TKafkaDescribeGroupsActor::Handle(NMetadata::NProvider::TEvManagerPrepared: void TKafkaDescribeGroupsActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("KQP session created"); + YDB_LOG_DEBUG("KQP session created", + {LogPrefix()}); if (!Kqp->HandleCreateSessionResponse(ev, ctx)) { SendFailResponse(EKafkaErrors::BROKER_NOT_AVAILABLE, "Failed to create KQP session"); Die(ctx); @@ -69,9 +74,11 @@ void TKafkaDescribeGroupsActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::T } void TKafkaDescribeGroupsActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Received query response from KQP DescribeGroups request"); + YDB_LOG_DEBUG("Received query response from KQP DescribeGroups request", + {LogPrefix()}); if (auto error = GetErrorFromYdbResponse(ev)) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse(EKafkaErrors::BROKER_NOT_AVAILABLE, *error); Die(ctx); return; @@ -80,14 +87,17 @@ void TKafkaDescribeGroupsActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, } void TKafkaDescribeGroupsActor::Handle(TEvents::TEvWakeup::TPtr&, const TActorContext& ctx) { - KAFKA_LOG_W("Sending fail response because of request timeout " << WAIT_REQUESTS_SECONDS << " sec"); + YDB_LOG_WARN("Sending fail response because of request timeout sec", + {LogPrefix()}, + {"timeout", WAIT_REQUESTS_SECONDS}); Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, BuildResponse(), EKafkaErrors::REQUEST_TIMED_OUT)); Die(ctx); } void TKafkaDescribeGroupsActor::Die(const TActorContext &ctx) { - KAFKA_LOG_D("Dying."); + YDB_LOG_DEBUG("Dying", + {LogPrefix()}); if (Kqp) { Kqp->CloseKqpSession(ctx); } @@ -95,15 +105,20 @@ void TKafkaDescribeGroupsActor::Die(const TActorContext &ctx) { } void TKafkaDescribeGroupsActor::StartKqpSession(const TActorContext& ctx) { - KAFKA_LOG_D("Sending create session request to KQP for database " << DatabasePath); + YDB_LOG_DEBUG("Sending create session request to KQP for database", + {LogPrefix()}, + {"databasePath", DatabasePath}); Kqp->SendCreateSessionRequest(ctx); } void TKafkaDescribeGroupsActor::HandleSelectResponse(const NKqp::TEvKqp::TEvQueryResponse& response, const TActorContext& ctx) { - KAFKA_LOG_D("Handling Select Response for DescribeGroups. SELECT result size: " << response.Record.GetResponse().GetYdbResults().size()); + YDB_LOG_DEBUG("Handling Select Response for DescribeGroups. SELECT result", + {LogPrefix()}, + {"size", response.Record.GetResponse().GetYdbResults().size()}); if (response.Record.GetResponse().GetYdbResults().size() != 2) { - TString errorMessage = TStringBuilder() << "KQP returned wrong number of result sets on SELECT query. Expected 2, got " << response.Record.GetResponse().GetYdbResults().size() << "."; - KAFKA_LOG_W(errorMessage); + YDB_LOG_WARN("KQP returned wrong number of result sets size!=2 on SELECT query", + {LogPrefix()}, + {"size", response.Record.GetResponse().GetYdbResults().size()}); return; } ParseGroupDescriptionMetadata(response); @@ -115,9 +130,11 @@ void TKafkaDescribeGroupsActor::HandleSelectResponse(const NKqp::TEvKqp::TEvQuer } void TKafkaDescribeGroupsActor::ParseGroupDescriptionMetadata(const NKqp::TEvKqp::TEvQueryResponse& response) { - KAFKA_LOG_D("Parsing Groups metadata"); + YDB_LOG_DEBUG("Parsing Groups metadata", + {LogPrefix()}); ParseGroupMetadata(response); - KAFKA_LOG_D("Parsing Members metadata"); + YDB_LOG_DEBUG("Parsing Members metadata", + {LogPrefix()}); ParseMembersMetadata(response); } @@ -181,7 +198,9 @@ NYdb::TParams TKafkaDescribeGroupsActor::BuildSelectParams() { params.AddParam("$Database").Utf8(DatabasePath).Build(); auto& groupIds = params.AddParam("$GroupIds").BeginList(); - KAFKA_LOG_D(TStringBuilder() << "Groups count: " << DescribeGroupsRequestData->Groups.size()); + YDB_LOG_DEBUG("Groups", + {LogPrefix()}, + {"count", DescribeGroupsRequestData->Groups.size()}); for (auto& groupId: DescribeGroupsRequestData->Groups) { groupIds.AddListItem().Utf8(*groupId); @@ -245,9 +264,14 @@ void TKafkaDescribeGroupsActor::SendFailResponse(EKafkaErrors errorCode, const s GroupIdToDescription[*groupId].ErrorCode = errorCode; } if (errorMessage.has_value()) { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode << ". Reason: " << errorMessage); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}, + {"reason", errorMessage}); } else { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}); } Send(Context->ConnectionId, diff --git a/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.h b/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.h index 08b93e21223..1211fdb18fe 100644 --- a/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_describe_groups_actor.h @@ -54,8 +54,11 @@ public: void Bootstrap(const NActors::TActorContext& ctx); -TStringBuilder LogPrefix() const { - return TStringBuilder() << "KafkaDescribeGroupsActor{DatabasePath=" << DatabasePath << "}: "; +NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaDescribeGroupsActor"}, + {"selfId", SelfId()}, + {"databasePath", DatabasePath}); } struct TDescribeGroupsKqpQuery { diff --git a/ydb/core/kafka_proxy/actors/kafka_fetch_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_fetch_actor.cpp index 14192f8392f..d7dc773c313 100644 --- a/ydb/core/kafka_proxy/actors/kafka_fetch_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_fetch_actor.cpp @@ -17,6 +17,8 @@ #include "actors.h" #include "kafka_fetch_actor.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -53,8 +55,11 @@ void TKafkaFetchActor::Bootstrap(const NActors::TActorContext& ctx) { void TKafkaFetchActor::SendFetchRequests(const TActorContext& ctx) { Response->Responses.resize(FetchRequestData->Topics.size()); - KAFKA_LOG_D(TStringBuilder() << "Fetch actor: New request. DatabasePath: " << Context->DatabasePath - << " MaxWaitMs: " << FetchRequestData->MaxWaitMs << " MaxBytes: " << FetchRequestData->MaxBytes); + YDB_LOG_DEBUG("Fetch actor: New request", + {LogPrefix()}, + {"databasePath", Context->DatabasePath}, + {"maxWaitMs", FetchRequestData->MaxWaitMs}, + {"maxBytes", FetchRequestData->MaxBytes}); for (size_t topicIndex = 0; topicIndex < Response->Responses.size(); topicIndex++) { auto partPQRequests = PrepareFetchRequestData(topicIndex); auto ruPerRequest = topicIndex == 0 && Context->Config.GetMeteringV2Enabled(); @@ -90,10 +95,12 @@ TVector<NKikimr::NPQ::TPartitionFetchRequest> TKafkaFetchActor::PrepareFetchRequ topicKafkaResponse.Partitions.resize(topicKafkaRequest.Partitions.size()); for (size_t partIndex = 0; partIndex < topicKafkaRequest.Partitions.size(); partIndex++) { auto& partKafkaRequest = topicKafkaRequest.Partitions[partIndex]; - KAFKA_LOG_D(TStringBuilder() << "Fetch actor: New request. Topic: " << topicKafkaRequest.Topic.value() - << " Partition: " << partKafkaRequest.Partition - << " FetchOffset: " << partKafkaRequest.FetchOffset - << " PartitionMaxBytes: " << partKafkaRequest.PartitionMaxBytes); + YDB_LOG_DEBUG("Fetch actor: New request", + {LogPrefix()}, + {"topic", topicKafkaRequest.Topic.value()}, + {"partition", partKafkaRequest.Partition}, + {"fetchOffset", partKafkaRequest.FetchOffset}, + {"partitionMaxBytes", partKafkaRequest.PartitionMaxBytes}); auto& partPQRequest = partPQRequests[partIndex]; partPQRequest.Topic = NormalizePath(Context->DatabasePath, topicKafkaRequest.Topic.value()); // FIXME(savnik): handle empty topic partPQRequest.Partition = partKafkaRequest.Partition; @@ -126,7 +133,8 @@ size_t TKafkaFetchActor::CheckTopicIndex(const NKikimr::TEvPQ::TEvFetchResponse: Y_DEBUG_ABORT_UNLESS(topicIt != TopicIndexes.end()); if (topicIt == TopicIndexes.end()) { - KAFKA_LOG_ERROR("Fetch actor: Received unexpected TEvFetchResponse. Ignoring. Expect malformed/incompled fetch reply."); + YDB_LOG_ERROR("Fetch actor: Received unexpected TEvFetchResponse. Ignoring. Expect malformed/incompled fetch reply", + {LogPrefix()}); return std::numeric_limits<size_t>::max(); } @@ -139,7 +147,11 @@ void TKafkaFetchActor::HandleErrorResponse(const NKikimr::TEvPQ::TEvFetchRespons for (auto& partitionResponse : topicResponse.Partitions) { partitionResponse.ErrorCode = code; } - KAFKA_LOG_ERROR("Fetch actor: Failed to get responses for topic: " << topicResponse.Topic << ". Code: " << ev->Get()->Status << ". Reason: " << ev->Get()->Message); + YDB_LOG_ERROR("Fetch actor: Failed to get responses", + {LogPrefix()}, + {"topic", topicResponse.Topic}, + {"code", ev->Get()->Status}, + {"reason", ev->Get()->Message}); } void TKafkaFetchActor::HandleSuccessResponse(const NKikimr::TEvPQ::TEvFetchResponse::TPtr& ev, TFetchResponseData::TFetchableTopicResponse& topicResponse, const TActorContext& ctx) { @@ -155,10 +167,12 @@ void TKafkaFetchActor::HandleSuccessResponse(const NKikimr::TEvPQ::TEvFetchRespo partKafkaResponse.ErrorCode = ConvertErrorCode(partPQResponse.GetReadResult().GetErrorCode()); if (partPQResponse.GetReadResult().GetErrorCode() != NPersQueue::NErrorCode::EErrorCode::OK) { - KAFKA_LOG_D("Fetch actor: Failed to get responses for topic: " << topicResponse.Topic << - ", partition: " << partPQResponse.GetPartition() << - ". Code: " << static_cast<size_t>(partPQResponse.GetReadResult().GetErrorCode()) << - ". Reason: " + partPQResponse.GetReadResult().GetErrorReason()); + YDB_LOG_DEBUG("Fetch actor: Failed to get responses", + {LogPrefix()}, + {"topic", topicResponse.Topic}, + {"partition", partPQResponse.GetPartition()}, + {"code", static_cast<size_t>(partPQResponse.GetReadResult().GetErrorCode())}, + {"reason", partPQResponse.GetReadResult().GetErrorReason()}); } partKafkaResponse.HighWatermark = partPQResponse.GetReadResult().GetMaxOffset(); @@ -206,9 +220,14 @@ void TKafkaFetchActor::FillRecordsBatch(const NKikimrClient::TPersQueueFetchResp //recordsBatch.Attributes https://kafka.apache.org/documentation/#recordbatch recordsBatch.BatchLength = recordsBatch.Size(TKafkaRecordBatch::MessageMeta::PresentVersions.Max) - BatchFirstTwoFieldsSize; - KAFKA_LOG_D("Fetch actor: RecordBatch info. BaseOffset: " << recordsBatch.BaseOffset << ", LastOffsetDelta: " << recordsBatch.LastOffsetDelta << - ", BaseTimestamp: " << recordsBatch.BaseTimestamp << ", MaxTimestamp: " << recordsBatch.MaxTimestamp << - ", BaseSequence: " << recordsBatch.BaseSequence << ", BatchLength: " << recordsBatch.BatchLength); + YDB_LOG_DEBUG("Fetch actor: RecordBatch info", + {LogPrefix()}, + {"baseOffset", recordsBatch.BaseOffset}, + {"lastOffsetDelta", recordsBatch.LastOffsetDelta}, + {"baseTimestamp", recordsBatch.BaseTimestamp}, + {"maxTimestamp", recordsBatch.MaxTimestamp}, + {"baseSequence", recordsBatch.BaseSequence}, + {"batchLength", recordsBatch.BatchLength}); kafkaBatchRecords += WriteKafkaRecordBatch(recordsBatch); recordsBatch = TKafkaRecordBatch(); @@ -246,15 +265,17 @@ void TKafkaFetchActor::FillRecordsBatch(const NKikimrClient::TPersQueueFetchResp const bool isKafkaBatch = dataChunk.HasCodec() && dataChunk.GetCodec() == KafkaBatchCodec(); - KAFKA_LOG_D("Fetch actor: Kafka record candidate. ResultsCount=" << partPQResponse.GetReadResult().GetResult().size() - << ", Offset=" << result.GetOffset() - << ", IsBatch=" << result.GetIsBatch() - << ", MessageCount=" << result.GetLogicalMessageCount() - << ", ChunkType=" << static_cast<int>(dataChunk.GetChunkType()) - << ", Codec=" << (dataChunk.HasCodec() ? static_cast<int>(dataChunk.GetCodec()) : -1) - << ", KafkaBatchCodec=" << static_cast<int>(KafkaBatchCodec()) - << ", DataSize=" << dataChunk.GetData().size() - << ", IsKafkaBatch=" << isKafkaBatch); + YDB_LOG_DEBUG("Fetch actor: Kafka record candidate", + {LogPrefix()}, + {"resultsCount", partPQResponse.GetReadResult().GetResult().size()}, + {"offset", result.GetOffset()}, + {"isBatch", result.GetIsBatch()}, + {"messageCount", result.GetLogicalMessageCount()}, + {"chunkType", static_cast<int>(dataChunk.GetChunkType())}, + {"codec", (dataChunk.HasCodec() ? static_cast<int>(dataChunk.GetCodec()) : -1)}, + {"kafkaBatchCodec", static_cast<int>(KafkaBatchCodec())}, + {"dataSize", dataChunk.GetData().size()}, + {"isKafkaBatch", isKafkaBatch}); if (isKafkaBatch) { flushRecordsBatch(); diff --git a/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.cpp index d9b37601029..ef280c01e53 100644 --- a/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.cpp @@ -2,6 +2,8 @@ #include <ydb/core/kafka_proxy/kafka_events.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -9,10 +11,10 @@ NActors::IActor* CreateKafkaFindCoordinatorActor(const TContext::TPtr context, c return new TKafkaFindCoordinatorActor(context, correlationId, message); } -TString TKafkaFindCoordinatorActor::LogPrefix() { - TStringBuilder sb; - sb << "TKafkaFindCoordinatorActor " << SelfId().ToString() << ": "; - return sb; +NActors::NStructuredLog::TStructuredMessage TKafkaFindCoordinatorActor::LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaFindCoordinatorActor"}, + {"selfId", SelfId()}); } void TKafkaFindCoordinatorActor::Bootstrap(const NActors::TActorContext& ctx) { @@ -34,7 +36,9 @@ void TKafkaFindCoordinatorActor::SendResponseOkAndDie(const TString& host, i32 p TFindCoordinatorResponseData::TPtr response = std::make_shared<TFindCoordinatorResponseData>(); for (auto coordinatorKey: Message->CoordinatorKeys) { - KAFKA_LOG_I("FIND_COORDINATOR incoming request for group# " << coordinatorKey); + YDB_LOG_INFO("FIND_COORDINATOR incoming request", + {LogPrefix()}, + {"group", coordinatorKey}); TFindCoordinatorResponseData::TCoordinator coordinator; coordinator.ErrorCode = NONE_ERROR; @@ -51,7 +55,11 @@ void TKafkaFindCoordinatorActor::SendResponseOkAndDie(const TString& host, i32 p response->Port = port; response->NodeId = nodeId; - KAFKA_LOG_D("FIND_COORDINATOR response. Host#: " << host << ", Port#: " << port << ", NodeId# " << nodeId); + YDB_LOG_DEBUG("FIND_COORDINATOR response", + {LogPrefix()}, + {"host", host}, + {"port", port}, + {"nodeId", nodeId}); Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, response, static_cast<EKafkaErrors>(response->ErrorCode))); Die(ctx); @@ -61,7 +69,9 @@ void TKafkaFindCoordinatorActor::SendResponseFailAndDie(EKafkaErrors error, cons TFindCoordinatorResponseData::TPtr response = std::make_shared<TFindCoordinatorResponseData>(); for (auto coordinatorKey: Message->CoordinatorKeys) { - KAFKA_LOG_CRIT("FIND_COORDINATOR request failed. Reason# " << message); + YDB_LOG_CRIT("FIND_COORDINATOR request failed", + {LogPrefix()}, + {"reason", message}); TFindCoordinatorResponseData::TCoordinator coordinator; coordinator.ErrorCode = error; diff --git a/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.h b/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.h index 81c3c588fb8..77e5c39135c 100644 --- a/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_find_coordinator_actor.h @@ -22,7 +22,7 @@ private: void SendResponseOkAndDie(const TString& host, i32 port, ui64 nodeId, const NActors::TActorContext& ctx); void SendResponseFailAndDie(EKafkaErrors error, const TString& message, const NActors::TActorContext& ctx); - TString LogPrefix(); + NActors::NStructuredLog::TStructuredMessage LogPrefix(); private: const TContext::TPtr Context; diff --git a/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.cpp index 9da00537985..a64ca707f26 100644 --- a/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.cpp @@ -21,6 +21,8 @@ #include <util/datetime/base.h> #include <regex> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { NActors::IActor* CreateKafkaInitProducerIdActor(const TContext::TPtr context, const ui64 correlationId, const TMessagePtr<TInitProducerIdRequestData>& message) { @@ -60,7 +62,8 @@ namespace NKafka { return; } Kqp = std::make_unique<TKqpTxHelper>(Context->ResourceDatabasePath); - KAFKA_LOG_D("Bootstrapping actor for transactional producer. Sending init table request to KQP."); + YDB_LOG_DEBUG("Bootstrapping actor for transactional producer. Sending init table request to KQP", + {LogPrefix()}); if (Context->ResourceDatabasePath == AppData(ctx)->TenantName) { Kqp->SendInitTableRequest(ctx, NKikimr::NGRpcProxy::V1::TTransactionalProducersInitManager::GetInstant()); } else { @@ -86,7 +89,8 @@ namespace NKafka { } void TKafkaInitProducerIdActor::Handle(NMetadata::NProvider::TEvManagerPrepared::TPtr&, const TActorContext& ctx) { - KAFKA_LOG_D("Received TEvManagerPrepared. Sending create session request to KQP."); + YDB_LOG_DEBUG("Received TEvManagerPrepared. Sending create session request to KQP", + {LogPrefix()}); Kqp->SendCreateSessionRequest(ctx); } @@ -102,7 +106,10 @@ namespace NKafka { void TKafkaInitProducerIdActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { if (ev->Cookie != KqpReqCookie) { - KAFKA_LOG_CRIT(TStringBuilder() << "Unexpected cookie in TEvQueryResponse. Expected: " << KqpReqCookie << ", Actual: " << ev->Cookie << "."); + YDB_LOG_CRIT("Unexpected cookie in TEvQueryResponse", + {LogPrefix()}, + {"expected", KqpReqCookie}, + {"actual", ev->Cookie}); SendResponseFail(EKafkaErrors::BROKER_NOT_AVAILABLE, "Failed to send request to producer_state table"); Die(ctx); return; @@ -112,7 +119,9 @@ namespace NKafka { auto status = record.GetYdbStatus(); if (status == Ydb::StatusIds::ABORTED) { if (CurrentTxAbortRetryNumber < TX_ABORT_RETRY_MAX_COUNT) { - KAFKA_LOG_ERROR(TStringBuilder() << "Retry after tx aborted. CurrentTxAbortRetryNumber# " << static_cast<int>(CurrentTxAbortRetryNumber)); + YDB_LOG_ERROR("Retry after tx aborted", + {LogPrefix()}, + {"currentTxAbortRetryNumber", static_cast<int>(CurrentTxAbortRetryNumber)}); RequestFullRetry(ctx); return; } @@ -150,7 +159,8 @@ namespace NKafka { } void TKafkaInitProducerIdActor::Die(const TActorContext& ctx) { - KAFKA_LOG_D("Pass away."); + YDB_LOG_DEBUG("Pass away", + {LogPrefix()}); if (Kqp) { Kqp->CloseKqpSession(ctx); } @@ -158,13 +168,16 @@ namespace NKafka { } void TKafkaInitProducerIdActor::StartTxProducerInitCycle(const TActorContext& ctx) { - KAFKA_LOG_D("Beginning transaction"); + YDB_LOG_DEBUG("Beginning transaction", + {LogPrefix()}); Kqp->BeginTransaction(++KqpReqCookie, ctx); LastSentToKqpRequest = EInitProducerIdKqpRequests::BEGIN_TRANSACTION; } void TKafkaInitProducerIdActor::HandleQueryResponseFromKqp(NKqp::TEvKqp::TEvQueryResponse::TPtr ev, const TActorContext& ctx) { - KAFKA_LOG_D("Handle kqp response for " << GetAsStr(LastSentToKqpRequest) << " request"); + YDB_LOG_DEBUG("Handle kqp response for request", + {LogPrefix()}, + {"lastSentToKqpRequest", GetAsStr(LastSentToKqpRequest)}); try { switch (LastSentToKqpRequest) { @@ -184,7 +197,8 @@ namespace NKafka { SendInsertRequest(ctx); break; default: - KAFKA_LOG_ERROR("Unknown EInitProducerIdKqpRequests"); + YDB_LOG_ERROR("Unknown EInitProducerIdKqpRequests", + {LogPrefix()}); Die(ctx); break; } @@ -275,14 +289,17 @@ namespace NKafka { // send responses methods void TKafkaInitProducerIdActor::SendResponseFail(EKafkaErrors error, const TString& message) { - KAFKA_LOG_ERROR(TStringBuilder() << "request failed. reason# " << message); + YDB_LOG_ERROR("Request failed", + {LogPrefix()}, + {"reason", message}); auto response = std::make_shared<TInitProducerIdResponseData>(); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, response, error)); } void TKafkaInitProducerIdActor::SendSuccessfullResponseForTxProducer(const TProducerState& producerState, const TActorContext& ctx) { - KAFKA_LOG_D("Sending succesfull response for transactional producer"); + YDB_LOG_DEBUG("Sending succesfull response for transactional producer", + {LogPrefix()}); auto response = std::make_shared<TInitProducerIdResponseData>(); response->ErrorCode = EKafkaErrors::NONE_ERROR; response->ProducerId = producerState.ProducerId; @@ -293,7 +310,8 @@ namespace NKafka { } void TKafkaInitProducerIdActor::SendSaveTxnProducerStateRequest(const TProducerState& producerState) { - KAFKA_LOG_D("Sending save txn producer state request"); + YDB_LOG_DEBUG("Sending save txn producer state request", + {LogPrefix()}); Send(NKafka::MakeTransactionsServiceID(SelfId().NodeId()), new TEvKafka::TEvSaveTxnProducerRequest( producerState.TransactionalId, @@ -370,8 +388,10 @@ namespace NKafka { ); } - TString TKafkaInitProducerIdActor::LogPrefix() { - return "InitProducerId actor: "; + NActors::NStructuredLog::TStructuredMessage TKafkaInitProducerIdActor::LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaInitProducerIdActor"}, + {"selfId", SelfId()}); } TString TKafkaInitProducerIdActor::GetAsStr(EInitProducerIdKqpRequests request) { diff --git a/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.h b/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.h index 0b6971797ad..f9c80cecd54 100644 --- a/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_init_producer_id_actor.h @@ -21,7 +21,7 @@ namespace NKafka { enum EInitProducerIdKqpRequests : ui8 { NO_REQUEST = 0, - + SELECT, INSERT, UPDATE, @@ -41,7 +41,7 @@ namespace NKafka { const ui64 CorrelationId; const std::optional<TString> TransactionalId; const std::optional<i32> TransactionTimeoutMs; - + // kqp related staff std::unique_ptr<NKafka::TKqpTxHelper> Kqp; ui64 KqpReqCookie = 0; @@ -70,7 +70,7 @@ namespace NKafka { void RequestFullRetry(const TActorContext& ctx); // event from KafkaTransactionCoordinator actor about successful or unsuccessful save of producer new state void Handle(NKafka::TEvKafka::TEvSaveTxnProducerResponse::TPtr& ev, const TActorContext& ctx); - + void Die(const TActorContext& ctx); @@ -103,7 +103,7 @@ namespace NKafka { EKafkaErrors KqpStatusToKafkaError(Ydb::StatusIds::StatusCode status); std::optional<TProducerState> ParseProducerState(NKqp::TEvKqp::TEvQueryResponse::TPtr ev); TString GetYqlWithTableName(const TString& templateStr); - TString LogPrefix(); + NActors::NStructuredLog::TStructuredMessage LogPrefix(); TString GetAsStr(EInitProducerIdKqpRequests request); }; diff --git a/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.cpp index 0c9caf508ab..73e40bf4423 100644 --- a/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.cpp @@ -16,6 +16,8 @@ #include "kafka_list_groups_actor.h" #include "kafka_state_name_to_int.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -33,7 +35,8 @@ void TKafkaListGroupsActor::Bootstrap(const NActors::TActorContext& ctx) { } Become(&TKafkaListGroupsActor::StateWork); } else { - KAFKA_LOG_ERROR("No EnableKafkaNativeBalancing FeatureFlag set."); + YDB_LOG_ERROR("No EnableKafkaNativeBalancing FeatureFlag set", + {LogPrefix()}); TListGroupsResponseData consumerGroupsResponseWithError; Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, @@ -43,12 +46,14 @@ void TKafkaListGroupsActor::Bootstrap(const NActors::TActorContext& ctx) { } void TKafkaListGroupsActor::Handle(NMetadata::NProvider::TEvManagerPrepared::TPtr&, const TActorContext& ctx) { - KAFKA_LOG_D("Received TEvManagerPrepared. Sending create session request to KQP."); + YDB_LOG_DEBUG("Received TEvManagerPrepared. Sending create session request to KQP", + {LogPrefix()}); StartKqpSession(ctx); } void TKafkaListGroupsActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("KQP session created"); + YDB_LOG_DEBUG("KQP session created", + {LogPrefix()}); if (!Kqp->HandleCreateSessionResponse(ev, ctx)) { SendFailResponse(EKafkaErrors::BROKER_NOT_AVAILABLE, "Failed to create KQP session"); Die(ctx); @@ -59,9 +64,11 @@ void TKafkaListGroupsActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& } void TKafkaListGroupsActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Received query response from KQP ListGroups request"); + YDB_LOG_DEBUG("Received query response from KQP ListGroups request", + {LogPrefix()}); if (auto error = GetErrorFromYdbResponse(ev)) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse(EKafkaErrors::BROKER_NOT_AVAILABLE, error->data()); Die(ctx); return; @@ -71,10 +78,14 @@ void TKafkaListGroupsActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, con } void TKafkaListGroupsActor::HandleSelectResponse(const NKqp::TEvKqp::TEvQueryResponse& response, const TActorContext& ctx) { - KAFKA_LOG_D("Handling Select Response " << response.Record.GetResponse().GetYdbResults().size()); + YDB_LOG_DEBUG("Handling Select Response", + {LogPrefix()}, + {"responseResults", response.Record.GetResponse().GetYdbResults().size()}); if (response.Record.GetResponse().GetYdbResults().size() != 1) { TString errorMessage = TStringBuilder() << "KQP returned wrong number of result sets on SELECT query. Expected 1, got " << response.Record.GetResponse().GetYdbResults().size() << "."; - KAFKA_LOG_W(errorMessage); + YDB_LOG_WARN("KQP returned wrong number of result sets size!=1 on SELECT query", + {LogPrefix()}, + {"responseResults", response.Record.GetResponse().GetYdbResults().size()}); SendFailResponse(EKafkaErrors::BROKER_NOT_AVAILABLE, errorMessage); Die(ctx); return; @@ -88,7 +99,8 @@ void TKafkaListGroupsActor::HandleSelectResponse(const NKqp::TEvKqp::TEvQueryRes } void TKafkaListGroupsActor::Die(const TActorContext &ctx) { - KAFKA_LOG_D("Dying."); + YDB_LOG_DEBUG("Dying", + {LogPrefix()}); if (Kqp) { Kqp->CloseKqpSession(ctx); } @@ -96,12 +108,15 @@ void TKafkaListGroupsActor::Die(const TActorContext &ctx) { } void TKafkaListGroupsActor::StartKqpSession(const TActorContext& ctx) { - KAFKA_LOG_D("Sending create session request to KQP for database " << DatabasePath); + YDB_LOG_DEBUG("Sending create session request to KQP for database", + {LogPrefix()}, + {"databasePath", DatabasePath}); Kqp->SendCreateSessionRequest(ctx); } TListGroupsResponseData TKafkaListGroupsActor::ParseGroupsMetadata(const NKqp::TEvKqp::TEvQueryResponse& response) { - KAFKA_LOG_D("Parsing KQP response"); + YDB_LOG_DEBUG("Parsing KQP response", + {LogPrefix()}); TListGroupsResponseData listGroupsResponse; NYdb::TResultSetParser parser(response.Record.GetResponse().GetYdbResults(0)); @@ -159,7 +174,9 @@ std::shared_ptr<TListGroupsResponseData> TKafkaListGroupsActor::BuildResponse(TL }; void TKafkaListGroupsActor::SendToKqpConsumerGroupsRequest(const TActorContext& ctx) { - KAFKA_LOG_W("Sending select request to KQP for database " << DatabasePath); + YDB_LOG_WARN("Sending select request to KQP for database", + {LogPrefix()}, + {"databasePath", DatabasePath}); Kqp->SendYqlRequest( GetYqlWithTableNames(ListGroupsRequestData->StatesFilter.size() > 0 ? SELECT_GROUPS_WITH_FILTER : @@ -173,9 +190,14 @@ void TKafkaListGroupsActor::SendToKqpConsumerGroupsRequest(const TActorContext& void TKafkaListGroupsActor::SendFailResponse(EKafkaErrors errorCode, const std::optional<TString>& errorMessage = std::nullopt) { if (errorMessage.has_value()) { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode << ". Reason: " << errorMessage); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}, + {"reason", errorMessage}); } else { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}); } TListGroupsResponseData consumerGroupsResponseWithError; diff --git a/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.h b/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.h index db6b2dce4e4..44714737b19 100644 --- a/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_list_groups_actor.h @@ -53,9 +53,11 @@ public: void Bootstrap(const NActors::TActorContext& ctx); - -TStringBuilder LogPrefix() const { - return TStringBuilder() << "KafkaListGroupsActor{DatabasePath=" << DatabasePath << "}: "; +NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaListGroupsActor"}, + {"selfId", SelfId()}, + {"databasePath", DatabasePath}); } private: diff --git a/ydb/core/kafka_proxy/actors/kafka_list_offsets_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_list_offsets_actor.cpp index da9019c8c8c..a9c45e08e5e 100644 --- a/ydb/core/kafka_proxy/actors/kafka_list_offsets_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_list_offsets_actor.cpp @@ -9,6 +9,8 @@ #include "kafka_list_offsets_actor.h" #include "kafka_topic_offsets_actor.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -64,7 +66,10 @@ void TKafkaListOffsetsActor::HandleMissingTopicName(const TListOffsetsRequestDat } TActorId TKafkaListOffsetsActor::SendOffsetsRequest(const TListOffsetsRequestData::TListOffsetsTopic& topic, const NActors::TActorContext&) { - KAFKA_LOG_D("ListOffsets actor: Get offsets for topic '" << topic.Name << "' for user " << GetUsernameOrAnonymous(Context)); + YDB_LOG_DEBUG("ListOffsets actor: Get offsets for topic for user", + {LogPrefix()}, + {"topicName", topic.Name}, + {"userName", GetUsernameOrAnonymous(Context)}); TEvKafka::TGetOffsetsRequest offsetsRequest; offsetsRequest.Topic = NormalizePath(Context->DatabasePath, topic.Name.value()); @@ -85,7 +90,8 @@ void TKafkaListOffsetsActor::Handle(TEvKafka::TEvTopicOffsetsResponse::TPtr& ev, Y_DEBUG_ABORT_UNLESS(it != TopicsRequestsInfo.end()); if (it == TopicsRequestsInfo.end()) { - KAFKA_LOG_CRIT("ListOffsets actor: received unexpected TEvTopicOffsetsResponse. Ignoring."); + YDB_LOG_CRIT("ListOffsets actor: received unexpected TEvTopicOffsetsResponse. Ignoring", + {LogPrefix()}); return RespondIfRequired(ctx); } @@ -104,7 +110,8 @@ void TKafkaListOffsetsActor::Handle(TEvKafka::TEvTopicOffsetsResponse::TPtr& ev, if (ev->Get()->Status == Ydb::StatusIds::SUCCESS) { auto it = responseFromPQPartitionsMap.find(partitionRequestInfo.PartitionId); if (it == responseFromPQPartitionsMap.end()) { - KAFKA_LOG_CRIT("ListOffsets actor: partition not found. Expect malformed/incompled reply"); + YDB_LOG_CRIT("ListOffsets actor: partition not found. Expect malformed/incompled reply", + {LogPrefix()}); continue; } auto& responseFromPQPartition = it->second; diff --git a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp index a6a8843c0e2..14f27c53907 100644 --- a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp @@ -8,6 +8,8 @@ #include <ydb/core/persqueue/public/list_topics/list_all_topics_actor.h> #include <ydb/services/persqueue_v1/actors/schema_actors.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { using namespace NKikimr; using namespace NKikimr::NGRpcProxy::V1; @@ -61,8 +63,11 @@ void TKafkaMetadataActor::SendDiscoveryRequest() { void TKafkaMetadataActor::HandleDiscoveryError(TEvDiscovery::TEvError::TPtr& ev) { PendingResponses--; HaveError = true; - KAFKA_LOG_ERROR("Port discovery failed for database '" << Context->DatabasePath << "' with error '" << ev->Get()->Error - << ", request " << CorrelationId); + YDB_LOG_ERROR("Port discovery failed for database with error request", + {LogPrefix()}, + {"databasePath", Context->DatabasePath}, + {"error", ev->Get()->Error}, + {"correlationId", CorrelationId}); RespondIfRequired(ActorContext()); } @@ -90,7 +95,9 @@ void TKafkaMetadataActor::ProcessDiscoveryData(TEvDiscovery::TEvDiscoveryData::T ok = leResponse.operation().result().UnpackTo(&leResult); } if (!ok) { - KAFKA_LOG_ERROR("Port discovery failed, unable to parse discovery response for request " << CorrelationId); + YDB_LOG_ERROR("Port discovery failed, unable to parse discovery response for request", + {LogPrefix()}, + {"correlationId", CorrelationId}); HaveError = true; return; } @@ -145,7 +152,10 @@ void TKafkaMetadataActor::AddTopic(const TString& topic, ui64 index) { } TActorId TKafkaMetadataActor::SendTopicRequest(const TString& topic) { - KAFKA_LOG_D("Describe partitions locations for topic '" << topic << "' for user '" << GetUsernameOrAnonymous(Context) << "'"); + YDB_LOG_DEBUG("Describe partitions locations for topic for user", + {LogPrefix()}, + {"topic", topic}, + {"userName", GetUsernameOrAnonymous(Context)}); TGetPartitionsLocationRequest locationRequest{}; locationRequest.Topic = NormalizePath(Context->DatabasePath, topic); @@ -237,12 +247,14 @@ void TKafkaMetadataActor::HandleLocationResponse(TEvLocationResponse::TPtr ev, c Y_DEBUG_ABORT_UNLESS(!actorIter->second.empty()); if (actorIter.IsEnd()) { - KAFKA_LOG_CRIT("Got unexpected location response, ignoring. Expect malformed/incompled reply"); + YDB_LOG_CRIT("Got unexpected location response, ignoring. Expect malformed/incompled reply", + {LogPrefix()}); return RespondIfRequired(ctx); } if (actorIter->second.empty()) { - KAFKA_LOG_CRIT("Corrupted state (empty actorId in mapping). Ignored location response, expect incomplete reply"); + YDB_LOG_CRIT("Corrupted state (empty actorId in mapping). Ignored location response, expect incomplete reply", + {LogPrefix()}); return RespondIfRequired(ctx); } @@ -250,20 +262,27 @@ void TKafkaMetadataActor::HandleLocationResponse(TEvLocationResponse::TPtr ev, c auto& topic = Response->Topics[index]; Ydb::StatusIds::StatusCode status = locationResponse->Status; if (status == Ydb::StatusIds::SUCCESS) { - KAFKA_LOG_D("Describe topic '" << topic.Name << "' location finishied successful"); + YDB_LOG_DEBUG("Describe topic location finishied successful", + {LogPrefix()}, + {"topicName", topic.Name}); PendingTopicResponses.emplace(index, locationResponse); } else if (status == Ydb::StatusIds::SCHEME_ERROR && Message->AllowAutoTopicCreation && Context->Config.GetAutoCreateTopicsEnable() && TopicСreationAttempts.find(*topic.Name) == TopicСreationAttempts.end() ) { - KAFKA_LOG_D("Sending create topic'" << topic.Name << "' request"); + YDB_LOG_DEBUG("Sending create topic' request", + {LogPrefix()}, + {"topicName", topic.Name}); TopicСreationAttempts.insert(*topic.Name); PendingResponses++; SendCreateTopicsRequest(*topic.Name, index, ctx); } else { - KAFKA_LOG_ERROR("Describe topic '" << topic.Name << "' location finishied with error: Code=" - << locationResponse->Status << ", Issues=" << locationResponse->Issues.ToOneLineString()); + YDB_LOG_ERROR("Describe topic location finishied with error", + {LogPrefix()}, + {"topicName", topic.Name}, + {"code", locationResponse->Status}, + {"issues", locationResponse->Issues.ToOneLineString()}); AddTopicError(topic, ConvertErrorCode(locationResponse->Status)); } } @@ -352,7 +371,9 @@ void TKafkaMetadataActor::RespondIfRequired(const TActorContext& ctx) { auto topicNodes = CheckTopicNodes(ev.Get()); if (topicNodes.empty()) { // Already tried YDB discovery. Throw error - KAFKA_LOG_ERROR("Could not discovery kafka port for topic '" << topic.Name); + YDB_LOG_ERROR("Could not discovery kafka port for topic", + {LogPrefix()}, + {"topicName", topic.Name}); AddTopicError(topic, EKafkaErrors::LISTENER_NOT_FOUND); } else { AddTopicResponse(topic, ev.Get(), topicNodes); @@ -371,8 +392,10 @@ void TKafkaMetadataActor::RespondIfRequired(const TActorContext& ctx) { Respond(); } -TString TKafkaMetadataActor::LogPrefix() const { - return TStringBuilder() << "TKafkaMetadataActor " << SelfId() << " "; +NStructuredLog::TStructuredMessage TKafkaMetadataActor::LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaMetadataActor"}, + {"selfId", SelfId()}); } } // namespace NKafka diff --git a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h index 176881707e1..d7ac9c8f928 100644 --- a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h @@ -73,7 +73,8 @@ private: } } - TString LogPrefix() const; + + NStructuredLog::TStructuredMessage LogPrefix() const; private: const TContext::TPtr Context; diff --git a/ydb/core/kafka_proxy/actors/kafka_metrics_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_metrics_actor.cpp index 91ca26c7e3d..f8b830cf8e8 100644 --- a/ydb/core/kafka_proxy/actors/kafka_metrics_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_metrics_actor.cpp @@ -24,8 +24,10 @@ namespace NKafka { TBase::Become(&TKafkaMetricsActor::StateWork); } - TStringBuilder LogPrefix() const { - return {}; + NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaMetricsActor"}, + {"selfId", SelfId()}); } private: diff --git a/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.cpp index 658098285f1..6ff8facfb1c 100644 --- a/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.cpp @@ -1,5 +1,7 @@ #include "kafka_offset_commit_actor.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -7,12 +9,15 @@ NActors::IActor* CreateKafkaOffsetCommitActor(const TContext::TPtr context, cons return new TKafkaOffsetCommitActor(context, correlationId, message); } -TString TKafkaOffsetCommitActor::LogPrefix() { - return "TKafkaOffsetCommitActor"; +NActors::NStructuredLog::TStructuredMessage TKafkaOffsetCommitActor::LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaOffsetCommitActor"}, + {"selfId", SelfId()}); } void TKafkaOffsetCommitActor::Die(const TActorContext& ctx) { - KAFKA_LOG_D("PassAway"); + YDB_LOG_DEBUG("PassAway", + {LogPrefix()}); ctx.Send(AuthInitActor, new TEvents::TEvPoisonPill()); for (const auto& tabletToPipePair: TabletIdToPipe) { NTabletPipe::CloseClient(ctx, tabletToPipePair.second); @@ -24,7 +29,9 @@ void TKafkaOffsetCommitActor::Die(const TActorContext& ctx) { } void TKafkaOffsetCommitActor::Handle(NKikimr::NGRpcProxy::V1::TEvPQProxy::TEvCloseSession::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_CRIT("Auth failed. reason# " << ev->Get()->Reason); + YDB_LOG_CRIT("Auth failed", + {LogPrefix()}, + {"reason", ev->Get()->Reason}); Error = ConvertErrorCode(ev->Get()->ErrorCode); if (Error == GROUP_ID_NOT_FOUND && Context->Config.GetAutoCreateConsumersEnable()) { for (auto topicReq: Message->Topics) { @@ -91,9 +98,13 @@ void TKafkaOffsetCommitActor::SendFailedForAllPartitions(EKafkaErrors error, con void TKafkaOffsetCommitActor::Handle(NKikimr::NReplication::TEvYdbProxy::TEvAlterTopicResponse::TPtr& ev, const TActorContext& ctx) { NYdb::TStatus& result = ev->Get()->Result; if (result.GetStatus() == NYdb::EStatus::SUCCESS) { - KAFKA_LOG_D("Handling TEvAlterTopicResponse. Status: " << result.GetStatus() << "\n"); + YDB_LOG_DEBUG("Handling TEvAlterTopicResponse. \n", + {LogPrefix()}, + {"status", result.GetStatus()}); } else { - KAFKA_LOG_I("Handling TEvAlterTopicResponse. Status: " << result.GetStatus() << "\n"); + YDB_LOG_INFO("Handling TEvAlterTopicResponse. \n", + {LogPrefix()}, + {"status", result.GetStatus()}); } PendingResponses--; if (result.GetStatus() != NYdb::EStatus::ALREADY_EXISTS && result.GetStatus() != NYdb::EStatus::SUCCESS) { @@ -108,13 +119,16 @@ void TKafkaOffsetCommitActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev TEvTabletPipe::TEvClientConnected *msg = ev->Get(); if (msg->Status != NKikimrProto::OK) { - KAFKA_LOG_CRIT("Pipe to tablet is dead. status# " << ev->Get()->Status); + YDB_LOG_CRIT("Pipe to tablet is dead", + {LogPrefix()}, + {"status", ev->Get()->Status}); ProcessPipeProblem(msg->TabletId, ctx); } } void TKafkaOffsetCommitActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_CRIT("Pipe to tablet is destroyed"); + YDB_LOG_CRIT("Pipe to tablet is destroyed", + {LogPrefix()}); ProcessPipeProblem(ev->Get()->TabletId, ctx); } @@ -134,8 +148,10 @@ void TKafkaOffsetCommitActor::ProcessPipeProblem(ui64 tabletId, const TActorCont } void TKafkaOffsetCommitActor::SendGenerationCheckRequest(const TActorContext& ctx) { - KAFKA_LOG_D("Sending generation check KQP request for group# " << Message->GroupId.value() - << ", generationId# " << Message->GenerationId); + YDB_LOG_DEBUG("Sending generation check KQP request", + {LogPrefix()}, + {"group", Message->GroupId.value()}, + {"generationId", Message->GenerationId}); NYdb::TParamsBuilder params; params.AddParam("$ConsumerGroup").Utf8(*Message->GroupId).Build(); @@ -150,9 +166,10 @@ void TKafkaOffsetCommitActor::SendGenerationCheckRequest(const TActorContext& ct void TKafkaOffsetCommitActor::Handle(NKikimr::NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; if (record.GetYdbStatus() != Ydb::StatusIds::SUCCESS) { - KAFKA_LOG_CRIT("Generation check KQP query failed." - << " group# " << Message->GroupId.value() - << " status# " << record.GetYdbStatus()); + YDB_LOG_CRIT("Generation check KQP query failed", + {LogPrefix()}, + {"group", Message->GroupId.value()}, + {"status", record.GetYdbStatus()}); Error = UNKNOWN_SERVER_ERROR; SendFailedForAllPartitions(Error, ctx); return; @@ -174,22 +191,28 @@ void TKafkaOffsetCommitActor::Handle(NKikimr::NKqp::TEvKqp::TEvQueryResponse::TP auto tableGeneration = parser.ColumnParser("generation").GetUint64(); if (tableGeneration != static_cast<ui64>(Message->GenerationId)) { - KAFKA_LOG_I("Generation mismatch for group# " << Message->GroupId.value() - << ". Expected# " << Message->GenerationId - << ", got# " << tableGeneration); + YDB_LOG_INFO("Generation mismatch", + {LogPrefix()}, + {"group", Message->GroupId.value()}, + {"expected", Message->GenerationId}, + {"got", tableGeneration}); Error = ILLEGAL_GENERATION; SendFailedForAllPartitions(Error, ctx); return; } - KAFKA_LOG_D("Generation check passed for group# " << Message->GroupId.value() - << ", generation# " << tableGeneration); + YDB_LOG_DEBUG("Generation check passed", + {LogPrefix()}, + {"group", Message->GroupId.value()}, + {"generation", tableGeneration}); SendCommits(ctx); } void TKafkaOffsetCommitActor::Handle(NGRpcProxy::V1::TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Auth success. Topics count: " << ev->Get()->TopicAndTablets.size()); + YDB_LOG_DEBUG("Auth success. Topics", + {LogPrefix()}, + {"count", ev->Get()->TopicAndTablets.size()}); TopicAndTablets = std::move(ev->Get()->TopicAndTablets); if (Message->GenerationId == -1) { @@ -203,7 +226,8 @@ void TKafkaOffsetCommitActor::Handle(NGRpcProxy::V1::TEvPQProxy::TEvAuthResultOk void TKafkaOffsetCommitActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { if (!Kqp->HandleCreateSessionResponse(ev, ctx)) { - KAFKA_LOG_ERROR("Failed to create KQP session"); + YDB_LOG_ERROR("Failed to create KQP session", + {LogPrefix()}); Error = EKafkaErrors::UNKNOWN_SERVER_ERROR; SendFailedForAllPartitions(Error, ctx); return; @@ -256,10 +280,12 @@ void TKafkaOffsetCommitActor::SendCommits(const TActorContext& ctx) { } PendingResponses++; - KAFKA_LOG_D("Send commit request for group# " << Message->GroupId.value() << - ", topic# " << topicIt->second.TopicNameConverter->GetPrimaryPath() << - ", partition# " << partitionRequest.PartitionIndex << - ", offset# " << partitionRequest.CommittedOffset); + YDB_LOG_DEBUG("Send commit request", + {LogPrefix()}, + {"group", Message->GroupId.value()}, + {"topic", topicIt->second.TopicNameConverter->GetPrimaryPath()}, + {"partition", partitionRequest.PartitionIndex}, + {"offset", partitionRequest.CommittedOffset}); TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); req->Record.Swap(&request); @@ -278,7 +304,10 @@ void TKafkaOffsetCommitActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const requestInfo->second.Done = true; if (ev->Get()->Record.GetErrorCode() != NPersQueue::NErrorCode::OK) { - KAFKA_LOG_CRIT("Commit offset error. status# " << EErrorCode_Name(ev->Get()->Record.GetErrorCode()) << ", reason# " << ev->Get()->Record.GetErrorReason()); + YDB_LOG_CRIT("Commit offset error", + {LogPrefix()}, + {"status", EErrorCode_Name(ev->Get()->Record.GetErrorCode())}, + {"reason", ev->Get()->Record.GetErrorReason()}); } AddPartitionResponse(ConvertErrorCode(NGRpcProxy::V1::ConvertOldCode(ev->Get()->Record.GetErrorCode())), requestInfo->second.TopicName, requestInfo->second.PartitionId, ctx); @@ -330,7 +359,8 @@ void TKafkaOffsetCommitActor::SendAuthRequest(const NActors::TActorContext& ctx) auto topicsToConverter = topicHandler->GetReadTopicsList(topicsToResolve, false, Context->DatabasePath); if (!topicsToConverter.IsValid) { - KAFKA_LOG_CRIT("Commit offsets failed. reason# topicsToConverter is not valid"); + YDB_LOG_CRIT("Commit offsets failed. topicsToConverter is not valid", + {LogPrefix()}); Error = INVALID_REQUEST; SendFailedForAllPartitions(Error, ctx); return; diff --git a/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.h b/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.h index 1f9992243f4..7178d4716ff 100644 --- a/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_offset_commit_actor.h @@ -53,11 +53,13 @@ public: void Bootstrap(const NActors::TActorContext& ctx); private: - TString LogPrefix(); + NActors::NStructuredLog::TStructuredMessage LogPrefix(); void Die(const TActorContext& ctx) override; STATEFN(StateWork) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(NGRpcProxy::V1::TEvPQProxy::TEvAuthResultOk, Handle); HFunc(TEvPersQueue::TEvResponse, Handle); diff --git a/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.cpp index 4ffbde3fc8f..518cb6d783a 100644 --- a/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.cpp @@ -10,6 +10,8 @@ #include "ydb/services/persqueue_v1/actors/schema_actors.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { NKikimr::NGRpcProxy::V1::TDescribeTopicActorSettings ConsumerOffsetSettings(std::shared_ptr<TSet<TString>> consumers, std::shared_ptr<TSet<ui32>>& partitions) { NKikimr::NGRpcProxy::V1::TDescribeTopicActorSettings settings { @@ -58,14 +60,18 @@ class TTopicOffsetActor: public NKikimr::NGRpcProxy::V1::TPQInternalSchemaActor< void Bootstrap(const NActors::TActorContext& ctx) override { Y_UNUSED(ctx); - KAFKA_LOG_D("Get commited offsets for topic '" << OriginalTopicName - << "' for user '" << UserSID << "'"); + YDB_LOG_DEBUG("Get commited offsets for topic for user", + {LogPrefix()}, + {"originalTopicName", OriginalTopicName}, + {"userSID", UserSID}); SendDescribeProposeRequest(); Become(&TTopicOffsetActor::StateWork); }; - TStringBuilder LogPrefix() const { - return TStringBuilder() << "KafkaTopicOffsetActor: "; + NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaTopicOffsetActor"}, + {"selfId", SelfId()}); } void StateWork(TAutoPtr<IEventHandle>& ev) { @@ -84,11 +90,13 @@ class TTopicOffsetActor: public NKikimr::NGRpcProxy::V1::TPQInternalSchemaActor< const TActorContext& ctx) override { Y_UNUSED(ctx); - KAFKA_LOG_D("Error raised for '" << OriginalTopicName << "'" - << " for user '" << UserSID << "'." - << " Error: '" << error << "'," - << " ErrorCode: '" << static_cast<int>(errorCode) << "'," - << " StatusCode: '" << status<< "'."); + YDB_LOG_DEBUG("Error raised for for user Error: ErrorCode: StatusCode", + {LogPrefix()}, + {"originalTopicName", OriginalTopicName}, + {"userSID", UserSID}, + {"error", error}, + {"errorCode", static_cast<int>(errorCode)}, + {"status", status}); THolder<TEvKafka::TEvCommitedOffsetsResponse> response(new TEvKafka::TEvCommitedOffsetsResponse()); response->TopicName = OriginalTopicName; @@ -142,8 +150,11 @@ class TTopicOffsetActor: public NKikimr::NGRpcProxy::V1::TPQInternalSchemaActor< void HandleCacheNavigateResponse(NKikimr::TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) override { const auto& response = ev->Get()->Request.Get()->ResultSet.front(); - KAFKA_LOG_D("TEvNavigateKeySetResult received for topic '" << OriginalTopicName - << "' for user '" << UserSID << "'. PQGroupInfo is present: " << (response.PQGroupInfo.Get() != nullptr)); + YDB_LOG_DEBUG("TEvNavigateKeySetResult received for topic for user PQGroupInfo is", + {LogPrefix()}, + {"originalTopicName", OriginalTopicName}, + {"userSID", UserSID}, + {"present", (response.PQGroupInfo.Get() != nullptr)}); if (!response.PQGroupInfo) { THolder<TEvKafka::TEvCommitedOffsetsResponse> response(new TEvKafka::TEvCommitedOffsetsResponse()); response->TopicName = OriginalTopicName; @@ -173,8 +184,10 @@ class TTopicOffsetActor: public NKikimr::NGRpcProxy::V1::TPQInternalSchemaActor< } void Reply(const TActorContext& ctx) override { - KAFKA_LOG_D("Replying for topic '" << OriginalTopicName - << "' for user '" << UserSID << "'" << " with status NONE_ERROR"); + YDB_LOG_DEBUG("Replying for topic for user with status NONE_ERROR", + {LogPrefix()}, + {"originalTopicName", OriginalTopicName}, + {"userSID", UserSID}); THolder<TEvKafka::TEvCommitedOffsetsResponse> response(new TEvKafka::TEvCommitedOffsetsResponse()); response->TopicName = OriginalTopicName; response->Status = NONE_ERROR; @@ -199,7 +212,9 @@ NActors::IActor* CreateKafkaOffsetFetchActor(const TContext::TPtr context, const void TKafkaOffsetFetchActor::Bootstrap(const NActors::TActorContext& ctx) { // If API level <= 7, Groups would be empty. In this case we convert message to level 8 and process it uniformely later - KAFKA_LOG_D("New request for user " << GetUsernameOrAnonymous(Context)); + YDB_LOG_DEBUG("New request for user", + {LogPrefix()}, + {"userName", GetUsernameOrAnonymous(Context)}); if (Message->Groups.empty()) { TOffsetFetchRequestData::TOffsetFetchRequestGroup group; @@ -230,7 +245,8 @@ void TKafkaOffsetFetchActor::Bootstrap(const NActors::TActorContext& ctx) { // topics for such groups will be retrieved from the table Kqp = std::make_unique<TKqpTxHelper>(Context->ResourceDatabasePath); Kqp->SendCreateSessionRequest(ctx); - KAFKA_LOG_D("Creating KQP Session"); + YDB_LOG_DEBUG("Creating KQP Session", + {LogPrefix()}); } else { FillMapWithGroupRequests(); for (const auto& topicToEntities : TopicToEntities) { @@ -334,9 +350,13 @@ void TKafkaOffsetFetchActor::Handle(const TEvKafka::TEvResponse::TPtr& ev, const void TKafkaOffsetFetchActor::Handle(NKikimr::NReplication::TEvYdbProxy::TEvAlterTopicResponse::TPtr& ev, const TActorContext& ctx) { NYdb::TStatus& result = ev->Get()->Result; if (result.GetStatus() == NYdb::EStatus::SUCCESS) { - KAFKA_LOG_D("Handling TEvAlterTopicResponse. Status: " << result.GetStatus() << "\n"); + YDB_LOG_DEBUG("Handling TEvAlterTopicResponse. \n", + {LogPrefix()}, + {"status", result.GetStatus()}); } else { - KAFKA_LOG_I("Handling TEvAlterTopicResponse. Status: " << result.GetStatus() << "\n"); + YDB_LOG_INFO("Handling TEvAlterTopicResponse. \n", + {LogPrefix()}, + {"status", result.GetStatus()}); } if (result.GetStatus() != NYdb::EStatus::SUCCESS) { InflyTopics--; @@ -369,10 +389,12 @@ void TKafkaOffsetFetchActor::Handle(NKikimr::NReplication::TEvYdbProxy::TEvAlter } void TKafkaOffsetFetchActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Got KQP CreateSession response"); + YDB_LOG_DEBUG("Got KQP CreateSession response", + {LogPrefix()}); if (!Kqp->HandleCreateSessionResponse(ev, ctx)) { Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, std::make_shared<TOffsetFetchResponseData>(), EKafkaErrors::UNKNOWN_SERVER_ERROR)); - KAFKA_LOG_D("KQP Session Error"); + YDB_LOG_DEBUG("KQP Session Error", + {LogPrefix()}); return; } NYdb::TParamsBuilder params = BuildFetchAssignmentsParams(GroupsToFetch); @@ -381,12 +403,15 @@ void TKafkaOffsetFetchActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr void NKafka::TKafkaOffsetFetchActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { std::vector<std::pair<std::optional<TString>, TConsumerProtocolAssignment>> assignments; - KAFKA_LOG_D("Received KQP response"); + YDB_LOG_DEBUG("Received KQP response", + {LogPrefix()}); ParseGroupsAssignments(ev, assignments); if (assignments.empty()) { auto response = GetOffsetFetchResponse(); - KAFKA_LOG_D("Sending response to user " << GetUsernameOrAnonymous(Context)); + YDB_LOG_DEBUG("Sending response to user", + {LogPrefix()}, + {"userName", GetUsernameOrAnonymous(Context)}); Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, response, static_cast<EKafkaErrors>(response->ErrorCode))); Die(ctx); return; @@ -586,11 +611,17 @@ TOffsetFetchResponseData::TOffsetFetchResponseGroup::TOffsetFetchResponseTopics partition.ErrorCode = NONE_ERROR; } else { partition.ErrorCode = RESOURCE_NOT_FOUND; - KAFKA_LOG_ERROR("Group " << groupId << " not found for topic " << topicName); + YDB_LOG_ERROR("Group not found for topic", + {LogPrefix()}, + {"groupId", groupId}, + {"topicName", topicName}); } } else { partition.ErrorCode = RESOURCE_NOT_FOUND; - KAFKA_LOG_ERROR("Partition " << requestPartition << " not found for topic " << topicName); + YDB_LOG_ERROR("Partition not found for topic", + {LogPrefix()}, + {"requestPartition", requestPartition}, + {"topicName", topicName}); } topic.Partitions.push_back(partition); } @@ -627,7 +658,8 @@ void TKafkaOffsetFetchActor::FillMapWithGroupRequests() { } void NKafka::TKafkaOffsetFetchActor::Die(const TActorContext &ctx) { - KAFKA_LOG_D("Dying."); + YDB_LOG_DEBUG("Dying", + {LogPrefix()}); for (const TActorId& actorId : DependantActors) { Send(actorId, new TEvents::TEvPoisonPill()); } diff --git a/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.h b/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.h index b7eeb7dbd14..d76036ae710 100644 --- a/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_offset_fetch_actor.h @@ -99,8 +99,12 @@ public: void ReplyError(const TActorContext& ctx); void Die(const TActorContext &ctx); - TStringBuilder LogPrefix() const { - return TStringBuilder() << "TKafkaOffsetFetchActor{GroupId=" << Message->GroupId.value() << ",DatabasePath=" << DatabasePath << "}: "; + NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaOffsetFetchActor"}, + {"selfId", SelfId()}, + {"groupId", Message->GroupId.value()}, + {"databasePath", DatabasePath}); } private: diff --git a/ydb/core/kafka_proxy/actors/kafka_produce_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_produce_actor.cpp index 416f0576899..697bb562952 100644 --- a/ydb/core/kafka_proxy/actors/kafka_produce_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_produce_actor.cpp @@ -13,6 +13,8 @@ #include <limits> #include <util/string/join.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { namespace { @@ -46,14 +48,18 @@ bool CanStoreRawRecords(const TKafkaBatchHeader& header, bool batchingEnabled) { EKafkaErrors ValidateBatchCompression(const TKafkaBatchHeader& header, bool batchingEnabled) { if (!IsRecordBatchV2OrNewer(header) && !IsRawRecordBatch(header)) { - KAFKA_LOG_ERROR("Compressed legacy Kafka record batches are not supported. Magic=" - << header.Magic << ", compressionType=" << static_cast<int>(CompressionType(header))); + YDB_LOG_ERROR("Compressed legacy Kafka record batches are not supported", + {LogPrefix()}, + {"magic", header.Magic}, + {"compressionType", static_cast<int>(CompressionType(header))}); return EKafkaErrors::UNSUPPORTED_COMPRESSION_TYPE; } if (!batchingEnabled && !IsRawRecordBatch(header)) { - KAFKA_LOG_ERROR("Compressed Kafka record batches are not supported when topic messages batching is disabled. Magic=" - << header.Magic << ", compressionType=" << static_cast<int>(CompressionType(header))); + YDB_LOG_ERROR("Compressed Kafka record batches are not supported when topic messages batching is disabled", + {LogPrefix()}, + {"magic", header.Magic}, + {"compressionType", static_cast<int>(CompressionType(header))}); return EKafkaErrors::UNSUPPORTED_COMPRESSION_TYPE; } @@ -110,22 +116,28 @@ NKikimrPQClient::TDataChunk MakeDataChunk(const TKafkaRecord& record) { return proto; } -TString TKafkaProduceActor::LogPrefix() { - TStringBuilder sb; - sb << "TKafkaProduceActor " << SelfId() << " State: "; +NStructuredLog::TStructuredMessage TKafkaProduceActor::LogPrefix() { + TString state; + auto stateFunc = CurrentStateFunc(); if (stateFunc == &TKafkaProduceActor::StateInit) { - sb << "Init "; + state = "Init"; } else if (stateFunc == &TKafkaProduceActor::StateWork) { - sb << "Work "; + state = "Work"; } else { - sb << "Unknown "; + state = "Unknown"; } - return sb; + + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaProduceActor"}, + {"selfId", SelfId()}, + {"state", state}); } void TKafkaProduceActor::LogEvent(IEventHandle& ev) { - KAFKA_LOG_T("Produce actor: Received event: " << ev.GetTypeName()); + YDB_LOG_TRACE("Produce actor: Received", + {LogPrefix()}, + {"event", ev.GetTypeName()}); } void TKafkaProduceActor::SendMetrics(const TString& topicName, size_t delta, const TString& name, const TActorContext& ctx) { @@ -147,7 +159,8 @@ void TKafkaProduceActor::Bootstrap(const NActors::TActorContext& /*ctx*/) { } void TKafkaProduceActor::Handle(TEvKafka::TEvWakeup::TPtr /*request*/, const TActorContext& ctx) { - KAFKA_LOG_T("Produce actor: Wakeup"); + YDB_LOG_TRACE("Produce actor: Wakeup", + {LogPrefix()}); SendResults(ctx); CleanTopics(ctx); @@ -155,11 +168,13 @@ void TKafkaProduceActor::Handle(TEvKafka::TEvWakeup::TPtr /*request*/, const TAc Schedule(WAKEUP_INTERVAL, new TEvKafka::TEvWakeup()); - KAFKA_LOG_T("Produce actor: Wakeup was completed successfully"); + YDB_LOG_TRACE("Produce actor: Wakeup was completed successfully", + {LogPrefix()}); } void TKafkaProduceActor::PassAway() { - KAFKA_LOG_D("Produce actor: PassAway"); + YDB_LOG_DEBUG("Produce actor: PassAway", + {LogPrefix()}); for(const auto& [_, partitionWriters] : NonTransactionalWriters) { for(const auto& [_, w] : partitionWriters) { @@ -174,7 +189,8 @@ void TKafkaProduceActor::PassAway() { TActorBootstrapped::PassAway(); - KAFKA_LOG_T("Produce actor: PassAway was completed successfully"); + YDB_LOG_TRACE("Produce actor: PassAway was completed successfully", + {LogPrefix()}); } void TKafkaProduceActor::CleanTopics(const TActorContext& ctx) { @@ -190,7 +206,8 @@ void TKafkaProduceActor::CleanTopics(const TActorContext& ctx) { } void TKafkaProduceActor::CleanWriters(const TActorContext& ctx) { - KAFKA_LOG_T("Produce actor: CleanWriters"); + YDB_LOG_TRACE("Produce actor: CleanWriters", + {LogPrefix()}); const auto earliestAllowedTs = ctx.Now() - WRITER_EXPIRATION_INTERVAL; for (auto& [topicPath, partitionWriters] : NonTransactionalWriters) { @@ -213,11 +230,15 @@ void TKafkaProduceActor::CleanWriters(const TActorContext& ctx) { } } - KAFKA_LOG_T("Produce actor: CleanWriters was completed successfully"); + YDB_LOG_TRACE("Produce actor: CleanWriters was completed successfully", + {LogPrefix()}); } void TKafkaProduceActor::CleanWriter(const TTopicPartition& topicPartition, const TActorId& writerId) { - KAFKA_LOG_D("Produce actor: Destroing inactive PartitionWriter. Topic='" << topicPartition.TopicPath << "', Partition=" << topicPartition.PartitionId); + YDB_LOG_DEBUG("Produce actor: Destroing inactive PartitionWriter. Topic='", + {LogPrefix()}, + {"topicPath", topicPartition.TopicPath}, + {"partition", topicPartition.PartitionId}); Send(writerId, new TEvents::TEvPoison()); } @@ -231,7 +252,9 @@ void TKafkaProduceActor::HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResu for (auto& info : navigate->ResultSet) { if (NSchemeCache::TSchemeCacheNavigate::EStatus::Ok == info.Status) { auto topicPath = CanonizePath(NKikimr::JoinPath(info.Path)); - KAFKA_LOG_D("Produce actor: Received topic '" << topicPath << "' description"); + YDB_LOG_DEBUG("Produce actor: Received topic description", + {LogPrefix()}, + {"topicPath", topicPath}); TopicsForInitialization.erase(topicPath); auto& topic = Topics[topicPath]; @@ -242,7 +265,9 @@ void TKafkaProduceActor::HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResu topic.ExpirationTime = now + TOPIC_OK_EXPIRATION_INTERVAL; topic.PartitionChooser = CreatePartitionChooser(info.PQGroupInfo->Description); } else { - KAFKA_LOG_W("Produce actor: Unauthorized PRODUCE to topic '" << topicPath << "'"); + YDB_LOG_WARN("Produce actor: Unauthorized PRODUCE to topic", + {LogPrefix()}, + {"topicPath", topicPath}); topic.Status = UNAUTHORIZED; topic.ExpirationTime = now + TOPIC_UNATHORIZED_EXPIRATION_INTERVAL; } @@ -254,7 +279,9 @@ void TKafkaProduceActor::HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResu } for(auto& topicPath : TopicsForInitialization) { - KAFKA_LOG_D("Produce actor: Topic '" << topicPath << "' not found"); + YDB_LOG_DEBUG("Produce actor: Topic not found", + {LogPrefix()}, + {"topicPath", topicPath}); auto& topicInfo = Topics[topicPath]; topicInfo.Status = NOT_FOUND; topicInfo.ExpirationTime = now + TOPIC_NOT_FOUND_EXPIRATION_INTERVAL; @@ -264,14 +291,17 @@ void TKafkaProduceActor::HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResu Become(&TKafkaProduceActor::StateWork); - KAFKA_LOG_T("Produce actor: HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr) was completed successfully"); + YDB_LOG_TRACE("Produce actor: HandleInit(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr) was completed successfully", + {LogPrefix()}); ProcessRequests(ctx); } void TKafkaProduceActor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TPtr& ev, const TActorContext& ctx) { auto& path = ev->Get()->Path; - KAFKA_LOG_I("Produce actor: Topic '" << path << "' was deleted"); + YDB_LOG_INFO("Produce actor: Topic was deleted", + {LogPrefix()}, + {"path", path}); auto it = NonTransactionalWriters.find(path); if (it != NonTransactionalWriters.end()) { @@ -297,7 +327,9 @@ void TKafkaProduceActor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyDeleted::TP void TKafkaProduceActor::Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev, const TActorContext& ctx) { auto* e = ev->Get(); auto& path = e->Path; - KAFKA_LOG_I("Produce actor: Topic '" << path << "' was updated"); + YDB_LOG_INFO("Produce actor: Topic was updated", + {LogPrefix()}, + {"path", path}); auto& topic = Topics[path]; if (topic.Status == UNAUTHORIZED) { @@ -459,7 +491,9 @@ std::pair<EKafkaErrors, THolder<TEvPartitionWriter::TEvWriteRequest>> Convert( TString str; const bool res = proto.SerializeToString(&str); if (!res) { - KAFKA_LOG_ERROR("Produce actor: Failed to serialize TDataChunk to string: " << proto.DebugString()); + YDB_LOG_ERROR("Produce actor: Failed to serialize TDataChunk to string", + {LogPrefix()}, + {"message", proto.DebugString()}); return {EKafkaErrors::INVALID_RECORD, nullptr}; } @@ -514,7 +548,9 @@ std::pair<EKafkaErrors, THolder<TEvPartitionWriter::TEvWriteRequest>> Convert( // set seqno if (enableKafkaDeduplication && batch.BaseSequence < 0) { - KAFKA_LOG_ERROR("Idempotent producer enabled and batch base sequence is less then zero: " << batch.BaseSequence); + YDB_LOG_ERROR("Idempotent producer enabled and batch base sequence is less then zero", + {LogPrefix()}, + {"baseSequence", batch.BaseSequence}); return {EKafkaErrors::INVALID_RECORD, nullptr}; } w->SetSeqNo(GetRecordSeqNo(batch, batchIndex, record)); @@ -547,7 +583,8 @@ size_t PartsCount(const TMessagePtr<TProduceRequestData>& r) { void TKafkaProduceActor::ProcessRequest(TPendingRequest::TPtr pendingRequest, const TActorContext& ctx) { auto r = pendingRequest->Request->Get()->Request; - KAFKA_LOG_D("Processing request"); + YDB_LOG_DEBUG("Processing request", + {LogPrefix()}); pendingRequest->Results.resize(PartsCount(r)); pendingRequest->StartTime = ctx.Now(); @@ -574,7 +611,9 @@ void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvWriteAccepted::TPtr reque auto it = Cookies.find(cookie); if (it == Cookies.end()) { - KAFKA_LOG_W("Produce actor: Received TEvWriteAccepted with unexpected cookie " << cookie); + YDB_LOG_WARN("Produce actor: Received TEvWriteAccepted with unexpected cookie", + {LogPrefix()}, + {"cookie", cookie}); return; } @@ -585,22 +624,31 @@ void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvWriteAccepted::TPtr reque if (expectedCookies.empty()) { ProcessRequests(ctx); } else { - KAFKA_LOG_W("Still in accepting after receive TEvPartitionWriter::TEvWriteAccepted cause cookies are expected: " << JoinSeq(", ", expectedCookies)); + YDB_LOG_WARN("Still in accepting after receive TEvPartitionWriter::TEvWriteAccepted cause cookies are expected", + {LogPrefix()}, + {"expected", JoinSeq(", ", expectedCookies)}); } } void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvInitResult::TPtr request, const TActorContext& /*ctx*/) { - KAFKA_LOG_D("Produce actor: Init " << request->Get()->ToString()); + YDB_LOG_DEBUG("Produce actor: Init", + {LogPrefix()}, + {"request", request->Get()->ToString()}); if (!request->Get()->IsSuccess()) { auto sender = request->Sender; if (WriterDied(sender, EKafkaErrors::UNKNOWN_SERVER_ERROR, request->Get()->GetError().Reason)) { - KAFKA_LOG_D("Produce actor: Received TEvPartitionWriter::TEvInitResult for " << sender << " with error: " << request->Get()->GetError().Reason); + YDB_LOG_DEBUG("Produce actor: Received TEvPartitionWriter::TEvInitResult for with error", + {LogPrefix()}, + {"sender", sender}, + {"error", request->Get()->GetError().Reason}); return; } - KAFKA_LOG_D("Produce actor: Received TEvPartitionWriter::TEvInitResult with unexpected writer " << sender); + YDB_LOG_DEBUG("Produce actor: Received TEvPartitionWriter::TEvInitResult with unexpected writer", + {LogPrefix()}, + {"sender", sender}); } } @@ -608,11 +656,15 @@ void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvDisconnected::TPtr reques auto sender = request->Sender; if (WriterDied(sender, EKafkaErrors::NOT_LEADER_OR_FOLLOWER, TStringBuilder() << "Partition writer " << sender << " disconnected")) { - KAFKA_LOG_D("Produce actor: Received TEvPartitionWriter::TEvDisconnected for " << sender); + YDB_LOG_DEBUG("Produce actor: Received TEvPartitionWriter::TEvDisconnected", + {LogPrefix()}, + {"sender", sender}); return; } - KAFKA_LOG_D("Produce actor: Received TEvPartitionWriter::TEvDisconnected with unexpected writer " << sender); + YDB_LOG_DEBUG("Produce actor: Received TEvPartitionWriter::TEvDisconnected with unexpected writer", + {LogPrefix()}, + {"sender", sender}); } bool TKafkaProduceActor::WriterDied(const TActorId& writerId, EKafkaErrors errorCode, TStringBuf errorMessage) { @@ -671,11 +723,15 @@ bool TKafkaProduceActor::WriterDied(const TActorId& writerId, EKafkaErrors error void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvWriteResponse::TPtr request, const TActorContext& ctx) { auto r = request->Get(); auto cookie = r->Record.GetPartitionResponse().GetCookie(); - KAFKA_LOG_T("Handling TEvPartitionWriter::TEvWriteResponse with cookie " << cookie); + YDB_LOG_TRACE("Handling TEvPartitionWriter::TEvWriteResponse with cookie", + {LogPrefix()}, + {"cookie", cookie}); auto it = Cookies.find(cookie); if (it == Cookies.end()) { - KAFKA_LOG_W("Produce actor: Received TEvWriteResponse with unexpected cookie " << cookie); + YDB_LOG_WARN("Produce actor: Received TEvWriteResponse with unexpected cookie", + {LogPrefix()}, + {"cookie", cookie}); return; } auto& cookieInfo = it->second; @@ -710,7 +766,9 @@ void TKafkaProduceActor::Handle(TEvPartitionWriter::TEvWriteResponse::TPtr reque if (cookieInfo.Request->WaitResultCookies.empty()) { SendResults(ctx); } else { - KAFKA_LOG_T("Skipping sending results in Handle TEvPartitionWriter::TEvWriteResponse. WaitResultCookies=" << JoinSeq(", ", cookieInfo.Request->WaitResultCookies)); + YDB_LOG_TRACE("Skipping sending results in Handle TEvPartitionWriter::TEvWriteResponse", + {LogPrefix()}, + {"waitResultCookies", JoinSeq(", ", cookieInfo.Request->WaitResultCookies)}); } Cookies.erase(cookie); @@ -730,7 +788,10 @@ EKafkaErrors Convert(TEvPartitionWriter::TEvWriteResponse::EErrorCode value) { void TKafkaProduceActor::SendResults(const TActorContext& ctx) { auto expireTime = ctx.Now() - REQUEST_EXPIRATION_INTERVAL; - KAFKA_LOG_T("Produce actor: Sending results. QueueSize= " << PendingRequests.size() << ", ExpirationTime=" << expireTime); + YDB_LOG_TRACE("Produce actor: Sending results", + {LogPrefix()}, + {"queueSize", PendingRequests.size()}, + {"expirationTime", expireTime}); // We send the results in the order of receipt of the request while (!PendingRequests.empty()) { @@ -740,7 +801,10 @@ void TKafkaProduceActor::SendResults(const TActorContext& ctx) { bool expired = expireTime > pendingRequest->StartTime; if (!expired && !pendingRequest->WaitResultCookies.empty()) { - KAFKA_LOG_T("Skipping sending results. Expired=" << expired << " WaitResultCookies=" << JoinSeq(", ", pendingRequest->WaitResultCookies)); + YDB_LOG_TRACE("Skipping sending results", + {LogPrefix()}, + {"expired", expired}, + {"waitResultCookies", JoinSeq(", ", pendingRequest->WaitResultCookies)}); return; } @@ -748,7 +812,10 @@ void TKafkaProduceActor::SendResults(const TActorContext& ctx) { auto correlationId = pendingRequest->Request->Get()->CorrelationId; EKafkaErrors metricsErrorCode = EKafkaErrors::NONE_ERROR; - KAFKA_LOG_D("Produce actor: Send result for correlation=" << correlationId << ". Expired=" << expired); + YDB_LOG_DEBUG("Produce actor: Send result", + {LogPrefix()}, + {"correlation", correlationId}, + {"expired", expired}); const auto topicsCount = request->TopicData.size(); auto response = std::make_shared<TProduceResponseData>(); @@ -770,15 +837,18 @@ void TKafkaProduceActor::SendResults(const TActorContext& ctx) { size_t recordsCount = result.RecordsCount; partitionResponse.Index = partitionData.Index; if (EKafkaErrors::NONE_ERROR != result.ErrorCode) { - KAFKA_LOG_ERROR("Produce actor: Partition result with error: ErrorCode=" << static_cast<int>(result.ErrorCode) - << ", ErrorMessage=" << result.ErrorMessage << ", #01"); + YDB_LOG_ERROR("Produce actor: Partition result with error: #01", + {LogPrefix()}, + {"errorCode", static_cast<int>(result.ErrorCode)}, + {"errorMessage", result.ErrorMessage}); partitionResponse.ErrorCode = result.ErrorCode; metricsErrorCode = result.ErrorCode; partitionResponse.ErrorMessage = result.ErrorMessage; SendMetrics(TStringBuilder() << topicData.Name, recordsCount, "failed_messages", ctx); } else if (expired) { - KAFKA_LOG_ERROR("Partition write expired."); + YDB_LOG_ERROR("Partition write expired", + {LogPrefix()}); SendMetrics(TStringBuilder() << topicData.Name, recordsCount, "failed_messages", ctx); partitionResponse.ErrorCode = EKafkaErrors::REQUEST_TIMED_OUT; metricsErrorCode = EKafkaErrors::REQUEST_TIMED_OUT; @@ -786,7 +856,8 @@ void TKafkaProduceActor::SendResults(const TActorContext& ctx) { } else { auto* msg = result.Value->Get(); if (msg->IsSuccess()) { - KAFKA_LOG_T("Produce actor: Partition result success."); + YDB_LOG_TRACE("Produce actor: Partition result success", + {LogPrefix()}); partitionResponse.ErrorCode = EKafkaErrors::NONE_ERROR; auto& writeResults = msg->Record.GetPartitionResponse().GetCmdWriteResult(); if (!writeResults.empty()) { @@ -796,11 +867,11 @@ void TKafkaProduceActor::SendResults(const TActorContext& ctx) { partitionResponse.BaseOffset = writeResults.at(0).GetOffset(); } } else { - KAFKA_LOG_ERROR("Produce actor: Partition result with error: ErrorCode=" - << static_cast<int>(Convert(msg->GetError().Code)) - << ", ErrorMessage=" << msg->GetError().Reason - << ", Error from writer=" << static_cast<int>(msg->Record.GetErrorCode()) - << ", #02"); + YDB_LOG_ERROR("Produce actor: Partition result with error: Error #02", + {LogPrefix()}, + {"errorCode", static_cast<int>(Convert(msg->GetError().Code))}, + {"errorMessage", msg->GetError().Reason}, + {"fromWriter", static_cast<int>(msg->Record.GetErrorCode())}); SendMetrics(TStringBuilder() << topicData.Name, recordsCount, "failed_messages", ctx); if (msg->Record.GetErrorCode() == NPersQueue::NErrorCode::KAFKA_INVALID_PRODUCER_EPOCH) { @@ -846,7 +917,9 @@ void TKafkaProduceActor::ProcessInitializationRequests(const TActorContext& ctx) auto request = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); for(auto& topicPath : TopicsForInitialization) { - KAFKA_LOG_D("Produce actor: Describe topic '" << topicPath << "'"); + YDB_LOG_DEBUG("Produce actor: Describe topic", + {LogPrefix()}, + {"topicPath", topicPath}); NSchemeCache::TSchemeCacheNavigate::TEntry entry; entry.Path = NKikimr::SplitPath(topicPath); entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpList; @@ -864,7 +937,8 @@ void TKafkaProduceActor::RecreatePartitionWriterAndRetry(ui64 cookie, const TAct auto it = Cookies.find(cookie); if (it != Cookies.end()) { auto& cookieInfo = it->second; - KAFKA_LOG_D("Transaction was committed. Retrying produce request as a part of next transaction"); + YDB_LOG_DEBUG("Transaction was committed. Retrying produce request as a part of next transaction", + {LogPrefix()}); auto txnIt = TransactionalWriters.find({cookieInfo.TopicPath, cookieInfo.PartitionId}); if (txnIt != TransactionalWriters.end()) { Send(txnIt->second.ActorId, new TEvents::TEvPoison()); @@ -932,7 +1006,9 @@ void TKafkaProduceActor::SendWriteRequest(const TProduceRequestData::TTopicProdu parsedRecords = std::move(parsed); result.RecordsCount = parsedRecords.RecordsCount; } catch (const yexception& e) { - KAFKA_LOG_ERROR("Failed to read Kafka records metadata: " << e.what()); + YDB_LOG_ERROR("Failed to read Kafka records", + {LogPrefix()}, + {"metadata", e.what()}); result.ErrorCode = EKafkaErrors::INVALID_RECORD; return; } @@ -960,7 +1036,10 @@ void TKafkaProduceActor::SendWriteRequest(const TProduceRequestData::TTopicProdu pendingRequest->WaitResultCookies.insert(ownCookie); ruPerRequest = false; - KAFKA_LOG_T("Sending TEvPartitionWriter::TEvWriteRequest to " << writer.second << " with cookie " << ownCookie); + YDB_LOG_TRACE("Sending TEvPartitionWriter::TEvWriteRequest to with cookie", + {LogPrefix()}, + {"writerSecond", writer.second}, + {"ownCookie", ownCookie}); Send(writer.second, std::move(ev)); result.ErrorCode = NONE_ERROR; } else { @@ -983,14 +1062,19 @@ void TKafkaProduceActor::SendWriteRequest(const TProduceRequestData::TTopicProdu } if (result.ErrorCode != EKafkaErrors::NONE_ERROR) { - KAFKA_LOG_ERROR("Write request failed with error " << result.ErrorCode << " and message " << result.ErrorMessage); + YDB_LOG_ERROR("Write request failed with error and message", + {LogPrefix()}, + {"errorCode", result.ErrorCode}, + {"errorMessage", result.ErrorMessage}); } } std::pair<TKafkaProduceActor::ETopicStatus, TActorId> TKafkaProduceActor::PartitionWriter(const TTopicPartition& topicPartition, const TProducerInstanceId& producerInstanceId, const TMaybe<TString>& transactionalId, const TActorContext& ctx) { auto it = Topics.find(topicPartition.TopicPath); if (it == Topics.end()) { - KAFKA_LOG_ERROR("Produce actor: Internal error: topic '" << topicPartition.TopicPath << "' isn`t initialized"); + YDB_LOG_ERROR("Produce actor: Internal error: topic isn`t initialized", + {LogPrefix()}, + {"topicPath", topicPartition.TopicPath}); return { NOT_FOUND, TActorId{} }; } @@ -1055,7 +1139,12 @@ std::pair<TKafkaProduceActor::ETopicStatus, TActorId> TKafkaProduceActor::GetOrC } std::pair<TKafkaProduceActor::ETopicStatus, TActorId> TKafkaProduceActor::CreateTransactionalWriter(const TTopicPartition& topicPartition, const TTopicInfo& topicInfo, const TProducerInstanceId& producerInstanceId, const TString& transactionalId, const TActorContext& ctx) { - KAFKA_LOG_D("Created transactional writer for producerId=" << producerInstanceId.Id << " and producerEpoch=" << producerInstanceId.Epoch << " for topic-partition " << topicPartition.TopicPath << ":" << topicPartition.PartitionId); + YDB_LOG_DEBUG("Created transactional writer for and for topic-partition", + {LogPrefix()}, + {"producerId", producerInstanceId.Id}, + {"producerEpoch", producerInstanceId.Epoch}, + {"topicPath", topicPartition.TopicPath}, + {"partitionId", topicPartition.PartitionId}); auto* partition = topicInfo.PartitionChooser->GetPartition(topicPartition.PartitionId); if (!partition) { return { NOT_FOUND, TActorId{} }; diff --git a/ydb/core/kafka_proxy/actors/kafka_produce_actor.h b/ydb/core/kafka_proxy/actors/kafka_produce_actor.h index 963aa52bf90..753f5c23089 100644 --- a/ydb/core/kafka_proxy/actors/kafka_produce_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_produce_actor.h @@ -20,7 +20,7 @@ using namespace NKikimrClient; // Each request can contain data for writing to several topics, and in each topic to several partitions. // When a request to write to an unknown topic arrives, the actor changes the state to Init until it receives // information about all the topics needed to process the request. -// +// // Requests are processed in parallel, but it is guaranteed that the recording order will be preserved. // The order of responses to requests is also guaranteed. // @@ -116,7 +116,7 @@ private: std::pair<ETopicStatus, TActorId> PartitionWriter(const TTopicPartition& topicPartition, const TProducerInstanceId& producerInstanceId, const TMaybe<TString>& transactionalId, const TActorContext& ctx); bool WriterDied(const TActorId& writerId, EKafkaErrors errorCode, TStringBuf errorMessage); - TString LogPrefix(); + NStructuredLog::TStructuredMessage LogPrefix(); void LogEvent(IEventHandle& ev); void SendMetrics(const TString& topicName, size_t delta, const TString& name, const TActorContext& ctx); diff --git a/ydb/core/kafka_proxy/actors/kafka_read_session_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_read_session_actor.cpp index b56b9f2d1e4..a1dea83bff4 100644 --- a/ydb/core/kafka_proxy/actors/kafka_read_session_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_read_session_actor.cpp @@ -1,6 +1,8 @@ #include "kafka_read_session_actor.h" #include "kafka_read_session_utils.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { static constexpr TDuration WAKEUP_INTERVAL = TDuration::Seconds(1); static constexpr TDuration LOCK_PARTITION_DELAY = TDuration::Seconds(3); @@ -15,14 +17,16 @@ void TKafkaReadSessionActor::Bootstrap(const NActors::TActorContext&) { Become(&TKafkaReadSessionActor::StateWork); } -TString TKafkaReadSessionActor::LogPrefix() { - TStringBuilder sb; - sb << "TKafkaReadSessionActor " << (Session == "" ? SelfId().ToString() : Session) << ": "; - return sb; +NStructuredLog::TStructuredMessage TKafkaReadSessionActor::LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaReadSessionActor"}, + {"selfId", SelfId()}, + {"session", Session}); } void TKafkaReadSessionActor::Die(const TActorContext& ctx) { - KAFKA_LOG_D("PassAway"); + YDB_LOG_DEBUG("PassAway", + {LogPrefix()}); for (auto& [topicName, topicInfo] : TopicsInfo) { NTabletPipe::CloseClient(ctx, topicInfo.PipeClient); } @@ -31,7 +35,8 @@ void TKafkaReadSessionActor::Die(const TActorContext& ctx) { void TKafkaReadSessionActor::HandleWakeup(TEvKafka::TEvWakeup::TPtr, const TActorContext& ctx) { if (CheckHeartbeatIsExpired()) { - KAFKA_LOG_D("Heartbeat expired"); + YDB_LOG_DEBUG("Heartbeat expired", + {LogPrefix()}); CloseReadSession(ctx); return; } @@ -241,7 +246,9 @@ void TKafkaReadSessionActor::SendJoinGroupResponseOk(const TActorContext&, ui64 } void TKafkaReadSessionActor::SendJoinGroupResponseFail(const TActorContext&, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_CRIT("JOIN_GROUP failed. reason# " << message); + YDB_LOG_CRIT("JOIN_GROUP failed", + {LogPrefix()}, + {"reason", message}); TJoinGroupResponseData::TPtr response = std::make_shared<TJoinGroupResponseData>(); response->ErrorCode = error; @@ -269,7 +276,9 @@ void TKafkaReadSessionActor::SendSyncGroupResponseOk(const TActorContext& ctx, u } void TKafkaReadSessionActor::SendSyncGroupResponseFail(const TActorContext&, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_CRIT("SYNC_GROUP failed. reason# " << message); + YDB_LOG_CRIT("SYNC_GROUP failed", + {LogPrefix()}, + {"reason", message}); TSyncGroupResponseData::TPtr response = std::make_shared<TSyncGroupResponseData>(); response->ErrorCode = error; @@ -287,7 +296,9 @@ void TKafkaReadSessionActor::SendLeaveGroupResponseOk(const TActorContext&, ui64 } void TKafkaReadSessionActor::SendLeaveGroupResponseFail(const TActorContext&, ui64 corellationId, EKafkaErrors error, TString message) { - KAFKA_LOG_CRIT("LEAVE_GROUP failed. reason# " << message); + YDB_LOG_CRIT("LEAVE_GROUP failed", + {LogPrefix()}, + {"reason", message}); TLeaveGroupResponseData::TPtr response = std::make_shared<TLeaveGroupResponseData>(); response->ErrorCode = error; @@ -303,7 +314,9 @@ void TKafkaReadSessionActor::SendHeartbeatResponseOk(const TActorContext&, ui64 void TKafkaReadSessionActor::SendHeartbeatResponseFail(const TActorContext&, ui64 corellationId, EKafkaErrors error, TString message) { THeartbeatResponseData::TPtr response = std::make_shared<THeartbeatResponseData>(); - KAFKA_LOG_CRIT("HEARTBEAT failed. reason# " << message); + YDB_LOG_CRIT("HEARTBEAT failed", + {LogPrefix()}, + {"reason", message}); response->ErrorCode = error; Send(Context->ConnectionId, new TEvKafka::TEvResponse(corellationId, response, error)); } @@ -337,7 +350,9 @@ bool TKafkaReadSessionActor::TryFillTopicsToRead(const TMessagePtr<TJoinGroupReq if (result.has_value()) { for (auto topic: result->Topics) { if (topic.has_value()) { - KAFKA_LOG_D("JOIN_GROUP requested topic to read: " << topic); + YDB_LOG_DEBUG("JOIN_GROUP requested topic", + {LogPrefix()}, + {"toRead", topic}); auto normalizedTopicName = NormalizePath(Context->DatabasePath, topic.value()); OriginalTopicNames[normalizedTopicName] = topic.value(); @@ -352,7 +367,9 @@ bool TKafkaReadSessionActor::TryFillTopicsToRead(const TMessagePtr<TJoinGroupReq TConsumerProtocolAssignment TKafkaReadSessionActor::BuildAssignmentAndInformBalancerIfRelease(const TActorContext& ctx) { TConsumerProtocolAssignment assignment; - KAFKA_LOG_D("SYNC_GROUP topics to assign count: " << TopicPartitions.size()); + YDB_LOG_DEBUG("SYNC_GROUP topics to assign", + {LogPrefix()}, + {"count", TopicPartitions.size()}); for (auto& [topicName, partitions] : TopicPartitions) { auto topicInfoIt = TopicsInfo.find(topicName); @@ -378,9 +395,13 @@ TConsumerProtocolAssignment TKafkaReadSessionActor::BuildAssignmentAndInformBala } partitions.ToRelease.clear(); - KAFKA_LOG_D("SYNC_GROUP partitions assigned: " << finalPartitionsToRead.size()); + YDB_LOG_DEBUG("SYNC_GROUP partitions", + {LogPrefix()}, + {"assigned", finalPartitionsToRead.size()}); for (auto part: finalPartitionsToRead) { - KAFKA_LOG_D("SYNC_GROUP assigned partition number: " << part); + YDB_LOG_DEBUG("SYNC_GROUP assigned partition", + {LogPrefix()}, + {"number", part}); topicPartition.Partitions.push_back(part); partitions.ReadingNow.emplace(part); } @@ -466,7 +487,9 @@ void TKafkaReadSessionActor::HandleBalancerError(TEvPersQueue::TEvError::TPtr& e } void TKafkaReadSessionActor::HandleAuthOk(NGRpcProxy::V1::TEvPQProxy::TEvAuthResultOk::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("JOIN_GROUP auth success. Topics count: " << ev->Get()->TopicAndTablets.size()); + YDB_LOG_DEBUG("JOIN_GROUP auth success. Topics", + {LogPrefix()}, + {"count", ev->Get()->TopicAndTablets.size()}); for (const auto& [name, t] : ev->Get()->TopicAndTablets) { auto internalName = t.TopicNameConverter->GetInternalName(); @@ -506,7 +529,9 @@ TActorId TKafkaReadSessionActor::CreatePipeClient(ui64 tabletId, const TActorCon } void TKafkaReadSessionActor::RegisterBalancerSession(const TString& topic, const TActorId& pipe, const TVector<ui32>& groups, const TActorContext& ctx) { - KAFKA_LOG_I("register session: topic# " << topic ); + YDB_LOG_INFO("Register session", + {LogPrefix()}, + {"topic", topic}); auto request = MakeHolder<TEvPersQueue::TEvRegisterReadSession>(); auto& req = request->Record; @@ -524,7 +549,10 @@ void TKafkaReadSessionActor::RegisterBalancerSession(const TString& topic, const void TKafkaReadSessionActor::HandleLockPartition(TEvPersQueue::TEvLockPartition::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; - KAFKA_LOG_D("partition lock is coming from PQRB topic# " << record.GetTopic() << ", partition# " << record.GetPartition()); + YDB_LOG_DEBUG("Partition lock is coming from PQRB", + {LogPrefix()}, + {"topic", record.GetTopic()}, + {"partition", record.GetPartition()}); Y_ABORT_UNLESS(record.GetSession() == Session); Y_ABORT_UNLESS(record.GetClientId() == GroupId); @@ -537,9 +565,10 @@ void TKafkaReadSessionActor::HandleLockPartition(TEvPersQueue::TEvLockPartition: auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(path)); if (converterIter == FullPathToConverter.end()) { - KAFKA_LOG_I("ignored ev lock topic# " << record.GetTopic() - << ", partition# " << record.GetPartition() - << ", reason# path not recognized"); + YDB_LOG_INFO("Ignored ev lock path not recognized", + {LogPrefix()}, + {"topic", record.GetTopic()}, + {"partition", record.GetPartition()}); return; } @@ -547,9 +576,10 @@ void TKafkaReadSessionActor::HandleLockPartition(TEvPersQueue::TEvLockPartition: auto topicInfoIt = TopicsInfo.find(topicName); if (topicInfoIt == TopicsInfo.end() || (topicInfoIt->second.PipeClient != ActorIdFromProto(record.GetPipeClient()))) { - KAFKA_LOG_I("ignored ev lock topic# " << record.GetTopic() - << ", partition# " << record.GetPartition() - << ", reason# topic is unknown"); + YDB_LOG_INFO("Ignored ev lock topic is unknown", + {LogPrefix()}, + {"topic", record.GetTopic()}, + {"partition", record.GetPartition()}); return; } @@ -562,7 +592,10 @@ void TKafkaReadSessionActor::HandleLockPartition(TEvPersQueue::TEvLockPartition: void TKafkaReadSessionActor::HandleReleasePartition(TEvPersQueue::TEvReleasePartition::TPtr& ev, const TActorContext& ctx) { const auto& record = ev->Get()->Record; const ui32 group = record.HasGroup() ? record.GetGroup() : 0; - KAFKA_LOG_D("partition release is coming from PQRB topic# " << record.GetTopic() << ", group# " << group); + YDB_LOG_DEBUG("Partition release is coming from PQRB", + {LogPrefix()}, + {"topic", record.GetTopic()}, + {"group", group}); Y_ABORT_UNLESS(record.GetSession() == Session); Y_ABORT_UNLESS(record.GetClientId() == GroupId); @@ -574,8 +607,9 @@ void TKafkaReadSessionActor::HandleReleasePartition(TEvPersQueue::TEvReleasePart Y_ABORT_UNLESS(topicInfoIt != TopicsInfo.end()); if (topicInfoIt->second.PipeClient != ActorIdFromProto(record.GetPipeClient())) { - KAFKA_LOG_I("ignored ev release topic# " << record.GetTopic() - << ", reason# topic is unknown"); + YDB_LOG_INFO("Ignored ev release topic is unknown", + {LogPrefix()}, + {"topic", record.GetTopic()}); return; } @@ -622,13 +656,17 @@ void TKafkaReadSessionActor::HandleReleasePartition(TEvPersQueue::TEvReleasePart } } - KAFKA_LOG_I("ignored ev release topic# " << record.GetTopic() - << ", reason# partition " << partitionToRelease << " isn`t locked"); + YDB_LOG_INFO("Ignored ev release partition isn`t locked", + {LogPrefix()}, + {"topic", record.GetTopic()}, + {"partitionToRelease", partitionToRelease}); } void TKafkaReadSessionActor::InformBalancerAboutPartitionRelease(const TString& topic, ui64 partition, const TActorContext& ctx) { - KAFKA_LOG_I("released topic# " << topic - << ", partition# " << partition); + YDB_LOG_INFO("Released", + {LogPrefix()}, + {"topic", topic}, + {"partition", partition}); auto request = MakeHolder<TEvPersQueue::TEvPartitionReleased>(); auto topicIt = TopicsInfo.find(topic); diff --git a/ydb/core/kafka_proxy/actors/kafka_read_session_actor.h b/ydb/core/kafka_proxy/actors/kafka_read_session_actor.h index 3ca986a6bf7..aa2b89b32ab 100644 --- a/ydb/core/kafka_proxy/actors/kafka_read_session_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_read_session_actor.h @@ -92,7 +92,7 @@ public: private: using TActorContext = NActors::TActorContext; - TString LogPrefix(); + NStructuredLog::TStructuredMessage LogPrefix(); void Die(const TActorContext& ctx) override; diff --git a/ydb/core/kafka_proxy/actors/kafka_read_session_proxy.cpp b/ydb/core/kafka_proxy/actors/kafka_read_session_proxy.cpp index 96ab74a505e..16ab825f3e2 100644 --- a/ydb/core/kafka_proxy/actors/kafka_read_session_proxy.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_read_session_proxy.cpp @@ -2,6 +2,8 @@ #include "kafka_read_session_utils.h" #include "kafka_balancer_actor.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { KafkaReadSessionProxyActor::KafkaReadSessionProxyActor(const TContext::TPtr context, ui64 cookie) @@ -19,14 +21,18 @@ template<bool handlePending, typename TRequest> void KafkaReadSessionProxyActor::DoHandle(TRequest& ev, const TString& event) { if constexpr (handlePending) { if (Context->ReadSession.PendingBalancingMode.has_value()) { - KAFKA_LOG_D("DoHandle " << event << " with pending balance mode"); + YDB_LOG_DEBUG("DoHandle with pending balance mode", + {LogPrefix()}, + {"event", event}); auto response = CreateChangeResponse(*ev->Get()->Request); Send(Context->ConnectionId, new TEvKafka::TEvResponse(ev->Get()->CorrelationId, response, EKafkaErrors::REBALANCE_IN_PROGRESS)); return; } } - KAFKA_LOG_D("DoHandle " << event); + YDB_LOG_DEBUG("DoHandle", + {LogPrefix()}, + {"event", event}); switch (Context->ReadSession.BalancingMode) { case EBalancingMode::Native: Register(new TKafkaBalancerActor(Context, 0, ev->Get()->CorrelationId, ev->Get()->Request)); @@ -40,10 +46,13 @@ void KafkaReadSessionProxyActor::DoHandle(TRequest& ev, const TString& event) { } void KafkaReadSessionProxyActor::Handle(TEvKafka::TEvJoinGroupRequest::TPtr& ev) { - KAFKA_LOG_D("Handle TEvKafka::TEvJoinGroupRequest"); + YDB_LOG_DEBUG("Handle TEvKafka::TEvJoinGroupRequest", + {LogPrefix()}); Context->ReadSession.BalancingMode = Context->ReadSession.PendingBalancingMode.value_or(GetBalancingMode(*ev->Get()->Request)); Context->ReadSession.PendingBalancingMode.reset(); - KAFKA_LOG_D("Balancing mode: " << Context->ReadSession.BalancingMode); + YDB_LOG_DEBUG("Balancing", + {LogPrefix()}, + {"mode", Context->ReadSession.BalancingMode}); DoHandle<false>(ev, "TEvKafka::TEvJoinGroupRequest"); } @@ -61,7 +70,8 @@ void KafkaReadSessionProxyActor::Handle(TEvKafka::TEvLeaveGroupRequest::TPtr& ev } void KafkaReadSessionProxyActor::Handle(TEvKafka::TEvFetchRequest::TPtr& ev) { - KAFKA_LOG_D("Handle TEvKafka::TEvFetchRequest"); + YDB_LOG_DEBUG("Handle TEvKafka::TEvFetchRequest", + {LogPrefix()}); Register(CreateKafkaFetchActor(Context, ev->Get()->CorrelationId, ev->Get()->Request)); } diff --git a/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp index a34d01f29e7..cb9a5d7042f 100644 --- a/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.cpp @@ -12,6 +12,8 @@ #include <ydb/library/login/sasl/scram.h> #include <ydb/services/persqueue_v1/actors/persqueue_utils.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { @@ -244,8 +246,9 @@ void TKafkaSaslAuthActor::SendResponse() { responseToClient->AuthBytes = ToRawBytes(responseToClient->AuthBytesStr); responseToClient->ErrorMessage = ""; - KAFKA_LOG_D("Authentication first step finished." - << " FirstServerMessage='" << AuthResponse << "'"); + YDB_LOG_DEBUG("Authentication first step finished. FirstServerMessage='", + {LogPrefix()}, + {"authResponse", AuthResponse}); auto evResponse = std::make_unique<TEvKafka::TEvResponse>(CorrelationId, responseToClient, EKafkaErrors::NONE_ERROR); Send(Context->ConnectionId, evResponse.release()); @@ -265,7 +268,9 @@ void TKafkaSaslAuthActor::SendResponseAndDie(EKafkaErrors errorCode, Ydb::Status TStringBuilder authenticationFailureReason; authenticationFailureReason << (errorMessage ? " " + errorMessage : "") << (details ? " " + details : ""); - KAFKA_LOG_ERROR("Authentication failure." << authenticationFailureReason); + YDB_LOG_ERROR("Authentication failure", + {LogPrefix()}, + {"authenticationFailureReason", authenticationFailureReason}); const auto& securityConfig = AppData()->DomainsConfig.GetSecurityConfig(); TStringBuilder responseErrorMessage; responseErrorMessage << "Authentication failure."; @@ -278,12 +283,14 @@ void TKafkaSaslAuthActor::SendResponseAndDie(EKafkaErrors errorCode, Ydb::Status auto authResult = new TEvKafka::TEvAuthResult(EAuthSteps::FAILED, evResponse, errorMessage); Send(Context->ConnectionId, authResult); } else { - KAFKA_LOG_D("Authentication success. Database='" << DatabasePath << "', " - << "FolderId='" << FolderId << "', " - << "ServiceAccountId='" << ServiceAccountId << "', " - << "DatabaseId='" << DatabaseId << "', " - << "Coordinator='" << Coordinator << "', " - << "ResourcePath='" << ResourcePath << "'"); + YDB_LOG_DEBUG("Authentication success. Database=' FolderId=' ServiceAccountId=' DatabaseId=' Coordinator=' ResourcePath='", + {LogPrefix()}, + {"databasePath", DatabasePath}, + {"folderId", FolderId}, + {"serviceAccountId", ServiceAccountId}, + {"databaseId", DatabaseId}, + {"coordinator", Coordinator}, + {"resourcePath", ResourcePath}); responseToClient->ErrorMessage = ""; auto evResponse = std::make_shared<TEvKafka::TEvResponse>(CorrelationId, responseToClient, errorCode); diff --git a/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.h b/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.h index 9788d073e4a..6e67085e391 100644 --- a/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_sasl_auth_actor.h @@ -34,7 +34,9 @@ public: private: STATEFN(StateWork) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(TEvKafka::TEvAuthRequest, HandleAuthRequest); HFunc(TEvKafka::TEvMtlsAuthRequest, HandleMtlsAuthRequest); @@ -43,7 +45,9 @@ private: } STATEFN(StateResolveDatabase) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleNavigate); CFunc(TEvents::TEvPoison::EventType, Die); @@ -51,7 +55,9 @@ private: } STATEFN(StateResolveSharedDatabase) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleNavigate); CFunc(TEvents::TEvPoison::EventType, Die); @@ -59,7 +65,9 @@ private: } STATEFN(StateSaslPlainLogin) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(NSasl::TEvSasl::TEvSaslPlainLoginResponse, HandleLoginResult); HFunc(NSasl::TEvSasl::TEvSaslPlainLdapLoginResponse, HandleLoginResult); @@ -69,7 +77,9 @@ private: } STATEFN(StateSaslScramLogin) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { hFunc(NSasl::TEvSasl::TEvSaslScramFirstServerResponse, HandleFirstLoginResponse); HFunc(TEvKafka::TEvAuthRequest, HandleAuthRequest); @@ -80,7 +90,9 @@ private: } STATEFN(StateTicketResolve) { - KAFKA_LOG_T("Received event: " << (*ev.Get()).GetTypeName()); + YDB_LOG_TRACE_COMP(NKikimrServices::KAFKA_PROXY, "Received", + {LogPrefix()}, + {"event", (*ev.Get()).GetTypeName()}); switch (ev->GetTypeRewrite()) { HFunc(TEvTicketParser::TEvAuthorizeTicketResult, Handle); CFunc(TEvents::TEvPoison::EventType, Die); diff --git a/ydb/core/kafka_proxy/actors/kafka_transaction_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_transaction_actor.cpp index 66f26739d61..d69ce77c21d 100644 --- a/ydb/core/kafka_proxy/actors/kafka_transaction_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_transaction_actor.cpp @@ -8,6 +8,8 @@ #include <util/generic/cast.h> #include <regex> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + #define VALIDATE_PRODUCER_IN_REQUEST(ErrorResponseType) \ if (!ProducerInRequestIsValid(ev->Get()->Request)) { \ auto& kafkaRequest = ev->Get()->Request; \ @@ -22,7 +24,8 @@ if (!ProducerInRequestIsValid(ev->Get()->Request)) { \ namespace NKafka { void TTransactionActor::Handle(TEvKafka::TEvAddPartitionsToTxnRequest::TPtr& ev, const TActorContext&){ - KAFKA_LOG_D("Received ADD_PARTITIONS_TO_TXN request"); + YDB_LOG_DEBUG("Received ADD_PARTITIONS_TO_TXN request", + {LogPrefix()}); VALIDATE_PRODUCER_IN_REQUEST(TAddPartitionsToTxnResponseData); for (auto& topicInRequest : ev->Get()->Request->Topics) { @@ -37,13 +40,15 @@ namespace NKafka { // in YDB Topics we store offsets in table and do not need this extra action. // Thus we can just ignore this request. void TTransactionActor::Handle(TEvKafka::TEvAddOffsetsToTxnRequest::TPtr& ev, const TActorContext&) { - KAFKA_LOG_D("Received ADD_OFFSETS_TO_TXN request"); + YDB_LOG_DEBUG("Received ADD_OFFSETS_TO_TXN request", + {LogPrefix()}); VALIDATE_PRODUCER_IN_REQUEST(TAddOffsetsToTxnResponseData); SendOkResponse<TAddOffsetsToTxnResponseData>(ev); } void TTransactionActor::Handle(TEvKafka::TEvTxnOffsetCommitRequest::TPtr& ev, const TActorContext&) { - KAFKA_LOG_D("Received TXN_OFFSET_COMMIT request"); + YDB_LOG_DEBUG("Received TXN_OFFSET_COMMIT request", + {LogPrefix()}); VALIDATE_PRODUCER_IN_REQUEST(TTxnOffsetCommitResponseData); // save offsets for future use @@ -79,7 +84,8 @@ namespace NKafka { 7. Close KQP session, send to coordinator TEvTransactionActorDied, die. */ void TTransactionActor::Handle(TEvKafka::TEvEndTxnRequest::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Received END_TXN request"); + YDB_LOG_DEBUG("Received END_TXN request", + {LogPrefix()}); VALIDATE_PRODUCER_IN_REQUEST(TEndTxnResponseData); bool txnAborted = !ev->Get()->Request->Committed; @@ -103,7 +109,8 @@ namespace NKafka { } void TTransactionActor::Handle(NKqp::TEvKqp::TEvCreateSessionResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("KQP session created"); + YDB_LOG_DEBUG("KQP session created", + {LogPrefix()}); if (!Kqp->HandleCreateSessionResponse(ev, ctx)) { SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::BROKER_NOT_AVAILABLE, "Failed to create KQP session"); Die(ctx); @@ -116,9 +123,13 @@ namespace NKafka { } void TTransactionActor::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr& ev, const TActorContext& ctx) { - KAFKA_LOG_D("Received query response from KQP for " << GetAsStr(LastSentToKqpRequest) << " request"); + YDB_LOG_DEBUG("Received query response from KQP for request", + {LogPrefix()}, + {"lastSentToKqpRequest", GetAsStr(LastSentToKqpRequest)}); if (auto error = GetErrorFromYdbResponse(ev)) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}, + {"error", error}); SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::BROKER_NOT_AVAILABLE, error->data()); Die(ctx); return; @@ -141,12 +152,16 @@ namespace NKafka { void TTransactionActor::StartKqpSession(const TActorContext& ctx) { Kqp = std::make_unique<TKqpTxHelper>(AppData(ctx)->TenantName); - KAFKA_LOG_D("Sending create session request to KQP for database " << DatabasePath); + YDB_LOG_DEBUG("Sending create session request to KQP for database", + {LogPrefix()}, + {"databasePath", DatabasePath}); Kqp->SendCreateSessionRequest(ctx); } void TTransactionActor::SendToKqpValidationRequests(const TActorContext& ctx) { - KAFKA_LOG_D("Sending select request to KQP for database " << DatabasePath); + YDB_LOG_DEBUG("Sending select request to KQP for database", + {LogPrefix()}, + {"databasePath", DatabasePath}); Kqp->SendYqlRequest( GetYqlWithTablesNames(), BuildSelectParams(), @@ -170,9 +185,14 @@ namespace NKafka { template<class ErrorResponseType, class EventType> void TTransactionActor::SendFailResponse(TAutoPtr<TEventHandle<EventType>>& evHandle, EKafkaErrors errorCode, const TString& errorMessage) { if (errorMessage) { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode << ". Reason: " << errorMessage); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}, + {"reason", errorMessage}); } else { - KAFKA_LOG_W("Sending fail response with error code: " << errorCode); + YDB_LOG_WARN("Sending fail response with error", + {LogPrefix()}, + {"code", errorCode}); } auto response = NKafkaTransactions::BuildResponse<ErrorResponseType>(evHandle->Get()->Request, errorCode); @@ -182,14 +202,19 @@ namespace NKafka { template<class ResponseType, class EventType> void TTransactionActor::SendOkResponse(TAutoPtr<TEventHandle<EventType>>& evHandle) { auto& kafkaRequest = evHandle->Get()->Request; - KAFKA_LOG_D("Sending OK response to " << evHandle->Get()->ConnectionId << " with correlationId " << evHandle->Get()->CorrelationId << " and transactionalId " << TransactionalId); + YDB_LOG_DEBUG("Sending OK response to with correlationId and transactionalId", + {LogPrefix()}, + {"connectionId", evHandle->Get()->ConnectionId}, + {"correlationId", evHandle->Get()->CorrelationId}, + {"transactionalId", TransactionalId}); std::shared_ptr<ResponseType> response = NKafkaTransactions::BuildResponse<ResponseType>(kafkaRequest, EKafkaErrors::NONE_ERROR); Send(evHandle->Get()->ConnectionId, new TEvKafka::TEvResponse(evHandle->Get()->CorrelationId, response, EKafkaErrors::NONE_ERROR)); } // helper methods void TTransactionActor::Die(const TActorContext &ctx) { - KAFKA_LOG_D("Dying."); + YDB_LOG_DEBUG("Dying", + {LogPrefix()}); if (Kqp) { Kqp->CloseKqpSession(ctx); } @@ -294,7 +319,8 @@ namespace NKafka { int expectedResultsSize = OffsetsToCommit.empty() ? 1 : 2; // if there were no consumer in transactions we do not send request to consumer table if (expectedResultsSize != resultsSize) { TString error = TStringBuilder() << "KQP returned wrong number of result sets on SELECT query. Expected " << expectedResultsSize << ", got " << resultsSize << "."; - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::BROKER_NOT_AVAILABLE, error); Die(ctx); return; @@ -306,13 +332,15 @@ namespace NKafka { producerState = ParseProducerState(response); } catch (const yexception& y) { TString error = TStringBuilder() << "Error parsing producer state response from KQP. Reason: " << y.what(); - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::BROKER_NOT_AVAILABLE, error); Die(ctx); return; } if (auto error = GetErrorInProducerState(producerState)) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::PRODUCER_FENCED, error->data()); Die(ctx); return; @@ -322,28 +350,32 @@ namespace NKafka { // parse and validate consumers std::unordered_map<TString, i32> consumerGenerationsByName = ParseConsumersGenerations(response); if (auto error = GetErrorInConsumersStates(consumerGenerationsByName)) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::PRODUCER_FENCED, error->data()); Die(ctx); return; } } - KAFKA_LOG_D("Validated producer and consumers states. Everything is alright, adding kafka operations to transaction."); + YDB_LOG_DEBUG("Validated producer and consumers states. Everything is alright, adding kafka operations to transaction", + {LogPrefix()}); auto kqpTxnId = response.Record.GetResponse().GetTxMeta().id(); // finally everything is valid and we can add kafka operations to transaction and attempt to commit SendAddKafkaOperationsToTxRequest(kqpTxnId); } void TTransactionActor::HandleAddKafkaOperationsResponse(const TString& kqpTransactionId, const TActorContext& ctx) { - KAFKA_LOG_D("Successfully added kafka operations to transaction. Committing transaction."); + YDB_LOG_DEBUG("Successfully added kafka operations to transaction. Committing transaction", + {LogPrefix()}); Kqp->SetTxId(kqpTransactionId); Kqp->CommitTx(++KqpCookie, ctx); LastSentToKqpRequest = EKafkaTxnKqpRequests::COMMIT; } void TTransactionActor::HandleCommitResponse(const TActorContext& ctx) { - KAFKA_LOG_D("Successfully committed transaction. Sending ok and dying."); + YDB_LOG_DEBUG("Successfully committed transaction. Sending ok and dying", + {LogPrefix()}); SendOkResponse<TEndTxnResponseData>(EndTxnRequestPtr); Die(ctx); } diff --git a/ydb/core/kafka_proxy/actors/kafka_transaction_actor.h b/ydb/core/kafka_proxy/actors/kafka_transaction_actor.h index 2daa546b98c..ace54800325 100644 --- a/ydb/core/kafka_proxy/actors/kafka_transaction_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_transaction_actor.h @@ -48,8 +48,15 @@ namespace NKafka { TxnTimeoutMs(txnTimeoutMs), CreatedAt(TAppData::TimeProvider->Now()) {}; - TStringBuilder LogPrefix() const { - return TStringBuilder() << "KafkaTransactionActor{TransactionalId=" << TransactionalId << "; ProducerId=" << ProducerInstanceId.Id << "; ProducerEpoch=" << ProducerInstanceId.Epoch << "}: "; + + NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaTransactionActor"}, + {"selfId", SelfId()}, + {"transactionalId", TransactionalId}, + {"producerId", ProducerInstanceId.Id}, + {"producerEpoch", ProducerInstanceId.Epoch}, + {"databasePath", DatabasePath}); } private: @@ -65,7 +72,9 @@ namespace NKafka { HFunc(TEvents::TEvPoison, Handle); } } catch (const yexception& y) { - KAFKA_LOG_CRIT("Critical error happened. Reason: " << y.what()); + YDB_LOG_CRIT_COMP(NKikimrServices::KAFKA_PROXY, "Critical error happened", + {LogPrefix()}, + {"reason", y.what()}); if (EndTxnRequestPtr) { SendFailResponse<TEndTxnResponseData>(EndTxnRequestPtr, EKafkaErrors::UNKNOWN_SERVER_ERROR, y.what()); } diff --git a/ydb/core/kafka_proxy/kafka_connection.cpp b/ydb/core/kafka_proxy/kafka_connection.cpp index 2685c3dd1df..fdf2f498e1c 100644 --- a/ydb/core/kafka_proxy/kafka_connection.cpp +++ b/ydb/core/kafka_proxy/kafka_connection.cpp @@ -11,9 +11,14 @@ #include "actors/actors.h" #include "kafka_connection.h" #include "kafka_events.h" + + #include <ydb/core/kafka_proxy/kafka_log_impl.h> + #include "kafka_metrics.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { using namespace NActors; @@ -126,11 +131,14 @@ public: Become(&TKafkaConnection::StateAccepting); Schedule(InactivityTimeout, InactivityEvent = new TEvPollerReady(nullptr, false, false)); - KAFKA_LOG_I("incoming connection opened " << Address); + YDB_LOG_INFO("Incoming connection opened", + {LogPrefix()}, + {"address", Address}); } void PassAway() override { - KAFKA_LOG_D("PassAway"); + YDB_LOG_DEBUG("PassAway", + {LogPrefix()}); ConnectionEstablished = false; if (ProduceActorId) { @@ -149,7 +157,9 @@ public: protected: void LogEvent(IEventHandle& ev) { - KAFKA_LOG_T("Received event: " << ev.GetTypeName()); + YDB_LOG_TRACE("Received", + {LogPrefix()}, + {"event", ev.GetTypeName()}); } void SetNonBlock() noexcept { @@ -157,7 +167,8 @@ protected: } void Shutdown() { - KAFKA_LOG_D("Shutdown"); + YDB_LOG_DEBUG("Shutdown", + {LogPrefix()}); PendingRequests.clear(); PendingRequestsQueue.clear(); @@ -168,7 +179,9 @@ protected: } ssize_t SocketSend(const void* data, size_t size) { - KAFKA_LOG_T("SocketSend Size=" << size); + YDB_LOG_TRACE("SocketSend", + {LogPrefix()}, + {"size", size}); return Socket->Send(data, size); } @@ -184,18 +197,24 @@ protected: return Socket->GetRawSocket(); } - TString LogPrefix() const { - TStringBuilder sb; - sb << "TKafkaConnection " << SelfId() << "(#" << GetRawSocket() << "," << Address->ToString() << ") State: "; + NStructuredLog::TStructuredMessage LogPrefix() const { + TString state; + auto stateFunc = CurrentStateFunc(); if (stateFunc == &TKafkaConnection::StateConnected) { - sb << "Connected "; + state = "Connected"; } else if (stateFunc == &TKafkaConnection::StateAccepting) { - sb << "Accepting "; + state = "Accepting"; } else { - sb << "Unknown "; + state = "Unknown"; } - return sb; + + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TKafkaConnection"}, + {"selfId", SelfId()}, + {"state", state}, + {"rawSocket", GetRawSocket()}, + {"address", Address->ToString()}); } void SendRequestMetrics(const TActorContext& ctx) { @@ -241,7 +260,9 @@ protected: HFunc(TEvKafka::TEvResponse, Handle); sFunc(NActors::TEvents::TEvPoison, PassAway); default: - KAFKA_LOG_ERROR("TKafkaConnection: Unexpected " << ev.Get()->GetTypeName()); + YDB_LOG_ERROR("TKafkaConnection: Unexpected", + {LogPrefix()}, + {"typeName", ev.Get()->GetTypeName()}); } } @@ -397,12 +418,17 @@ protected: } bool ProcessRequest(const TActorContext& ctx) { - KAFKA_LOG_D("process message: ApiKey=" << Request->Header.RequestApiKey << ", ExpectedSize=" << Request->ExpectedSize - << ", Size=" << Request->Size); + YDB_LOG_DEBUG("Process message", + {LogPrefix()}, + {"apiKey", Request->Header.RequestApiKey}, + {"expectedSize", Request->ExpectedSize}, + {"size", Request->Size}); auto apiKeyNameIt = EApiKeyNames.find(static_cast<EApiKey>(Request->Header.RequestApiKey)); if (apiKeyNameIt == EApiKeyNames.end()) { - KAFKA_LOG_ERROR("Unsupported message: ApiKey=" << Request->Header.RequestApiKey); + YDB_LOG_ERROR("Unsupported message", + {LogPrefix()}, + {"apiKey", Request->Header.RequestApiKey}); PassAway(); return false; } @@ -418,7 +444,8 @@ protected: } if (IsTransactionalApiKey(Request->Header.RequestApiKey) && !TransactionsEnabled()) { - KAFKA_LOG_ERROR("Transactional API keys are not enabled. To enable them set \"EnableKafkaTransactions\" feature flag to true in cluster configuration."); + YDB_LOG_ERROR("Transactional API keys are not enabled. To enable them set \"EnableKafkaTransactions\" feature flag to true in cluster configuration", + {LogPrefix()}); PassAway(); return false; } @@ -524,7 +551,9 @@ protected: break; default: - KAFKA_LOG_ERROR("Unsupported message: ApiKey=" << Request->Header.RequestApiKey); + YDB_LOG_ERROR("Unsupported message", + {LogPrefix()}, + {"apiKey", Request->Header.RequestApiKey}); PassAway(); return false; } @@ -545,7 +574,9 @@ protected: void Handle(TEvKafka::TEvReadSessionInfo::TPtr readInfo, const TActorContext& /*ctx*/) { auto r = readInfo->Get(); - KAFKA_LOG_D("Initializing GroupId=" << r->GroupId); + YDB_LOG_DEBUG("Initializing", + {LogPrefix()}, + {"groupId", r->GroupId}); Context->GroupId = r->GroupId; } @@ -557,7 +588,9 @@ protected: if (IsSslActive && NKikimr::AppData()->KafkaProxyConfig.GetMtlsEnable()) { MtlsAuthStage = AUTH_FAILED; } - KAFKA_LOG_ERROR(event->Error); + YDB_LOG_ERROR("", + {LogPrefix()}, + {"error", event->Error}); Reply(event->ClientResponse->CorrelationId, event->ClientResponse->Response, event->ClientResponse->ErrorCode, ctx); CloseConnection = true; return; @@ -569,7 +602,9 @@ protected: responseToClient->ErrorCode = kafkaError; TString errorMessage = TStringBuilder() << Context->SaslMechanism << " authentication mechanism is disabled, because mTLS flag is on. Turn of mTLS in configuration to use this mechanism."; responseToClient->ErrorMessage = errorMessage; - KAFKA_LOG_D(errorMessage); + YDB_LOG_DEBUG("Dump logPrefix, errorMessage", + {LogPrefix()}, + {"errorMessage", errorMessage}); std::shared_ptr<TEvKafka::TEvResponse> errorResponse = std::make_shared<TEvKafka::TEvResponse>(event->ClientResponse->CorrelationId, responseToClient, kafkaError); Reply(event->ClientResponse->CorrelationId, errorResponse->Response, kafkaError, ctx); @@ -588,7 +623,9 @@ protected: Context->IsServerless = event->IsServerless; Context->ResourceDatabasePath = event->ResourceDatabasePath ? NKikimr::CanonizePath(event->ResourceDatabasePath) : Context->DatabasePath; - KAFKA_LOG_D("Authentication successful. SID=" << Context->UserToken->GetUserSID()); + YDB_LOG_DEBUG("Authentication successful", + {LogPrefix()}, + {"SID", Context->UserToken->GetUserSID()}); if (Context->SaslMechanism != "MTLS") { Reply(event->ClientResponse->CorrelationId, event->ClientResponse->Response, event->ClientResponse->ErrorCode, ctx); } else { @@ -607,7 +644,8 @@ protected: auto errorResponse = std::make_shared<TEvKafka::TEvResponse>(event->ClientResponse->CorrelationId, responseToClient, kafkaError); TString errorMessage = TStringBuilder() << event->SaslMechanism << " authentication mechanism is disabled, because mTLS flag is on. Turn of mTLS in configuration to use this mechanism."; - KAFKA_LOG_D(errorMessage); + YDB_LOG_DEBUG(errorMessage, + {LogPrefix()}); Reply(event->ClientResponse->CorrelationId, errorResponse->Response, kafkaError, ctx); CloseConnection = true; return; @@ -616,7 +654,8 @@ protected: Reply(event->ClientResponse->CorrelationId, event->ClientResponse->Response, event->ClientResponse->ErrorCode, ctx); auto authStep = event->AuthStep; if (authStep == EAuthSteps::FAILED) { - KAFKA_LOG_ERROR(event->Error); + YDB_LOG_ERROR(event->Error, + {LogPrefix()}); CloseConnection = true; return; } @@ -636,7 +675,9 @@ protected: void Reply(const ui64 correlationId, TApiMessage::TPtr response, EKafkaErrors errorCode, const TActorContext& ctx) { auto it = PendingRequests.find(correlationId); if (it == PendingRequests.end()) { - KAFKA_LOG_ERROR("Unexpected correlationId " << correlationId); + YDB_LOG_ERROR("Unexpected correlationId", + {LogPrefix()}, + {"correlationId", correlationId}); return; } @@ -651,7 +692,9 @@ protected: } void OnRequestProcessed(const Msg::TPtr& request) { - KAFKA_LOG_T("Request with correlationId " << request->Header.CorrelationId << " processed. Erasing it from PendingRequests and PendingRequestsQueue"); + YDB_LOG_TRACE("Request with correlationId processed. Erasing it from PendingRequests and PendingRequestsQueue", + {LogPrefix()}, + {"correlationId", request->Header.CorrelationId}); InflightSize -= request->ExpectedSize; PendingRequests.erase(request->Header.CorrelationId); PendingRequestsQueue.pop_front(); @@ -660,9 +703,13 @@ protected: bool ProcessReplyQueue(const TActorContext& ctx) { while(!PendingRequestsQueue.empty()) { auto& request = PendingRequestsQueue.front(); - KAFKA_LOG_T("Processing reply queue for request with correlationId " << request->Header.CorrelationId); + YDB_LOG_TRACE("Processing reply queue for request with correlationId", + {LogPrefix()}, + {"correlationId", request->Header.CorrelationId}); if (request->Response.get() == nullptr) { - KAFKA_LOG_T("Response for request with correlationId " << request->Header.CorrelationId << " is empty."); + YDB_LOG_TRACE("Response for request with correlationId is empty", + {LogPrefix()}, + {"correlationId", request->Header.CorrelationId}); break; } @@ -681,7 +728,11 @@ protected: } bool Reply(const TRequestHeaderData* header, const TApiMessage* reply, const TString method, const TInstant requestStartTime, EKafkaErrors errorCode, const TActorContext& ctx) { - KAFKA_LOG_T("Building reply for method " << method << " and correlationId " << header->CorrelationId << " with error code: " << errorCode); + YDB_LOG_TRACE("Building reply for method and correlationId with error", + {LogPrefix()}, + {"method", method}, + {"correlationId", header->CorrelationId}, + {"code", errorCode}); TKafkaVersion headerVersion = ResponseHeaderVersion(header->RequestApiKey, header->RequestApiVersion); TKafkaVersion version = header->RequestApiVersion; @@ -706,19 +757,28 @@ protected: // PollerReady means that poller polled socket ready status if (res == -EAGAIN || res == -EWOULDBLOCK) { RetryingWriteToSocket = true; - KAFKA_LOG_D("Socket is busy. Buffer queue size: " << BufferedWriter.GetBuffersDeque().size() << ". Waiting for PollerReady event"); + YDB_LOG_DEBUG("Socket is busy. Buffer queue Waiting for PollerReady event", + {LogPrefix()}, + {"size", BufferedWriter.GetBuffersDeque().size()}); RequestPoller(); return false; } else if (res < 0) { ythrow yexception() << "Error during flush of the written to socket data. Error code: " << strerror(-res) << " (" << res << ")"; } else { - KAFKA_LOG_D("Sent reply: ApiKey=" << header->RequestApiKey << ", Version=" << version << ", Correlation=" << responseHeader.CorrelationId << ", Size=" << size); + YDB_LOG_DEBUG("Sent reply", + {LogPrefix()}, + {"apiKey", header->RequestApiKey}, + {"version", version}, + {"correlation", responseHeader.CorrelationId}, + {"size", size}); } } catch(const yexception& e) { - KAFKA_LOG_ERROR("error on processing response: ApiKey=" << reply->ApiKey() - << ", Version=" << version - << ", CorrelationId=" << header->CorrelationId - << ", Error=" << e.what()); + YDB_LOG_ERROR("Error on processing response", + {LogPrefix()}, + {"apiKey", reply->ApiKey()}, + {"version", version}, + {"correlationId", header->CorrelationId}, + {"error", e.what()}); PassAway(); return false; } @@ -730,7 +790,9 @@ protected: if (IsSslRequired && !IsSslActive) { int res = Socket->TryUpgradeToSecure(NKikimrServices::KAFKA_PROXY, ServerCreds); if (res < 0) { - KAFKA_LOG_ERROR("connection closed - error in UpgradeToSecure: " << strerror(-res)); + YDB_LOG_ERROR("Connection closed - error in UpgradeToSecure", + {LogPrefix()}, + {"error", strerror(-res)}); PassAway(); return false; } @@ -740,7 +802,10 @@ protected: } bool DoRead(const TActorContext& ctx) { - KAFKA_LOG_T("DoRead: Demand=" << Demand.Length << ", Step=" << static_cast<i32>(Step)); + YDB_LOG_TRACE("DoRead", + {LogPrefix()}, + {"demand", Demand.Length}, + {"step", static_cast<i32>(Step)}); for (;;) { while (Demand) { ssize_t received = 0; @@ -750,11 +815,14 @@ protected: } else if (-res == EINTR) { continue; } else if (!res) { - KAFKA_LOG_I("connection closed"); + YDB_LOG_INFO("Connection closed", + {LogPrefix()}); PassAway(); return false; } else if (res < 0) { - KAFKA_LOG_I("connection closed - error in recv: " << strerror(-res)); + YDB_LOG_INFO("Connection closed - error", + {LogPrefix()}, + {"error", strerror(-res)}); PassAway(); return false; } @@ -775,18 +843,24 @@ protected: case SIZE_PREPARE: NormalizeNumber(Request->ExpectedSize); if (Request->ExpectedSize < 0) { - KAFKA_LOG_ERROR("Wrong message size. Size: " << Request->ExpectedSize); + YDB_LOG_ERROR("Wrong message size", + {LogPrefix()}, + {"size", Request->ExpectedSize}); PassAway(); return false; } if ((ui64)Request->ExpectedSize > Context->Config.GetMaxMessageSize()) { - KAFKA_LOG_ERROR("message is big. Size: " << Request->ExpectedSize << ". MaxSize: " - << Context->Config.GetMaxMessageSize()); + YDB_LOG_ERROR("Message is big", + {LogPrefix()}, + {"size", Request->ExpectedSize}, + {"maxSize", Context->Config.GetMaxMessageSize()}); PassAway(); return false; } if (static_cast<size_t>(Request->ExpectedSize) < HeaderSize) { - KAFKA_LOG_ERROR("message is small. Size: " << Request->ExpectedSize); + YDB_LOG_ERROR("Message is small", + {LogPrefix()}, + {"size", Request->ExpectedSize}); PassAway(); return false; } @@ -798,12 +872,17 @@ protected: case INFLIGHT_CHECK: if (!Context->Authenticated() && !PendingRequestsQueue.empty()) { // Allow only one message to be processed at a time for non-authenticated users - KAFKA_LOG_ERROR("DoRead: failed inflight check: there are " << PendingRequestsQueue.size() << " pending requests and user is not authnicated. Only one paraller request is allowed for a non-authenticated user."); + YDB_LOG_ERROR("DoRead: failed inflight check: there are pending requests and user is not authnicated. Only one paraller request is allowed for a non-authenticated user", + {LogPrefix()}, + {"pendingRequestsQueue", PendingRequestsQueue.size()}); return true; } if (InflightSize + Request->ExpectedSize > Context->Config.GetMaxInflightSize()) { // We limit the size of processed messages so as not to exceed the size of available memory - KAFKA_LOG_ERROR("DoRead: failed inflight check: InflightSize + Request->ExpectedSize=" << InflightSize + Request->ExpectedSize << " > Context->Config.GetMaxInflightSize=" << Context->Config.GetMaxInflightSize()); + YDB_LOG_ERROR("DoRead: failed inflight check: InflightSize + >", + {LogPrefix()}, + {"expectedSize", InflightSize + Request->ExpectedSize}, + {"getMaxInflightSize", Context->Config.GetMaxInflightSize()}); return true; } InflightSize += Request->ExpectedSize; @@ -812,7 +891,9 @@ protected: [[fallthrough]]; case HEADER_READ: - KAFKA_LOG_T("start read header. ExpectedSize=" << Request->ExpectedSize); + YDB_LOG_TRACE("Start read header", + {LogPrefix()}, + {"expectedSize", Request->ExpectedSize}); Request->Buffer->Resize(HeaderSize); Demand = TReadDemand(Request->Buffer->Data(), HeaderSize); @@ -830,12 +911,16 @@ protected: NormalizeNumber(Request->CorrelationId); if (PendingRequests.contains(Request->CorrelationId)) { - KAFKA_LOG_ERROR("CorrelationId " << Request->CorrelationId << " already processing"); + YDB_LOG_ERROR("CorrelationId already processing", + {LogPrefix()}, + {"correlationId", Request->CorrelationId}); PassAway(); return false; } if (!Context->Authenticated() && RequireAuthentication(static_cast<EApiKey>(Request->ApiKey))) { - KAFKA_LOG_ERROR("unauthenticated request: ApiKey=" << Request->ApiKey); + YDB_LOG_ERROR("Unauthenticated request", + {LogPrefix()}, + {"apiKey", Request->ApiKey}); PassAway(); return false; } @@ -845,7 +930,9 @@ protected: [[fallthrough]]; case MESSAGE_READ: - KAFKA_LOG_T("start read new message. ExpectedSize=" << Request->ExpectedSize); + YDB_LOG_TRACE("Start read new message", + {LogPrefix()}, + {"expectedSize", Request->ExpectedSize}); Request->Buffer->Resize(Request->ExpectedSize); Demand = TReadDemand(Request->Buffer->Data() + HeaderSize, Request->ExpectedSize - HeaderSize); @@ -856,10 +943,18 @@ protected: case MESSAGE_PROCESS: Request->StartTime = TInstant::Now(); if constexpr (DEBUG_ENABLED) { - KAFKA_LOG_D("received message. ApiKey=" << Request->ApiKey << ", Version=" << Request->ApiVersion << ", CorrelationId=" << Request->CorrelationId - << ", Data=" << Hex(Request->Buffer->Begin(), Request->Buffer->End())); + YDB_LOG_DEBUG("Received message", + {LogPrefix()}, + {"apiKey", Request->ApiKey}, + {"version", Request->ApiVersion}, + {"correlationId", Request->CorrelationId}, + {"data", Hex(Request->Buffer->Begin(), Request->Buffer->End())}); } else { - KAFKA_LOG_D("received message. ApiKey=" << Request->ApiKey << ", Version=" << Request->ApiVersion << ", CorrelationId=" << Request->CorrelationId); + YDB_LOG_DEBUG("Received message", + {LogPrefix()}, + {"apiKey", Request->ApiKey}, + {"version", Request->ApiVersion}, + {"correlationId", Request->CorrelationId}); } TKafkaReadable readable(*Request->Buffer); @@ -871,10 +966,12 @@ protected: Request->Header.Read(readable, RequestHeaderVersion(Request->ApiKey, Request->ApiVersion)); Request->Message->Read(readable, Request->ApiVersion); } catch(const yexception& e) { - KAFKA_LOG_ERROR("error on processing message: ApiKey=" << Request->ApiKey - << ", Version=" << Request->ApiVersion - << ", CorrelationId=" << Request->CorrelationId - << ", Error=" << e.what()); + YDB_LOG_ERROR("Error on processing message", + {LogPrefix()}, + {"apiKey", Request->ApiKey}, + {"version", Request->ApiVersion}, + {"correlationId", Request->CorrelationId}, + {"error", e.what()}); PassAway(); return false; } @@ -882,7 +979,8 @@ protected: Step = SIZE_READ; if (IsSslActive && NKikimr::AppData()->KafkaProxyConfig.GetMtlsEnable() && MtlsAuthStage != MtlsAuthStages::AUTH_SUCCESSFUL) { - KAFKA_LOG_D("Mtls authentication was not successful."); + YDB_LOG_DEBUG("Mtls authentication was not successful", + {LogPrefix()}); return false; } @@ -910,14 +1008,17 @@ protected: if (sslHandshakeResult != SslHandshakeErrors::ERROR_WANT_READ && sslHandshakeResult != SslHandshakeErrors::ERROR_WANT_WRITE && sslHandshakeResult != SslHandshakeErrors::ERROR_NONE) { - KAFKA_LOG_D("Error in ssl handshake, ssl ErrorCode=" << sslHandshakeResult); + YDB_LOG_DEBUG("Error in ssl handshake, ssl", + {LogPrefix()}, + {"errorCode", sslHandshakeResult}); PassAway(); return; } if (sslHandshakeResult == SslHandshakeErrors::ERROR_NONE) { TSslHelpers::TSslHolder<X509> cert = Socket->GetSslClientCert(); if (!cert) { - KAFKA_LOG_ERROR("No cert was received from client during ssl handshake for mTLS authentication."); + YDB_LOG_ERROR("No cert was received from client during ssl handshake for mTLS authentication", + {LogPrefix()}); PassAway(); return; } @@ -943,7 +1044,8 @@ protected: if (event->Get() == InactivityEvent) { const TDuration passed = TDuration::Seconds(std::abs(InactivityTimer.Passed())); if (passed >= InactivityTimeout) { - KAFKA_LOG_D("connection closed by inactivity timeout"); + YDB_LOG_DEBUG("Connection closed by inactivity timeout", + {LogPrefix()}); return PassAway(); // timeout } else { Schedule(InactivityTimeout - passed, InactivityEvent = new TEvPollerReady(nullptr, false, false)); @@ -951,21 +1053,32 @@ protected: } } if (event->Get()->Write && !BufferedWriter.Empty()) { - KAFKA_LOG_D("Retrying flush. Buffer queue size: " << BufferedWriter.GetBuffersDeque().size()); + YDB_LOG_DEBUG("Retrying flush. Buffer queue", + {LogPrefix()}, + {"size", BufferedWriter.GetBuffersDeque().size()}); ssize_t res = BufferedWriter.flush(); if (res == -EAGAIN || res == -EWOULDBLOCK) { - KAFKA_LOG_D("Socket is busy during retry. Buffer queue size: " << BufferedWriter.GetBuffersDeque().size() << ". Waiting for PollerReady event"); + YDB_LOG_DEBUG("Socket is busy during retry. Buffer queue Waiting for PollerReady event", + {LogPrefix()}, + {"size", BufferedWriter.GetBuffersDeque().size()}); RequestPoller(); return; } else if (res < 0) { - KAFKA_LOG_ERROR("connection closed - error in FlushOutput: " << strerror(-res) << ". Buffer queue size: " << BufferedWriter.GetBuffersDeque().size()); + YDB_LOG_ERROR("Connection closed - error Buffer queue", + {LogPrefix()}, + {"flushOutput", strerror(-res)}, + {"size", BufferedWriter.GetBuffersDeque().size()}); PassAway(); return; } else if (res > 0 && BufferedWriter.Empty()) { // we successfuly retried sending the response RetryingWriteToSocket = false; auto& request = PendingRequestsQueue.front(); auto& header = request->Header; - KAFKA_LOG_D("Sent reply (after retry): ApiKey=" << header.RequestApiKey << ", Version=" << header.RequestApiVersion << ", Correlation=" << header.CorrelationId); + YDB_LOG_DEBUG("Sent reply (after retry)", + {LogPrefix()}, + {"apiKey", header.RequestApiKey}, + {"version", header.RequestApiVersion}, + {"correlation", header.CorrelationId}); OnRequestProcessed(request); ProcessReplyQueue(ctx); @@ -976,7 +1089,8 @@ protected: } if (CloseConnection && BufferedWriter.Empty()) { - KAFKA_LOG_D("connection closed"); + YDB_LOG_DEBUG("Connection closed", + {LogPrefix()}); return PassAway(); } @@ -1003,7 +1117,9 @@ protected: sFunc(TEvKafka::TEvKillReadSession, HandleKillReadSession); sFunc(NActors::TEvents::TEvPoison, PassAway); default: - KAFKA_LOG_ERROR("TKafkaConnection: Unexpected " << ev.Get()->GetTypeName()); + YDB_LOG_ERROR("TKafkaConnection: Unexpected", + {LogPrefix()}, + {"typeName", ev.Get()->GetTypeName()}); } } }; diff --git a/ydb/core/kafka_proxy/kafka_log_impl.h b/ydb/core/kafka_proxy/kafka_log_impl.h index c6ba1e8e5dc..df427e3d00a 100644 --- a/ydb/core/kafka_proxy/kafka_log_impl.h +++ b/ydb/core/kafka_proxy/kafka_log_impl.h @@ -6,14 +6,6 @@ namespace NKafka { -inline TString LogPrefix() { return {}; } +inline NActors::NStructuredLog::TStructuredMessage LogPrefix() { return {}; } } - -#define KAFKA_LOG_T(stream) LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_D(stream) LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_I(stream) LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_W(stream) LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_NOTICE(stream) LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_ERROR(stream) LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) -#define KAFKA_LOG_CRIT(stream) LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::KAFKA_PROXY, LogPrefix() << stream) diff --git a/ydb/core/kafka_proxy/kafka_transactions_coordinator.cpp b/ydb/core/kafka_proxy/kafka_transactions_coordinator.cpp index d950b37cb80..dc7fb3e1339 100644 --- a/ydb/core/kafka_proxy/kafka_transactions_coordinator.cpp +++ b/ydb/core/kafka_proxy/kafka_transactions_coordinator.cpp @@ -3,6 +3,8 @@ #include "actors/txn_actor_response_builder.h" #include <ydb/core/kqp/common/simple/services.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KAFKA_PROXY + namespace NKafka { // Handles new transactional_id+producer_id+producer_epoch: // 1. validates that producer is not a zombie (in case of parallel init_producer_requests) @@ -49,7 +51,8 @@ namespace NKafka { void TTransactionsCoordinator::Handle(NMetadata::NProvider::TEvManagerPrepared::TPtr&, const TActorContext&) { TablesInited++; if (TablesInited == TABLES_COUNT) { - KAFKA_LOG_D("All tables are prepared"); + YDB_LOG_DEBUG("All tables are prepared", + {LogPrefix()}); } }; @@ -59,28 +62,35 @@ namespace NKafka { if (it == ProducersByTransactionalId.end() || it->second.Id != deadActorProducerState) { // new actor was already registered, we can just ignore this event - KAFKA_LOG_D("Received TEvTransactionActorDied for transactionalId " << ev->Get()->TransactionalId << " but producer has already been reinitialized with new epoch or deleted. Ignoring this event"); + YDB_LOG_DEBUG("Received TEvTransactionActorDied for transactionalId but producer has already been reinitialized with new epoch or deleted. Ignoring this event", + {LogPrefix()}, + {"transactionalId", ev->Get()->TransactionalId}); } else { - KAFKA_LOG_D("Received TEvTransactionActorDied for transactionalId " << ev->Get()->TransactionalId - << " and producerId " << ev->Get()->ProducerState.Id - << " and producerEpoch " << ev->Get()->ProducerState.Epoch - << ". Erasing info about this actor."); + YDB_LOG_DEBUG("Received TEvTransactionActorDied for transactionalId and producerId and producerEpoch Erasing info about this actor", + {LogPrefix()}, + {"transactionalId", ev->Get()->TransactionalId}, + {"producerStateId", ev->Get()->ProducerState.Id}, + {"producerStateEpoch", ev->Get()->ProducerState.Epoch}); // erase info about actor to prevent future zombie requests from this producer TxnActorByTransactionalId.erase(ev->Get()->TransactionalId); } }; void TTransactionsCoordinator::Handle(TEvents::TEvPoison::TPtr&, const TActorContext& ctx) { - KAFKA_LOG_D("Got poison pill, killing all transaction actors"); + YDB_LOG_DEBUG("Got poison pill, killing all transaction actors", + {LogPrefix()}); for (auto& [transactionalId, txnActorId]: TxnActorByTransactionalId) { ctx.Send(txnActorId, new TEvents::TEvPoison()); - KAFKA_LOG_D("Sent poison pill to transaction actor for transactionalId " << transactionalId); + YDB_LOG_DEBUG("Sent poison pill to transaction actor for transactionalId", + {LogPrefix()}, + {"transactionalId", transactionalId}); } PassAway(); }; void TTransactionsCoordinator::PassAway() { - KAFKA_LOG_D("Killing myself"); + YDB_LOG_DEBUG("Killing myself", + {LogPrefix()}); TBase::PassAway(); }; @@ -90,7 +100,10 @@ namespace NKafka { template<class ErrorResponseType, class EventType> void TTransactionsCoordinator::HandleTransactionalRequest(TAutoPtr<TEventHandle<EventType>>& evHandle, const TActorContext& ctx) { EventType* ev = evHandle->Get(); - KAFKA_LOG_D("Received message for transactionalId " << *ev->Request->TransactionalId << " and ApiKey " << ev->Request->ApiKey()); + YDB_LOG_DEBUG("Received message for transactionalId and ApiKey", + {LogPrefix()}, + {"transactionalId", *ev->Request->TransactionalId}, + {"apiKey", ev->Request->ApiKey()}); // create helper struct to simplify methods interaction auto txnRequest = TTransactionalRequest( @@ -108,7 +121,8 @@ namespace NKafka { template<class ErrorResponseType, class RequestType> void TTransactionsCoordinator::SendProducerFencedResponse(TMessagePtr<RequestType> kafkaRequest, const TString& error, const TTransactionalRequest& txnRequestDetails) { - KAFKA_LOG_W(error); + YDB_LOG_WARN(error, + {LogPrefix()}); std::shared_ptr<ErrorResponseType> response = NKafkaTransactions::BuildResponse<ErrorResponseType>(kafkaRequest, EKafkaErrors::PRODUCER_FENCED); Send(txnRequestDetails.ConnectionId, new TEvKafka::TEvResponse(txnRequestDetails.CorrelationId, response, EKafkaErrors::PRODUCER_FENCED)); }; @@ -125,9 +139,17 @@ namespace NKafka { txnActorId = ctx.Register(new TTransactionActor(*ev->Request->TransactionalId, {ev->Request->ProducerId, ev->Request->ProducerEpoch}, ev->DatabasePath, producerInstance.TxnTimeoutMs, ev->ResourceDatabasePath ? ev->ResourceDatabasePath : ev->DatabasePath)); TxnActorByTransactionalId[*ev->Request->TransactionalId] = txnActorId; - KAFKA_LOG_D("Registered TTransactionActor with id " << txnActorId << " for transactionalId " << *ev->Request->TransactionalId << " and ApiKey " << ev->Request->ApiKey()); + YDB_LOG_DEBUG("Registered TTransactionActor with id for transactionalId and ApiKey", + {LogPrefix()}, + {"txnActorId", txnActorId}, + {"transactionalId", *ev->Request->TransactionalId}, + {"apiKey", ev->Request->ApiKey()}); } - KAFKA_LOG_D("Forwarded message to TTransactionActor with id " << txnActorId << " for transactionalId " << *ev->Request->TransactionalId << " and ApiKey " << ev->Request->ApiKey()); + YDB_LOG_DEBUG("Forwarded message to TTransactionActor with id for transactionalId and ApiKey", + {LogPrefix()}, + {"txnActorId", txnActorId}, + {"transactionalId", *ev->Request->TransactionalId}, + {"apiKey", ev->Request->ApiKey()}); TAutoPtr<IEventHandle> tmpPtr = evHandle.Release(); ctx.Forward(tmpPtr, txnActorId); }; diff --git a/ydb/core/kafka_proxy/kafka_transactions_coordinator.h b/ydb/core/kafka_proxy/kafka_transactions_coordinator.h index ff982afbdb6..4dc147ba157 100644 --- a/ydb/core/kafka_proxy/kafka_transactions_coordinator.h +++ b/ydb/core/kafka_proxy/kafka_transactions_coordinator.h @@ -51,8 +51,10 @@ namespace NKafka { TBase::Become(&TTransactionsCoordinator::StateWork); } - TStringBuilder LogPrefix() const { - return TStringBuilder() << "KafkaTransactionsCoordinator "; + NStructuredLog::TStructuredMessage LogPrefix() const { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "KafkaTransactionsCoordinator"}, + {"selfId", SelfId()}); } void PassAway() override; diff --git a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp b/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp index ec839ca3fd9..3e440fd148b 100644 --- a/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp +++ b/ydb/core/kqp/gateway/behaviour/resource_pool_classifier/checker.cpp @@ -323,6 +323,29 @@ private: return; } + if (!featureFlags.GetEnableHasPredicatesInResourcePoolClassifiers()) { + for (const auto& object : PatchedObjects) { + const auto& configMap = object.GetConfigJson().GetMap(); + if (configMap.count("has_full_scan") || configMap.count("has_path") || + configMap.count("has_stream") || configMap.count("has_app_name")) { + FailAndPassAway("Advanced resource pool classifier properties are disabled. " + "Please contact your system administrator to enable it"); + return; + } + } + } + + if (!featureFlags.GetEnableRejectActionInResourcePoolClassifiers()) { + for (const auto& object : PatchedObjects) { + const auto& configMap = object.GetConfigJson().GetMap(); + if (configMap.count("action")) { + FailAndPassAway("Resource pool classifier action property is disabled. " + "Please contact your system administrator to enable it"); + return; + } + } + } + FeatureFlagChecked = true; ValidateExistence(); ValidatePools(); diff --git a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp index cd7e2a8f400..df938397094 100644 --- a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp +++ b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_query_builder.cpp @@ -165,6 +165,26 @@ TVector<TExprNode::TPtr> TPhysicalQueryBuilder::BuildPhysicalStageGraph() { TExprNode::TPtr stage; if (Graph.IsSourceStageRowType(id)) { stage = Stages.at(id); + // Want to build materialize for ranges. + // TODO: Actually old optimizer has some machinery to compute ranges during compilation for some cases, for + // example when `LiteralRange` is defined, but currenlty we put any case in separate tx. Performance improvement is possible here. + auto rowSettingsPtr = FindNode(stage, [](const TExprNode::TPtr& node) { return !!TMaybeNode<TKqpReadRangesSourceSettings>(node); }); + if (rowSettingsPtr) { + auto rowSettings = TExprBase(rowSettingsPtr).Cast<TKqpReadRangesSourceSettings>(); + if (!rowSettings.RangesExpr().Maybe<TCoVoid>()) { + const auto materializeResult = BuildMaterialize(rowSettings.RangesExpr().Ptr()); + // clang-fomrat off + const auto newRowSettings = Build<TKqpReadRangesSourceSettings>(ctx, rowSettingsPtr->Pos()) + .Table(rowSettings.Table()) + .Columns(rowSettings.Columns()) + .Settings(rowSettings.Settings()) + .RangesExpr(materializeResult) + .ExplainPrompt(rowSettings.ExplainPrompt()) + .Done().Ptr(); + // clang-format on + stage = ctx.ReplaceNode(std::move(stage), rowSettings.Ref(), newRowSettings); + } + } } else { TVector<TExprNode::TPtr> stageInputConnections; TVector<TExprNode::TPtr> stageInputArgs; diff --git a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp index 53efaa30b8f..bcb8695fa2c 100644 --- a/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp +++ b/ydb/core/kqp/opt/rbo/physical_conversion/kqp_rbo_physical_source_builder.cpp @@ -34,6 +34,8 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { for (const auto& column : Read->Columns) { columns.push_back(Ctx.NewAtom(Pos, column)); } + // Extract ranges. + TExprNode::TPtr ranges = Read->GetRanges() ? Read->GetRanges() : Build<TCoVoid>(Ctx, Pos).Done().Ptr(); switch (Read->GetTableStorageType()) { case NYql::EStorageType::RowStorage: { @@ -48,7 +50,7 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { .Add(columns) .Build() .Settings<TCoNameValueTupleList>().Build() - .RangesExpr<TCoVoid>().Build() + .RangesExpr(ranges) .ExplainPrompt<TCoNameValueTupleList>().Build() .Build() .Done().Ptr(); @@ -108,7 +110,6 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { settings.SequentialInFlight = 1; } - TExprNode::TPtr ranges = Read->GetRanges() ? Read->GetRanges() : Build<TCoVoid>(Ctx, Pos).Done().Ptr(); // clang-format off auto olapRead = Build<TKqpBlockReadOlapTableRanges>(Ctx, Pos) .Table(Read->TableCallable) @@ -139,7 +140,7 @@ TExprNode::TPtr TPhysicalSourceBuilder::BuildPhysicalOp() { break; } default: - Y_ENSURE(false, "Unsupported table source type"); + Y_ENSURE(false, "Unsupported table source type."); } YQL_CLOG(TRACE, CoreDq) << "[NEW RBO Physical source] " << KqpExprToPrettyString(TExprBase(source), Ctx); diff --git a/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp b/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp index 80a8978fc6b..3614c0aa3b0 100644 --- a/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp +++ b/ydb/core/kqp/opt/rbo/rules/push_ranges.cpp @@ -123,8 +123,8 @@ bool IsSuitableToExtractAndPushRanges(const TIntrusivePtr<IOperator>& input) { } const auto read = CastOperator<TOpRead>(maybeRead); - // Currently supported only for cs. - return !read->GetRanges() && read->GetTableStorageType() == NYql::EStorageType::ColumnStorage; + const auto tableType = read->GetTableStorageType(); + return !read->GetRanges() && (tableType == NYql::EStorageType::ColumnStorage || tableType == NYql::EStorageType::RowStorage); } TPredicateExtractorSettings PrepareExtractorSettings(TKqpOptimizeContext& kqpCtx) { diff --git a/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp b/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp index 2935c169ebe..e7b464503f5 100644 --- a/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp +++ b/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp @@ -350,7 +350,7 @@ public: limits.RemoteChannelInflightBytes = config.GetRemoteChannelInflightBytes(); limits.RemoteSessionInflightBytes = config.GetRemoteSessionInflightBytes(); limits.ReconciliationCount = config.GetReconciliationCount(); - limits.CleanupPeriod = TDuration::MilliSeconds(std::max<ui64>(config.GetCleanupPeriodMs(), 200)); + limits.CleanupPeriod = TDuration::MilliSeconds(config.GetCleanupPeriodMs()); limits.IdlePingPeriod = TDuration::MilliSeconds(config.GetIdlePingPeriodMs()); limits.IdleDestroyPeriod = TDuration::MilliSeconds(config.GetIdleDestroyPeriodMs()); } else { // deprecated diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index ac0d8afcd2e..1b13e6dea22 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -1007,7 +1007,7 @@ public: } bool sent = false; - const auto result = classifier->PostCompileClassify(*QueryState->PreparedQuery); + const auto result = classifier->PostCompileClassify(*QueryState->PreparedQuery, *QueryState->UserRequestContext); using TError = std::optional<std::pair<Ydb::StatusIds::StatusCode, TString>>; auto error = std::visit(TOverloaded { diff --git a/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp b/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp index d6935ee8eaa..1c272f4dbc6 100644 --- a/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp +++ b/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp @@ -58,6 +58,8 @@ struct TWorkerSettings { int MinMessageSize = 10; int MaxMessageSize = 10000; bool EarlyFinish = false; + int PauseMessageIndex = -1; + int PauseDelayMs = 0; }; struct TFailureSettings { @@ -67,12 +69,13 @@ struct TFailureSettings { int Discovery = 0; }; -class TWorkerActor : public NActors::TActor<TWorkerActor> { +template <typename TDerived> +class TWorkerActor : public NActors::TActor<TDerived> { public: - TWorkerActor(std::shared_ptr<IDqChannelService> service, TEvTestPrivate::ERole role, ui32 channelId, const TWorkerSettings& settings) - : TActor(&TThis::StateFunc) + TWorkerActor(const TString& logPrefix, std::shared_ptr<IDqChannelService> service, ui32 channelId, const TWorkerSettings& settings) + : NActors::TActor<TDerived>(&TWorkerActor::StateFunc) + , LogPrefix(logPrefix) , Service(service) - , Role(role) , ChannelId(channelId) , Settings(settings) {} @@ -86,98 +89,38 @@ public: } } - void Run() { - if (!Started) { - switch (Role) { - case TEvTestPrivate::ERole::Producer: { - TChannelFullInfo info(ChannelId, SelfId(), PeerId, 0, 1, TCollectStatsLevel::None); - Buffer = Service->GetOutputBuffer(info, nullptr, nullptr); - break; - } - case TEvTestPrivate::ERole::Consumer: { - TChannelFullInfo info(ChannelId, PeerId, SelfId(), 0, 1, TCollectStatsLevel::None); - Buffer = Service->GetInputBuffer(info, nullptr); - break; - } - } - Started = true; - } - - switch (Role) { - case TEvTestPrivate::ERole::Producer: { - if (Buffer->IsFinished()) { - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, "TEST FINISHED SelfId=" << SelfId() << ", ChannelId=" << ChannelId); - Send(RunnerId, new TEvTestPrivate::TEvFinished(Role, false)); - PassAway(); - return; - } - while (Buffer->GetFillLevel() == EDqFillLevel::NoLimit && MessageIndex <= Settings.MessageCount) { - if (MessageIndex == Settings.MessageCount) { - Buffer->SendFinish(); - } else { - auto bytes = Settings.MinMessageSize + RandomNumber<ui64>(Settings.MaxMessageSize - Settings.MinMessageSize); - Buffer->Push(TDataChunk(NYql::TChunkedBuffer(TString(bytes, 'a')), 1, false)); - } - MessageIndex++; - } - break; - } - case TEvTestPrivate::ERole::Consumer: { - TDataChunk data; - while (MessageIndex < Settings.MessageCount && Buffer->Pop(data)) { - MessageIndex++; - } - if (Settings.EarlyFinish && MessageIndex == Settings.MessageCount) { - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, "TEST EARLY FINISH SelfId=" << SelfId() << ", ChannelId=" << ChannelId); - Buffer->EarlyFinish(); - MessageIndex++; - } - if (MessageIndex <= Settings.MessageCount && Buffer->Pop(data)) { - MessageIndex++; - } - if (Buffer->IsFinished()) { - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, "TEST FINISHED SelfId=" << SelfId() << ", ChannelId=" << ChannelId - << ", Role=" << (Role == TEvTestPrivate::ERole::Producer ? "Producer" : "Consumer")); - Send(RunnerId, new TEvTestPrivate::TEvFinished(Role, false)); - PassAway(); - } - break; - } - } - } + virtual void Run() = 0; - void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr&) { + virtual void HandleWakeup(NActors::TEvents::TEvWakeup::TPtr&) { Run(); } - void HandleResume(TEvDqCompute::TEvResumeExecution::TPtr&) { + virtual void HandleResume(TEvDqCompute::TEvResumeExecution::TPtr&) { Run(); } - void HandleStart(TEvTestPrivate::TEvStart::TPtr& ev) { + virtual void HandleStart(TEvTestPrivate::TEvStart::TPtr& ev) { RunnerId = ev->Sender; PeerId = ev->Get()->PeerId; - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, - "TEST WORKER START SelfId=" << SelfId() << ", ChannelId=" << ChannelId << ", PeerId=" << PeerId << ", Role=" << (Role == TEvTestPrivate::ERole::Producer ? "Producer" : "Consumer") - ); + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST START SelfId=" << this->SelfId() << ", ChannelId=" << ChannelId << ", PeerId=" << PeerId); if (Settings.StartDelayMs) { - Schedule(TDuration::MilliSeconds(RandomNumber<ui64>(Settings.StartDelayMs) + 1), new NActors::TEvents::TEvWakeup()); + this->Schedule(TDuration::MilliSeconds(RandomNumber<ui64>(Settings.StartDelayMs) + 1), new NActors::TEvents::TEvWakeup()); } else { Run(); } } - void HandleAbort(NYql::NDq::TEvDq::TEvAbortExecution::TPtr& ev) { + virtual void HandleAbort(NYql::NDq::TEvDq::TEvAbortExecution::TPtr& ev) { LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, - "TEST WORKER ABORT SelfId=" << SelfId() << ", ChannelId=" << ChannelId << ", " << ev->Get()->GetIssues().ToOneLineString() + LogPrefix << "TEST ABORT SelfId=" << this->SelfId() << ", ChannelId=" << ChannelId << ", " << ev->Get()->GetIssues().ToOneLineString() ); - Send(RunnerId, new TEvTestPrivate::TEvFinished(Role, true)); - PassAway(); + this->Send(RunnerId, new TEvTestPrivate::TEvFinished(TEvTestPrivate::ERole::Producer, true)); + this->PassAway(); } + TString LogPrefix; std::shared_ptr<IDqChannelService> Service; std::shared_ptr<IChannelBuffer> Buffer; - TEvTestPrivate::ERole Role; ui32 ChannelId; NActors::TActorId PeerId; NActors::TActorId RunnerId; @@ -186,105 +129,246 @@ public: bool Started = false; }; -Y_UNIT_TEST_SUITE(Channels20) { - - void LoadTest(int count, bool local, const TWorkerSettings& producerSettings, const TWorkerSettings& consumerSettings, const TFailureSettings& failureSettings = TFailureSettings{}) { +class TProducerActor : public TWorkerActor<TProducerActor> { +public: + TProducerActor(std::shared_ptr<IDqChannelService> service, ui32 channelId, const TWorkerSettings& settings) + : TWorkerActor("PROD ", service, channelId, settings) + {} - TKikimrSettings settings; - settings.NodeCount = local ? 1 : 2; - settings.LogSettings = TTestLogSettings().AddLogPriority(NKikimrServices::KQP_CHANNELS, NActors::NLog::EPriority::PRI_TRACE); - settings.LogSettings->DefaultLogPriority = NActors::NLog::EPriority::PRI_CRIT; - TKikimrRunner kikimr(settings); - auto& runtime = *kikimr.GetTestServer().GetRuntime(); - runtime.SetUseRealInterconnect(); + void Run() override { + if (!Started) { + TChannelFullInfo info(ChannelId, SelfId(), PeerId, 0, 1, TCollectStatsLevel::None); + Buffer = Service->GetOutputBuffer(info, nullptr, nullptr); + Started = true; + } + if (Buffer->IsFinished()) { + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST FINISHED SelfId=" << SelfId() << ", ChannelId=" << ChannelId); + Send(RunnerId, new TEvTestPrivate::TEvFinished(TEvTestPrivate::ERole::Producer, false)); + PassAway(); + return; + } + while (Buffer->GetFillLevel() == EDqFillLevel::NoLimit && MessageIndex < Settings.MessageCount) { + if (Settings.PauseMessageIndex == MessageIndex) { + if (!ResumeTime) { + ResumeTime = TInstant::Now() + TDuration::MilliSeconds(Settings.PauseDelayMs); + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST PAUSED SelfId=" << SelfId() << ", ChannelId=" << ChannelId); + } + if (TInstant::Now() < ResumeTime) { + Schedule(ResumeTime, new NActors::TEvents::TEvWakeup()); + return; + } else { + ResumeTime = TInstant::Zero(); + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST RESUMED SelfId=" << SelfId() << ", ChannelId=" << ChannelId); + } + } + auto bytes = Settings.MinMessageSize + RandomNumber<ui64>(Settings.MaxMessageSize - Settings.MinMessageSize); + Buffer->Push(TDataChunk(NYql::TChunkedBuffer(TString(bytes, 'a')), 1, false)); + MessageIndex++; + } + if (MessageIndex == Settings.MessageCount) { + Buffer->SendFinish(); + } + } - auto control0 = runtime.AllocateEdgeActor(0); - auto control1 = local ? control0 : runtime.AllocateEdgeActor(1); + TInstant ResumeTime; +}; - std::shared_ptr<TDqChannelService> service0; - std::shared_ptr<TDqChannelService> service1; +class TConsumerActor : public TWorkerActor<TConsumerActor> { +public: + TConsumerActor(std::shared_ptr<IDqChannelService> service, ui32 channelId, const TWorkerSettings& settings) + : TWorkerActor("CONS ", service, channelId, settings) + {} - ui32 nodeIndex0 = 0; - ui32 nodeIndex1 = local ? 0 : 1; + void Run() override { + if (!Started) { + TChannelFullInfo info(ChannelId, PeerId, SelfId(), 0, 1, TCollectStatsLevel::None); + Buffer = Service->GetInputBuffer(info, nullptr); + Started = true; + } + TDataChunk data; + while (MessageIndex < Settings.MessageCount && Buffer->Pop(data)) { + MessageIndex++; + } + if (Settings.EarlyFinish && MessageIndex == Settings.MessageCount) { + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST EARLY FINISH SelfId=" << SelfId() << ", ChannelId=" << ChannelId); + Buffer->EarlyFinish(); + MessageIndex++; + } + if (MessageIndex <= Settings.MessageCount && Buffer->Pop(data)) { + MessageIndex++; + } + if (Buffer->IsFinished()) { + LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST FINISHED SelfId=" << SelfId() << ", ChannelId=" << ChannelId); + Send(RunnerId, new TEvTestPrivate::TEvFinished(TEvTestPrivate::ERole::Consumer, false)); + PassAway(); + } + } +}; - runtime.Send(MakeChannelServiceActorID(runtime.GetNodeId(0)), control0, new TEvPrivate::TEvServiceLookup(), nodeIndex0); - auto serviceReply = runtime.GrabEdgeEvent<TEvPrivate::TEvServiceReply>(control0)->Release(); - service0 = serviceReply->Service; +struct TLoadTest { - if (local) { - service1 = service0; - } else { - runtime.Send(MakeChannelServiceActorID(runtime.GetNodeId(1)), control1, new TEvPrivate::TEvServiceLookup(), nodeIndex1); - auto serviceReply = runtime.GrabEdgeEvent<TEvPrivate::TEvServiceReply>(control1)->Release(); - service1 = serviceReply->Service; + virtual void Prepare() { + settings.NodeCount = Local ? 1 : 2; + settings.LogSettings = TTestLogSettings().AddLogPriority(NKikimrServices::KQP_CHANNELS, NActors::NLog::EPriority::PRI_TRACE); + settings.LogSettings->DefaultLogPriority = NActors::NLog::EPriority::PRI_CRIT; + if (Local) { + NodeIndex1 = NodeIndex0; } + } - THashSet<NActors::TActorId> actors; + virtual void Init() { + Runner = std::make_unique<TKikimrRunner>(settings); + Runtime = Runner->GetTestServer().GetRuntime(); + Runtime->SetUseRealInterconnect(); - std::atomic<int> dataCount = 0; - NActors::TTestActorRuntime::TEventObserverHolder dataHolder; - if (failureSettings.Data) { - dataHolder = runtime.AddObserver<TEvDqCompute::TEvChannelDataV2>([&](TEvDqCompute::TEvChannelDataV2::TPtr& event) { - if (dataCount.fetch_add(1) == failureSettings.Data) { - dataCount.fetch_sub(failureSettings.Data); - event.Reset(); - } - }); + Control0 = Runtime->AllocateEdgeActor(0); + Control1 = Local ? Control0 : Runtime->AllocateEdgeActor(1); + + Runtime->Send(MakeChannelServiceActorID(Runtime->GetNodeId(0)), Control0, new TEvPrivate::TEvServiceLookup(), NodeIndex0); + auto serviceReply = Runtime->GrabEdgeEvent<TEvPrivate::TEvServiceReply>(Control0)->Release(); + Service0 = serviceReply->Service; + + if (Local) { + Service1 = Service0; + } else { + Runtime->Send(MakeChannelServiceActorID(Runtime->GetNodeId(1)), Control1, new TEvPrivate::TEvServiceLookup(), NodeIndex1); + auto serviceReply = Runtime->GrabEdgeEvent<TEvPrivate::TEvServiceReply>(Control1)->Release(); + Service1 = serviceReply->Service; } + } - for (auto i = 0; i < count; i ++) { + virtual void Start() { + for (auto i = 0; i < Count; i ++) { auto channelId = i + 1; if ((i & 1) == 0) { - auto producer = runtime.Register(new TWorkerActor(service0, TEvTestPrivate::ERole::Producer, channelId, producerSettings), nodeIndex0); - auto consumer = runtime.Register(new TWorkerActor(service1, TEvTestPrivate::ERole::Consumer, channelId, consumerSettings), nodeIndex1); - runtime.Send(consumer, control1, new TEvTestPrivate::TEvStart(producer), nodeIndex1, true); - runtime.Send(producer, control0, new TEvTestPrivate::TEvStart(consumer), nodeIndex0, true); - actors.insert(producer); - actors.insert(consumer); + auto producer = Runtime->Register(new TProducerActor(Service0, channelId, ProducerSettings), NodeIndex0); + auto consumer = Runtime->Register(new TConsumerActor(Service1, channelId, ConsumerSettings), NodeIndex1); + Runtime->Send(consumer, Control1, new TEvTestPrivate::TEvStart(producer), NodeIndex1, true); + Runtime->Send(producer, Control0, new TEvTestPrivate::TEvStart(consumer), NodeIndex0, true); + Actors.insert(producer); + Actors.insert(consumer); } else { - auto producer = runtime.Register(new TWorkerActor(service1, TEvTestPrivate::ERole::Producer, channelId, producerSettings), nodeIndex1); - auto consumer = runtime.Register(new TWorkerActor(service0, TEvTestPrivate::ERole::Consumer, channelId, consumerSettings), nodeIndex0); - runtime.Send(consumer, control0, new TEvTestPrivate::TEvStart(producer), nodeIndex0, true); - runtime.Send(producer, control1, new TEvTestPrivate::TEvStart(consumer), nodeIndex1, true); - actors.insert(producer); - actors.insert(consumer); + auto producer = Runtime->Register(new TProducerActor(Service1, channelId, ProducerSettings), NodeIndex1); + auto consumer = Runtime->Register(new TConsumerActor(Service0, channelId, ConsumerSettings), NodeIndex0); + Runtime->Send(consumer, Control0, new TEvTestPrivate::TEvStart(producer), NodeIndex0, true); + Runtime->Send(producer, Control1, new TEvTestPrivate::TEvStart(consumer), NodeIndex1, true); + Actors.insert(producer); + Actors.insert(consumer); } } + } - int finishCount[2][2] = {{0, 0}, {0, 0}}; - int errorCount = 0; + virtual void Wait() { try { - for (auto i = 0; i < count; i++) { - auto msg0 = runtime.GrabEdgeEvent<TEvTestPrivate::TEvFinished>(control0, TDuration::Seconds(10)); - actors.erase(msg0->Sender); - finishCount[nodeIndex0][msg0->Get()->Role]++; - errorCount += msg0->Get()->Error; - auto msg1 = runtime.GrabEdgeEvent<TEvTestPrivate::TEvFinished>(control1, TDuration::Seconds(10)); - actors.erase(msg1->Sender); - finishCount[nodeIndex1][msg1->Get()->Role]++; - errorCount += msg1->Get()->Error; + for (auto i = 0; i < Count; i++) { + auto msg0 = Runtime->GrabEdgeEvent<TEvTestPrivate::TEvFinished>(Control0, TDuration::Seconds(10)); + Actors.erase(msg0->Sender); + FinishCount[NodeIndex0][msg0->Get()->Role]++; + ErrorCount += msg0->Get()->Error; + auto msg1 = Runtime->GrabEdgeEvent<TEvTestPrivate::TEvFinished>(Control1, TDuration::Seconds(10)); + Actors.erase(msg1->Sender); + FinishCount[NodeIndex1][msg1->Get()->Role]++; + ErrorCount += msg1->Get()->Error; } } catch (NActors::TEmptyEventQueueException&) { - if (!actors.empty()) { + if (!Actors.empty()) { TStringBuilder builder; builder << "NOT FINISHED ACTORS "; - for (auto actorId : actors) { + for (auto actorId : Actors) { builder << ' ' << actorId; } UNIT_ASSERT_C(false, builder); } } + } - if (local) { - UNIT_ASSERT_VALUES_EQUAL(finishCount[0][TEvTestPrivate::ERole::Producer], count); - UNIT_ASSERT_VALUES_EQUAL(finishCount[0][TEvTestPrivate::ERole::Consumer], count); + virtual void Check() { + if (Local) { + UNIT_ASSERT_VALUES_EQUAL(FinishCount[0][TEvTestPrivate::ERole::Producer], Count); + UNIT_ASSERT_VALUES_EQUAL(FinishCount[0][TEvTestPrivate::ERole::Consumer], Count); } else { - UNIT_ASSERT_VALUES_EQUAL(finishCount[0][TEvTestPrivate::ERole::Producer], (count + 1) / 2); - UNIT_ASSERT_VALUES_EQUAL(finishCount[0][TEvTestPrivate::ERole::Consumer], count / 2); - UNIT_ASSERT_VALUES_EQUAL(finishCount[1][TEvTestPrivate::ERole::Producer], count / 2); - UNIT_ASSERT_VALUES_EQUAL(finishCount[1][TEvTestPrivate::ERole::Consumer], (count + 1) / 2); + UNIT_ASSERT_VALUES_EQUAL(FinishCount[0][TEvTestPrivate::ERole::Producer], (Count + 1) / 2); + UNIT_ASSERT_VALUES_EQUAL(FinishCount[0][TEvTestPrivate::ERole::Consumer], Count / 2); + UNIT_ASSERT_VALUES_EQUAL(FinishCount[1][TEvTestPrivate::ERole::Producer], Count / 2); + UNIT_ASSERT_VALUES_EQUAL(FinishCount[1][TEvTestPrivate::ERole::Consumer], (Count + 1) / 2); } - UNIT_ASSERT_VALUES_EQUAL(errorCount, 0); + UNIT_ASSERT_VALUES_EQUAL(ErrorCount, 0); + } + + virtual void Run() { + Prepare(); + Init(); + Start(); + Wait(); + Check(); + } + + int Count = 1; + bool Local = true; + ui32 NodeIndex0 = 0; + ui32 NodeIndex1 = 1; + TKikimrSettings settings; + std::unique_ptr<TKikimrRunner> Runner; + NActors::TTestActorRuntime* Runtime; + std::shared_ptr<TDqChannelService> Service0; + std::shared_ptr<TDqChannelService> Service1; + NActors::TActorId Control0; + NActors::TActorId Control1; + TWorkerSettings ProducerSettings; + TWorkerSettings ConsumerSettings; + THashSet<NActors::TActorId> Actors; + int ErrorCount = 0; + int FinishCount[2][2] = {{0, 0}, {0, 0}}; +}; + +struct TReconTest : public TLoadTest { + + void Prepare() override { + TLoadTest::Prepare(); + settings.AppConfig.MutableTableServiceConfig()->MutableDqChannelConfig()->SetCleanupPeriodMs(20); + settings.AppConfig.MutableTableServiceConfig()->MutableDqChannelConfig()->SetIdlePingPeriodMs(10); + } + + void Start() override { + for (auto i = 0; i < Count; i ++) { + auto channelId = i + 1; + if ((i & 1) == 0) { + auto producerSettings = ProducerSettings; + producerSettings.PauseMessageIndex = (channelId + producerSettings.MessageCount / 2) % producerSettings.MessageCount; + producerSettings.PauseDelayMs = 50; + auto producer = Runtime->Register(new TProducerActor(Service0, channelId, producerSettings), NodeIndex0); + auto consumer = Runtime->Register(new TConsumerActor(Service1, channelId, ConsumerSettings), NodeIndex1); + Runtime->Send(consumer, Control1, new TEvTestPrivate::TEvStart(producer), NodeIndex1, true); + Runtime->Send(producer, Control0, new TEvTestPrivate::TEvStart(consumer), NodeIndex0, true); + Actors.insert(producer); + Actors.insert(consumer); + } else { + auto producerSettings = ProducerSettings; + producerSettings.PauseMessageIndex = channelId; + producerSettings.PauseDelayMs = 50; + auto producer = Runtime->Register(new TProducerActor(Service1, channelId, producerSettings), NodeIndex1); + auto consumer = Runtime->Register(new TConsumerActor(Service0, channelId, ConsumerSettings), NodeIndex0); + Runtime->Send(consumer, Control0, new TEvTestPrivate::TEvStart(producer), NodeIndex0, true); + Runtime->Send(producer, Control1, new TEvTestPrivate::TEvStart(consumer), NodeIndex1, true); + Actors.insert(producer); + Actors.insert(consumer); + } + } + } + +}; + +Y_UNIT_TEST_SUITE(Channels20) { + + void LoadTest(int count, bool local, const TWorkerSettings& producerSettings, const TWorkerSettings& consumerSettings, const TFailureSettings& = TFailureSettings{}) { + TLoadTest test; + + test.Count = count; + test.Local = local; + test.ProducerSettings = producerSettings; + test.ConsumerSettings = consumerSettings; + + test.Run(); } void LoadTest(int count, bool local, const TWorkerSettings& settings = TWorkerSettings{}, const TFailureSettings& failureSettings = TFailureSettings{}) { @@ -326,4 +410,15 @@ Y_UNIT_TEST_SUITE(Channels20) { Y_UNIT_TEST(MissedData) { LoadTest(100, false, TWorkerSettings{ .MessageCount = 100 }, TWorkerSettings{ .MessageCount = 100 }, TFailureSettings{ .Data = 10 }); } + + Y_UNIT_TEST(Reconciliation) { + TReconTest test; + + test.Count = 1; + test.Local = false; + test.ProducerSettings.MessageCount = 100; + test.ConsumerSettings.MessageCount = 100; + + test.Run(); + } } diff --git a/ydb/core/kqp/ut/federated_query/common/common.cpp b/ydb/core/kqp/ut/federated_query/common/common.cpp index de1411b5a50..0903c656efb 100644 --- a/ydb/core/kqp/ut/federated_query/common/common.cpp +++ b/ydb/core/kqp/ut/federated_query/common/common.cpp @@ -99,6 +99,10 @@ std::shared_ptr<TKikimrRunner> MakeKikimrRunner( if (appFlags.GetEnableColumnStore()) { featureFlags.SetEnableColumnStore(true); } + + if (appFlags.GetEnableHasPredicatesInResourcePoolClassifiers()) { + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(true); + } } if (!appConfig) { diff --git a/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp b/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp index 7d5ae2fdc07..3d3b992c865 100644 --- a/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp @@ -73,7 +73,9 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // The topic path lands in tx.Tables (B walk). Y_UNIT_TEST_F(DirectTopicMatches, TStreamingTestFixture) { InternalInitFederatedQuerySetupFactory = true; - SetupAppConfig().MutableFeatureFlags()->SetEnableTopicsSqlIoOperations(true); + auto& featureFlags = *SetupAppConfig().MutableFeatureFlags(); + featureFlags.SetEnableTopicsSqlIoOperations(true); + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(true); constexpr TStringBuf topicName = "test_topic"; CreateTopic(std::string(topicName), std::nullopt, /*local*/ true); @@ -96,7 +98,9 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // direct topic; path is `<table>/<stream>`. Y_UNIT_TEST_F(CdcStreamMatches, TStreamingTestFixture) { InternalInitFederatedQuerySetupFactory = true; - SetupAppConfig().MutableFeatureFlags()->SetEnableTopicsSqlIoOperations(true); + auto& featureFlags = *SetupAppConfig().MutableFeatureFlags(); + featureFlags.SetEnableTopicsSqlIoOperations(true); + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(true); ExecSchemeQuery(RejectClassifierDdl( "hp_cdc", "/Root/t_cdc/cf")); @@ -129,6 +133,7 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // Uses a loopback Ydb-typed EDS pointing at our own runner + a local topic // as the target object (SetupMockPqGateway + real local PQ serves reads). Y_UNIT_TEST_F(FederatedEdsPathMatches, TStreamingTestFixture) { + SetupAppConfig().MutableFeatureFlags()->SetEnableHasPredicatesInResourcePoolClassifiers(true); SetupMockPqGateway(); constexpr TStringBuf topicName = "eds_topic"; CreateTopic(std::string(topicName), std::nullopt, /*local*/ true); @@ -162,6 +167,7 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // appears in TDqPqTopicSource.TopicPath — the (D) walk's unique-coverage // case verified on real cluster). Y_UNIT_TEST_F(FederatedTopicRemoteNameMatches, TStreamingTestFixture) { + SetupAppConfig().MutableFeatureFlags()->SetEnableHasPredicatesInResourcePoolClassifiers(true); SetupMockPqGateway(); constexpr TStringBuf topicName = "remote_by_name_topic"; CreateTopic(std::string(topicName), std::nullopt, /*local*/ true); @@ -196,6 +202,8 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { // KindExternalTable — External Table on top of an ObjectStorage EDS. // Both the ET path and the underlying EDS path land in tx.Tables (B walk). Y_UNIT_TEST_F(ExternalTableMatches, TStreamingTestFixture) { + SetupAppConfig().MutableFeatureFlags()->SetEnableHasPredicatesInResourcePoolClassifiers(true); + ExecSchemeQuery(RejectClassifierDdl( "hp_et", "/Root/et_s3")); diff --git a/ydb/core/kqp/ut/federated_query/datastreams/streaming_ddl_ut.cpp b/ydb/core/kqp/ut/federated_query/datastreams/streaming_ddl_ut.cpp index fed9351e487..b512a370adb 100644 --- a/ydb/core/kqp/ut/federated_query/datastreams/streaming_ddl_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/datastreams/streaming_ddl_ut.cpp @@ -1445,6 +1445,167 @@ Y_UNIT_TEST_SUITE(KqpStreamingQueriesDdl) { ReadTopicMessage(outputTopicName, "message-2-value-2", disposition); } + Y_UNIT_TEST_F(StreamingQueryPrecomputeRecalculationOnRetry, TStreamingTestFixture) { + const auto pqGateway = SetupMockPqGateway(); + + constexpr char inputTopicName[] = "streamingQueryPrecomputeRecalculationOnRetryInputTopic"; + constexpr char outputTopicName[] = "streamingQueryPrecomputeRecalculationOnRetryOutputTopic"; + constexpr char pqSourceName[] = "pqSourceName"; + CreateTopic(inputTopicName); + CreateTopic(outputTopicName); + CreatePqSource(pqSourceName); + + constexpr char tableName[] = "oltpTable"; + ExecQuery(fmt::format(R"( + CREATE TABLE `{table_name}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + );)", + "table_name"_a = tableName + )); + + ExecQuery(fmt::format(R"( + UPSERT INTO `{table_name}` + (Key, Value) + VALUES + (1, "value-1");)", + "table_name"_a = tableName + )); + + constexpr char queryName[] = "streamingQuery"; + ExecQuery(fmt::format(R"( + CREATE STREAMING QUERY `{query_name}` AS + DO BEGIN + $r = SELECT Value FROM `{table_name}`; + + INSERT INTO `{pq_source}`.`{output_topic}` + SELECT + Unwrap(Data || "-" || $r) + FROM `{pq_source}`.`{input_topic}` + END DO;)", + "query_name"_a = queryName, + "pq_source"_a = pqSourceName, + "input_topic"_a = inputTopicName, + "output_topic"_a = outputTopicName, + "table_name"_a = tableName + )); + CheckScriptExecutionsCount(1, 1); + Sleep(TDuration::Seconds(1)); + + const auto readSession = pqGateway->WaitReadSession(inputTopicName); + readSession->AddDataReceivedEvent(0, "message-1"); + pqGateway->WaitWriteSession(outputTopicName)->ExpectMessage("message-1-value-1"); + + ExecQuery(fmt::format(R"( + UPSERT INTO `{table_name}` + (Key, Value) + VALUES + (1, "value-2");)", + "table_name"_a = tableName + )); + + readSession->AddCloseSessionEvent(EStatus::UNAVAILABLE, {NIssue::TIssue("Test pq session failure")}); + pqGateway->WaitReadSession(inputTopicName)->AddDataReceivedEvent(1, "message-2"); + pqGateway->WaitWriteSession(outputTopicName)->ExpectMessage("message-2-value-2"); + } + + Y_UNIT_TEST_F(StreamingQueryWithDifferentPrecomputeTypes, TStreamingTestFixture) { + constexpr char oltpTableName[] = "oltpTable"; + constexpr char olapTableName[] = "olapTable"; + ExecQuery(fmt::format(R"( + CREATE TABLE `{oltp_table_name}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ); + CREATE TABLE `{olap_table_name}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ) WITH ( + STORE = COLUMN + );)", + "oltp_table_name"_a = oltpTableName, + "olap_table_name"_a = olapTableName + )); + + ExecQuery(fmt::format(R"( + UPSERT INTO `{oltp_table_name}` + (Key, Value) + VALUES + (1, "value-1"), + (2, "value-1"); + UPSERT INTO `{olap_table_name}` + (Key, Value) + VALUES + (1, "value-1"), + (2, "value-1");)", + "oltp_table_name"_a = oltpTableName, + "olap_table_name"_a = olapTableName + )); + + constexpr char sourceBucket[] = "test_streaming_query_with_s3_join"; + constexpr char objectContent[] = R"( +{"Key": 1, "Value": "value-1"} +{"Key": 2, "Value": "value-1"})"; + CreateBucketWithObject(sourceBucket, "path/test_object.json", objectContent); + + constexpr char pqSourceName[] = "pqSourceName"; + constexpr char s3SourceName[] = "s3Source"; + CreatePqSource(pqSourceName); + CreateS3Source(sourceBucket, s3SourceName); + + constexpr char externalTableName[] = "externalTable"; + ExecQuery(fmt::format(R"( + CREATE EXTERNAL TABLE `{external_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL + ) WITH ( + DATA_SOURCE = "{external_source}", + LOCATION = "path/test_object.json", + FORMAT = "json_each_row" + );)", + "external_table"_a = externalTableName, + "external_source"_a = s3SourceName + )); + + constexpr char inputTopicName[] = "streamingQueryWithDifferentPrecomputeTypesInputTopicName"; + constexpr char outputTopicName[] = "streamingQueryWithDifferentPrecomputeTypesOutputTopicName"; + CreateTopic(inputTopicName); + CreateTopic(outputTopicName); + + for (const auto& sourceName : {oltpTableName, olapTableName, externalTableName}) { + constexpr char queryName[] = "streamingQuery"; + + ExecQuery(fmt::format(R"( + CREATE OR REPLACE STREAMING QUERY `{query_name}` AS + DO BEGIN + $precompute_agg = SELECT CAST(MAX(Key) AS String) FROM `{precompute_source}`; + $precompute_limit = SELECT Value FROM `{precompute_source}` LIMIT 1; + $empty_precompute = SELECT Value FROM `{precompute_source}` WHERE Key = 3; + + INSERT INTO `{pq_source}`.`{output_topic}` + SELECT + Unwrap(Data || $precompute_agg || $precompute_limit || ($empty_precompute ?? "<null>")) + FROM `{pq_source}`.`{input_topic}`; + END DO;)", + "query_name"_a = queryName, + "pq_source"_a = pqSourceName, + "precompute_source"_a = sourceName, + "input_topic"_a = inputTopicName, + "output_topic"_a = outputTopicName + )); + + Sleep(TDuration::Seconds(1)); + + const auto disposition = TInstant::Now(); + auto message = TStringBuilder() << "test_message" << sourceName; + WriteTopicMessage(inputTopicName, message); + ReadTopicMessage(outputTopicName, message << "2value-1<null>", disposition); + } + } + Y_UNIT_TEST_F(StreamingQueryUnderSecureScriptExecutions, TStreamingTestFixture) { auto& appConfig = SetupAppConfig(); appConfig.MutableFeatureFlags()->SetEnableSecureScriptExecutions(true); diff --git a/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp index 383e457683b..7d367abc7d4 100644 --- a/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp +++ b/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp @@ -161,9 +161,9 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { )"); ExecQuery(kikimr, UseQueryService, TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_OPTIONS, SCHEME_NEED_ACTUALIZATION=`true`);"); - csController->WaitActualization(TDuration::Seconds(10)); // Make the just-actualized portions (with index data) visible to the following scan. AdvancePlanStep(kikimr); + csController->WaitActualization(TDuration::Seconds(10)); { auto it = tableClient .StreamExecuteScanQuery(R"( @@ -219,8 +219,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { auto settings = TKikimrSettings().SetColumnShardAlterObjectEnabled(true); settings.AppConfig.MutableFeatureFlags()->SetEnableLocalIndexAsSchemeObject(LocalIndexAsSchemeObject); Variator::ToExecutor(Variator::SingleScript(scriptChunkDetailsMinMaxWithLocalDBStorage)).Execute(settings); - } - + } + TString scriptChunkDetailsMinMaxWithBSStorage = R"( STOP_COMPACTION ------ @@ -244,7 +244,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { READ: SELECT COALESCE(sum(CAST(ChunkDetails = "{\"min\":\"x\",\"max\":\"x\"}" as Uint32) ), 0) = count(ChunkDetails) FROM `/Root/ColumnTable/.sys/primary_index_stats` WHERE EntityName="field_mm"; EXPECTED: [[%true]] )"; - + Y_UNIT_TEST(ChunkDetailsMinMaxWithBSStorage, ELocalIndexAsSchemeObject) { const bool LocalIndexAsSchemeObject = (Arg<0>() == ELocalIndexAsSchemeObject::SchemeObjectEnabled); auto settings = TKikimrSettings().SetColumnShardAlterObjectEnabled(true); @@ -309,14 +309,14 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { assertDDLQueryNotOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_mm2` LOCAL USING min_max ON(`value`); )"); - + assertDDLQueryOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom` LOCAL USING bloom_filter ON(`value`); )"); assertDDLQueryNotOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom2` LOCAL USING bloom_filter ON(`value`); )"); - + assertDDLQueryOk(R"( ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom_ngram` LOCAL USING bloom_ngram_filter ON(`value`) WITH (ngram_size = 3, false_positive_probability = 0.01, case_sensitive = true); @@ -326,7 +326,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { ALTER TABLE `/Root/test_cannot_have_two_indexes_on_one_column` ADD INDEX `value_bloom_ngram2` LOCAL USING bloom_ngram_filter ON(`value`) WITH (ngram_size = 3, false_positive_probability = 0.01, case_sensitive = true); )"); - + assertDDLQueryNotOk(R"( CREATE TABLE `/Root/two_minmax_in_create` ( `key` Int32 NOT NULL, @@ -360,7 +360,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { )"); } - + TString scriptMinMaxIndexOnInsertEnabled = R"( STOP_COMPACTION @@ -695,9 +695,9 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { )"); NYdb::TStatus status = runDDLQuery(R"( - ALTER TABLE `/Root/minmax_test_table_doesnt_exist` ADD INDEX `value_mm` LOCAL USING min_max ON(`value`); + ALTER TABLE `/Root/minmax_test_table_doesnt_exist` ADD INDEX `value_mm` LOCAL USING min_max ON(`value`); )"); - + UNIT_ASSERT_C(!status.IsSuccess(), status.GetIssues().ToString()); } @@ -917,7 +917,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { runDMLQuery(R"( $data1 = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); - UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT + UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT CAST(item AS Int32) AS `key`, "Value_" || CAST(item+1 AS String) AS `value_str`, CAST(item+1 AS Utf8) AS `value_utf`, @@ -928,7 +928,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { runDMLQuery(R"( $data1 = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); - UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT + UPSERT INTO `/Root/minmax_test_appropriate_storage_location` (`key`, `value_str`, `value_utf`, `value_int`, `value_ts`) SELECT CAST(item AS Int32) AS `key`, "Value_" || CAST(item AS String) AS `value_str`, CAST(item AS Utf8) AS `value_utf`, @@ -939,7 +939,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { csController->WaitCompactions(TDuration::Seconds(5)); - using TQueryResult = TVector<TString>; + using TQueryResult = TVector<TString>; auto runDMLQueryTyped = [&](TString text) -> TQueryResult { auto columns = runDMLQuery(text); @@ -950,7 +950,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { } return res; }; - + { TQueryResult tierNamesStr = runDMLQueryTyped(R"( SELECT TierName FROM `/Root/minmax_test_appropriate_storage_location/.sys/primary_index_stats` WHERE EntityName == "value_str_mm"; @@ -958,7 +958,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { UNIT_ASSERT_VALUES_UNEQUAL_C(tierNamesStr.size(), 0, "portions are not min_max indexed with String column type"); for (const auto& tierName: tierNamesStr) { UNIT_ASSERT_VALUES_EQUAL_C(tierName, "__DEFAULT", "min_max index must store its data is BS when building over String column"); - } + } } { TQueryResult tierNamesUtf8 = runDMLQueryTyped(R"( @@ -2038,9 +2038,9 @@ Y_UNIT_TEST(RenameLocalBloomIndex, EUseQueryService) { )"); ExecQuery(kikimr, UseQueryService, TStringBuilder() << "ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_OPTIONS, SCHEME_NEED_ACTUALIZATION=`true`);"); - csController->WaitActualization(TDuration::Seconds(10)); // Make the just-actualized portions (with index data) visible to the following scan. AdvancePlanStep(kikimr); + csController->WaitActualization(TDuration::Seconds(10)); { auto it = tableClient .StreamExecuteScanQuery(R"( diff --git a/ydb/core/kqp/ut/olap/types/json_ut.cpp b/ydb/core/kqp/ut/olap/types/json_ut.cpp index 50649a8d967..15210705b18 100644 --- a/ydb/core/kqp/ut/olap/types/json_ut.cpp +++ b/ydb/core/kqp/ut/olap/types/json_ut.cpp @@ -1699,6 +1699,29 @@ Y_UNIT_TEST_SUITE(KqpOlapJsonNativeScalars) { Variator::ToExecutor(Variator::SingleScript(script)).Execute(); } + // A full cycle test for doubles near max magnitude. The stored value stays exact; reading back renders + // it with fewer digits (17 in, 16 out) on BinaryJson level. + Y_UNIT_TEST(CompactionNearMaxDouble) { + const TString script = TStringBuilder() << NativeTableSetup() << R"( + DATA: + REPLACE INTO `/Root/ColumnTable` (Col1, Col2) VALUES (1u, JsonDocument('{"n" : 1.7976931348623157e308}')), + (2u, JsonDocument('{"n" : -1.7976931348623157e308}')) + ------ + READ: SELECT * FROM `/Root/ColumnTable` ORDER BY Col1; + EXPECTED: [[1u;["{\"n\":1.797693134862316e+308}"]];[2u;["{\"n\":-1.797693134862316e+308}"]]] + ------ + DATA: + REPLACE INTO `/Root/ColumnTable` (Col1, Col2) VALUES (3u, JsonDocument('{"n" : 1.5}')) + ------ + ONE_COMPACTION + ------ + READ: SELECT * FROM `/Root/ColumnTable` ORDER BY Col1; + EXPECTED: [[1u;["{\"n\":1.797693134862316e+308}"]];[2u;["{\"n\":-1.797693134862316e+308}"]];[3u;["{\"n\":1.5}"]]] + ------ + )" << NativeValueTypeCheck(EValueType::Double); + Variator::ToExecutor(Variator::SingleScript(script)).Execute(); + } + Y_UNIT_TEST(CompactionDivergentTypesFallback) { const TString script = TStringBuilder() << NativeTableSetup() << R"( DATA: diff --git a/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp b/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp index 84a7ba07175..8fc3e67375b 100644 --- a/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp +++ b/ydb/core/kqp/ut/rbo/kqp_rbo_yql_ut.cpp @@ -2021,9 +2021,7 @@ Y_UNIT_TEST_SUITE(KqpRboYql) { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); auto ast = *result.GetStats()->GetAst(); - if (columnTables) { - UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); - } + UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); result = session.ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx(), NYdb::NQuery::TExecuteQuerySettings().ExecMode(NQuery::EExecMode::Execute)) .ExtractValueSync(); @@ -2072,10 +2070,7 @@ Y_UNIT_TEST_SUITE(KqpRboYql) { .ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); auto ast = *result.GetStats()->GetAst(); - - if (columnTables) { - UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); - } + UNIT_ASSERT_C(ast.find("RangeFinalize") != TString::npos, "Ranges not pushed"); auto params = paramsVector[i]; // clang-format off diff --git a/ydb/core/kqp/workload_service/kqp_has_stream_matcher.cpp b/ydb/core/kqp/workload_service/kqp_has_stream_matcher.cpp new file mode 100644 index 00000000000..5b6a0b5ee9d --- /dev/null +++ b/ydb/core/kqp/workload_service/kqp_has_stream_matcher.cpp @@ -0,0 +1,14 @@ +#include "kqp_has_stream_matcher.h" +#include <ydb/core/kqp/common/kqp_user_request_context.h> + +namespace NKikimr::NKqp::NWorkload { + +bool MatchesStream(const std::optional<bool>& hasStream, + const TUserRequestContext& userRequestContext) { + if (!hasStream) { + return true; + } + return *hasStream ? userRequestContext.IsStreamingQuery : !userRequestContext.IsStreamingQuery; +} + +} // namespace NKikimr::NKqp::NWorkload diff --git a/ydb/core/kqp/workload_service/kqp_has_stream_matcher.h b/ydb/core/kqp/workload_service/kqp_has_stream_matcher.h new file mode 100644 index 00000000000..c11e867734c --- /dev/null +++ b/ydb/core/kqp/workload_service/kqp_has_stream_matcher.h @@ -0,0 +1,14 @@ +#pragma once + +#include <ydb/core/kqp/common/kqp_user_request_context.h> + +#include <optional> + + +namespace NKikimr::NKqp::NWorkload { + +/// Returns true if the classifier should accept the query based on the HAS_STREAM filter. +bool MatchesStream(const std::optional<bool>& hasStream, + const TUserRequestContext& userRequestContext); + +} // namespace NKikimr::NKqp::NWorkload diff --git a/ydb/core/kqp/workload_service/kqp_query_classifier.cpp b/ydb/core/kqp/workload_service/kqp_query_classifier.cpp index 244cdab017c..88bdf504efa 100644 --- a/ydb/core/kqp/workload_service/kqp_query_classifier.cpp +++ b/ydb/core/kqp/workload_service/kqp_query_classifier.cpp @@ -1,5 +1,6 @@ #include "kqp_has_full_scan_matcher.h" #include "kqp_has_path_matcher.h" +#include "kqp_has_stream_matcher.h" #include "kqp_query_classifier.h" #include "kqp_workload_service.h" @@ -69,7 +70,7 @@ public: } [[nodiscard]] - TPostCompileClassifyResult PostCompileClassify(const TPreparedQueryHolder& preparedQuery) override { + TPostCompileClassifyResult PostCompileClassify(const TPreparedQueryHolder& preparedQuery, const TUserRequestContext& userRequestContext) override { Y_VALIDATE(ClassifierView, "Post compile classify without configuration"); Y_VALIDATE(PreClassifyResult.has_value() && std::holds_alternative<TPendingCompilation>(*PreClassifyResult), "Post compile classify requires TPendingCompilation from pre-classification"); @@ -83,7 +84,7 @@ public: continue; } - if (!MatchesDynamic(settings, preparedQuery)) { + if (!MatchesDynamic(settings, preparedQuery, userRequestContext)) { continue; } @@ -179,7 +180,7 @@ private: } bool NeedsPreparedQuery(const NResourcePool::TClassifierSettings& settings) const { - return settings.HasFullScan.has_value() || settings.HasPath.has_value(); + return settings.HasFullScan.has_value() || settings.HasPath.has_value() || settings.HasStream.has_value(); } /// @@ -188,9 +189,10 @@ private: /// - Involves SQL analysis, plan building, or computations. /// - Depends on actual query structure and execution characteristics. /// - bool MatchesDynamic(const NResourcePool::TClassifierSettings& settings, const TPreparedQueryHolder& preparedQuery) const { + bool MatchesDynamic(const NResourcePool::TClassifierSettings& settings, const TPreparedQueryHolder& preparedQuery, const TUserRequestContext& userRequestContext) const { return MatchesFullScan(settings.HasFullScan, preparedQuery.GetPhysicalQuery()) - && MatchesPath(settings.HasPath, preparedQuery.GetQueryTables(), preparedQuery.GetPhysicalQuery()); + && MatchesPath(settings.HasPath, preparedQuery.GetQueryTables(), preparedQuery.GetPhysicalQuery()) + && MatchesStream(settings.HasStream, userRequestContext); } const TResourcePoolEntry* FindPool(const TString& poolId) const { diff --git a/ydb/core/kqp/workload_service/kqp_query_classifier.h b/ydb/core/kqp/workload_service/kqp_query_classifier.h index bdbab934f22..808a30f45c2 100644 --- a/ydb/core/kqp/workload_service/kqp_query_classifier.h +++ b/ydb/core/kqp/workload_service/kqp_query_classifier.h @@ -11,6 +11,8 @@ namespace NKikimr::NKqp { +struct TUserRequestContext; + struct TClassifyContext { const TString PoolId; const TString AppName; @@ -77,7 +79,7 @@ public: /// Refines classification once the query plan is available [[nodiscard]] - virtual TPostCompileClassifyResult PostCompileClassify(const TPreparedQueryHolder& preparedQuery) = 0; + virtual TPostCompileClassifyResult PostCompileClassify(const TPreparedQueryHolder& preparedQuery, const TUserRequestContext& userRequestContext) = 0; /// Get the current classification state [[nodiscard]] diff --git a/ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h b/ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h index ec531244a68..df4ff58edd0 100644 --- a/ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h +++ b/ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h @@ -1,6 +1,7 @@ #pragma once #include <ydb/core/base/path.h> +#include <ydb/core/kqp/common/kqp_user_request_context.h> #include <ydb/core/kqp/gateway/behaviour/resource_pool_classifier/snapshot.h> #include <ydb/core/kqp/workload_service/kqp_query_classifier.h> #include <ydb/core/kqp/query_data/kqp_prepared_query.h> @@ -24,6 +25,7 @@ inline TResourcePoolClassifierConfig MakeClassifierConfig( std::optional<TString> hasAppName = std::nullopt, std::optional<TString> hasFullScan = std::nullopt, std::optional<TString> hasPath = std::nullopt, + std::optional<bool> hasStream = std::nullopt, std::optional<TString> action = std::nullopt) { NJson::TJsonValue json(NJson::JSON_MAP); @@ -40,6 +42,9 @@ inline TResourcePoolClassifierConfig MakeClassifierConfig( if (hasPath) { json["has_path"] = *hasPath; } + if (hasStream) { + json["has_stream"] = *hasStream ? "true" : "false"; + } if (action) { json["action"] = *action; } @@ -93,6 +98,7 @@ struct TClassifyTestCase { std::optional<TString> ClassifierHasAppName; std::optional<TString> ClassifierHasFullScan; std::optional<TString> ClassifierHasPath; + std::optional<bool> ClassifierHasStream; std::optional<TString> ClassifierAction; TString ContextAppName; @@ -109,6 +115,7 @@ struct TClassifyTestCase { std::optional<TString> HasAppName; std::optional<TString> HasFullScan; std::optional<TString> HasPath; + std::optional<bool> HasStream; std::optional<TString> Action; }; std::vector<TExtraClassifier> ExtraClassifiers; @@ -117,12 +124,12 @@ struct TClassifyTestCase { std::vector<TResourcePoolClassifierConfig> configs; configs.push_back(MakeClassifierConfig( TEST_DB, "c_main", Rank, ResourcePool, - ClassifierMemberName, ClassifierHasAppName, ClassifierHasFullScan, ClassifierHasPath, ClassifierAction)); + ClassifierMemberName, ClassifierHasAppName, ClassifierHasFullScan, ClassifierHasPath, ClassifierHasStream, ClassifierAction)); for (const auto& extra : ExtraClassifiers) { configs.push_back(MakeClassifierConfig( TEST_DB, extra.Name, extra.Rank, extra.ResourcePool, - extra.MemberName, extra.HasAppName, extra.HasFullScan, extra.HasPath, extra.Action)); + extra.MemberName, extra.HasAppName, extra.HasFullScan, extra.HasPath, extra.HasStream, extra.Action)); } auto classifierSnap = MakeClassifierSnapshot(std::move(configs)); @@ -183,7 +190,8 @@ struct TClassifyTestCase { } TPreparedQueryHolder holder(proto.release(), nullptr, /*noFillTables=*/true); - return classifier->PostCompileClassify(holder); + TUserRequestContext userRequestContext; + return classifier->PostCompileClassify(holder, userRequestContext); } /// @@ -204,7 +212,21 @@ struct TClassifyTestCase { tx->AddTables()->MutableId()->SetPath(queryTablePath); TPreparedQueryHolder holder(proto.release(), nullptr, /*noFillTables=*/true); - return classifier->PostCompileClassify(holder); + TUserRequestContext userRequestContext; + return classifier->PostCompileClassify(holder, userRequestContext); + } + + NWorkload::IQueryClassifier::TPostCompileClassifyResult RunPostClassifyForStream( + bool isStreamingQuery) const + { + auto classifier = BuildClassifier(); + (void)classifier->PreCompileClassify(); + + auto proto = std::make_unique<NKikimrKqp::TPreparedQuery>(); + TPreparedQueryHolder holder(proto.release(), nullptr, /*noFillTables=*/true); + TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = isStreamingQuery; + return classifier->PostCompileClassify(holder, userRequestContext); } }; diff --git a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.cpp b/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.cpp index b0cb6c030d7..db35abb6bb3 100644 --- a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.cpp +++ b/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.cpp @@ -302,6 +302,8 @@ private: featureFlags.SetEnableResourcePoolsCounters(true); featureFlags.SetEnableStreamingQueries(Settings_.EnableStreamingQueries_); featureFlags.SetEnableResourcePoolsScheduler(Settings_.EnableResourcePoolsScheduler_); + featureFlags.SetEnableHasPredicatesInResourcePoolClassifiers(Settings_.EnableHasPredicatesInResourcePoolClassifiers_); + featureFlags.SetEnableRejectActionInResourcePoolClassifiers(Settings_.EnableRejectActionInResourcePoolClassifiers_); auto& queryServiceConfig = *appConfig.MutableQueryServiceConfig(); queryServiceConfig.SetAllExternalDataSourcesAreAvailable(true); diff --git a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h b/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h index 47b84ed1ef7..1b6b2d020bd 100644 --- a/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h +++ b/ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.h @@ -81,6 +81,8 @@ struct TYdbSetupSettings { FLUENT_SETTING_DEFAULT(bool, EnableMetadataObjectsOnServerless, true); FLUENT_SETTING_DEFAULT(bool, EnableExternalDataSourcesOnServerless, true); FLUENT_SETTING_DEFAULT(bool, EnableStreamingQueries, true); + FLUENT_SETTING_DEFAULT(bool, EnableHasPredicatesInResourcePoolClassifiers, true); + FLUENT_SETTING_DEFAULT(bool, EnableRejectActionInResourcePoolClassifiers, true); FLUENT_SETTING(NKikimrConfig::TWorkloadManagerConfig, WorkloadManagerConfig); FLUENT_SETTING_DEFAULT(i32, DedicatedDiskQuota, -1); diff --git a/ydb/core/kqp/workload_service/ut/kqp_has_stream_matcher_ut.cpp b/ydb/core/kqp/workload_service/ut/kqp_has_stream_matcher_ut.cpp new file mode 100644 index 00000000000..12f77152c6b --- /dev/null +++ b/ydb/core/kqp/workload_service/ut/kqp_has_stream_matcher_ut.cpp @@ -0,0 +1,52 @@ +#include <ydb/core/kqp/workload_service/kqp_has_stream_matcher.h> + +#include <library/cpp/testing/unittest/registar.h> + + +namespace NKikimr::NKqp { + + +Y_UNIT_TEST_SUITE(TStreamMatcherPredicate) { + + + Y_UNIT_TEST(NulloptAcceptsAny) { + // No filter — every query passes regardless of streaming ops. + { + TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = false; + UNIT_ASSERT(NWorkload::MatchesStream(std::nullopt, userRequestContext)); + } + { + TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = true; + UNIT_ASSERT(NWorkload::MatchesStream(std::nullopt, userRequestContext)); + } + } + + Y_UNIT_TEST(TrueMatchesStreamingQuery) { + TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = true; + + UNIT_ASSERT(NWorkload::MatchesStream(true, userRequestContext)); + } + + Y_UNIT_TEST(TrueDoesNotMatchNonStreamingQuery) { + TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = false; + UNIT_ASSERT(!NWorkload::MatchesStream(true, userRequestContext)); + } + + Y_UNIT_TEST(FalseMatchesNonStreamingQuery) { + TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = false; + UNIT_ASSERT(NWorkload::MatchesStream(false, userRequestContext)); + } + + Y_UNIT_TEST(FalseDoesNotMatchStreamingQuery) { + TUserRequestContext userRequestContext; + userRequestContext.IsStreamingQuery = true; + UNIT_ASSERT(!NWorkload::MatchesStream(false, userRequestContext)); + } +} + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/workload_service/ut/kqp_has_stream_ut.cpp b/ydb/core/kqp/workload_service/ut/kqp_has_stream_ut.cpp new file mode 100644 index 00000000000..2674ecbd350 --- /dev/null +++ b/ydb/core/kqp/workload_service/ut/kqp_has_stream_ut.cpp @@ -0,0 +1,61 @@ +#include <ydb/core/kqp/workload_service/ut/common/kqp_query_classifier_ut_common.h> + +#include <library/cpp/testing/unittest/registar.h> + + +namespace NKikimr::NKqp { + +using namespace NWorkload; + + +namespace { + +TString GetPostPoolId(const IQueryClassifier::TPostCompileClassifyResult& result) { + UNIT_ASSERT_C(std::holds_alternative<IQueryClassifier::TResolvedPoolId>(result), + TStringBuilder() << "Expected TResolvedPoolId, got variant index " << result.index()); + return std::get<IQueryClassifier::TResolvedPoolId>(result).PoolId; +} + +} // anonymous namespace + + +Y_UNIT_TEST_SUITE(TQueryClassifierHasStream) { + + Y_UNIT_TEST(ShouldTriggerPendingCompilation) { + TClassifyTestCase tc; + tc.ClassifierHasStream = true; + auto result = tc.RunPreClassify(); + UNIT_ASSERT_C(std::holds_alternative<IQueryClassifier::TPendingCompilation>(result), + TStringBuilder() << "Expected TPendingCompilation, got variant index " << result.index()); + } + + Y_UNIT_TEST(TrueMatchesStreamingQuery) { + TClassifyTestCase tc; + tc.ClassifierHasStream = true; + auto result = tc.RunPostClassifyForStream(true); + UNIT_ASSERT_VALUES_EQUAL(GetPostPoolId(result), "pool_target"); + } + + Y_UNIT_TEST(TrueDoesNotMatchNonStreamingQuery) { + TClassifyTestCase tc; + tc.ClassifierHasStream = true; + auto result = tc.RunPostClassifyForStream(false); + UNIT_ASSERT_VALUES_EQUAL(GetPostPoolId(result), "default"); + } + + Y_UNIT_TEST(FalseMatchesNonStreamingQuery) { + TClassifyTestCase tc; + tc.ClassifierHasStream = false; + auto result = tc.RunPostClassifyForStream(false); + UNIT_ASSERT_VALUES_EQUAL(GetPostPoolId(result), "pool_target"); + } + + Y_UNIT_TEST(FalseDoesNotMatchStreamingQuery) { + TClassifyTestCase tc; + tc.ClassifierHasStream = false; + auto result = tc.RunPostClassifyForStream(true); + UNIT_ASSERT_VALUES_EQUAL(GetPostPoolId(result), "default"); + } +} + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/workload_service/ut/kqp_query_classifier_ut.cpp b/ydb/core/kqp/workload_service/ut/kqp_query_classifier_ut.cpp index e7716bb7b73..bccbbd71df6 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_query_classifier_ut.cpp +++ b/ydb/core/kqp/workload_service/ut/kqp_query_classifier_ut.cpp @@ -32,7 +32,7 @@ public: return EState::PreCompileDone; } - TPostCompileClassifyResult PostCompileClassify(const TPreparedQueryHolder&) override { + TPostCompileClassifyResult PostCompileClassify(const TPreparedQueryHolder&, const TUserRequestContext&) override { PostCompileCalled = true; return PostClassifyResult; } diff --git a/ydb/core/kqp/workload_service/ut/kqp_stream_query_classification_ut.cpp b/ydb/core/kqp/workload_service/ut/kqp_stream_query_classification_ut.cpp index e13717696b9..45ccc9a364f 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_stream_query_classification_ut.cpp +++ b/ydb/core/kqp/workload_service/ut/kqp_stream_query_classification_ut.cpp @@ -14,7 +14,9 @@ using namespace NYdb; namespace { TIntrusivePtr<NWorkload::IYdbSetup> MakeStreamingYdb() { - return NWorkload::TYdbSetupSettings().Create([](auto) {}); + return NWorkload::TYdbSetupSettings() + .EnableHasPredicatesInResourcePoolClassifiers(true) + .Create([](auto) {}); } void CreateTopic(TIntrusivePtr<NWorkload::IYdbSetup> ydb, TString name) { @@ -68,6 +70,46 @@ Y_UNIT_TEST_SUITE(StreamingQueryClassification) { } } + Y_UNIT_TEST(TestClassifierMatchesStreamingQuery) { + auto ydb = MakeStreamingYdb(); + + const TString& poolId = "streaming_pool"; + CreateTopic(ydb, "input_topic"); + CreateTopic(ydb, "output_topic"); + + { + const auto& result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL )" << poolId << R"( WITH ( + CONCURRENT_QUERY_LIMIT = 10, + QUEUE_SIZE = 100, + TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 10, + QUERY_CPU_LIMIT_PERCENT_PER_NODE = 1 + ); + CREATE RESOURCE POOL CLASSIFIER streaming_classifier WITH ( + RESOURCE_POOL=")" << poolId << R"(", + HAS_STREAM = "true" + ); + )"); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToOneLineString()); + } + ydb->WaitForClassifierPropagation(); + + { + const auto& result = ydb->ExecuteQuery(R"( + CREATE STREAMING QUERY MyStreamingQuery + AS DO BEGIN + INSERT INTO output_topic SELECT * FROM input_topic; + END DO + )"); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToOneLineString()); + ydb->WaitPoolState({.DelayedRequests = 0, .RunningRequests = 1}, poolId); + + } + { + std::this_thread::sleep_for(5s); + ydb->WaitPoolState({.DelayedRequests = 0, .RunningRequests = 1}, poolId); + } + } } } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/workload_service/ut/kqp_workload_service_ut.cpp b/ydb/core/kqp/workload_service/ut/kqp_workload_service_ut.cpp index c011a23cf45..85490c90c74 100644 --- a/ydb/core/kqp/workload_service/ut/kqp_workload_service_ut.cpp +++ b/ydb/core/kqp/workload_service/ut/kqp_workload_service_ut.cpp @@ -11,6 +11,18 @@ using namespace NWorkload; using namespace NYdb; +TIntrusivePtr<IYdbSetup> MakeYdbWithAdvancedDisabled() { + return TYdbSetupSettings() + .EnableHasPredicatesInResourcePoolClassifiers(false) + .Create(); +} + +TIntrusivePtr<IYdbSetup> MakeYdbWithRejectActionDisabled() { + return TYdbSetupSettings() + .EnableRejectActionInResourcePoolClassifiers(false) + .Create(); +} + void StartTestConcurrentQueryLimit(const ui64 activeCountLimit, const ui64 queueSize, const ui64 nodeCount = 1) { auto ydb = TYdbSetupSettings() .NodeCount(nodeCount) @@ -2065,4 +2077,154 @@ Y_UNIT_TEST_SUITE(DefaultPoolSettings) { +Y_UNIT_TEST_SUITE(HasPredicatesClassifierFeatureFlagDisabled) { + + Y_UNIT_TEST(CreateWithHasFullScanIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_full_scan WITH ( + RESOURCE_POOL="default", + HAS_FULL_SCAN='/Root/.*', + RANK=100 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithHasPathIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_has_path WITH ( + RESOURCE_POOL="default", + HAS_PATH='/Root/mydb/.*', + RANK=200 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithHasStreamIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_has_stream WITH ( + RESOURCE_POOL="default", + HAS_STREAM='true', + RANK=300 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithHasAppNameIsRejected) { + auto ydb = MakeYdbWithAdvancedDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_has_app_name WITH ( + RESOURCE_POOL="default", + HAS_APP_NAME='myapp', + RANK=350 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(AlterWithHasFullScanIsRejected) { + // Create a basic classifier first (no advanced props) — should succeed. + auto ydb = MakeYdbWithAdvancedDisabled(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_basic WITH ( + RESOURCE_POOL="default", + RANK=500 + ); + )"); + + // ALTER to add HAS_FULL_SCAN — should fail. + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + ALTER RESOURCE POOL CLASSIFIER cl_basic SET ( + HAS_FULL_SCAN='/Root/.*' + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Advanced resource pool classifier properties"); + } + + Y_UNIT_TEST(CreateWithoutAdvancedPropsSucceeds) { + // Plain classifier must still work even when the flag is off. + auto ydb = MakeYdbWithAdvancedDisabled(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_plain WITH ( + RESOURCE_POOL="default", + RANK=600 + ); + )"); + // If we get here without an exception, the creation succeeded. + } +} + +Y_UNIT_TEST_SUITE(HasPredicatesFeatureFlagEnabled) { + + Y_UNIT_TEST(CreateWithHasFullScanSucceeds) { + auto ydb = TYdbSetupSettings().Create(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_full_scan_ok WITH ( + RESOURCE_POOL="default", + HAS_FULL_SCAN='/Root/.*', + RANK=100 + ); + )"); + } +} + +Y_UNIT_TEST_SUITE(RejectActionClassifierFeatureFlagDisabled) { + + Y_UNIT_TEST(CreateWithActionRejectIsRejected) { + auto ydb = MakeYdbWithRejectActionDisabled(); + + auto result = ydb->ExecuteQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_action_reject WITH ( + ACTION="reject", + RANK=400 + ); + )", TQueryRunnerSettings().PoolId(NResourcePool::DEFAULT_POOL_ID)); + + UNIT_ASSERT_VALUES_UNEQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToOneLineString(), + "Resource pool classifier action property is disabled"); + } +} + +Y_UNIT_TEST_SUITE(RejectActionFeatureFlagEnabled) { + + Y_UNIT_TEST(CreateWithActionRejectSucceeds) { + auto ydb = TYdbSetupSettings().Create(); + + ydb->ExecuteSchemeQuery(TStringBuilder() << R"( + CREATE RESOURCE POOL CLASSIFIER cl_reject_ok WITH ( + ACTION="reject", + RANK=200 + ); + )"); + } +} + } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/workload_service/ut/ya.make b/ydb/core/kqp/workload_service/ut/ya.make index de52737c141..97b51dfe873 100644 --- a/ydb/core/kqp/workload_service/ut/ya.make +++ b/ydb/core/kqp/workload_service/ut/ya.make @@ -18,6 +18,8 @@ SRCS( kqp_has_path_matcher_ut.cpp kqp_has_path_ut.cpp kqp_stream_query_classification_ut.cpp + kqp_has_stream_matcher_ut.cpp + kqp_has_stream_ut.cpp kqp_member_name_ut.cpp kqp_query_classifier_match_ut.cpp kqp_query_classifier_ut.cpp diff --git a/ydb/core/kqp/workload_service/ya.make b/ydb/core/kqp/workload_service/ya.make index d02845691d1..ba5c5bd0056 100644 --- a/ydb/core/kqp/workload_service/ya.make +++ b/ydb/core/kqp/workload_service/ya.make @@ -3,6 +3,7 @@ LIBRARY() SRCS( kqp_has_full_scan_matcher.cpp kqp_has_path_matcher.cpp + kqp_has_stream_matcher.cpp kqp_query_classifier.cpp kqp_workload_service.cpp ) diff --git a/ydb/core/mind/bscontroller/self_heal.cpp b/ydb/core/mind/bscontroller/self_heal.cpp index 13bd7c56fb4..4dfee75874b 100644 --- a/ydb/core/mind/bscontroller/self_heal.cpp +++ b/ydb/core/mind/bscontroller/self_heal.cpp @@ -181,6 +181,7 @@ namespace NKikimr::NBsController { auto *cmd = record.MutableReassignGroupDisk(); VDiskIDFromVDiskID(*VDiskToReplace, cmd->MutableVDiskId()); cmd->SetConvertToDonor(DonorMode); + cmd->SetAllowUnusableDisks(true); cmd->SetIsSelfHealReasonDecommit(IsSelfHealReasonDecommit); cmd->SetFromSelfHeal(true); Send(MakeBlobStorageNodeWardenID(SelfId().NodeId()), ev.release()); diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index e71178384a4..ad1d1c15efe 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -9736,7 +9736,6 @@ Y_UNIT_TEST_SUITE(THiveTest) { } Y_UNIT_TEST(TestLockedTabletVolatileStateDependsOnMetricsFlag) { - return; // https://github.com/ydb-platform/ydb/issues/46172 for (bool lockedTabletsSendMetrics : {false, true}) { const ui64 hiveTablet = MakeDefaultHiveID(); const ui64 testerTablet = MakeTabletID(false, 1); @@ -9783,7 +9782,12 @@ Y_UNIT_TEST_SUITE(THiveTest) { : NKikimrHive::ETabletVolatileState::TABLET_VOLATILE_STATE_STOPPED, tabletState(tabletIdExt)); - RebootTablet(runtime, hiveTablet, runtime.AllocateEdgeActor(0)); + runtime.Register(CreateTabletKiller(hiveTablet)); + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvLocal::EvStatus, runtime.GetNodeCount()); + runtime.DispatchEvents(options); + } SendLockTabletExecution(runtime, hiveTablet, tabletIdExt, 0, NKikimrProto::OK, owner, 60000, true); diff --git a/ydb/core/mind/hive/leader_tablet_info.h b/ydb/core/mind/hive/leader_tablet_info.h index d4c06eb6444..7d268747719 100644 --- a/ydb/core/mind/hive/leader_tablet_info.h +++ b/ydb/core/mind/hive/leader_tablet_info.h @@ -111,38 +111,10 @@ public: return !NeedToReleaseFromParent && State == ETabletState::ReadyToWork && !IsBootingSuppressed(); } - bool IsReadyToBoot() const { - return IsReadyToWork() && TTabletInfo::IsReadyToBoot(); - } - - bool IsReadyToStart(TInstant now) const { - return IsReadyToWork() && TTabletInfo::IsReadyToStart(now); - } - bool IsReadyToBlockStorage() const { return State == ETabletState::BlockStorage; } - bool IsStarting() const { - return IsReadyToWork() && TTabletInfo::IsStarting(); - } - - bool IsStartingOnNode(TNodeId nodeId) const { - return IsReadyToWork() && TTabletInfo::IsStartingOnNode(nodeId); - } - - bool IsRunning() const { - return IsReadyToWork() && TTabletInfo::IsRunning(); - } - - bool IsAlive() const { - return IsReadyToWork() && TTabletInfo::IsAlive(); - } - - bool IsAliveOnLocal(const TActorId& local) const { - return IsReadyToWork() && TTabletInfo::IsAliveOnLocal(local); - } - bool IsDeleting() const { return State == ETabletState::Deleting; } diff --git a/ydb/core/mind/hive/tablet_info.cpp b/ydb/core/mind/hive/tablet_info.cpp index 55ec6e8442a..9fa41bcd477 100644 --- a/ydb/core/mind/hive/tablet_info.cpp +++ b/ydb/core/mind/hive/tablet_info.cpp @@ -150,17 +150,28 @@ void TTabletInfo::ChangeVolatileState(EVolatileState state) { VolatileStateChangeTime = TActivationContext::Now(); } +bool TTabletInfo::IsReadyToWork() const { + if (IsLeader()) { + return AsLeader().IsReadyToWork(); + } + return true; +} + +bool TTabletInfo::IsReadyToBoot() const { + return IsReadyToWork() && NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STOPPED; +} + bool TTabletInfo::IsReadyToStart(TInstant now) const { if (IsFollower()) { if (!GetLeader().IsRunning()) { return false; } } - return NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING && now >= PostponedStart; + return IsReadyToWork() && NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING && now >= PostponedStart; } bool TTabletInfo::IsStarting() const { - return NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING; + return IsReadyToWork() && NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING; } bool TTabletInfo::IsStartingOnNode(TNodeId nodeId) const { @@ -168,28 +179,31 @@ bool TTabletInfo::IsStartingOnNode(TNodeId nodeId) const { } bool TTabletInfo::IsRunning() const { - return Node != nullptr && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING; + return IsReadyToWork() && Node != nullptr && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING; } bool TTabletInfo::IsBooting() const { - return VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING; + return IsReadyToWork() && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_BOOTING; } bool TTabletInfo::IsAlive() const { - return Node != nullptr && + return IsReadyToWork() && + Node != nullptr && (VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING); } bool TTabletInfo::CanBeAlive() const { - return Node != nullptr && + return IsReadyToWork() && + Node != nullptr && (VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_UNKNOWN); // KIKIMR-12558 } bool TTabletInfo::IsAliveOnLocal(const TActorId& local) const { - return Node != nullptr + return IsReadyToWork() + && Node != nullptr && Node->Local == local && (VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STARTING || VolatileState == EVolatileState::TABLET_VOLATILE_STATE_RUNNING diff --git a/ydb/core/mind/hive/tablet_info.h b/ydb/core/mind/hive/tablet_info.h index c88c635e08a..2edbab742dd 100644 --- a/ydb/core/mind/hive/tablet_info.h +++ b/ydb/core/mind/hive/tablet_info.h @@ -204,10 +204,8 @@ public: TString FamilyString() const; void ChangeVolatileState(EVolatileState state); - bool IsReadyToBoot() const { - return NodeId == 0 && VolatileState == EVolatileState::TABLET_VOLATILE_STATE_STOPPED; - } - + bool IsReadyToWork() const; + bool IsReadyToBoot() const; bool IsReadyToStart(TInstant now) const; bool IsStarting() const; bool IsStartingOnNode(TNodeId nodeId) const; diff --git a/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.cpp b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.cpp new file mode 100644 index 00000000000..fc45a2b3871 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.cpp @@ -0,0 +1,104 @@ +#include "block_range_field.h" + +namespace NYdb::NBS::NBlockStore { + +//////////////////////////////////////////////////////////////////////////////// + +void TBlockRangeField::Add(TBlockRange64 range) +{ + // Non-overlapping ranges sorted by End are also sorted by Start, so we + // can iterate forward and stop early. + + // Find first existing interval with End >= range.Start - 1 (adjacent or + // overlapping on the left side of the new range). + // When range.Start == 0, "range.Start - 1" would underflow → start from + // begin() to cover all intervals. + auto it = (range.Start > 0) + ? Intervals.lower_bound( + TBlockRange64::MakeClosedInterval(0, range.Start - 1)) + : Intervals.begin(); + + ui64 mergedStart = range.Start; + ui64 mergedEnd = range.End; + + while (it != Intervals.end()) { + // For non-overlapping ranges sorted by End (= sorted by Start), we can + // stop when the next interval starts strictly after mergedEnd + 1. + // Guard against overflow when mergedEnd == MaxIndex: in that case every + // possible Start is <= mergedEnd, so no early exit is possible. + if (mergedEnd != TBlockRange64::MaxIndex && it->Start > mergedEnd + 1) { + break; + } + + mergedStart = Min(mergedStart, it->Start); + mergedEnd = Max(mergedEnd, it->End); + it = Intervals.erase(it); + } + + Intervals.insert(TBlockRange64::MakeClosedInterval(mergedStart, mergedEnd)); +} + +void TBlockRangeField::Remove(TBlockRange64 range) +{ + if (Intervals.empty()) { + return; + } + + // Find first interval with End >= range.Start (could overlap with range). + auto it = Intervals.lower_bound( + TBlockRange64::MakeClosedInterval(0, range.Start)); + + while (it != Intervals.end()) { + // Since Start is monotonically increasing (non-overlapping + sorted by + // End), stop once Start is past range.End. + if (it->Start > range.End) { + break; + } + + const TBlockRange64 existing = *it; + it = Intervals.erase(it); + + // Keep the left tail if the existing interval starts before + // range.Start. + if (existing.Start < range.Start) { + Intervals.insert(TBlockRange64::MakeClosedInterval( + existing.Start, + range.Start - 1)); + } + + // Keep the right tail if the existing interval ends after range.End. + if (existing.End > range.End) { + Intervals.insert( + TBlockRange64::MakeClosedInterval(range.End + 1, existing.End)); + break; + } + } +} + +bool TBlockRangeField::Overlaps(TBlockRange64 other) const +{ + if (Intervals.empty()) { + return false; + } + + // First interval with End >= other.Start. + auto it = Intervals.lower_bound( + TBlockRange64::MakeClosedInterval(0, other.Start)); + + if (it == Intervals.end()) { + return false; + } + + return it->Overlaps(other); +} + +void TBlockRangeField::Enumerate(TEnumerateFunc func) const +{ + for (const auto& range: Intervals) { + func(range); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore diff --git a/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.h b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.h new file mode 100644 index 00000000000..2ca4f886a41 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field.h @@ -0,0 +1,39 @@ +#pragma once + +#include "public.h" + +#include "block_range.h" + +#include <util/generic/set.h> + +namespace NYdb::NBS::NBlockStore { + +//////////////////////////////////////////////////////////////////////////////// + +class TBlockRangeField +{ +public: + using TEnumerateFunc = std::function<void(TBlockRange64 item)>; + + void Add(TBlockRange64 range); + void Remove(TBlockRange64 range); + + [[nodiscard]] bool Overlaps(TBlockRange64 other) const; + + void Enumerate(TEnumerateFunc func) const; + +private: + struct TBlockRangeComparator + { + bool operator()(TBlockRange64 a, TBlockRange64 b) const + { + return a.End < b.End; + } + }; + + TSet<TBlockRange64, TBlockRangeComparator> Intervals; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore diff --git a/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field_ut.cpp new file mode 100644 index 00000000000..cb848e43b97 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/common/block_range_field_ut.cpp @@ -0,0 +1,323 @@ +#include "block_range_field.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/vector.h> + +namespace NYdb::NBS::NBlockStore { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +TVector<TBlockRange64> Collect(const TBlockRangeField& field) +{ + TVector<TBlockRange64> result; + field.Enumerate([&](TBlockRange64 r) { result.push_back(r); }); + return result; +} + +TBlockRange64 R(ui64 start, ui64 end) +{ + return TBlockRange64::MakeClosedInterval(start, end); +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TBlockRangeFieldTest) +{ + // ------------------------------------------------------------------------- + // Add – basic + + Y_UNIT_TEST(AddSingleRange) + { + TBlockRangeField f; + f.Add(R(10, 20)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(10, 20), v[0]); + } + + Y_UNIT_TEST(AddTwoNonAdjacentRanges) + { + TBlockRangeField f; + f.Add(R(0, 5)); + f.Add(R(10, 15)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(2u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 5), v[0]); + UNIT_ASSERT_VALUES_EQUAL(R(10, 15), v[1]); + } + + Y_UNIT_TEST(AddAdjacentRangesMerged) + { + TBlockRangeField f; + f.Add(R(0, 5)); + f.Add(R(6, 10)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 10), v[0]); + } + + Y_UNIT_TEST(AddOverlappingRangesMerged) + { + TBlockRangeField f; + f.Add(R(0, 10)); + f.Add(R(5, 15)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 15), v[0]); + } + + Y_UNIT_TEST(AddCoveredByExistingIsNoop) + { + TBlockRangeField f; + f.Add(R(0, 100)); + f.Add(R(10, 20)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 100), v[0]); + } + + Y_UNIT_TEST(AddCoversMultipleRanges) + { + TBlockRangeField f; + f.Add(R(0, 5)); + f.Add(R(10, 15)); + f.Add(R(20, 25)); + // New range covers all three and the gaps between them. + f.Add(R(0, 25)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 25), v[0]); + } + + Y_UNIT_TEST(AddMergesOnBothSides) + { + TBlockRangeField f; + f.Add(R(0, 5)); + f.Add(R(10, 15)); + // Bridge the gap. + f.Add(R(5, 10)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 15), v[0]); + } + + Y_UNIT_TEST(AddSameRangeTwice) + { + TBlockRangeField f; + f.Add(R(3, 7)); + f.Add(R(3, 7)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(3, 7), v[0]); + } + + // ------------------------------------------------------------------------- + // Remove – basic + + Y_UNIT_TEST(RemoveFromEmpty) + { + TBlockRangeField f; + f.Remove(R(0, 10)); // must not crash + UNIT_ASSERT(Collect(f).empty()); + } + + Y_UNIT_TEST(RemoveExact) + { + TBlockRangeField f; + f.Add(R(0, 10)); + f.Remove(R(0, 10)); + UNIT_ASSERT(Collect(f).empty()); + } + + Y_UNIT_TEST(RemoveFromMiddle) + { + TBlockRangeField f; + f.Add(R(0, 20)); + f.Remove(R(5, 10)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(2u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 4), v[0]); + UNIT_ASSERT_VALUES_EQUAL(R(11, 20), v[1]); + } + + Y_UNIT_TEST(RemoveLeftPart) + { + TBlockRangeField f; + f.Add(R(0, 20)); + f.Remove(R(0, 9)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(10, 20), v[0]); + } + + Y_UNIT_TEST(RemoveRightPart) + { + TBlockRangeField f; + f.Add(R(0, 20)); + f.Remove(R(10, 20)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 9), v[0]); + } + + Y_UNIT_TEST(RemoveNonOverlapping) + { + TBlockRangeField f; + f.Add(R(10, 20)); + f.Remove(R(30, 40)); // no overlap, no change + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(10, 20), v[0]); + } + + Y_UNIT_TEST(RemoveSeveralRanges) + { + TBlockRangeField f; + f.Add(R(0, 5)); + f.Add(R(10, 15)); + f.Add(R(20, 25)); + f.Remove(R(3, 22)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(2u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 2), v[0]); + UNIT_ASSERT_VALUES_EQUAL(R(23, 25), v[1]); + } + + // ------------------------------------------------------------------------- + // Overlaps + + Y_UNIT_TEST(OverlapsOnEmpty) + { + TBlockRangeField f; + UNIT_ASSERT(!f.Overlaps(R(0, 100))); + } + + Y_UNIT_TEST(OverlapsExact) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(f.Overlaps(R(10, 20))); + } + + Y_UNIT_TEST(OverlapsPartialLeft) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(f.Overlaps(R(5, 12))); + } + + Y_UNIT_TEST(OverlapsPartialRight) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(f.Overlaps(R(15, 30))); + } + + Y_UNIT_TEST(OverlapsCovering) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(f.Overlaps(R(0, 100))); + } + + Y_UNIT_TEST(OverlapsNoOverlapBefore) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(!f.Overlaps(R(0, 9))); + } + + Y_UNIT_TEST(OverlapsNoOverlapAfter) + { + TBlockRangeField f; + f.Add(R(10, 20)); + UNIT_ASSERT(!f.Overlaps(R(21, 30))); + } + + Y_UNIT_TEST(OverlapsAdjacentNotOverlapping) + { + TBlockRangeField f; + f.Add(R(10, 20)); + // [9,9] touches start but doesn't overlap. + UNIT_ASSERT(!f.Overlaps(R(5, 9))); + // [21,21] touches end but doesn't overlap. + UNIT_ASSERT(!f.Overlaps(R(21, 25))); + } + + // ------------------------------------------------------------------------- + // Edge / boundary cases + + Y_UNIT_TEST(AddStartingAtZero) + { + TBlockRangeField f; + f.Add(R(0, 0)); + f.Add(R(1, 5)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 5), v[0]); + } + + Y_UNIT_TEST(RemoveSingleBlock) + { + TBlockRangeField f; + f.Add(R(0, 4)); + f.Remove(R(2, 2)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(2u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 1), v[0]); + UNIT_ASSERT_VALUES_EQUAL(R(3, 4), v[1]); + } + + Y_UNIT_TEST(ManyFragmentsAfterRemoves) + { + TBlockRangeField f; + f.Add(R(0, 99)); + // Remove every even block to create 50 gaps. + for (ui64 i = 0; i < 100; i += 2) { + f.Remove(R(i, i)); + } + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(50u, v.size()); + for (ui64 i = 0; i < 50; ++i) { + ui64 odd = i * 2 + 1; + UNIT_ASSERT_VALUES_EQUAL(R(odd, odd), v[i]); + } + } + + Y_UNIT_TEST(AddRestoresAfterRemoves) + { + TBlockRangeField f; + f.Add(R(0, 99)); + for (ui64 i = 0; i < 100; i += 2) { + f.Remove(R(i, i)); + } + // Adding back should merge everything. + for (ui64 i = 0; i < 100; i += 2) { + f.Add(R(i, i)); + } + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(1u, v.size()); + UNIT_ASSERT_VALUES_EQUAL(R(0, 99), v[0]); + } + + Y_UNIT_TEST(EnumerateOrderedByStart) + { + TBlockRangeField f; + f.Add(R(50, 60)); + f.Add(R(10, 20)); + f.Add(R(30, 40)); + auto v = Collect(f); + UNIT_ASSERT_VALUES_EQUAL(3u, v.size()); + UNIT_ASSERT(v[0].Start < v[1].Start); + UNIT_ASSERT(v[1].Start < v[2].Start); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore diff --git a/ydb/core/nbs/cloud/blockstore/libs/common/ut/ya.make b/ydb/core/nbs/cloud/blockstore/libs/common/ut/ya.make index 74e5274eeec..8140f03c6d5 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/common/ut/ya.make +++ b/ydb/core/nbs/cloud/blockstore/libs/common/ut/ya.make @@ -4,6 +4,7 @@ INCLUDE(${ARCADIA_ROOT}/ydb/core/nbs/cloud/storage/core/tests/recipes/small.inc) SRCS( block_range_algorithms_ut.cpp + block_range_field_ut.cpp block_range_map_ut.cpp block_range_ut.cpp diff --git a/ydb/core/nbs/cloud/blockstore/libs/common/ya.make b/ydb/core/nbs/cloud/blockstore/libs/common/ya.make index e94dec6092d..6c7742a0e16 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/common/ya.make +++ b/ydb/core/nbs/cloud/blockstore/libs/common/ya.make @@ -2,6 +2,7 @@ LIBRARY() SRCS( block_range_algorithms.cpp + block_range_field.cpp block_range_map.cpp block_range.cpp thread_checker.cpp diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp index c124311c3d5..f56d55568fb 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp @@ -112,14 +112,18 @@ CreateWaitSessionCbForSyncWithPBuffer( } if (auto self = weakSelf.lock()) { - childSpan->Event("ConnectionReady"); + NWilson::TTraceId traceId; + if (childSpan) { + childSpan->Event("ConnectionReady"); + traceId = childSpan->GetTraceId(); + } self->SyncWithPBuffer( vChunkIndex, pbufferHostIndex, ddiskHostIndex, segments, - childSpan->GetTraceId()) + traceId) .Subscribe([promise = std::move(promise)] // (const TDBGFlushResponseFuture& f) mutable { promise.SetValue(f.GetValue()); }); @@ -127,8 +131,8 @@ CreateWaitSessionCbForSyncWithPBuffer( for (size_t i = 0; i < segments.size(); ++i) { flushResponse.Errors.push_back(MakeError(E_CANCELLED)); } + promise.SetValue(std::move(flushResponse)); } - promise.SetValue(std::move(flushResponse)); }; return cb; @@ -315,14 +319,18 @@ TDirectBlockGroup::ReadBlocksFromDDisk( } if (auto self = weakSelf.lock()) { - childSpan->Event("ConnectionReady"); + NWilson::TTraceId traceId; + if (childSpan) { + childSpan->Event("ConnectionReady"); + traceId = childSpan->GetTraceId(); + } self->ReadBlocksFromDDisk( vChunkIndex, hostIndex, range, guardedSglist, - childSpan->GetTraceId()) + traceId) .Subscribe([promise = std::move(promise)] // (const TDBGReadBlocksResponseFuture& f) mutable { promise.SetValue(UnsafeExtractValue(f)); }); @@ -519,14 +527,18 @@ TDirectBlockGroup::WriteBlocksToDDisk( return; } if (auto self = weakSelf.lock()) { - childSpan->Event("ConnectionReady"); + NWilson::TTraceId traceId; + if (childSpan) { + childSpan->Event("ConnectionReady"); + traceId = childSpan->GetTraceId(); + } self->WriteBlocksToDDisk( vChunkIndex, hostIndex, range, guardedSglist, - childSpan->GetTraceId()) + traceId) .Subscribe([promise = std::move(promise)] // (const TDBGWriteBlocksResponseFuture& f) mutable { promise.SetValue(f.GetValue()); }); @@ -1211,9 +1223,11 @@ NThreading::TFuture<TListPBufferResponse> TDirectBlockGroup::ListPBuffers( } const auto& connection = PBufferConnections[hostIndex]; + // Hold a local copy of the connect future, + // do not put an address of changeable field into a wait. + auto connectFuture = connection.GetFuture(); // Switch co-routine context if needed. - const NProto::TError& connectError = - Executor->WaitFor(connection.GetFuture()); + const NProto::TError& connectError = Executor->WaitFor(connectFuture); if (HasError(connectError)) { return MakeFuture(TListPBufferResponse{.Error = connectError}); } @@ -1551,18 +1565,16 @@ void TDirectBlockGroup::OnConnectionEstablished( // Unblock waiters on ConnectFuture with the error. connection.ConnectPromise.SetValue(error); return; - // TODO (future phase): handle non-BLOCKED connect errors - // (ERROR/unavailability) via reconnect. - } else if (IsInitialized()) { + } else { LOG_ERROR( *ActorSystem, NKikimrServices::NBS_PARTITION, - "%s connection failed for %s (post-init): %s", + "%s connection failed %s: %s", LogTitle.GetWithTime().c_str(), PrintHostIndex(hostIndex).c_str(), FormatError(error).c_str()); - } else { - Y_ABORT("Unhandled branch of connect error"); + ReEstablishConnection(connectionType, hostIndex); + return; } // ConnectPromise resolves both "connection ready" and "session ready" in @@ -1579,12 +1591,16 @@ void TDirectBlockGroup::OnConnectionEstablished( } } -void TDirectBlockGroup::ReEstablishDDiskConnection( - THostIndex hostIndex, - TDuration reconnectDelay) +void TDirectBlockGroup::ReEstablishConnection( + EConnectionType connectionType, + THostIndex hostIndex) { Y_ABORT_UNLESS(ExecutorThreadChecker.Check()); - Y_ABORT_UNLESS(hostIndex < DDiskConnections.size()); + auto& connections = connectionType == EConnectionType::DDisk + ? DDiskConnections + : PBufferConnections; + Y_ABORT_UNLESS(hostIndex < connections.size()); + TDDiskConnection& connection = connections[hostIndex]; if (BlockedGenerationDetected) { LOG_WARN( @@ -1595,13 +1611,14 @@ void TDirectBlockGroup::ReEstablishDDiskConnection( return; } - DDiskConnections[hostIndex].ResetSession(); + connection.ResetSession(); + TDuration reconnectDelay = Oracle.GetHostReconnectDelay(hostIndex); Schedule( reconnectDelay, - [hostIndex, weakSelf = weak_from_this()]() + [hostIndex, weakSelf = weak_from_this(), connectionType]() { if (auto self = weakSelf.lock()) { - self->DoEstablishConnection(hostIndex, EConnectionType::DDisk); + self->DoEstablishConnection(hostIndex, connectionType); } }); } @@ -1619,9 +1636,9 @@ void TDirectBlockGroup::OnNodeDisconnected(THostIndex hostIndex, ui32 nodeId) nodeId); Oracle.OnDDiskDisconnected(hostIndex, TInstant::Now()); + // OnNodeDisconnected may be called only for DDisk - TDuration reconnectDelay = Oracle.GetDDiskReconnectDelay(hostIndex); - ReEstablishDDiskConnection(hostIndex, reconnectDelay); + ReEstablishConnection(EConnectionType::DDisk, hostIndex); } bool TDirectBlockGroup::HasPBufferQuorum() const diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.h index 602c15a9f72..366615e0718 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.h +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.h @@ -194,9 +194,9 @@ private: THostIndex hostIndex, ui64 seqNo, const NKikimrBlobStorage::NDDisk::TEvConnectResult& result); - void ReEstablishDDiskConnection( - THostIndex hostIndex, - TDuration reconnectDelay); + void ReEstablishConnection( + EConnectionType connectionType, + THostIndex hostIndex); void OnNodeDisconnected(THostIndex hostIndex, ui32 nodeId); [[nodiscard]] bool HasPBufferQuorum() const; diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl_ut.cpp index 79af0b61f78..2ce680d6282 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl_ut.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl_ut.cpp @@ -979,6 +979,199 @@ Y_UNIT_TEST_SUITE(TDirectBlockGroupTest) UNIT_ASSERT_VALUES_EQUAL(connectsBefore, connectsAfter); } + // A non-BLOCKED connect failure on DDisk (e.g. ERROR/unavailable) must NOT + // be terminal: the session is reset and a fresh Connect is retried with the + // next seq_no. + Y_UNIT_TEST_F(ShouldReconnectDDiskOnNonBlockedConnectError, TDBGFixture) + { + auto executor = MakeExecutor(); + auto transport = std::make_unique<TStorageTransportMock>(); + auto* transportPtr = transport.get(); + + const auto& ddisks = transportPtr->GetDDiskIds(); + // host[0] connect is deferred; hosts 1..4 connect immediately -> the + // quorum (3 of 5) is reached without host[0]. + auto pendingConnect0 = + transportPtr->SetPendingConnect(EConnectionType::DDisk, ddisks[0]); + + auto dbg = MakeDirectBlockGroup(executor, std::move(transport)); + auto initialReady = RunAndGetInitialReady(dbg, false); + WaitReady(executor, initialReady); + + // Only the first (initial) connect attempt was issued for host[0]. + const size_t connectsBefore = + transportPtr + ->GetConnectCredentials(EConnectionType::DDisk, ddisks[0]) + .size(); + UNIT_ASSERT_VALUES_EQUAL(1, connectsBefore); + + // Arm the next (reconnect) Connect for host[0] before delivering the + // failure. + auto reconnectPending = + transportPtr->SetPendingConnect(EConnectionType::DDisk, ddisks[0]); + + // Resolve the in-flight connect with a non-BLOCKED error. + pendingConnect0.SetValue(TStorageTransportMock::MakeConnectResult( + 1, + NKikimrBlobStorage::NDDisk::TReplyStatus::ERROR)); + + // Drain OnConnectionEstablished (queues the reconnect) then the + // reconnect itself (issues a fresh Connect). + DrainExecutor(executor); + DrainExecutor(executor); + + // A reconnect Connect was issued and carries the next seq_no (=2). + const auto credentials = transportPtr->GetConnectCredentials( + EConnectionType::DDisk, + ddisks[0]); + UNIT_ASSERT_VALUES_EQUAL(connectsBefore + 1, credentials.size()); + UNIT_ASSERT_VALUES_EQUAL(2, credentials.back().DDiskSessionSeqNo); + + // No suicide on a non-BLOCKED connect error. + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + { + const auto state = + GetBlockedDetected(executor, dbg, 0, WaitTimeout); + UNIT_ASSERT(!state.DDiskSessionBroken); + UNIT_ASSERT(!state.BlockedGenerationDetected); + } + + // The retry succeeds -> the session is confirmed at seq_no=2. + reconnectPending.SetValue(TStorageTransportMock::MakeConnectResult()); + DrainExecutor(executor); + + const auto seqNos = ReadAllDDiskSeqNos(executor, dbg, WaitTimeout); + UNIT_ASSERT_VALUES_EQUAL(2, seqNos[0]); + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + } + + // A non-BLOCKED connect failure on PBuffer is retried the same way. PBuffer + // has no session/lock, so only the reconnect (a fresh Connect) is asserted, + // and no suicide happens. + Y_UNIT_TEST_F(ShouldReconnectPBufferOnNonBlockedConnectError, TDBGFixture) + { + auto executor = MakeExecutor(); + auto transport = std::make_unique<TStorageTransportMock>(); + auto* transportPtr = transport.get(); + + const auto& pbuffers = transportPtr->GetPBufferIds(); + // PBuffer[0] connect is deferred; the rest connect immediately -> the + // PBuffer quorum is reached without PBuffer[0]. All DDisks connect + // immediately -> locked quorum is reached too. + auto pendingConnect0 = transportPtr->SetPendingConnect( + EConnectionType::PBuffer, + pbuffers[0]); + + auto dbg = MakeDirectBlockGroup(executor, std::move(transport)); + auto initialReady = RunAndGetInitialReady(dbg, false); + WaitReady(executor, initialReady); + + // Only the first (initial) connect attempt was issued for PBuffer[0]. + const size_t connectsBefore = + transportPtr + ->GetConnectCredentials(EConnectionType::PBuffer, pbuffers[0]) + .size(); + UNIT_ASSERT_VALUES_EQUAL(1, connectsBefore); + + // Arm the next (reconnect) Connect for PBuffer[0] before delivering the + // failure. + auto reconnectPending = transportPtr->SetPendingConnect( + EConnectionType::PBuffer, + pbuffers[0]); + + // Resolve the in-flight PBuffer connect with a non-BLOCKED error. + pendingConnect0.SetValue(TStorageTransportMock::MakeConnectResult( + 1, + NKikimrBlobStorage::NDDisk::TReplyStatus::ERROR)); + + // Drain OnConnectionEstablished (queues the reconnect) then the + // reconnect itself (issues a fresh Connect). + DrainExecutor(executor); + DrainExecutor(executor); + + // A reconnect Connect was issued for PBuffer[0]. + const size_t connectsAfter = + transportPtr + ->GetConnectCredentials(EConnectionType::PBuffer, pbuffers[0]) + .size(); + UNIT_ASSERT_VALUES_EQUAL(connectsBefore + 1, connectsAfter); + + // A PBuffer connect error never triggers a suicide. + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + + // The retry succeeds without any crash/suicide. + reconnectPending.SetValue(TStorageTransportMock::MakeConnectResult()); + DrainExecutor(executor); + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + } + + // Startup must not fail permanently on a non-BLOCKED connect + // error: initial-ready stays unresolved while the DDisk connects keep + // failing, and resolves once the retried connects reach the locked quorum. + Y_UNIT_TEST_F( + ShouldReachInitialReadyAfterTransientConnectError, + TDBGFixture) + { + auto executor = MakeExecutor(); + auto transport = std::make_unique<TStorageTransportMock>(); + auto* transportPtr = transport.get(); + + const auto& ddisks = transportPtr->GetDDiskIds(); + // Defer every DDisk connect so we control when the quorum is reached. + // PBuffers connect immediately -> PBuffer quorum is satisfied. + TVector<TStorageTransportMock::TConnectPromise> firstConnects; + for (const auto& ddiskId: ddisks) { + firstConnects.push_back(transportPtr->SetPendingConnect( + EConnectionType::DDisk, + ddiskId)); + } + + auto dbg = MakeDirectBlockGroup(executor, std::move(transport)); + auto initialReady = RunAndGetInitialReady(dbg, false); + + DrainExecutor(executor); + UNIT_ASSERT(!initialReady.HasValue()); + + // Arm the reconnect Connect for the first three DDisks (the quorum). + constexpr size_t quorum = QuorumDirectBlockGroupHostCount; + TVector<TStorageTransportMock::TConnectPromise> reconnects; + for (size_t i = 0; i < quorum; ++i) { + reconnects.push_back(transportPtr->SetPendingConnect( + EConnectionType::DDisk, + ddisks[i])); + } + + // First attempt fails on all three with a non-BLOCKED error. + for (size_t i = 0; i < quorum; ++i) { + firstConnects[i].SetValue(TStorageTransportMock::MakeConnectResult( + 1, + NKikimrBlobStorage::NDDisk::TReplyStatus::ERROR)); + } + // Drain OnConnectionEstablished + queued reconnects, then the + // reconnects. + DrainExecutor(executor); + DrainExecutor(executor); + + // Still not ready: no DDisk session is locked yet. + UNIT_ASSERT(!initialReady.HasValue()); + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + + // The retried connects succeed -> the locked quorum is reached and + // startup completes. + for (auto& reconnect: reconnects) { + reconnect.SetValue(TStorageTransportMock::MakeConnectResult()); + } + WaitReady(executor, initialReady); + + // The recovered sessions are confirmed at seq_no=2 (attempt + retry), + // and no suicide happened. + const auto seqNos = ReadAllDDiskSeqNos(executor, dbg, WaitTimeout); + for (size_t i = 0; i < quorum; ++i) { + UNIT_ASSERT_VALUES_EQUAL(2, seqNos[i]); + } + UNIT_ASSERT_VALUES_EQUAL(0, Service->BlockedGenerationCount); + } + // A PBuffer error must NOT trigger the suicide: the BLOCKED check is // only wired into DDisk-addressed responses. Y_UNIT_TEST_F(ShouldNotSuicideOnPBufferError, TDBGFixture) diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp index 4e0ce88de08..bf0941a86ce 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp @@ -38,7 +38,7 @@ void TOracleMock::OnDDiskDisconnected(THostIndex hostIndex, TInstant now) Y_UNUSED(hostIndex, now); } -TDuration TOracleMock::GetDDiskReconnectDelay(THostIndex hostIndex) +TDuration TOracleMock::GetHostReconnectDelay(THostIndex hostIndex) { Y_UNUSED(hostIndex); return TDuration::MilliSeconds(1); diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h index c9d535839ed..214edafc69f 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h @@ -41,7 +41,7 @@ struct TOracleMock: public IOracle void OnDDiskDisconnected(THostIndex hostIndex, TInstant now) override; void OnDDiskConnected(THostIndex hostIndex, TInstant now) override; - TDuration GetDDiskReconnectDelay(THostIndex hostIndex) override; + TDuration GetHostReconnectDelay(THostIndex hostIndex) override; [[nodiscard]] THostIndex SelectBestPBufferHost( THostMask hosts, diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp index d79c836bf0f..c29e14c7ce7 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp @@ -285,7 +285,7 @@ void TOracle::OnDDiskConnected(THostIndex hostIndex, TInstant now) HostsReconnectDelays[hostIndex].Reset(); } -TDuration TOracle::GetDDiskReconnectDelay(THostIndex hostIndex) +TDuration TOracle::GetHostReconnectDelay(THostIndex hostIndex) { return HostsReconnectDelays[hostIndex].GetDelayAndIncrease(); } diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h index 4e68e4eb54d..b78a53f216c 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h @@ -71,7 +71,7 @@ public: virtual void OnDDiskDisconnected(THostIndex hostIndex, TInstant now) = 0; virtual void OnDDiskConnected(THostIndex hostIndex, TInstant now) = 0; - virtual TDuration GetDDiskReconnectDelay(THostIndex hostIndex) = 0; + virtual TDuration GetHostReconnectDelay(THostIndex hostIndex) = 0; // Picks the best host (by lowest inflight count) out of the provided set // of hosts. Ties are broken uniformly at random. @@ -135,7 +135,7 @@ public: void OnDDiskDisconnected(THostIndex hostIndex, TInstant now) override; void OnDDiskConnected(THostIndex hostIndex, TInstant now) override; - [[nodiscard]] TDuration GetDDiskReconnectDelay( + [[nodiscard]] TDuration GetHostReconnectDelay( THostIndex hostIndex) override; [[nodiscard]] THostIndex SelectBestPBufferHost( diff --git a/ydb/core/persqueue/deferred_publish/finalize_publication_actor.cpp b/ydb/core/persqueue/deferred_publish/finalize_publication_actor.cpp index 5890aa4f953..713fd40ff7a 100644 --- a/ydb/core/persqueue/deferred_publish/finalize_publication_actor.cpp +++ b/ydb/core/persqueue/deferred_publish/finalize_publication_actor.cpp @@ -21,6 +21,10 @@ namespace { using namespace NKikimr::NKqp; +TString MetadataUserToken() { + return NACLib::TSystemUsers::Metadata().SerializeAsString(); +} + TString MakeDeletePublicationSql() { return TStringBuilder() << R"( -- TFinalizePublicationActor::MakeDeletePublicationSql @@ -111,13 +115,11 @@ public: TString database, ui64 intPublicationId, EFinalizePublicationOp op, - TString userToken, TString callerSid) : ReplyTo(replyTo) , Database(std::move(database)) , IntPublicationId(intPublicationId) , Op(op) - , UserToken(std::move(userToken)) , CallerSid(std::move(callerSid)) {} @@ -158,19 +160,11 @@ private: Become(&TFinalizePublicationActor::StateDeleteOnly); } - void SetCreateSessionIdentity(TEvKqp::TEvCreateSessionRequest& ev) { - if (!UserToken.empty()) { - NACLibProto::TUserToken token; - if (token.ParseFromString(UserToken)) { - ev.Record.SetUserSID(token.GetUserSID()); - } - } - } - void StartKqpSession() { auto ev = MakeHolder<TEvKqp::TEvCreateSessionRequest>(); ev->Record.MutableRequest()->SetDatabase(Database); - SetCreateSessionIdentity(*ev); + // Registry and topic finalize ops run as metadata@system, like Begin/List query actors. + ev->Record.SetUserSID(BUILTIN_ACL_METADATA); Send(MakeKqpProxyID(SelfId().NodeId()), ev.Release()); Step = EStep::KqpCreateSession; Become(&TFinalizePublicationActor::StateKqp); @@ -178,9 +172,7 @@ private: void SendKqpBeginTx() { auto ev = MakeHolder<TEvKqp::TEvQueryRequest>(); - if (!UserToken.empty()) { - ev->Record.SetUserToken(UserToken); - } + ev->Record.SetUserToken(MetadataUserToken()); ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_BEGIN_TX); ev->Record.MutableRequest()->MutableTxControl()->mutable_begin_tx()->mutable_serializable_read_write(); ev->Record.MutableRequest()->SetSessionId(KqpSessionId); @@ -197,9 +189,7 @@ private: .Build(); auto ev = MakeHolder<TEvKqp::TEvQueryRequest>(); - if (!UserToken.empty()) { - ev->Record.SetUserToken(UserToken); - } + ev->Record.SetUserToken(MetadataUserToken()); ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_EXECUTE); ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_SQL_DML); ev->Record.MutableRequest()->SetQuery(MakeDeletePublicationSql()); @@ -218,9 +208,7 @@ private: void SendKqpDeferredPublication() { auto ev = MakeHolder<TEvKqp::TEvQueryRequest>(); - if (!UserToken.empty()) { - ev->Record.SetUserToken(UserToken); - } + ev->Record.SetUserToken(MetadataUserToken()); ev->Record.MutableRequest()->SetType(NKikimrKqp::QUERY_TYPE_UNDEFINED); ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_TOPIC); ev->Record.MutableRequest()->SetDatabase(Database); @@ -235,9 +223,7 @@ private: void SendKqpCommit() { auto ev = MakeHolder<TEvKqp::TEvQueryRequest>(); - if (!UserToken.empty()) { - ev->Record.SetUserToken(UserToken); - } + ev->Record.SetUserToken(MetadataUserToken()); ev->Record.MutableRequest()->SetAction(NKikimrKqp::QUERY_ACTION_COMMIT_TX); ev->Record.MutableRequest()->MutableTxControl()->set_tx_id(TxId); ev->Record.MutableRequest()->MutableTxControl()->set_commit_tx(true); @@ -369,7 +355,6 @@ private: const TString Database; const ui64 IntPublicationId; const EFinalizePublicationOp Op; - const TString UserToken; const TString CallerSid; TMaybe<TListDestinationsData> ListDestinationsData; @@ -386,10 +371,9 @@ NActors::IActor* CreateFinalizePublicationActor( const TString& database, ui64 intPublicationId, EFinalizePublicationOp op, - const TString& userToken, const TString& callerSid) { - return new TFinalizePublicationActor(replyTo, database, intPublicationId, op, userToken, callerSid); + return new TFinalizePublicationActor(replyTo, database, intPublicationId, op, callerSid); } } // namespace NKikimr::NPQ::NDeferredPublish diff --git a/ydb/core/persqueue/deferred_publish/finalize_publication_actor.h b/ydb/core/persqueue/deferred_publish/finalize_publication_actor.h index b14130cd4e4..28c33b38dd2 100644 --- a/ydb/core/persqueue/deferred_publish/finalize_publication_actor.h +++ b/ydb/core/persqueue/deferred_publish/finalize_publication_actor.h @@ -11,7 +11,6 @@ NActors::IActor* CreateFinalizePublicationActor( const TString& database, ui64 intPublicationId, EFinalizePublicationOp op, - const TString& userToken, const TString& callerSid); } // namespace NKikimr::NPQ::NDeferredPublish diff --git a/ydb/core/persqueue/pqrb/mirror_describer.cpp b/ydb/core/persqueue/pqrb/mirror_describer.cpp index 47185c8846d..6cd1e3e3687 100644 --- a/ydb/core/persqueue/pqrb/mirror_describer.cpp +++ b/ydb/core/persqueue/pqrb/mirror_describer.cpp @@ -4,6 +4,8 @@ #include <google/protobuf/util/message_differencer.h> +#define YDB_LOG_THIS_FILE_COMPONENT Service + #define PQ_ENSURE(condition) AFL_ENSURE(condition)("topic", TopicName) using namespace NPersQueue; @@ -34,7 +36,8 @@ void TMirrorDescriber::StartInit(const TActorContext& ctx) { } void TMirrorDescriber::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { - LOG_N("killed"); + YDB_LOG_NOTICE("Killed", + {"logPrefix", NPQ_LOG_PREFIX}); CredentialsProvider = nullptr; Die(ctx); } @@ -44,10 +47,13 @@ void TMirrorDescriber::HandleChangeConfig(TEvPQ::TEvChangePartitionConfig::TPtr& Config, ev->Get()->Config.GetPartitionConfig().GetMirrorFrom() ); - LOG_D("got new config, equal with previous: " << equalConfigs); + YDB_LOG_DEBUG("Got new config, equal with", + {"logPrefix", NPQ_LOG_PREFIX}, + {"previous", equalConfigs}); if (!equalConfigs) { Config = ev->Get()->Config.GetPartitionConfig().GetMirrorFrom(); - LOG_I("changing config"); + YDB_LOG_INFO("Changing config", + {"logPrefix", NPQ_LOG_PREFIX}); StartInit(ctx); } } @@ -56,13 +62,17 @@ void TMirrorDescriber::HandleDescriptionResult(TEvPQ::TEvMirrorTopicDescription: DescribeTopicRequestInFlight = false; const auto& description = ev->Get()->Description; if (!description.has_value()) { - LOG_E("cannot describe topic " << description.error()); + YDB_LOG_ERROR("Cannot describe topic", + {"logPrefix", NPQ_LOG_PREFIX}, + {"error", description.error()}); ScheduleWithIncreasingTimeout<TEvents::TEvWakeup>(SelfId(), DescribeRetryTimeout, DESCRIBE_RETRY_TIMEOUT_MAX, ctx); return; } const NYdb::NTopic::TDescribeTopicResult& result = description.value(); if (!result.IsSuccess()) { - LOG_E("cannot describe topic " << result.GetIssues().ToString()); + YDB_LOG_ERROR("Cannot describe topic", + {"logPrefix", NPQ_LOG_PREFIX}, + {"issues", result.GetIssues()}); ScheduleWithIncreasingTimeout<TEvents::TEvWakeup>(SelfId(), DescribeRetryTimeout, DESCRIBE_RETRY_TIMEOUT_MAX, ctx); return; } @@ -71,14 +81,17 @@ void TMirrorDescriber::HandleDescriptionResult(TEvPQ::TEvMirrorTopicDescription: descr.SerializeTo(req); return req.ShortUtf8DebugString(); }; - LOG_T("topic description: " << debugTopicDescriptionString(description.value().GetTopicDescription())); + YDB_LOG_TRACE("Topic", + {"logPrefix", NPQ_LOG_PREFIX}, + {"description", debugTopicDescriptionString(description.value().GetTopicDescription())}); ctx.Send(TabletActorId, ev->Release()); ctx.Schedule(DESCRIBE_RETRY_TIMEOUT_MAX, new TEvents::TEvWakeup()); } void TMirrorDescriber::DescribeTopic(const TActorContext& ctx) { if (DescribeTopicRequestInFlight) { - LOG_I("description request already inflight."); + YDB_LOG_INFO("Description request already inflight", + {"logPrefix", NPQ_LOG_PREFIX}); return; } @@ -111,7 +124,8 @@ void TMirrorDescriber::DescribeTopic(const TActorContext& ctx) { void TMirrorDescriber::HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& /*ev*/, const TActorContext& ctx) { if (CredentialsRequestInFlight) { - LOG_W("credentials request already inflight."); + YDB_LOG_WARN("Credentials request already inflight", + {"logPrefix", NPQ_LOG_PREFIX}); return; } CredentialsProvider = nullptr; @@ -145,13 +159,17 @@ void TMirrorDescriber::HandleInitCredentials(TEvPQ::TEvInitCredentials::TPtr& /* void TMirrorDescriber::HandleCredentialsCreated(TEvPQ::TEvCredentialsCreated::TPtr& ev, const TActorContext& ctx) { CredentialsRequestInFlight = false; if (ev->Get()->Error) { - LOG_W("cannot initialize credentials provider: " << ev->Get()->Error.value()); + YDB_LOG_WARN("Cannot initialize credentials", + {"logPrefix", NPQ_LOG_PREFIX}, + {"provider", ev->Get()->Error.value()}); ScheduleWithIncreasingTimeout<TEvPQ::TEvInitCredentials>(SelfId(), CredentialsInitInterval, INIT_INTERVAL_MAX, ctx); return; } CredentialsProvider = ev->Get()->Credentials; - LOG_N("credentials provider created " << bool(CredentialsProvider)); + YDB_LOG_NOTICE("Credentials provider created", + {"logPrefix", NPQ_LOG_PREFIX}, + {"hasCredentialsProvider", bool(CredentialsProvider)}); CredentialsInitInterval = INIT_INTERVAL_START; ScheduleDescription(ctx); } diff --git a/ydb/core/persqueue/pqrb/partition_scale_manager.cpp b/ydb/core/persqueue/pqrb/partition_scale_manager.cpp index 12b49163dbc..517fe6591ba 100644 --- a/ydb/core/persqueue/pqrb/partition_scale_manager.cpp +++ b/ydb/core/persqueue/pqrb/partition_scale_manager.cpp @@ -6,6 +6,8 @@ #include <algorithm> #include <ranges> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + namespace NKikimr { namespace NPQ { @@ -35,9 +37,13 @@ void TPartitionScaleManager::HandleScaleStatusChange(const ui32 partitionId, NKi TMaybe<NKikimrPQ::TPartitionScaleParticipants> participants, TMaybe<TString> splitBoundary, const TActorContext& ctx) { - PQ_LOG_D("Handle HandleScaleStatusChange. Scale status: " << NKikimrPQ::EScaleStatus_Name(scaleStatus)); + YDB_LOG_DEBUG("Handle HandleScaleStatusChange. Scale", + {"logPrefix", LogPrefix()}, + {"status", NKikimrPQ::EScaleStatus_Name(scaleStatus)}); if (scaleStatus == NKikimrPQ::EScaleStatus::NEED_SPLIT) { - PQ_LOG_D("::HandleScaleStatusChange need to split partition " << partitionId); + YDB_LOG_DEBUG("::HandleScaleStatusChange need to split partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); TPartitionScaleOperationInfo op{ .PartitionId = partitionId, .PartitionScaleParticipants = std::move(participants), @@ -58,13 +64,15 @@ void TPartitionScaleManager::TrySendScaleRequest(const TActorContext& ctx) { auto splitMergeRequest = BuildScaleRequest(ctx); if (splitMergeRequest.Empty()) { - PQ_LOG_D("splitMergeRequest empty"); + YDB_LOG_DEBUG("SplitMergeRequest empty", + {"logPrefix", LogPrefix()}); return; } RequestInflight = true; RootPartitionsResetRequestInflight = !splitMergeRequest.SetBoundary.empty(); - PQ_LOG_D("send split request"); + YDB_LOG_DEBUG("Send split request", + {"logPrefix", LogPrefix()}); CurrentScaleRequest = ctx.Register(new TPartitionScaleRequest( TopicName, TopicPath, @@ -129,12 +137,12 @@ TPartitionScaleManager::TScaleRequest TPartitionScaleManager::BuildScaleRequest( auto mergesToApply = BuildMergeRequest(allowedSplitsCount); auto splitsToApply = BuildSplitRequest(allowedSplitsCount); - PQ_LOG_D(fmt::format("Scale request: #splits={}, #unprocessed={}, splitsLimit={}, #merges={}", - splitsToApply.Requests.size(), - splitsToApply.Unprocessed, - allowedSplitsCountLimit, - mergesToApply.Requests.size() - )); + YDB_LOG_DEBUG("Scale request", + {"logPrefix", LogPrefix()}, + {"splits", splitsToApply.Requests.size()}, + {"unprocessed", splitsToApply.Unprocessed}, + {"splitsLimit", allowedSplitsCountLimit}, + {"merges", mergesToApply.Requests.size()}); return { .Split = std::move(splitsToApply.Requests), @@ -166,19 +174,20 @@ TPartitionScaleManager::TRequests<TPartitionScaleManager::TPartitionBoundary> TP allowedSplitsCount -= cost; boundsToApply.push_back(std::move(part)); } else { - PQ_LOG_W(fmt::format("MaxActivePartitions ({}) is too low to recreate {} root partitions from the mirror source topic", - BalancerConfig.MaxActivePartitions, - RootPartitionsToCreate->size())); + YDB_LOG_WARN("MaxActivePartitions is too low to recreate root partitions from the mirror source topic", + {"logPrefix", LogPrefix()}, + {"maxActivePartitions", BalancerConfig.MaxActivePartitions}, + {"createPartitions", RootPartitionsToCreate->size()}); // don't send request at all, if there is not enough quota return { .Unprocessed = RootPartitionsToCreate->size(), }; } } - PQ_LOG_D(fmt::format("Set partition boundaries requsts: #modify={}, #create{}", - modifyPartitions, - createPartitions - )); + YDB_LOG_DEBUG("Set partition boundaries requsts", + {"logPrefix", LogPrefix()}, + {"modify", modifyPartitions}, + {"create", createPartitions}); } return { .Requests = std::move(boundsToApply), @@ -221,21 +230,29 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu const ui32 partitionId = splitParameters.PartitionId; if (MirroredFromSomewhere) { if (!AppData()->FeatureFlags.GetEnableMirroredTopicSplitMerge()) { - PQ_LOG_D("split request for mirrored topic is disabled. Partition# " << partitionId); + YDB_LOG_DEBUG("Split request for mirrored topic is disabled", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = false}; } if (!splitParameters.PartitionScaleParticipants.Defined()) { - PQ_LOG_NOTICE("split request for mirrored topic doesn't have prescribed partition ids. Partition# " << partitionId); + YDB_LOG_NOTICE("Split request for mirrored topic doesn't have prescribed partition ids", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } } const auto* node = PartitionGraph.GetPartition(partitionId); if (node == nullptr) { if (splitParameters.PartitionScaleParticipants.Defined()) { - PQ_LOG_NOTICE("attempt to split partition that was not created yet. Partition# " << partitionId); + YDB_LOG_NOTICE("Attempt to split partition that was not created yet", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = false}; } else { - PQ_LOG_ERROR("partition not found. Partition# " << partitionId); + YDB_LOG_ERROR("Partition not found", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } } @@ -244,18 +261,25 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu auto to = node->To; auto mid = splitParameters.SplitBoundary.GetOrElse(MiddleOf(from, to)); if (mid.empty()) { - PQ_LOG_ERROR("wrong partition key range. Can't get mid. Partition# " << partitionId); + YDB_LOG_ERROR("Wrong partition key range. Can't get mid", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } if (splitParameters.PartitionScaleParticipants.Defined() && splitParameters.PartitionScaleParticipants->AdjacentPartitionIdsSize() != 0) { - PQ_LOG_ERROR("split request cannot have adjacent partitions. Partition# " << partitionId); + YDB_LOG_ERROR("Split request cannot have adjacent partitions", + {"logPrefix", LogPrefix()}, + {"partition", partitionId}); return {.Split = Nothing(), .Remove = true}; } - PQ_LOG_D("partition split ranges. From# '" << ToHex(from) - << "'. To# '" << ToHex(to) << "'. Mid# '" << ToHex(mid) - << "'. Partition# " << partitionId); + YDB_LOG_DEBUG("Partition split ranges", + {"logPrefix", LogPrefix()}, + {"fromHex", ToHex(from)}, + {"toHex", ToHex(to)}, + {"midHex", ToHex(mid)}, + {"partition", partitionId}); TPartitionSplit split; split.set_partition(partitionId); @@ -264,7 +288,10 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu for (const auto& childPartitionId : splitParameters.PartitionScaleParticipants->GetChildPartitionIds()) { split.add_childpartitionids(childPartitionId); if (const auto* childNode = PartitionGraph.GetPartition(childPartitionId); childNode != nullptr) { - PQ_LOG_NOTICE(fmt::format("Child partition# {} already exists. Performing unordered split. Partition# {}", childPartitionId, partitionId)); + YDB_LOG_NOTICE("Child partition already exists. Performing unordered split", + {"logPrefix", LogPrefix()}, + {"childPartition", childPartitionId}, + {"partition", partitionId}); split.set_createrootlevelsibling(true); } } @@ -276,7 +303,10 @@ TPartitionScaleManager::TBuildSplitScaleRequestResult TPartitionScaleManager::Bu const auto& prescribedChildrenIds = splitParameters.PartitionScaleParticipants->GetChildPartitionIds(); if (!std::ranges::is_permutation(nodeChildrenIds, prescribedChildrenIds)) { const std::string mappingStr = fmt::format("([{}]->[{}])", fmt::join(nodeChildrenIds, ","), fmt::join(prescribedChildrenIds, ",")); - PQ_LOG_ERROR("trying to split partition into different set of children partitions " << mappingStr << ". Partition# " << partitionId); + YDB_LOG_ERROR("Trying to split partition into different set of children partitions", + {"logPrefix", LogPrefix()}, + {"mappingStr", mappingStr}, + {"partition", partitionId}); } } return {.Split = Nothing(), .Remove = true}; @@ -287,7 +317,9 @@ void TPartitionScaleManager::HandleScaleRequestResult(TPartitionScaleRequest::TE RequestInflight = false; LastResponseTime = ctx.Now(); auto result = ev->Get(); - PQ_LOG_D("HandleScaleRequestResult scale request result: " << result->Status); + YDB_LOG_DEBUG("HandleScaleRequestResult scale request", + {"logPrefix", LogPrefix()}, + {"result", result->Status}); if (result->Status == TEvTxUserProxy::TResultStatus::ExecComplete) { RequestTimeout = TDuration::Zero(); Backoff.Reset(); @@ -312,24 +344,27 @@ void TPartitionScaleManager::UpdateMirrorRootPartitionsSet() { NMirror::TMirrorGraphComparisonResult cmp = NMirror::ComparePartitionGraphs(PartitionGraph, MirrorTopicDescription->GetPartitions()); if (!cmp.RootPartitionsMismatch.has_value()) { - PQ_LOG_D("Topic has all root partitions from the source topic"); + YDB_LOG_DEBUG("Topic has all root partitions from the source topic", + {"logPrefix", LogPrefix()}); RootPartitionsToCreate.reset(); MirrorTopicError.reset(); return; } auto& rootPartitionsMismatch = cmp.RootPartitionsMismatch.value(); if (rootPartitionsMismatch.Error.has_value()) { - std::string msg = TStringBuilder() << "Incompatable configuration of root partitions between source and target topics:" << rootPartitionsMismatch.Error.value(); - PQ_LOG_ERROR(msg); + YDB_LOG_ERROR("Incompatable configuration of root partitions between source and target topics", + {"logPrefix", LogPrefix()}, + {"topics", rootPartitionsMismatch.Error.value()}); RootPartitionsToCreate.reset(); MirrorTopicError = std::move(*rootPartitionsMismatch.Error); return; } const size_t existingPartitions = std::ranges::count(rootPartitionsMismatch.AlterRootPartitions, NMirror::EPartitionAction::Modify, &NMirror::TPartitionWithBounds::Action); const size_t newPartitions = std::ranges::count(rootPartitionsMismatch.AlterRootPartitions, NMirror::EPartitionAction::Create, &NMirror::TPartitionWithBounds::Action); - PQ_LOG_I(fmt::format("Topic has less root partitions than the mirror source. New configuration has {}+{} partitions.", - existingPartitions, - newPartitions)); + YDB_LOG_INFO("Topic has less root partitions than the mirror source", + {"logPrefix", LogPrefix()}, + {"existing", existingPartitions}, + {"new", newPartitions}); RootPartitionsToCreate = std::move(rootPartitionsMismatch.AlterRootPartitions); MirrorTopicError.reset(); @@ -341,7 +376,8 @@ std::expected<void, std::string> TPartitionScaleManager::HandleMirrorTopicDescri } else { auto& description = ev->Get()->Description; if (!description.has_value() || !description.value().IsSuccess()) { - PQ_LOG_W("Ignoring invalid mirror source description"); + YDB_LOG_WARN("Ignoring invalid mirror source description", + {"logPrefix", LogPrefix()}); return {}; } MirrorTopicDescription.emplace(std::move(description->GetTopicDescription())); diff --git a/ydb/core/persqueue/pqrb/partition_scale_request.cpp b/ydb/core/persqueue/pqrb/partition_scale_request.cpp index e09adfe7b0e..30d1298c230 100644 --- a/ydb/core/persqueue/pqrb/partition_scale_request.cpp +++ b/ydb/core/persqueue/pqrb/partition_scale_request.cpp @@ -3,6 +3,8 @@ #include <ydb/core/protos/schemeshard/operations.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + namespace NKikimr { namespace NPQ { @@ -76,7 +78,8 @@ void TPartitionScaleRequest::FillProposeRequest(TEvTxUserProxy::TEvProposeTransa } logMessage << "."; } - PQ_LOG_D( logMessage); + YDB_LOG_DEBUG(logMessage, + {"logPrefix", LogPrefix()}); for(const auto& merge: Merges) { auto* newMerge = groupDescription.AddMerge(); @@ -124,7 +127,9 @@ void TPartitionScaleRequest::Handle(TEvTxUserProxy::TEvProposeTransactionStatus: for (auto& issue : ev->Get()->Record.GetIssues()) { issues << issue.ShortDebugString() + ", "; } - PQ_LOG_ERROR("TPartitionScaleRequest SchemaShard error when trying to execute a split request: " << issues); + YDB_LOG_ERROR("TPartitionScaleRequest SchemaShard error when trying to execute a split", + {"logPrefix", LogPrefix()}, + {"request", issues}); Send(ParentActorId, scaleRequestResult.release()); Die(ctx); } else { diff --git a/ydb/core/persqueue/pqrb/read_balancer.cpp b/ydb/core/persqueue/pqrb/read_balancer.cpp index 08ad274f8de..5eafeb10d21 100644 --- a/ydb/core/persqueue/pqrb/read_balancer.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer.cpp @@ -16,6 +16,8 @@ #include <library/cpp/string_utils/base64/base64.h> #include <library/cpp/random_provider/random_provider.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + #define PQ_ENSURE(condition) AFL_ENSURE(condition)("tablet_id", TabletID())("path", Path)("topic", Topic) namespace NKikimr { @@ -142,7 +144,9 @@ void TPersQueueReadBalancer::InitDone(const TActorContext &ctx) { } return s; }; - PQ_LOG_D(getInitLog()); + YDB_LOG_DEBUG("BALANCER INIT DONE dump logPrefix, getInitLog", + {"logPrefix", LogPrefix()}, + {"getInitLog", getInitLog()}); for (auto &ev : UpdateEvents) { ctx.Send(ctx.SelfID, ev.Release()); @@ -161,7 +165,8 @@ void TPersQueueReadBalancer::InitDone(const TActorContext &ctx) { } void TPersQueueReadBalancer::HandleWakeup(TEvents::TEvWakeup::TPtr& ev, const TActorContext &ctx) { - PQ_LOG_D("TPersQueueReadBalancer::HandleWakeup"); + YDB_LOG_DEBUG("TPersQueueReadBalancer::HandleWakeup", + {"logPrefix", LogPrefix()}); switch (ev->Get()->Tag) { case TPartitionScaleManager::TRY_SCALE_REQUEST_WAKE_UP_TAG: { @@ -209,9 +214,13 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvUpdateBalancerConfig::TPtr if (!WaitingResponse.empty()) { //got transaction infly WaitingResponse.push_back(ev->Sender); } else { //version already applied - PQ_LOG_D("BALANCER Topic " << Topic << "Tablet " << TabletID() - << " Config already applied version " << record.GetVersion() << " actor " << ev->Sender - << " txId " << record.GetTxId()); + YDB_LOG_DEBUG("BALANCER Topic Tablet Config already applied version actor txId", + {"logPrefix", LogPrefix()}, + {"topic", Topic}, + {"tabletID", TabletID()}, + {"version", record.GetVersion()}, + {"sender", ev->Sender}, + {"txId", record.GetTxId()}); THolder<TEvPersQueue::TEvUpdateConfigResponse> res{new TEvPersQueue::TEvUpdateConfigResponse}; res->Record.SetStatus(NKikimrPQ::OK); res->Record.SetTxId(record.GetTxId()); @@ -374,7 +383,9 @@ TStringBuilder TPersQueueReadBalancer::LogPrefix() const { void TPersQueueReadBalancer::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { auto tabletId = ev->Get()->TabletId; - PQ_LOG_D("TEvClientDestroyed " << tabletId); + YDB_LOG_DEBUG("TEvClientDestroyed", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}); ClosePipe(tabletId, ctx); RequestTabletIfNeeded(tabletId, ctx, true); @@ -391,7 +402,10 @@ void TPersQueueReadBalancer::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, ClosePipe(ev->Get()->TabletId, ctx); RequestTabletIfNeeded(ev->Get()->TabletId, ctx, true); - PQ_LOG_ERROR("TEvClientConnected Status " << ev->Get()->Status << ", TabletId " << tabletId); + YDB_LOG_ERROR("TEvClientConnected Status TabletId", + {"logPrefix", LogPrefix()}, + {"status", ev->Get()->Status}, + {"tabletId", tabletId}); return; } @@ -402,10 +416,16 @@ void TPersQueueReadBalancer::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, it->second.Generation = ev->Get()->Generation; it->second.NodeId = ev->Get()->ServerId.NodeId(); - PQ_LOG_D("TEvClientConnected TabletId " << tabletId << ", NodeId " << ev->Get()->ServerId.NodeId() << ", Generation " << ev->Get()->Generation); + YDB_LOG_DEBUG("TEvClientConnected TabletId NodeId Generation", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}, + {"nodeId", ev->Get()->ServerId.NodeId()}, + {"generation", ev->Get()->Generation}); } else - PQ_LOG_I("TEvClientConnected Pipe is not found, TabletId " << tabletId); + YDB_LOG_INFO("TEvClientConnected Pipe is not found, TabletId", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}); } void TPersQueueReadBalancer::ClosePipe(const ui64 tabletId, const TActorContext& ctx) @@ -453,7 +473,10 @@ void TPersQueueReadBalancer::RequestTabletIfNeeded(const ui64 tabletId, const TA StatsRequestTracker.Cookies[tabletId] = cookie; } - PQ_LOG_D("Send TEvPersQueue::TEvStatus TabletId: " << tabletId << " Cookie: " << cookie); + YDB_LOG_DEBUG("Send TEvPersQueue::TEvStatus", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}, + {"cookie", cookie}); NTabletPipe::SendData(ctx, pipeClient, new TEvPersQueue::TEvStatus("", true), cookie); } } @@ -531,11 +554,13 @@ void TPersQueueReadBalancer::CheckStat(const TActorContext& ctx) { UpdateCounters(ctx); TEvPersQueue::TEvPeriodicTopicStats* ev = GetStatsEvent(); - PQ_LOG_D("Send TEvPeriodicTopicStats PathId: " << PathId - << " Generation: " << Generation - << " StatsReportRound: " << StatsReportRound - << " DataSize: " << TopicMetricsHandler->GetTopicMetrics().TotalDataSize - << " UsedReserveSize: " << TopicMetricsHandler->GetTopicMetrics().TotalUsedReserveSize); + YDB_LOG_DEBUG("Send TEvPeriodicTopicStats", + {"logPrefix", LogPrefix()}, + {"pathId", PathId}, + {"generation", Generation}, + {"statsReportRound", StatsReportRound}, + {"dataSize", TopicMetricsHandler->GetTopicMetrics().TotalDataSize}, + {"usedReserveSize", TopicMetricsHandler->GetTopicMetrics().TotalUsedReserveSize}); NTabletPipe::SendData(ctx, GetPipeClient(SchemeShardId, ctx), ev); @@ -622,8 +647,12 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvGetPartitionsLocation::TPtr pResponse->SetNodeId(iter->second.NodeId.GetRef()); pResponse->SetGeneration(iter->second.Generation.GetRef()); - PQ_LOG_D("The partition location was added to response: TabletId " << tabletId << ", PartitionId " << partitionId - << ", NodeId " << pResponse->GetNodeId() << ", Generation " << pResponse->GetGeneration()); + YDB_LOG_DEBUG("The partition location was added to response: TabletId PartitionId NodeId Generation", + {"logPrefix", LogPrefix()}, + {"tabletId", tabletId}, + {"partitionId", partitionId}, + {"pResponseNodeId", pResponse->GetNodeId()}, + {"pResponseGeneration", pResponse->GetGeneration()}); return true; }; @@ -713,7 +742,10 @@ void TPersQueueReadBalancer::Handle(NSchemeShard::TEvSchemeShard::TEvSubDomainPa if (SchemeShardId == msg->SchemeShardId && (!SubDomainPathId || SubDomainPathId->OwnerId != msg->SchemeShardId)) { - PQ_LOG_D("Discovered subdomain " << msg->LocalPathId << " at RB " << TabletID()); + YDB_LOG_DEBUG("Discovered subdomain at RB", + {"logPrefix", LogPrefix()}, + {"localPathId", msg->LocalPathId}, + {"tabletID", TabletID()}); SubDomainPathId.emplace(msg->SchemeShardId, msg->LocalPathId); Execute(new TTxWriteSubDomainPathId(this), ctx); @@ -767,8 +799,11 @@ void TPersQueueReadBalancer::Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated .GetDomainState() .GetDiskQuotaExceeded(); - PQ_LOG_D("Discovered subdomain " << msg->PathId << " state, outOfSpace = " << outOfSpace - << " at RB " << TabletID()); + YDB_LOG_DEBUG("Discovered subdomain state, at RB", + {"logPrefix", LogPrefix()}, + {"pathId", msg->PathId}, + {"outOfSpace", outOfSpace}, + {"tabletID", TabletID()}); SubDomainOutOfSpace = outOfSpace; @@ -847,7 +882,9 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvGetReadSessionsInfo::TPtr& void TPersQueueReadBalancer::Handle(TEvPQ::TEvMLPConsumerStatus::TPtr& ev, const TActorContext& ctx) { Y_UNUSED(ctx); - PQ_LOG_D("Handle TEvPQ::TEvMLPConsumerStatus " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvMLPConsumerStatus", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->Record.ShortDebugString()}); MLPBalancer->Handle(ev); } @@ -874,13 +911,16 @@ void TPersQueueReadBalancer::Handle(TEvPersQueue::TEvBalancingUnsubscribe::TPtr& void TPersQueueReadBalancer::Handle(TEvPQ::TEvPartitionScaleStatusChanged::TPtr& ev, const TActorContext& ctx) { if (!SplitMergeEnabled(TabletConfig)) { - PQ_LOG_D("Skip TEvPartitionScaleStatusChanged: autopartitioning disabled."); + YDB_LOG_DEBUG("Skip TEvPartitionScaleStatusChanged: autopartitioning disabled", + {"logPrefix", LogPrefix()}); return; } auto& record = ev->Get()->Record; auto* node = PartitionGraph.GetPartition(record.GetPartitionId()); if (!node) { - PQ_LOG_D("Skip TEvPartitionScaleStatusChanged: partition " << record.GetPartitionId() << " not found."); + YDB_LOG_DEBUG("Skip TEvPartitionScaleStatusChanged: partition not found", + {"logPrefix", LogPrefix()}, + {"partitionId", record.GetPartitionId()}); return; } @@ -893,7 +933,8 @@ void TPersQueueReadBalancer::Handle(TEvPQ::TEvPartitionScaleStatusChanged::TPtr& ctx ); } else { - PQ_LOG_NOTICE("Skip TEvPartitionScaleStatusChanged: scale manager isn`t initialized."); + YDB_LOG_NOTICE("Skip TEvPartitionScaleStatusChanged: scale manager isn`t initialized", + {"logPrefix", LogPrefix()}); } } @@ -907,7 +948,8 @@ void TPersQueueReadBalancer::Handle(TPartitionScaleRequest::TEvPartitionScaleReq } void TPersQueueReadBalancer::Handle(TEvPQ::TEvMirrorTopicDescription::TPtr& ev, const TActorContext& ctx) { - PQ_LOG_D("Received TEvMirrorTopicDescription"); + YDB_LOG_DEBUG("Received TEvMirrorTopicDescription", + {"logPrefix", LogPrefix()}); if (!MirroringEnabled(TabletConfig)) { return; } @@ -928,13 +970,17 @@ void TPersQueueReadBalancer::BroadcastPartitionError(const TString& message, con } void TPersQueueReadBalancer::Handle(TEvPQ::TEvMLPGetPartitionRequest::TPtr& ev) { - PQ_LOG_D("Handle TEvPQ::TEvMLPGetPartitionRequest: " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvMLPGetPartitionRequest", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->Record.ShortDebugString()}); PendingMLPGetPartitionRequests.push_back(std::move(ev)); ProcessMLPGetPartitionRequests(ActorContext()); } void TPersQueueReadBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { - PQ_LOG_D("Handle TEvPQ::TEvMLPGetRuntimeAttributesRequest: " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle", + {"logPrefix", LogPrefix()}, + {"mlpGetRuntimeAttributesRequest", ev->Get()->Record.ShortDebugString()}); if (StatsRequestTracker.StatsReceived) { return MLPBalancer->Handle(ev); } diff --git a/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp b/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp index 98ba0856039..b91d5afe224 100644 --- a/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer__balancing.cpp @@ -3,8 +3,7 @@ #include <ydb/core/persqueue/public/utils.h> -#define DEBUG(message) - +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER namespace NKikimr::NPQ::NBalancing { @@ -164,17 +163,23 @@ TString TPartitionFamily::LogPrefix() const { void TPartitionFamily::Release(const TActorContext& ctx, ETargetStatus targetStatus) { if (Status != EStatus::Active) { - PQ_LOG_CRIT("releasing the family " << DebugStr() << " that isn't active"); + YDB_LOG_CRIT("Releasing the family that isn't active", + {"logPrefix", LogPrefix()}, + {"debugStr", DebugStr()}); return; } if (!Session) { - PQ_LOG_CRIT("releasing the family " << DebugStr() << " that does not have a session"); + YDB_LOG_CRIT("Releasing the family that does not have a session", + {"logPrefix", LogPrefix()}, + {"debugStr", DebugStr()}); return; } - PQ_LOG_I(" release partitions [" << JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end()) - << "]. Target status " << targetStatus); + YDB_LOG_INFO("Release partitions. Target status", + {"logPrefix", LogPrefix()}, + {"lockedPartitions", JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end())}, + {"targetStatus", targetStatus}); Status = EStatus::Releasing; TargetStatus = targetStatus; @@ -193,25 +198,35 @@ void TPartitionFamily::Release(const TActorContext& ctx, ETargetStatus targetSta bool TPartitionFamily::Unlock(const TActorId& sender, ui32 partitionId, const TActorContext& ctx) { if (!Session || Session->Pipe != sender) { - PQ_LOG_D("try unlock the partition " << partitionId << " from other sender"); + YDB_LOG_DEBUG("Try unlock the partition from other sender", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); return false; } if (Status != EStatus::Releasing) { - PQ_LOG_CRIT("try unlock partition " << partitionId << " but family status is " << Status); + YDB_LOG_CRIT("Try unlock partition but family status is", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"status", Status}); return false; } if (!LockedPartitions.erase(partitionId)) { - PQ_LOG_CRIT("try unlock partition " << partitionId << " but partition isn't locked." - << " Locked partitions are [" << JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end()) << "]"); + YDB_LOG_CRIT("Try unlock partition but partition isn't locked. Locked partitions are", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"lockedPartitions", JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end())}); return false; } --Session->ReleasingPartitionCount; if (!LockedPartitions.empty()) { - PQ_LOG_D("partition " << partitionId << " was unlocked but wait else [" << JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end()) << "]"); + YDB_LOG_DEBUG("Partition was unlocked, but wait", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"lockedPartitions", JoinRange(", ", LockedPartitions.begin(), LockedPartitions.end())}); return false; } @@ -238,7 +253,8 @@ bool TPartitionFamily::Reset(ETargetStatus targetStatus, const TActorContext& ct return false; case ETargetStatus::Free: - PQ_LOG_T(" is free."); + YDB_LOG_TRACE("Is free", + {"logPrefix", LogPrefix()}); Status = EStatus::Free; AfterRelease(); @@ -251,7 +267,8 @@ bool TPartitionFamily::Reset(ETargetStatus targetStatus, const TActorContext& ct auto it = Consumer.Families.find(MergeTo); if (it == Consumer.Families.end()) { - PQ_LOG_D(" has been released for merge but target family is not exists."); + YDB_LOG_DEBUG("Has been released for merge but target family is not exists", + {"logPrefix", LogPrefix()}); return true; } auto* targetFamily = it->second.get(); @@ -266,7 +283,8 @@ bool TPartitionFamily::Reset(ETargetStatus targetStatus, const TActorContext& ct } void TPartitionFamily::Destroy(const TActorContext&) { - PQ_LOG_D(" destroyed."); + YDB_LOG_DEBUG("Destroyed", + {"logPrefix", LogPrefix()}); if (Session) { Session->Families.erase(Id); @@ -301,11 +319,14 @@ void TPartitionFamily::AfterRelease() { void TPartitionFamily::StartReading(TSession& session, const TActorContext& ctx) { if (Status != EStatus::Free) { - PQ_LOG_CRIT("try start reading but the family status is " << Status); + YDB_LOG_CRIT("Try start reading but the family status is", + {"logPrefix", LogPrefix()}, + {"status", Status}); return; } - PQ_LOG_T("start reading"); + YDB_LOG_TRACE("Start reading", + {"logPrefix", LogPrefix()}); Status = EStatus::Active; @@ -327,7 +348,9 @@ void TPartitionFamily::StartReading(TSession& session, const TActorContext& ctx) } void TPartitionFamily::AttachePartitions(const std::vector<ui32>& partitions, const TActorContext& ctx) { - PQ_LOG_D("attaching partitions [" << JoinRange(", ", partitions.begin(), partitions.end()) << "]"); + YDB_LOG_DEBUG("Attaching partitions", + {"logPrefix", LogPrefix()}, + {"partitions", JoinRange(", ", partitions.begin(), partitions.end())}); std::unordered_set<ui32> existedPartitions; existedPartitions.insert(Partitions.begin(), Partitions.end()); @@ -376,13 +399,17 @@ void TPartitionFamily::AttachePartitions(const std::vector<ui32>& partitions, co } void TPartitionFamily::ActivatePartition(ui32 partitionId) { - PQ_LOG_D("activating partition " << partitionId); + YDB_LOG_DEBUG("Activating partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); ChangePartitionCounters(1, -1); } void TPartitionFamily::InactivatePartition(ui32 partitionId) { - PQ_LOG_D("inactivating partition " << partitionId); + YDB_LOG_DEBUG("Inactivating partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); ChangePartitionCounters(-1, 1); } @@ -401,7 +428,9 @@ void TPartitionFamily::InactivatePartition(ui32 partitionId) { } void TPartitionFamily::Merge(TPartitionFamily* other) { - PQ_LOG_D("merge family with " << other->DebugStr()); + YDB_LOG_DEBUG("Merge family with", + {"logPrefix", LogPrefix()}, + {"debug", other->DebugStr()}); Y_VERIFY(this != other); @@ -545,8 +574,12 @@ void TPartitionFamily::UpdateSpecialSessions() { void TPartitionFamily::LockPartition(ui32 partitionId, const TActorContext& ctx) { auto step = NextStep(); - PQ_LOG_I("lock partition " << partitionId << " for " << Session->DebugStr() - << " generation " << TabletGeneration() << " step " << step); + YDB_LOG_INFO("Lock partition for generation step", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"debug", Session->DebugStr()}, + {"tabletGeneration", TabletGeneration()}, + {"step", step}); ctx.Send(Session->Sender, MakeEvLockPartition(partitionId, step).release()); } @@ -636,7 +669,9 @@ ui32 TConsumer::NextStep() { void TConsumer::RegisterPartition(ui32 partitionId, const TActorContext& ctx) { auto [_, inserted] = Partitions.try_emplace(partitionId, TPartition()); if (inserted && IsReadable(partitionId)) { - PQ_LOG_D("register readable partition " << partitionId); + YDB_LOG_DEBUG("Register readable partition", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); CreateFamily({partitionId}, ctx); } @@ -666,7 +701,9 @@ TPartitionFamily* TConsumer::CreateFamily(std::vector<ui32>&& partitions, TParti UnreadableFamilies[id] = family; } - PQ_LOG_D("family created " << family->DebugStr()); + YDB_LOG_DEBUG("Family created", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); return family; } @@ -706,7 +743,10 @@ bool TConsumer::BreakUpFamily(TPartitionFamily* family, ui32 partitionId, bool d std::vector<TPartitionFamily*> newFamilies; if (!family->IsLonely()) { - PQ_LOG_D("break up " << family->DebugStr() << " partition=" << partitionId); + YDB_LOG_DEBUG("Break up", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}, + {"partition", partitionId}); std::unordered_set<ui32> partitions; partitions.insert(family->Partitions.begin(), family->Partitions.end()); @@ -786,7 +826,10 @@ bool TConsumer::BreakUpFamily(TPartitionFamily* family, ui32 partitionId, bool d } } } else { - PQ_LOG_D("can't break up " << family->DebugStr() << " because partition=" << partitionId << " is not root of family"); + YDB_LOG_DEBUG("Can't break up because is not root of family", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}, + {"partition", partitionId}); } } @@ -868,7 +911,9 @@ TPartitionFamily* TConsumer::FindFamily(ui32 partitionId) { } void TConsumer::RegisterReadingSession(TSession* session, const TActorContext& ctx) { - PQ_LOG_I("register reading session " << session->DebugStr()); + YDB_LOG_INFO("Register reading session", + {"logPrefix", LogPrefix()}, + {"debug", session->DebugStr()}); Sessions[session->Pipe] = session; @@ -952,7 +997,9 @@ void TConsumer::UnregisterReadingSession(TSession* session, const TActorContext& bool TConsumer::Unlock(const TActorId& sender, ui32 partitionId, const TActorContext& ctx) { auto* family = FindFamily(partitionId); if (!family) { - PQ_LOG_CRIT("unlocking the partition " << partitionId << " from unknown family."); + YDB_LOG_CRIT("Unlocking the partition from unknown family", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}); return false; } @@ -1034,7 +1081,10 @@ bool TConsumer::ProccessReadingFinished(ui32 partitionId, bool wasInactive, cons }); if (partition.NeedReleaseChildren()) { - PQ_LOG_D("Attache partitions [" << JoinRange(", ", newPartitions.begin(), newPartitions.end()) << "] to " << family->DebugStr()); + YDB_LOG_DEBUG("Attache partitions", + {"logPrefix", LogPrefix()}, + {"newPartitions", JoinRange(", ", newPartitions.begin(), newPartitions.end())}, + {"family", family->DebugStr()}); for (auto id : newPartitions) { if (family->CanAttach(std::vector{id})) { auto* node = GetPartitionGraph().GetPartition(id); @@ -1066,7 +1116,10 @@ bool TConsumer::ProccessReadingFinished(ui32 partitionId, bool wasInactive, cons } } } else { - PQ_LOG_D("Can't attache partition " << id << " to " << family->DebugStr()); + YDB_LOG_DEBUG("Can't attache partition", + {"logPrefix", LogPrefix()}, + {"id", id}, + {"family", family->DebugStr()}); } } } else { @@ -1083,19 +1136,28 @@ bool TConsumer::ProccessReadingFinished(ui32 partitionId, bool wasInactive, cons void TConsumer::StartReading(ui32 partitionId, const TActorContext& ctx) { if (!GetPartitionInfo(partitionId)) { - PQ_LOG_NOTICE("Reading of the partition " << partitionId << " was started by " << ConsumerName << " but partition has been deleted."); + YDB_LOG_NOTICE("Reading of the partition was started by but partition has been deleted", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } auto* partition = GetPartition(partitionId); if (!partition) { - PQ_LOG_NOTICE("Reading of the partition " << partitionId << " was started by " << ConsumerName << " but partition does not exist."); + YDB_LOG_NOTICE("Reading of the partition was started by but partition does not exist", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } auto wasInactive = partition->IsInactive(); if (partition->StartReading()) { - PQ_LOG_D("Reading of the partition " << partitionId << " was started by " << ConsumerName << ". We stop reading from child partitions."); + YDB_LOG_DEBUG("Reading of the partition was started by We stop reading from child partitions", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); auto* family = FindFamily(partitionId); if (!family) { @@ -1137,21 +1199,27 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: auto partitionId = r.GetPartitionId(); if (!IsReadable(partitionId)) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << ConsumerName - << " but the partition isn't readable"); + YDB_LOG_DEBUG("Reading of the partition was finished by but the partition isn't readable", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } auto* family = FindFamily(partitionId); if (!family) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << ConsumerName - << " but the partition hasn't family"); + YDB_LOG_DEBUG("Reading of the partition was finished by but the partition hasn't family", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } if (!family->Session) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << ConsumerName - << " but the partition hasn't reading session"); + YDB_LOG_DEBUG("Reading of the partition was finished by but the partition hasn't reading session", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", ConsumerName}); return; } @@ -1159,8 +1227,12 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: const bool wasInactive = partition.IsInactive(); if (partition.SetFinishedState(r.GetScaleAwareSDK(), r.GetStartedReadingFromEndOffset()) || wasInactive) { - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << r.GetConsumer() - << ", firstMessage=" << r.GetStartedReadingFromEndOffset() << ", " << GetSdkDebugString0(r.GetScaleAwareSDK())); + YDB_LOG_DEBUG("Reading of the partition was finished by", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumer", r.GetConsumer()}, + {"firstMessage", r.GetStartedReadingFromEndOffset()}, + {"scaleAwareSdk", GetSdkDebugString0(r.GetScaleAwareSDK())}); if (ProccessReadingFinished(partitionId, wasInactive, ctx)) { ScheduleBalance(ctx); @@ -1168,9 +1240,13 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: } else if (!partition.IsInactive()) { auto delay = std::min<size_t>(1ul << partition.Iteration, Balancer.GetLifetimeSeconds()); // TODO use split/merge time - PQ_LOG_D("Reading of the partition " << partitionId << " was finished by " << r.GetConsumer() - << ". Scheduled release of the partition for re-reading. Delay=" << delay << " seconds," - << " firstMessage=" << r.GetStartedReadingFromEndOffset() << ", " << GetSdkDebugString0(r.GetScaleAwareSDK())); + YDB_LOG_DEBUG("Reading of the partition was finished by Scheduled release of the partition for re-reading. seconds", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumer", r.GetConsumer()}, + {"delay", delay}, + {"firstMessage", r.GetStartedReadingFromEndOffset()}, + {"scaleAwareSdk", GetSdkDebugString0(r.GetScaleAwareSDK())}); ctx.Schedule(TDuration::Seconds(delay), new TEvPQ::TEvWakeupReleasePartition(ConsumerName, partitionId, partition.Cookie)); } @@ -1178,13 +1254,15 @@ void TConsumer::FinishReading(TEvPersQueue::TEvReadingPartitionFinishedRequest:: void TConsumer::ScheduleBalance(const TActorContext& ctx) { if (BalanceScheduled) { - PQ_LOG_T("rebalancing already was scheduled"); + YDB_LOG_TRACE("Rebalancing already was scheduled", + {"logPrefix", LogPrefix()}); return; } BalanceScheduled = true; - PQ_LOG_D("rebalancing was scheduled"); + YDB_LOG_DEBUG("Rebalancing was scheduled", + {"logPrefix", LogPrefix()}); ctx.Send(Balancer.TopicActor.SelfId(), new TEvPQ::TEvBalanceConsumer(ConsumerName)); } @@ -1254,9 +1332,14 @@ size_t GetMaxFamilySize(const std::unordered_map<size_t, const std::unique_ptr<T } void TConsumer::Balance(const TActorContext& ctx) { - PQ_LOG_D("balancing. Sessions=" << Sessions.size() << ", Families=" << Families.size() - << ", UnreadableFamilies=" << UnreadableFamilies.size() << " [" << DebugStr(UnreadableFamilies) - << "], RequireBalancing=" << FamiliesRequireBalancing.size() << " [" << DebugStr(FamiliesRequireBalancing) << "]"); + YDB_LOG_DEBUG("Balancing", + {"logPrefix", LogPrefix()}, + {"sessions", Sessions.size()}, + {"families", Families.size()}, + {"unreadableFamilies", UnreadableFamilies.size()}, + {"unreadableFamiliesDebug", DebugStr(UnreadableFamilies)}, + {"requireBalancing", FamiliesRequireBalancing.size()}, + {"familiesRequireBalancingDebug", DebugStr(FamiliesRequireBalancing)}); if (Sessions.empty()) { return; @@ -1270,7 +1353,9 @@ void TConsumer::Balance(const TActorContext& ctx) { continue; } if (!family->SpecialSessions.contains(family->Session->Pipe)) { - PQ_LOG_D("rebalance " << family->DebugStr() << " because exists the special session for it"); + YDB_LOG_DEBUG("Rebalance because exists the special session for it", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); family->Release(ctx); } } @@ -1293,7 +1378,9 @@ void TConsumer::Balance(const TActorContext& ctx) { } if (sit == sessions.end()) { - PQ_LOG_D("balancing of the " << family->DebugStr() << " failed because there are no suitable reading sessions."); + YDB_LOG_DEBUG("Balancing of the failed because there are no suitable reading sessions", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); continue; } @@ -1303,7 +1390,10 @@ void TConsumer::Balance(const TActorContext& ctx) { // Reorder sessions sessions.erase(sit); - PQ_LOG_D("balancing " << family->DebugStr() << " for " << session->DebugStr()); + YDB_LOG_DEBUG("Balancing", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}, + {"debug", session->DebugStr()}); family->StartReading(*session, ctx); // Reorder sessions @@ -1322,8 +1412,12 @@ void TConsumer::Balance(const TActorContext& ctx) { auto desiredFamilyCount = familyCount / commonSessions.size(); auto allowPlusOne = familyCount % commonSessions.size(); - PQ_LOG_D("start rebalancing. familyCount=" << familyCount << ", sessionCount=" << commonSessions.size() - << ", desiredFamilyCount=" << desiredFamilyCount << ", allowPlusOne=" << allowPlusOne); + YDB_LOG_DEBUG("Start rebalancing", + {"logPrefix", LogPrefix()}, + {"familyCount", familyCount}, + {"sessionCount", commonSessions.size()}, + {"desiredFamilyCount", desiredFamilyCount}, + {"allowPlusOne", allowPlusOne}); if (!OrderedSessions) { OrderedSessions.emplace(); @@ -1352,7 +1446,9 @@ void TConsumer::Balance(const TActorContext& ctx) { auto* family = it->second; if (!family->IsActive()) { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because it is not active."); + YDB_LOG_DEBUG("Skip balancing because it is not active", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); it = FamiliesRequireBalancing.erase(it); continue; @@ -1365,14 +1461,18 @@ void TConsumer::Balance(const TActorContext& ctx) { } if (family->Session->ActiveFamilyCount == 1) { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because it is considered a session that does not read anything else."); + YDB_LOG_DEBUG("Skip balancing because it is considered a session that does not read anything else", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); it = FamiliesRequireBalancing.erase(it); continue; } if (family->SpecialSessions.size() <= 1) { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because there are no other suitable reading sessions."); + YDB_LOG_DEBUG("Skip balancing because there are no other suitable reading sessions", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); it = FamiliesRequireBalancing.erase(it); continue; @@ -1394,14 +1494,18 @@ void TConsumer::Balance(const TActorContext& ctx) { family->Release(ctx); it = FamiliesRequireBalancing.erase(it); } else { - PQ_LOG_D("skip balancing " << family->DebugStr() << " because it is already being read by the best session."); + YDB_LOG_DEBUG("Skip balancing because it is already being read by the best session", + {"logPrefix", LogPrefix()}, + {"family", family->DebugStr()}); ++it; } } } auto duration = TInstant::Now() - startTime; - PQ_LOG_D("balancing duration: " << duration); + YDB_LOG_DEBUG("Balancing", + {"logPrefix", LogPrefix()}, + {"duration", duration}); } void TConsumer::Release(ui32 partitionId, const TActorContext& ctx) { @@ -1517,8 +1621,10 @@ const std::unordered_map<TActorId, std::unique_ptr<TSession>>& TBalancer::GetSes void TBalancer::UpdateConfig(const std::vector<ui32>& addedPartitions, const std::vector<ui32>& deletedPartitions, const TActorContext& ctx) { - PQ_LOG_D("updating configuration. Deleted partitions [" << JoinRange(", ", deletedPartitions.begin(), deletedPartitions.end()) - << "]. Added partitions [" << JoinRange(", ", addedPartitions.begin(), addedPartitions.end()) << "]"); + YDB_LOG_DEBUG("Updating configuration. Deleted partitions Added partitions", + {"logPrefix", LogPrefix()}, + {"deletedPartitions", JoinRange(", ", deletedPartitions.begin(), deletedPartitions.end())}, + {"addedPartitions", JoinRange(", ", addedPartitions.begin(), addedPartitions.end())}); for (auto partitionId : deletedPartitions) { for (auto& [_, consumer] : Consumers) { @@ -1544,14 +1650,19 @@ bool TBalancer::SetCommittedState(const TString& consumerName, ui32 partitionId, } if (!consumer->IsReadable(partitionId)) { - PQ_LOG_D("The offset of the partition " << partitionId << " was commited by " << consumerName - << " but the partition isn't readable"); + YDB_LOG_DEBUG("The offset of the partition was commited by but the partition isn't readable", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", consumerName}); return false; } auto wasInactive = consumer->IsInactive(partitionId); if (consumer->SetCommittedState(partitionId, generation, cookie)) { - PQ_LOG_D("The offset of the partition " << partitionId << " was commited by " << consumerName); + YDB_LOG_DEBUG("The offset of the partition was commited by", + {"logPrefix", LogPrefix()}, + {"partitionId", partitionId}, + {"consumerName", consumerName}); if (consumer->ProccessReadingFinished(partitionId, wasInactive, ctx)) { consumer->ScheduleBalance(ctx); @@ -1575,13 +1686,17 @@ void TBalancer::Handle(TEvPersQueue::TEvReadingPartitionStartedRequest::TPtr& ev auto pipeClient = ActorIdFromProto(r.GetPipeClient()); if (pipeClient && !Sessions.contains(pipeClient)) { - PQ_LOG_D("Received TEvReadingPartitionStartedRequest from unknown pipe " << pipeClient); + YDB_LOG_DEBUG("Received TEvReadingPartitionStartedRequest from unknown pipe", + {"logPrefix", LogPrefix()}, + {"pipeClient", pipeClient}); return; } auto consumer = GetConsumer(r.GetConsumer()); if (!consumer) { - PQ_LOG_D("Received TEvReadingPartitionStartedRequest from unknown consumer " << r.GetConsumer()); + YDB_LOG_DEBUG("Received TEvReadingPartitionStartedRequest from unknown consumer", + {"logPrefix", LogPrefix()}, + {"consumer", r.GetConsumer()}); return; } @@ -1593,13 +1708,17 @@ void TBalancer::Handle(TEvPersQueue::TEvReadingPartitionFinishedRequest::TPtr& e auto pipeClient = ActorIdFromProto(r.GetPipeClient()); if (pipeClient && !Sessions.contains(pipeClient)) { - PQ_LOG_D("Received TEvReadingPartitionFinishedRequest from unknown pipe " << pipeClient); + YDB_LOG_DEBUG("Received TEvReadingPartitionFinishedRequest from unknown pipe", + {"logPrefix", LogPrefix()}, + {"pipeClient", pipeClient}); return; } auto consumer = GetConsumer(r.GetConsumer()); if (!consumer) { - PQ_LOG_D("Received TEvReadingPartitionFinishedRequest from unknown consumer " << r.GetConsumer()); + YDB_LOG_DEBUG("Received TEvReadingPartitionFinishedRequest from unknown consumer", + {"logPrefix", LogPrefix()}, + {"consumer", r.GetConsumer()}); return; } @@ -1614,17 +1733,28 @@ void TBalancer::Handle(TEvPersQueue::TEvPartitionReleased::TPtr& ev, const TActo auto* partitionInfo = GetPartitionInfo(partitionId); if (!partitionInfo) { - PQ_LOG_CRIT("client " << r.GetClientId() << " pipe " << sender << " got deleted partition " << r); + YDB_LOG_CRIT("Client pipe got deleted partition", + {"logPrefix", LogPrefix()}, + {"clientId", r.GetClientId()}, + {"sender", sender}, + {"r", r}); return; } - PQ_LOG_I("client " << r.GetClientId() << " released partition from pipe " << sender - << " session " << r.GetSession() << " partition " << partitionId); + YDB_LOG_INFO("Client released partition from pipe session partition", + {"logPrefix", LogPrefix()}, + {"clientId", r.GetClientId()}, + {"sender", sender}, + {"session", r.GetSession()}, + {"partitionId", partitionId}); auto* consumer = GetConsumer(consumerName); if (!consumer) { - PQ_LOG_CRIT("client " << r.GetClientId() << " pipe " << sender - << " is not connected and got release partitions request for session " << r.GetSession()); + YDB_LOG_CRIT("Client pipe is not connected and got release partitions request for session", + {"logPrefix", LogPrefix()}, + {"clientId", r.GetClientId()}, + {"sender", sender}, + {"session", r.GetSession()}); return; } @@ -1646,11 +1776,17 @@ void TBalancer::Handle(TEvPQ::TEvWakeupReleasePartition::TPtr &ev, const TActorC } if (partition->Commited) { - PQ_LOG_D("skip releasing partition " << msg->PartitionId << " of consumer \"" << msg->Consumer << "\" by reading finished timeout because offset is commited"); + YDB_LOG_DEBUG("Skip releasing partition of consumer by reading finished timeout because offset is commited", + {"logPrefix", LogPrefix()}, + {"partitionId", msg->PartitionId}, + {"consumer", msg->Consumer}); return; } - PQ_LOG_I("releasing partition " << msg->PartitionId << " of consumer \"" << msg->Consumer << "\" by reading finished timeout"); + YDB_LOG_INFO("Releasing partition of consumer by reading finished timeout", + {"logPrefix", LogPrefix()}, + {"partitionId", msg->PartitionId}, + {"consumer", msg->Consumer}); consumer->Release(msg->PartitionId, ctx); } @@ -1666,22 +1802,31 @@ void TBalancer::Handle(TEvTabletPipe::TEvServerConnected::TPtr& ev, const TActor auto& session = it->second; ++session->ServerActors; - PQ_LOG_I("pipe " << sender << " connected; active server actors: " << session->ServerActors); + YDB_LOG_INFO("Pipe connected; active server", + {"logPrefix", LogPrefix()}, + {"sender", sender}, + {"actors", session->ServerActors}); } void TBalancer::Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev, const TActorContext& ctx) { - PQ_LOG_D("pipe " << ev->Get()->ClientId << " disconnected."); + YDB_LOG_DEBUG("Pipe disconnected", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}); Subscriptions.erase(ev->Get()->ClientId); auto it = Sessions.find(ev->Get()->ClientId); if (it == Sessions.end()) { - PQ_LOG_D("pipe " << ev->Get()->ClientId << " disconnected but there aren't sessions exists."); + YDB_LOG_DEBUG("Pipe disconnected but there aren't sessions exists", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}); return; } - PQ_LOG_I("pipe " << ev->Get()->ClientId << " disconnected; active server actors: " - << (it != Sessions.end() ? it->second->ServerActors : -1)); + YDB_LOG_INFO("Pipe disconnected; active server", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}, + {"actors", (it != Sessions.end() ? it->second->ServerActors : -1)}); auto& session = it->second; if (--(session->ServerActors) > 0) { @@ -1689,8 +1834,11 @@ void TBalancer::Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev, const TAc } if (!session->SessionName.empty()) { - PQ_LOG_NOTICE("pipe " << ev->Get()->ClientId << " client " - << session->ClientId << " disconnected session " << session->SessionName); + YDB_LOG_NOTICE("Pipe client disconnected session", + {"logPrefix", LogPrefix()}, + {"eventClientId", ev->Get()->ClientId}, + {"sessionClientId", session->ClientId}, + {"sessionName", session->SessionName}); auto* consumer = GetConsumer(session->ClientId); if (consumer) { @@ -1706,7 +1854,9 @@ void TBalancer::Handle(TEvTabletPipe::TEvServerDisconnected::TPtr& ev, const TAc Sessions.erase(it); } else { - PQ_LOG_I("pipe " << ev->Get()->ClientId << " disconnected no session"); + YDB_LOG_INFO("Pipe disconnected no session", + {"logPrefix", LogPrefix()}, + {"clientId", ev->Get()->ClientId}); Sessions.erase(it); } @@ -1717,27 +1867,37 @@ void TBalancer::Handle(TEvPersQueue::TEvRegisterReadSession::TPtr& ev, const TAc auto& consumerName = r.GetClientId(); TActorId pipe = ActorIdFromProto(r.GetPipeClient()); - PQ_LOG_NOTICE("consumer \"" << consumerName << "\" register session for pipe " << pipe << " session " << r.GetSession()); + YDB_LOG_NOTICE("Consumer register session for pipe session", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}, + {"pipe", pipe}, + {"session", r.GetSession()}); if (consumerName.empty()) { - PQ_LOG_CRIT("ignored the session registration with empty consumer name."); + YDB_LOG_CRIT("Ignored the session registration with empty consumer name", + {"logPrefix", LogPrefix()}); return; } if (r.GetSession().empty()) { - PQ_LOG_CRIT("ignored the session registration with empty session name."); + YDB_LOG_CRIT("Ignored the session registration with empty session name", + {"logPrefix", LogPrefix()}); return; } if (!pipe) { - PQ_LOG_CRIT("ignored the session registration with empty Pipe."); + YDB_LOG_CRIT("Ignored the session registration with empty Pipe", + {"logPrefix", LogPrefix()}); return; } auto jt = Sessions.find(pipe); if (jt == Sessions.end()) { - PQ_LOG_CRIT("client \"" << consumerName << "\" pipe " << pipe - << " is not connected and got register session request for session " << r.GetSession()); + YDB_LOG_CRIT("Client pipe is not connected and got register session request for session", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}, + {"pipe", pipe}, + {"session", r.GetSession()}); return; } @@ -1852,7 +2012,9 @@ void TBalancer::Handle(TEvPersQueue::TEvStatusResponse::TPtr& ev, const TActorCo } void TBalancer::ProcessPendingStats(const TActorContext& ctx) { - PQ_LOG_D("ProcessPendingStats. PendingUpdates size " << PendingUpdates.size()); + YDB_LOG_DEBUG("ProcessPendingStats. PendingUpdates size", + {"logPrefix", LogPrefix()}, + {"pendingUpdatesSize", PendingUpdates.size()}); GetPartitionGraph().Travers([&](ui32 id) { for (auto& d : PendingUpdates[id]) { @@ -1868,7 +2030,9 @@ void TBalancer::ProcessPendingStats(const TActorContext& ctx) { void TBalancer::Handle(TEvPersQueue::TEvBalancingSubscribe::TPtr& ev, const TActorContext& ctx) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPersQueue::TEvBalancingSubscribe " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPersQueue::TEvBalancingSubscribe", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); auto sender = ActorIdFromProto(record.GetSourceActor()); auto status = Consumers.contains(record.GetConsumer()) ? @@ -1880,7 +2044,9 @@ void TBalancer::Handle(TEvPersQueue::TEvBalancingSubscribe::TPtr& ev, const TAct void TBalancer::Handle(TEvPersQueue::TEvBalancingUnsubscribe::TPtr& ev, const TActorContext&) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPersQueue::TEvBalancingUnsubscribe " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPersQueue::TEvBalancingUnsubscribe", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); auto sender = ActorIdFromProto(record.GetSourceActor()); auto& consumer = record.GetConsumer(); diff --git a/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp b/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp index d62bd55c870..370de6c38d0 100644 --- a/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer__balancing_app.cpp @@ -3,9 +3,6 @@ #include <library/cpp/monlib/service/pages/templates.h> #include <ydb/core/persqueue/common/common_app.h> -#define DEBUG(message) - - namespace NKikimr::NPQ::NBalancing { void TBalancer::RenderApp(NApp::TNavigationBar& __navigationBar) const { diff --git a/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp b/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp index 61030f78738..2fdf743951d 100644 --- a/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp +++ b/ydb/core/persqueue/pqrb/read_balancer__mlp_balancing.cpp @@ -3,6 +3,8 @@ #include <util/generic/ymath.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PERSQUEUE_READ_BALANCER + namespace NKikimr::NPQ::NBalancing { namespace { @@ -192,7 +194,9 @@ void TMLPConsumer::Rebuild() { } } - PQ_LOG_D("Rebuild " << JoinSeq(",", PartitionsForBalancing) << " partitions for balancing"); + YDB_LOG_DEBUG("Rebuild partitions for balancing", + {"logPrefix", LogPrefix()}, + {"partitionsForBalancing", JoinSeq(",", PartitionsForBalancing)}); } const TMLPConsumer::TMetrics& TMLPConsumer::GetMetrics() const { @@ -212,7 +216,9 @@ TPrepareGetPartitionResponse TMLPBalancer::PrepareGetPartitionResponse( auto* consumerConfig = NPQ::GetConsumer(GetConfig(), consumerName); if (!consumerConfig) { - PQ_LOG_D("Consumer '" << consumerName << "' does not exist"); + YDB_LOG_DEBUG("Consumer does not exist", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); result.IsError = true; result.ErrorStatus = Ydb::StatusIds::SCHEME_ERROR; result.ErrorMessage = TStringBuilder() << "Consumer '" << consumerName << "' does not exist"; @@ -220,7 +226,9 @@ TPrepareGetPartitionResponse TMLPBalancer::PrepareGetPartitionResponse( } if (consumerConfig->GetType() != NKikimrPQ::TPQTabletConfig::CONSUMER_TYPE_MLP) { - PQ_LOG_D("Consumer '" << consumerName << "' is not MLP consumer"); + YDB_LOG_DEBUG("Consumer is not MLP consumer", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); result.IsError = true; result.ErrorStatus = Ydb::StatusIds::SCHEME_ERROR; result.ErrorMessage = TStringBuilder() << "Consumer '" << consumerName << "' is not MLP consumer"; @@ -270,14 +278,18 @@ void TMLPBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { const auto* consumerConfig = NPQ::GetConsumer(GetConfig(), consumerName); if (!consumerConfig) { - PQ_LOG_D("Consumer '" << consumerName << "' does not exist"); + YDB_LOG_DEBUG("Consumer does not exist", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); TopicActor.Send(ev->Sender, new TEvPQ::TEvMLPErrorResponse(Ydb::StatusIds::SCHEME_ERROR, TStringBuilder() << "Consumer '" << consumerName << "' does not exist"), 0, ev->Cookie); return; } if (consumerConfig->GetType() != NKikimrPQ::TPQTabletConfig::CONSUMER_TYPE_MLP) { - PQ_LOG_D("Consumer '" << consumerName << "' is not MLP consumer"); + YDB_LOG_DEBUG("Consumer is not MLP consumer", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); TopicActor.Send(ev->Sender, new TEvPQ::TEvMLPErrorResponse(Ydb::StatusIds::SCHEME_ERROR, TStringBuilder() << "Consumer '" << consumerName << "' is not MLP consumer"), 0, ev->Cookie); return; @@ -285,7 +297,9 @@ void TMLPBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { auto it = Consumers.find(consumerName); if (it == Consumers.end()) { - PQ_LOG_D("Consumer '" << consumerName << "' is not initialized"); + YDB_LOG_DEBUG("Consumer is not initialized", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); TopicActor.Send(ev->Sender, new TEvPQ::TEvMLPGetRuntimeAttributesResponse(0, 0, 0), 0, ev->Cookie); return; } @@ -302,7 +316,9 @@ void TMLPBalancer::Handle(TEvPQ::TEvMLPGetRuntimeAttributesRequest::TPtr& ev) { } void TMLPBalancer::Handle(TEvPersQueue::TEvStatusResponse::TPtr& ev, const TActorContext&) { - PQ_LOG_D("Handle TEvPersQueue::TEvStatusResponse " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPersQueue::TEvStatusResponse", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->Record.ShortDebugString()}); absl::flat_hash_map<TString, bool> mlpConsumers; for (const auto& consumer : GetConfig().GetConsumers()) { @@ -347,7 +363,9 @@ void TMLPBalancer::Handle(TEvPersQueue::TEvStatusResponse::TPtr& ev, const TActo void TMLPBalancer::Handle(TEvPQ::TEvReadingPartitionStatusRequest::TPtr& ev, const TActorContext&) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPQ::TEvReadingPartitionStatusRequest " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvReadingPartitionStatusRequest", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); SetUseForReading(record.GetConsumer(), record.GetPartitionId(), true, // reading is finished @@ -363,7 +381,9 @@ void TMLPBalancer::Handle(TEvPQ::TEvReadingPartitionStatusRequest::TPtr& ev, con void TMLPBalancer::Handle(TEvPQ::TEvMLPConsumerStatus::TPtr& ev) { auto& record = ev->Get()->Record; - PQ_LOG_D("Handle TEvPQ::TEvMLPConsumerStatus " << record.ShortDebugString()); + YDB_LOG_DEBUG("Handle TEvPQ::TEvMLPConsumerStatus", + {"logPrefix", LogPrefix()}, + {"ev", record.ShortDebugString()}); SetUseForReading(record.GetConsumer(), record.GetPartitionId(), std::nullopt, // reading is finished @@ -418,12 +438,16 @@ void TMLPBalancer::SetUseForReading(const TString& consumerName, ui64 cookie) { auto* consumerConfig = NPQ::GetConsumer(GetConfig(), consumerName); if (!consumerConfig) { - PQ_LOG_D("Consumer '" << consumerName << "' does not exist"); + YDB_LOG_DEBUG("Consumer does not exist", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); return; } if (consumerConfig->GetType() != NKikimrPQ::TPQTabletConfig::CONSUMER_TYPE_MLP) { - PQ_LOG_D("Consumer '" << consumerName << "' is not MLP consumer"); + YDB_LOG_DEBUG("Consumer is not MLP consumer", + {"logPrefix", LogPrefix()}, + {"consumerName", consumerName}); return; } diff --git a/ydb/core/persqueue/pqrb/read_balancer_log.h b/ydb/core/persqueue/pqrb/read_balancer_log.h index 96890d1d810..934eb09106e 100644 --- a/ydb/core/persqueue/pqrb/read_balancer_log.h +++ b/ydb/core/persqueue/pqrb/read_balancer_log.h @@ -7,12 +7,5 @@ namespace NKikimr::NPQ { inline TString LogPrefix() { return {}; } -#define PQ_LOG_T(stream) LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_D(stream) LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_I(stream) LOG_INFO_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_W(stream) LOG_WARN_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_NOTICE(stream) LOG_NOTICE_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_ERROR(stream) LOG_ERROR_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) -#define PQ_LOG_CRIT(stream) LOG_CRIT_S(*NActors::TlsActivationContext, NKikimrServices::PERSQUEUE_READ_BALANCER, LogPrefix() << stream) } // namespace NKikimr::NPQ diff --git a/ydb/core/persqueue/public/schema/schema_operation.cpp b/ydb/core/persqueue/public/schema/schema_operation.cpp index 0f98bb13e42..99ab505f3cf 100644 --- a/ydb/core/persqueue/public/schema/schema_operation.cpp +++ b/ydb/core/persqueue/public/schema/schema_operation.cpp @@ -177,10 +177,9 @@ private: } void ReplyErrorAndDie(Ydb::StatusIds::StatusCode errorCode, TString&& errorMessage) { - YDB_LOG_DEBUG("Dump NPQLOGPREFIX, replyErrorAndDie, errorMessage", + YDB_LOG_DEBUG(errorMessage, {"logPrefix", NPQ_LOG_PREFIX}, - {"replyErrorAndDie", errorCode}, - {"errorMessage", errorMessage}); + {"replyErrorAndDie", errorCode}); Send(ParentId, new TEvSchemaOperationResponse(errorCode, std::move(errorMessage)), 0, Cookie); PassAway(); } diff --git a/ydb/core/persqueue/writer/writer.cpp b/ydb/core/persqueue/writer/writer.cpp index f1ca11529d9..9a68f23d15a 100644 --- a/ydb/core/persqueue/writer/writer.cpp +++ b/ydb/core/persqueue/writer/writer.cpp @@ -318,7 +318,7 @@ class TPartitionWriter : public TActorBootstrapped<TPartitionWriter>, public TPa WriteId = NPQ::GetWriteId(record.GetResponse().GetTopicOperations()); - YDB_LOG_DEBUG("Dump LOGPREFIX, sessionId, txId, writeId", + YDB_LOG_DEBUG("", {"logPrefix", LOG_PREFIX}, {"sessionId", Opts.SessionId}, {"txId", Opts.TxId}, diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto index a83698da532..d3e00489294 100644 --- a/ydb/core/protos/blobstorage.proto +++ b/ydb/core/protos/blobstorage.proto @@ -1059,12 +1059,20 @@ message TNodeWardenServiceSet { optional uint64 TotalResponseBytesPerSecond = 6; // quota for generated responses at network layer } + message TFailureInjectionConfig { + optional double FailureProbability = 1; // positive values enable failure injection + optional bool IncludeStaticGroups = 2; // also inject failures in request to static groups + optional uint64 RandomSeed = 3; + optional NKikimrProto.EReplyStatus ErrorReplyStatus = 4 [default = ERROR]; + } + repeated TPDisk PDisks = 1; repeated TVDisk VDisks = 2; repeated NKikimrBlobStorage.TGroupInfo Groups = 3; repeated uint64 AvailabilityDomains = 4; optional TReplBrokerConfig ReplBrokerConfig = 5; optional bool EnableProxyMock = 7; + optional TFailureInjectionConfig FailureInjectionConfig = 8; } message TNodeWardenCache { diff --git a/ydb/core/protos/blobstorage_ddisk.proto b/ydb/core/protos/blobstorage_ddisk.proto index d5db486bc4c..b453c39f312 100644 --- a/ydb/core/protos/blobstorage_ddisk.proto +++ b/ydb/core/protos/blobstorage_ddisk.proto @@ -267,8 +267,11 @@ message TEvReadPersistentBufferResult { uint64 VChunkIndex = 4; uint32 OffsetInBytes = 5; uint32 SizeInBytes = 6; - // Reserved for checksum piggybacking: same raw XXH3_64(data) per MinSectorSize block of ReadResult - // payload as TEvWritePersistentBuffer.Checksums, no identity/salt mixed in; not filled yet. + // Same raw XXH3_64(data) per MinSectorSize block of ReadResult payload as + // TEvWritePersistentBuffer.Checksums, no identity/salt mixed in. Opt-in: populated only when the + // persisted record itself carries sender-supplied checksums (empty for legacy/checksum-less + // records). Forwarded verbatim by TWritePersistentBuffersRequestActor when re-replicating a + // record via TEvReadThenWritePersistentBuffers. repeated fixed64 Checksums = 7; } diff --git a/ydb/core/protos/blobstorage_distributed_config.proto b/ydb/core/protos/blobstorage_distributed_config.proto index e051fcb28b5..f0411a11f4d 100644 --- a/ydb/core/protos/blobstorage_distributed_config.proto +++ b/ydb/core/protos/blobstorage_distributed_config.proto @@ -198,6 +198,7 @@ message TEvNodeConfigInvokeOnRoot { bool IgnoreVSlotQuotaCheck = 6; bool IsSelfHealReasonDecommit = 7; bool FromSelfHeal = 8; + bool AllowUnusableDisks = 9; } // Regenerate configuration so the slain VDisk is no more reported as DESTROY one in the list. diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 01c75df650f..5c4c5453267 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -393,13 +393,25 @@ message TSelfManagementConfig { optional bool AutomaticBoxManagement = 21 [default = true]; // invoke BSC DefineHostConfig/DefineBox automatically optional bool AutomaticBootstrap = 22; // whether bootstrap should be performed automatically; PROHIBITED for production optional bool AutomaticStaticGroupManagement = 23; // whether distconf/SelfHeal can change static group on its behalf - optional bool AutomaticStateStorageManagement = 24; // the same for state storage - optional bool AutomaticStateStorageBoardManagement = 25; // the same for state storage board - optional bool AutomaticSchemeBoardManagement = 26; // the same for scheme board + optional bool AutomaticStateStorageManagement = 24 [default = true]; // the same for state storage; if disabled, SelfHeal must not change current state storage config + optional bool AutomaticStateStorageBoardManagement = 25 [default = true]; // the same for state storage board; if disabled, SelfHeal must not change current state storage board config + optional bool AutomaticSchemeBoardManagement = 26 [default = true]; // the same for scheme board; if disabled, SelfHeal must not change current scheme board config // node ids onto which vdisks of static groups may be relocated during self-heal; // empty list means no restriction repeated uint32 StaticGroupSelfHealAllowedNodes = 27; + + // node ids onto which state storage may be relocated during self-heal; + // empty list means no restriction + repeated uint32 StateStorageSelfHealAllowedNodes = 28; + + // node ids onto which state storage board may be relocated during self-heal; + // empty list means no restriction + repeated uint32 StateStorageBoardSelfHealAllowedNodes = 29; + + // node ids onto which state storage scheme board may be relocated during self-heal; + // empty list means no restriction + repeated uint32 SchemeBoardSelfHealAllowedNodes = 30; } message TBlobStorageFormatConfig { @@ -3105,6 +3117,7 @@ message TKafkaProxyConfig { optional bool EnableKafkaProxy = 1 [default = false]; optional int32 ListeningPort = 2 [default = 9092]; + optional string ListeningAddress = 20 [default = "::"]; // You can either specify the path to the certificate, which will contain both the certificate and the private key, // or specify them separately in parameters Cert and Key. diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index c05170d7d96..4eeb18b8486 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -353,4 +353,6 @@ message TFeatureFlags { // RequireRestart: the in-memory counter is rebuilt from all FileStores at SchemeShard start, // the counter can be re-enabled, which requires restarting the SchemeShard tablet. optional bool DisableFileStoreSSDSystemSpaceAccounting = 302 [default = false, (RequireRestart) = true]; + optional bool EnableHasPredicatesInResourcePoolClassifiers = 303 [default = false]; + optional bool EnableRejectActionInResourcePoolClassifiers = 304 [default = false]; } diff --git a/ydb/core/resource_pools/resource_pool_classifier_settings.cpp b/ydb/core/resource_pools/resource_pool_classifier_settings.cpp index c6d7cc211d7..aad6cde67ef 100644 --- a/ydb/core/resource_pools/resource_pool_classifier_settings.cpp +++ b/ydb/core/resource_pools/resource_pool_classifier_settings.cpp @@ -22,6 +22,14 @@ void TClassifierSettings::TParser::operator()(TString* setting) const { *setting = Value; } +void TClassifierSettings::TParser::operator()(std::optional<bool>* setting) const { + if (Value.empty()) { + setting->reset(); + } else { + *setting = FromString<bool>(Value); + } +} + void TClassifierSettings::TParser::operator()(std::optional<TString>* setting) const { if (Value.empty()) { setting->reset(); @@ -60,6 +68,13 @@ TString TClassifierSettings::TExtractor::operator()(TString* setting) const { return *setting; } +TString TClassifierSettings::TExtractor::operator()(std::optional<bool>* setting) const { + if (!*setting) { + return TString{}; + } + return **setting ? "true" : "false"; +} + TString TClassifierSettings::TExtractor::operator()(std::optional<TString>* setting) const { return setting->value_or(TString{}); } @@ -88,6 +103,7 @@ std::unordered_map<TString, TClassifierSettings::TProperty> TClassifierSettings: {"has_app_name", &HasAppName}, {"has_full_scan", &HasFullScan}, {"has_path", &HasPath}, + {"has_stream", &HasStream}, {"action", &Action} }; return properties; diff --git a/ydb/core/resource_pools/resource_pool_classifier_settings.h b/ydb/core/resource_pools/resource_pool_classifier_settings.h index 71eff8f0ccf..88d77b28911 100644 --- a/ydb/core/resource_pools/resource_pool_classifier_settings.h +++ b/ydb/core/resource_pools/resource_pool_classifier_settings.h @@ -17,11 +17,12 @@ enum class EClassifierAction { struct TClassifierSettings : public TSettingsBase { using TBase = TSettingsBase; - using TProperty = std::variant<i64*, TString*, std::optional<TString>*, std::optional<TRegexPredicate>*, std::optional<EClassifierAction>*>; + using TProperty = std::variant<i64*, TString*, std::optional<bool>*, std::optional<TString>*, std::optional<TRegexPredicate>*, std::optional<EClassifierAction>*>; struct TParser : public TBase::TParser { void operator()(i64* setting) const; void operator()(TString* setting) const; + void operator()(std::optional<bool>* setting) const; void operator()(std::optional<TString>* setting) const; void operator()(std::optional<TRegexPredicate>* setting) const; void operator()(std::optional<EClassifierAction>* setting) const; @@ -30,6 +31,7 @@ struct TClassifierSettings : public TSettingsBase { struct TExtractor : public TBase::TExtractor { TString operator()(i64* setting) const; TString operator()(TString* setting) const; + TString operator()(std::optional<bool>* setting) const; TString operator()(std::optional<TString>* setting) const; TString operator()(std::optional<TRegexPredicate>* setting) const; TString operator()(std::optional<EClassifierAction>* setting) const; @@ -46,6 +48,7 @@ struct TClassifierSettings : public TSettingsBase { std::optional<TString> HasAppName; std::optional<TRegexPredicate> HasFullScan; std::optional<TRegexPredicate> HasPath; + std::optional<bool> HasStream; std::optional<EClassifierAction> Action; }; diff --git a/ydb/core/resource_pools/resource_pool_classifier_settings_ut.cpp b/ydb/core/resource_pools/resource_pool_classifier_settings_ut.cpp index debd32c057f..ffd4dde207d 100644 --- a/ydb/core/resource_pools/resource_pool_classifier_settings_ut.cpp +++ b/ydb/core/resource_pools/resource_pool_classifier_settings_ut.cpp @@ -46,6 +46,7 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifierTest) { std::visit(TClassifierSettings::TParser{"my_app"}, propertiesMap["has_app_name"]); std::visit(TClassifierSettings::TParser{"/Root/db/orders_*"}, propertiesMap["has_full_scan"]); std::visit(TClassifierSettings::TParser{"/Root/db/archive/*"}, propertiesMap["has_path"]); + std::visit(TClassifierSettings::TParser{"true"}, propertiesMap["has_stream"]); UNIT_ASSERT(settings.HasAppName.has_value()); UNIT_ASSERT_VALUES_EQUAL(*settings.HasAppName, "my_app"); @@ -55,6 +56,9 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifierTest) { UNIT_ASSERT(settings.HasPath.has_value()); UNIT_ASSERT_VALUES_EQUAL(settings.HasPath->Pattern, "/Root/db/archive/*"); + + UNIT_ASSERT(settings.HasStream.has_value()); + UNIT_ASSERT_VALUES_EQUAL(settings.HasStream.value(), true); } Y_UNIT_TEST(SettingsExtracting) { @@ -78,11 +82,13 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifierTest) { std::visit(TClassifierSettings::TParser{"ydb-cli"}, propertiesMap["has_app_name"]); std::visit(TClassifierSettings::TParser{"/Root/db/orders_*"}, propertiesMap["has_full_scan"]); std::visit(TClassifierSettings::TParser{"/Root/db/archive/*"}, propertiesMap["has_path"]); + std::visit(TClassifierSettings::TParser{"false"}, propertiesMap["has_stream"]); TClassifierSettings::TExtractor extractor; UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_app_name"]), "ydb-cli"); UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_full_scan"]), "/Root/db/orders_*"); UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_path"]), "/Root/db/archive/*"); + UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_stream"]), "false"); } Y_UNIT_TEST(OptionalStringEmptyResets) { @@ -111,6 +117,7 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifierTest) { UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_app_name"]), ""); UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_full_scan"]), ""); UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_path"]), ""); + UNIT_ASSERT_VALUES_EQUAL(std::visit(extractor, propertiesMap["has_stream"]), ""); } Y_UNIT_TEST(ActionParsing) { diff --git a/ydb/core/sys_view/common/registry.h b/ydb/core/sys_view/common/registry.h index e74e88c1c20..06442dc8e7b 100644 --- a/ydb/core/sys_view/common/registry.h +++ b/ydb/core/sys_view/common/registry.h @@ -779,6 +779,7 @@ struct Schema : NIceDb::Schema { struct Action : Column<7, NScheme::NTypeIds::Utf8> {}; struct HasFullScan : Column<8, NScheme::NTypeIds::Utf8> {}; struct HasPath : Column<9, NScheme::NTypeIds::Utf8> {}; + struct HasStream : Column<10, NScheme::NTypeIds::Bool> {}; using TKey = TableKey<Name>; using TColumns = TableColumns< @@ -789,7 +790,8 @@ struct Schema : NIceDb::Schema { HasAppName, Action, HasFullScan, - HasPath>; + HasPath, + HasStream>; }; struct ShowCreate : Table<21> { diff --git a/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp b/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp index 53ddd92b2a4..f9e2c703a68 100644 --- a/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp +++ b/ydb/core/sys_view/resource_pool_classifiers/resource_pool_classifiers.cpp @@ -115,6 +115,9 @@ private: const auto& hasPath = config.GetConfigJson()["has_path"].GetString(); return TCell(hasPath.data(), hasPath.size()); }}); + insert({TSchema::HasStream::ColumnId, [] (const NKqp::TResourcePoolClassifierConfig& config) { + return TCell::Make<bool>(config.GetConfigJson()["has_stream"].GetBoolean()); + }}); } }; static TExtractorsMap extractors; diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index dc2bb371c58..ca57c2a7c37 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -1622,6 +1622,7 @@ namespace Tests { NKafka::TListenerSettings settings; settings.Port = Settings->AppConfig->GetKafkaProxyConfig().GetListeningPort(); + settings.Address = Settings->AppConfig->GetKafkaProxyConfig().GetListeningAddress(); bool ssl = false; if (Settings->AppConfig->GetKafkaProxyConfig().HasSslCertificate()) { ssl = true; diff --git a/ydb/core/tx/columnshard/backup/iscan/ut/ut_iscan.cpp b/ydb/core/tx/columnshard/backup/iscan/ut/ut_iscan.cpp index e9f51e5b10d..76de9107fc2 100644 --- a/ydb/core/tx/columnshard/backup/iscan/ut/ut_iscan.cpp +++ b/ydb/core/tx/columnshard/backup/iscan/ut/ut_iscan.cpp @@ -90,12 +90,15 @@ class TGrabActor: public TActorBootstrapped<TGrabActor> { std::deque<TAutoPtr<IEventHandle>> Inputs; TMutex Mutex; std::unique_ptr<NColumnShard::NBackup::TExportDriver> Driver; - std::unique_ptr<NTable::IScan> Exporter; + + // non-owning pointer to the exporter actor + NTable::IScan* Exporter = nullptr; public: - TRuntimePtr Runtime; + // Non-owning pointer to the actor runtime + TTestActorRuntime* Runtime; - TGrabActor(TRuntimePtr runtime) + TGrabActor(TTestActorRuntime* runtime) : Runtime(runtime) { } @@ -107,12 +110,14 @@ public: auto exportFactory = std::make_shared<TDataShardExportFactory>(); - Exporter = + std::unique_ptr<NTable::IScan> exporter = NColumnShard::NBackup::CreateIScanExportUploader(SelfId(), MakeBackupTask("test"), exportFactory.get(), columns, 0).DetachResult(); - UNIT_ASSERT(Exporter); + UNIT_ASSERT(exporter); + Exporter = exporter.get(); Driver = std::make_unique<NColumnShard::NBackup::TExportDriver>(TActorContext::ActorSystem(), SelfId()); auto initialState = Exporter->Prepare(Driver.get(), MakeSchema()); UNIT_ASSERT_VALUES_EQUAL(initialState.Scan, NTable::EScan::Feed); + Y_UNUSED(exporter.release()); NTable::TLead lead; auto seekState = Exporter->Seek(lead, 0); @@ -193,7 +198,7 @@ Y_UNIT_TEST_SUITE(IScan) { runtime->SetLogPriority(NKikimrServices::DATASHARD_BACKUP, NActors::NLog::PRI_DEBUG); SetupTabletServices(*runtime); - auto grabActor = new TGrabActor(runtime); + auto grabActor = new TGrabActor(runtime.get()); runtime->Register(grabActor); while (true) { diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.cpp b/ydb/core/tx/columnshard/engines/changes/compaction.cpp index 1ea9b91d814..e31512c860d 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction.cpp @@ -36,6 +36,7 @@ void TCompactColumnEngineChanges::DoStart(NColumnShard::TColumnShard& self) { self.BackgroundController.StartCompaction(GranuleMeta->GetPathId(), GetTaskIdentifier()); NeedGranuleStatusProvide = true; GranuleMeta->OnCompactionStarted(); + PortionsIndexSnapshot = GranuleMeta->GetPortionsIndex().GetPortionsSnapshot(); } void TCompactColumnEngineChanges::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) { diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.h b/ydb/core/tx/columnshard/engines/changes/compaction.h index a1ec7967289..66a009b93b4 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction.h @@ -3,6 +3,7 @@ #include <ydb/core/tx/columnshard/common/path_id.h> #include <ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.h> #include <ydb/core/tx/columnshard/engines/changes/with_appended.h> +#include <ydb/core/tx/columnshard/engines/storage/granule/portions_index.h> namespace NKikimr::NOlap { @@ -16,6 +17,7 @@ private: protected: std::vector<TPortionInfo::TConstPtr> SwitchedPortions; // Portions that would be replaced by new ones std::shared_ptr<TGranuleMeta> GranuleMeta; + NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot PortionsIndexSnapshot; virtual void DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) override; diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.cpp index d5a0f48761e..cf56253be88 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.cpp @@ -13,7 +13,7 @@ namespace NKikimr::NOlap::NCompaction::NSubColumns { std::shared_ptr<NArrow::NAccessor::IChunkedArray> TMergedBuilder::MaybeDictionaryEncode( const std::shared_ptr<NArrow::NAccessor::IChunkedArray>& accessor, const ui32 filledRecordsCount, const EValueType valueType) const { - if (!NArrow::NAccessor::NSubColumns::DictionaryApplicableForValueType(valueType)) { + if (!NArrow::NAccessor::NSubColumns::CanBeDictionaryEncoded(valueType)) { return accessor; } const auto enumerateNotNull = [&accessor](const auto& consumer) { @@ -85,12 +85,11 @@ void TMergedBuilder::Initialize() { switch (ResultColumnStats.GetAccessorType(i)) { case NArrow::NAccessor::IChunkedArray::EType::Array: ColumnBuilders.emplace_back( - TPlainRuntimeBuilder(NArrow::NAccessor::NSubColumns::GetArrowTypeForValueType(valueType)), valueType); + TEncodingPlainBuilder(NArrow::NAccessor::NSubColumns::GetCodecForValueType(valueType), 0, 0), valueType); break; case NArrow::NAccessor::IChunkedArray::EType::SparsedArray: - // Native scalars are never sparsed, so a sparsed column is always binary-backed. - AFL_VERIFY(valueType == EValueType::BinaryJson || valueType == EValueType::String)("value_type", (ui32)valueType); - ColumnBuilders.emplace_back(TSparsedBuilder(nullptr, 0, 0), valueType); + ColumnBuilders.emplace_back( + TEncodingSparsedBuilder(NArrow::NAccessor::NSubColumns::GetCodecForValueType(valueType), 0, 0), valueType); break; case NArrow::NAccessor::IChunkedArray::EType::Undefined: case NArrow::NAccessor::IChunkedArray::EType::SerializedChunkedArray: diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.h b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.h index 46730ba517c..ba053489d46 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/builder.h @@ -1,9 +1,8 @@ #pragma once #include "remap.h" -#include <ydb/core/formats/arrow/accessor/plain/accessor.h> -#include <ydb/core/formats/arrow/accessor/sparsed/accessor.h> #include <ydb/core/formats/arrow/accessor/sub_columns/accessor.h> +#include <ydb/core/formats/arrow/accessor/sub_columns/encoding_builders.h> #include <ydb/core/formats/arrow/accessor/sub_columns/settings.h> #include <ydb/core/tx/columnshard/engines/changes/compaction/abstract/merger.h> #include <ydb/core/tx/columnshard/engines/storage/chunks/column.h> @@ -12,7 +11,8 @@ namespace NKikimr::NOlap::NCompaction::NSubColumns { class TMergedBuilder { private: - using TSparsedBuilder = NArrow::NAccessor::TSparsedArray::TSparsedBuilder<arrow::BinaryType>; + using TEncodingSparsedBuilder = NArrow::NAccessor::NSubColumns::TEncodingSparsedBuilder; + using TEncodingPlainBuilder = NArrow::NAccessor::NSubColumns::TEncodingPlainBuilder; using TColumnsData = NArrow::NAccessor::NSubColumns::TColumnsData; using TOthersData = NArrow::NAccessor::NSubColumns::TOthersData; using TSettings = NArrow::NAccessor::NSubColumns::TSettings; @@ -29,106 +29,39 @@ private: const TSettings Settings; const TRemapColumns& Remapper; - // A plain array builder whose element type is chosen at construction type. - // Commonality with TPlainBuilder to be refactored. - class TPlainRuntimeBuilder { - private: - std::unique_ptr<arrow::ArrayBuilder> Builder; - std::optional<ui32> LastRecordIndex; - - void AppendGapNulls(const ui32 recordIndex) { - if (LastRecordIndex) { - AFL_VERIFY(*LastRecordIndex < recordIndex)("last", LastRecordIndex)("index", recordIndex); - NArrow::TStatusValidator::Validate(Builder->AppendNulls(recordIndex - *LastRecordIndex - 1)); - } else { - NArrow::TStatusValidator::Validate(Builder->AppendNulls(recordIndex)); - } - LastRecordIndex = recordIndex; - } - - public: - TPlainRuntimeBuilder(const std::shared_ptr<arrow::DataType>& type) - : Builder(NArrow::MakeBuilder(type)) - { - } - - void AddNativeValue(const ui32 recordIndex, const arrow::Array& array, const i64 position) { - AFL_VERIFY(Builder->type()->id() == array.type_id())("builder", Builder->type()->ToString())("array", array.type()->ToString()); - AppendGapNulls(recordIndex); - AFL_VERIFY(NArrow::Append(*Builder, array, position)); - } - - void AddBinaryValue(const ui32 recordIndex, const TStringBuf value) { - AFL_VERIFY(Builder->type()->id() == arrow::Type::BINARY)("type", Builder->type()->ToString()); - AppendGapNulls(recordIndex); - AFL_VERIFY(NArrow::Append<arrow::BinaryType>(*Builder, arrow::util::string_view(value.data(), value.size()))); - } - - std::shared_ptr<NArrow::NAccessor::IChunkedArray> Finish(const ui32 recordsCount) { - if (LastRecordIndex) { - AFL_VERIFY(*LastRecordIndex < recordsCount)("last", LastRecordIndex)("count", recordsCount); - NArrow::TStatusValidator::Validate(Builder->AppendNulls(recordsCount - *LastRecordIndex - 1)); - } else { - NArrow::TStatusValidator::Validate(Builder->AppendNulls(recordsCount)); - } - return std::make_shared<NArrow::NAccessor::TTrivialArray>(NArrow::FinishBuilder(std::move(Builder))); - } - }; - class TGeneralAccessorBuilder { private: - std::variant<TSparsedBuilder, TPlainRuntimeBuilder> Builder; + std::variant<TEncodingSparsedBuilder, TEncodingPlainBuilder> Builder; const EValueType TargetValueType; YDB_READONLY(ui32, FilledRecordsCount, 0); YDB_READONLY(ui64, FilledRecordsSize, 0); public: - TGeneralAccessorBuilder(TSparsedBuilder&& builder, const EValueType targetValueType) + TGeneralAccessorBuilder(TEncodingSparsedBuilder&& builder, const EValueType targetValueType) : Builder(std::move(builder)) , TargetValueType(targetValueType) { } - TGeneralAccessorBuilder(TPlainRuntimeBuilder&& builder, const EValueType targetValueType) + TGeneralAccessorBuilder(TEncodingPlainBuilder&& builder, const EValueType targetValueType) : Builder(std::move(builder)) , TargetValueType(targetValueType) { } - // Returns the number of value bytes actually stored, since a re-encoded value's stored size differs from its size in the source. + // Returns the number of value bytes actually stored, since a re-encoded value's stored size differs from its + // size in the source. Both builders share the same interface, so one generic visitor covers both. ui32 AddRecord(const ui32 recordIndex, const TGeneralIterator& it) { const bool passthrough = it.GetValueType() == TargetValueType; - - struct TVisitor { - const ui32 RecordIndex; - const TGeneralIterator& It; - const bool Passthrough; - - ui32 operator()(TSparsedBuilder& builder) const { - // Sparsed columns are always binary-backed, so the passthrough value is its storage bytes. - if (Passthrough) { - builder.AddRecord(RecordIndex, It.GetStorageView()); - return It.GetValueSize(); - } else { - const auto bj = It.GetValueAsBinaryJson(); - builder.AddRecord(RecordIndex, TStringBuf(bj.data(), bj.size())); - return bj.size(); - } - } - - ui32 operator()(TPlainRuntimeBuilder& builder) const { - if (Passthrough) { - builder.AddNativeValue(RecordIndex, It.GetArray(), It.GetLocalIndex()); - return It.GetValueSize(); - } else { - const auto bj = It.GetValueAsBinaryJson(); - builder.AddBinaryValue(RecordIndex, TStringBuf(bj.data(), bj.size())); - return bj.size(); - } + const ui32 storedSize = std::visit([&](auto& builder) -> ui32 { + if (passthrough) { + builder.AddArrayElement(recordIndex, it.GetArray(), it.GetLocalIndex()); + return it.GetValueSize(); } - }; - - const ui32 storedSize = std::visit(TVisitor{ recordIndex, it, passthrough }, Builder); + const auto bj = it.GetValueAsBinaryJson(); + builder.AddFromBinaryJson(recordIndex, bj); + return bj.size(); + }, Builder); ++FilledRecordsCount; FilledRecordsSize += storedSize; return storedSize; diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp index 60aabe3784b..372845d292a 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp @@ -27,12 +27,8 @@ void TSubColumnsMerger::DoStart(const std::vector<std::shared_ptr<NArrow::NAcces AFL_VERIFY(stats.size()); AFL_VERIFY(statRecordsCount); auto commonStats = TDictStats::Merge(stats, GetSettings(), statRecordsCount); - auto splitted = commonStats.SplitByVolume(GetSettings(), statRecordsCount); - ResultColumnStats = splitted.ExtractColumns(); + ResultColumnStats = commonStats.SelectSeparatedColumns(GetSettings(), statRecordsCount); ResultColumnStats->CreateJsonPathAccessorTrieCache(); - // YDB_LOG_ERROR("", - // {"columns", ResultColumnStats->DebugJson()}, - // {"others", splitted.ExtractOthers().DebugJson()}); RemapKeyIndex.RegisterColumnStats(*ResultColumnStats); for (auto&& i : OrderedIterators) { i.Start(); diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp index c1206d05bf0..07480a5a5fa 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/remap.cpp @@ -20,8 +20,7 @@ TRemapColumns::TOthersData::TFinishContext TRemapColumns::BuildRemapInfo( builder.Add(i.first, statsByKeyIndex[i.second].GetRecordsCount(), statsByKeyIndex[i.second].GetDataSize(), settings.IsSparsed(statsByKeyIndex[i.second].GetRecordsCount(), recordsCount) ? NArrow::NAccessor::IChunkedArray::EType::SparsedArray : NArrow::NAccessor::IChunkedArray::EType::Array, - // For now others always encode in BinaryJson - NArrow::NAccessor::NSubColumns::EValueType::BinaryJson); + NArrow::NAccessor::NSubColumns::OthersExplicitBinaryJson); remap[i.second] = idx++; } return TOthersData::TFinishContext(builder.Finish(), remap); diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index 318e1b56a3b..2ccdec52fb5 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -66,10 +66,11 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc std::vector<TReadPortionInfoWithBlobs> portions = TReadPortionInfoWithBlobs::RestorePortions(accessors, Blobs, context.SchemaVersions); THashSet<ui64> usedPortionIds; + AFL_VERIFY(PortionsIndexSnapshot); std::vector<std::shared_ptr<ISubsetToMerge>> currentToMerge; for (auto&& i : portions) { AFL_VERIFY(usedPortionIds.emplace(i.GetPortionInfo().GetPortionId()).second); - currentToMerge.emplace_back(std::make_shared<TReadPortionToMerge>(std::move(i), GranuleMeta)); + currentToMerge.emplace_back(std::make_shared<TReadPortionToMerge>(std::move(i), GranuleMeta, PortionsIndexSnapshot)); } const auto buildPortionsToMerge = [&](const std::vector<std::shared_ptr<ISubsetToMerge>>& toMerge, const bool useDeletion) { @@ -94,7 +95,8 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc subsetsCount > 1) { auto merged = BuildAppendedPortionsByChunks(context, buildPortionsToMerge(toMerge, false), resultFiltered, stats); if (merged.size()) { - appendedToMerge.emplace_back(std::make_shared<TWritePortionsToMerge>(std::move(merged), GranuleMeta)); + appendedToMerge.emplace_back( + std::make_shared<TWritePortionsToMerge>(std::move(merged), GranuleMeta, PortionsIndexSnapshot)); } toMerge.clear(); sumMemory = 0; @@ -110,7 +112,8 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc BuildAppendedPortionsByChunks(context, buildPortionsToMerge(toMerge, appendedToMerge.empty()), resultFiltered, stats); if (appendedToMerge.size()) { if (merged.size()) { - appendedToMerge.emplace_back(std::make_shared<TWritePortionsToMerge>(std::move(merged), GranuleMeta)); + appendedToMerge.emplace_back( + std::make_shared<TWritePortionsToMerge>(std::move(merged), GranuleMeta, PortionsIndexSnapshot)); } } else { context.Counters.OnCompactionCorrectMemory(totalSumMemory); diff --git a/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp b/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp index 9791471ee14..a67e03dc53d 100644 --- a/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp +++ b/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp @@ -33,7 +33,7 @@ std::shared_ptr<NArrow::TColumnFilter> ISubsetToMerge::BuildPortionFilter(const } } } - if (GranuleMeta->GetPortionsIndex().HasOlderIntervals(pInfo, portionsInUsage)) { + if (NGranule::NPortionsIndex::TPortionsIndex::HasOlderIntervals(*PortionsIndexSnapshot, pInfo, portionsInUsage)) { filterDeleted = NArrow::TColumnFilter::BuildAllowFilter(); } } @@ -80,9 +80,9 @@ ui64 TWritePortionsToMerge::GetColumnMaxChunkMemory() const { return result; } -TWritePortionsToMerge::TWritePortionsToMerge( - std::vector<TWritePortionInfoWithBlobsResult>&& portions, const std::shared_ptr<TGranuleMeta>& granuleMeta) - : TBase(granuleMeta) +TWritePortionsToMerge::TWritePortionsToMerge(std::vector<TWritePortionInfoWithBlobsResult>&& portions, + const std::shared_ptr<TGranuleMeta>& granuleMeta, const NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot& portionsIndexSnapshot) + : TBase(granuleMeta, portionsIndexSnapshot) , WritePortions(std::move(portions)) { ui32 idx = 0; diff --git a/ydb/core/tx/columnshard/engines/changes/merge_subset.h b/ydb/core/tx/columnshard/engines/changes/merge_subset.h index cafcf9207fd..228b48e2c73 100644 --- a/ydb/core/tx/columnshard/engines/changes/merge_subset.h +++ b/ydb/core/tx/columnshard/engines/changes/merge_subset.h @@ -6,6 +6,7 @@ #include <ydb/core/tx/columnshard/engines/portions/read_with_blobs.h> #include <ydb/core/tx/columnshard/engines/portions/write_with_blobs.h> #include <ydb/core/tx/columnshard/engines/scheme/versions/filtered_scheme.h> +#include <ydb/core/tx/columnshard/engines/storage/granule/portions_index.h> namespace NKikimr::NOlap::NCompaction { @@ -30,15 +31,19 @@ private: protected: const std::shared_ptr<TGranuleMeta> GranuleMeta; + const NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot PortionsIndexSnapshot; std::shared_ptr<NArrow::TColumnFilter> BuildPortionFilter(const std::optional<NKikimr::NOlap::TGranuleShardingInfo>& shardingActual, const std::shared_ptr<NArrow::TGeneralContainer>& batch, const TPortionInfo& pInfo, const THashSet<ui64>& portionsInUsage, const bool useDeletionFilter) const; public: - ISubsetToMerge(const std::shared_ptr<TGranuleMeta>& granule) + ISubsetToMerge( + const std::shared_ptr<TGranuleMeta>& granule, const NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot& portionsIndexSnapshot) : GranuleMeta(granule) + , PortionsIndexSnapshot(portionsIndexSnapshot) { AFL_VERIFY(GranuleMeta); + AFL_VERIFY(PortionsIndexSnapshot); } virtual ~ISubsetToMerge() = default; @@ -62,8 +67,9 @@ private: const bool useDeletionFilter) const override; public: - TReadPortionToMerge(TReadPortionInfoWithBlobs&& rPortion, const std::shared_ptr<TGranuleMeta>& granuleMeta) - : TBase(granuleMeta) + TReadPortionToMerge(TReadPortionInfoWithBlobs&& rPortion, const std::shared_ptr<TGranuleMeta>& granuleMeta, + const NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot& portionsIndexSnapshot) + : TBase(granuleMeta, portionsIndexSnapshot) , ReadPortion(std::move(rPortion)) { } @@ -89,7 +95,8 @@ private: virtual ui64 GetColumnMaxChunkMemory() const override; public: - TWritePortionsToMerge(std::vector<TWritePortionInfoWithBlobsResult>&& portions, const std::shared_ptr<TGranuleMeta>& granuleMeta); + TWritePortionsToMerge(std::vector<TWritePortionInfoWithBlobsResult>&& portions, const std::shared_ptr<TGranuleMeta>& granuleMeta, + const NGranule::NPortionsIndex::TPortionsIndex::TPortionsSnapshot& portionsIndexSnapshot); }; } // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp index e73e84a27d0..666f92d9c72 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp @@ -3,8 +3,8 @@ namespace NKikimr::NOlap::NGranule::NPortionsIndex { -bool TPortionsIndex::HasOlderIntervals(const TPortionInfo& inputPortion, const THashSet<ui64>& skipPortions) const { - for (auto&& [_, p] : Portions) { +bool TPortionsIndex::HasOlderIntervals(const TPortions& portions, const TPortionInfo& inputPortion, const THashSet<ui64>& skipPortions) { + for (const auto& p : portions) { if (p->GetPortionId() == inputPortion.GetPortionId()) { continue; } diff --git a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h index ede88dfa60b..22724e7f82b 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h @@ -10,6 +10,10 @@ class TGranuleMeta; namespace NKikimr::NOlap::NGranule::NPortionsIndex { class TPortionsIndex { +public: + using TPortions = std::vector<TPortionInfo::TConstPtr>; + using TPortionsSnapshot = std::shared_ptr<const TPortions>; + private: THashMap<ui64, std::shared_ptr<TPortionInfo>> Portions; const TGranuleMeta& Owner; @@ -31,7 +35,16 @@ public: AFL_VERIFY(Portions.erase(p->GetPortionId())); } - bool HasOlderIntervals(const TPortionInfo& inputPortion, const THashSet<ui64>& skipPortions) const; + TPortionsSnapshot GetPortionsSnapshot() const { + auto result = std::make_shared<TPortions>(); + result->reserve(Portions.size()); + for (const auto& [_, portion] : Portions) { + result->emplace_back(portion); + } + return result; + } + + static bool HasOlderIntervals(const TPortions& portions, const TPortionInfo& inputPortion, const THashSet<ui64>& skipPortions); }; } // namespace NKikimr::NOlap::NGranule::NPortionsIndex diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.cpp b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.cpp index bafe47aeaca..2da9da71ff4 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.cpp +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.cpp @@ -1,5 +1,7 @@ #include "local_partition_actor.h" -#include "logging.h" +#include <ydb/library/actors/core/log.h> + +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::LOCAL_YDB_PROXY namespace NKikimr::NReplication { @@ -17,6 +19,7 @@ TBaseLocalTopicPartitionActor::TBaseLocalTopicPartitionActor( void TBaseLocalTopicPartitionActor::Bootstrap() { LogPrefix = MakeLogPrefix(); + YDB_LOG_CREATE_CONTEXT(LogPrefix); DoDescribe(TopicPath); } @@ -34,7 +37,8 @@ TString TBaseLocalTopicPartitionActor::MakeAbsolutePath(TString path) const { void TBaseLocalTopicPartitionActor::DoDescribe(const TString& topicPath) { auto path = MakeAbsolutePath(topicPath); - LOG_D("Describe topic '" << path << "'"); + YDB_LOG_DEBUG("Describe topic", + {"path", path}); auto request = MakeHolder<TNavigate>(); request->DatabaseName = Database; @@ -45,7 +49,8 @@ void TBaseLocalTopicPartitionActor::DoDescribe(const TString& topicPath) { } void TBaseLocalTopicPartitionActor::Handle(TEvNavigateResult::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto& result = ev->Get()->Request; static const TString errorMarker = "LocalYdbProxy"; @@ -108,6 +113,8 @@ TSchemeCacheHelpers::TCheckFailFunc TBaseLocalTopicPartitionActor::LeaveOnError( } STATEFN(TBaseLocalTopicPartitionActor::StateDescribe) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateDescribe"}); switch (ev->GetTypeRewrite()) { hFunc(TEvNavigateResult, Handle); hFunc(TEvents::TEvWakeup, HandleOnDescribe); @@ -119,7 +126,8 @@ STATEFN(TBaseLocalTopicPartitionActor::StateDescribe) { } void TBaseLocalTopicPartitionActor::DoCreatePipe() { - LOG_T("Create pipe to " << PartitionTabletId); + YDB_LOG_TRACE("Create pipe", + {"partitionTabletId", PartitionTabletId}); Attempt = 0; CreatePipe(); @@ -133,7 +141,8 @@ void TBaseLocalTopicPartitionActor::CreatePipe() { } void TBaseLocalTopicPartitionActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto& msg = *ev->Get(); if (msg.Status != NKikimrProto::OK) { @@ -143,17 +152,20 @@ void TBaseLocalTopicPartitionActor::Handle(TEvTabletPipe::TEvClientConnected::TP return CreatePipe(); } - LOG_T("Pipe has been connected"); + YDB_LOG_TRACE("Pipe has been connected"); OnDescribeFinished(); } void TBaseLocalTopicPartitionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); OnError("Pipe destroyed"); } STATEFN(TBaseLocalTopicPartitionActor::StateCreatePipe) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateCreatePipe"}); switch (ev->GetTypeRewrite()) { hFunc(TEvTabletPipe::TEvClientConnected, Handle); hFunc(TEvTabletPipe::TEvClientDestroyed, Handle); diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.h b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.h index f7cf7721a8c..111a37ccee4 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.h +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_actor.h @@ -21,7 +21,7 @@ protected: virtual void OnError(const TString& error) = 0; virtual void OnFatalError(const TString& error) = 0; virtual STATEFN(OnInitEvent) = 0; - virtual TString MakeLogPrefix() = 0; + virtual NActors::NStructuredLog::TStructuredMessage MakeLogPrefix() = 0; }; class TBaseLocalTopicPartitionActor @@ -71,7 +71,7 @@ protected: TActorId PartitionPipeClient; size_t Attempt = 0; - TString LogPrefix; + NActors::NStructuredLog::TStructuredMessage LogPrefix; }; } diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_committer.cpp b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_committer.cpp index c0981576f55..f8c47e393ad 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_committer.cpp +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_committer.cpp @@ -1,11 +1,14 @@ #include "local_partition_actor.h" #include "local_proxy.h" -#include "logging.h" #include <ydb/core/persqueue/events/global.h> #include <ydb/core/persqueue/writer/common.h> #include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> +#include <ydb/library/actors/core/log.h> + +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::LOCAL_YDB_PROXY + namespace NKikimr::NReplication { class TLocalTopicPartitionCommitActor: public TBaseLocalTopicPartitionActor { @@ -41,8 +44,13 @@ protected: PassAway(); } - TString MakeLogPrefix() override { - return TStringBuilder() << "Committer[" << SelfId() << ":/" << Database << TopicPath <<" ] "; + NActors::NStructuredLog::TStructuredMessage MakeLogPrefix() override { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "LocalTopicPartitionCommitActor"}, + {"selfId", SelfId()}, + {"database", Database}, + {"topicPath", TopicPath}, + ); } static std::unique_ptr<TEvYdbProxy::TEvCommitOffsetResponse> MakeResponse(NYdb::EStatus status, const TString& error) { @@ -60,14 +68,15 @@ protected: private: void DoCommitOffset() { - LOG_T("DoCommit"); + YDB_LOG_TRACE("DoCommit"); NTabletPipe::SendData(SelfId(), PartitionPipeClient, MakeCommitRequest().release()); Become(&TLocalTopicPartitionCommitActor::StateCommitOffset); } void Handle(TEvPersQueue::TEvResponse::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); const auto& record = ev->Get()->Record; @@ -94,6 +103,8 @@ private: } STATEFN(StateCommitOffset) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateCommitOffset"}); switch (ev->GetTypeRewrite()) { hFunc(TEvPersQueue::TEvResponse, Handle); @@ -112,7 +123,8 @@ private: }; // TLocalTopicPartitionCommitActor void TLocalProxyActor::Handle(TEvYdbProxy::TEvCommitOffsetRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto [topicName, partitionId, consumerName, offset, settings] = std::move(ev->Get()->GetArgs()); RegisterWithSameMailbox(new TLocalTopicPartitionCommitActor( diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_reader.cpp b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_reader.cpp index 5bbd1f42f98..6037640f61f 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_reader.cpp +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_partition_reader.cpp @@ -1,12 +1,15 @@ #include "local_partition_actor.h" #include "local_proxy.h" -#include "logging.h" #include <ydb/core/persqueue/writer/common.h> #include <ydb/core/persqueue/events/global.h> #include <ydb/core/protos/grpc_pq_old.pb.h> #include <ydb/core/tx/replication/ydb_proxy/ydb_proxy.h> +#include <ydb/library/actors/core/log.h> + +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::LOCAL_YDB_PROXY + namespace NKikimr::NReplication { class TLocalTopicPartitionReaderActor: public TBaseLocalTopicPartitionActor { @@ -52,8 +55,12 @@ protected: PassAway(); } - TString MakeLogPrefix() override { - return TStringBuilder() << "Reader[" << SelfId() << ":/" << Database << TopicPath <<" ] "; + NActors::NStructuredLog::TStructuredMessage MakeLogPrefix() override { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "LocalTopicPartitionReaderActor"}, + {"selfId", SelfId()}, + {"database", Database}, + {"topicPath", TopicPath}); } static std::unique_ptr<TEvYdbProxy::TEvTopicReaderGone> MakeError(NYdb::EStatus status, const TString& error) { @@ -63,6 +70,8 @@ protected: } STATEFN(OnInitEvent) override { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "OnInitEvent"}); switch (ev->GetTypeRewrite()) { hFunc(TEvYdbProxy::TEvReadTopicRequest, HandleInit); hFunc(TEvYdbProxy::TEvCommitOffsetRequest, Handle); @@ -79,12 +88,14 @@ private: } void HandleInit(TEvYdbProxy::TEvReadTopicRequest::TPtr& ev) { - LOG_T("Handle on init " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle on init", + {"ev", ev->Get()->ToString()}); RequestsQueue.emplace_back(ev->Sender, ev->Cookie, GetSkipCommit(ev)); } void Handle(TEvYdbProxy::TEvCommitOffsetRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); } private: @@ -94,7 +105,8 @@ private: } void HandleOnInitOffset(TEvPersQueue::TEvResponse::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto& record = ev->Get()->Record; if (record.GetErrorCode() == NPersQueue::NErrorCode::INITIALIZING) { @@ -137,6 +149,8 @@ private: } STATEFN(StateInitOffset) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateInitOffset"}); switch (ev->GetTypeRewrite()) { hFunc(TEvPersQueue::TEvResponse, HandleOnInitOffset); hFunc(TEvents::TEvWakeup, HandleOnInitOffset); @@ -158,7 +172,8 @@ private: } void Handle(TEvYdbProxy::TEvReadTopicRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); HandleInit(ev); Handle(RequestsQueue.front()); @@ -204,6 +219,8 @@ private: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateWork"}); switch (ev->GetTypeRewrite()) { hFunc(TEvYdbProxy::TEvReadTopicRequest, Handle); hFunc(TEvYdbProxy::TEvCommitOffsetRequest, Handle); @@ -226,7 +243,8 @@ private: } void HandleOnWaitData(TEvPersQueue::TEvResponse::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); const auto& record = ev->Get()->Record; @@ -316,6 +334,8 @@ private: } STATEFN(StateWaitData) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateWaitData"}); switch (ev->GetTypeRewrite()) { hFunc(TEvPersQueue::TEvResponse, HandleOnWaitData); @@ -339,7 +359,8 @@ private: }; // TLocalTopicPartitionReaderActor void TLocalProxyActor::Handle(TEvYdbProxy::TEvCreateTopicReaderRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto args = std::move(ev->Get()->GetArgs()); auto& settings = std::get<TEvYdbProxy::TTopicReaderSettings>(args); diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.cpp b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.cpp index a087fb7cb92..2bbe23da69c 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.cpp +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.cpp @@ -1,12 +1,14 @@ #include "local_proxy.h" #include "local_proxy_request.h" -#include "logging.h" #include <ydb/core/grpc_services/service_scheme.h> #include <ydb/core/grpc_services/service_topic.h> +#include <ydb/library/actors/core/log.h> #include <ydb/library/yverify_stream/yverify_stream.h> #include <ydb/public/api/protos/ydb_table.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::LOCAL_YDB_PROXY + namespace NKikimr::NReplication { TLocalProxyActor::TLocalProxyActor(const TString& database) @@ -15,7 +17,10 @@ TLocalProxyActor::TLocalProxyActor(const TString& database) } void TLocalProxyActor::Bootstrap() { - LogPrefix = TStringBuilder() << "[" << SelfId() << ":" << Database << "] "; + LogPrefix = YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "LocalTopicPartitionReaderActor"}, + {"selfId", SelfId()}, + {"database", Database}); Become(&TLocalProxyActor::StateWork); } @@ -65,7 +70,8 @@ auto CreateCallback(std::shared_ptr<T>&& ctx) { } void TLocalProxyActor::Handle(TEvYdbProxy::TEvAlterTopicRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto [path, settings] = std::move(ev->Get()->GetArgs()); path = MakeLocalPath(path); @@ -97,7 +103,8 @@ void TLocalProxyActor::Handle(TEvYdbProxy::TEvAlterTopicRequest::TPtr& ev) { } void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribeTopicRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto [path, _] = std::move(ev->Get()->GetArgs()); path = MakeLocalPath(path); @@ -133,7 +140,8 @@ void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribeTopicRequest::TPtr& ev) { } void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribePathRequest::TPtr& ev) { - LOG_T("Handle " << ev->Get()->ToString()); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}); auto [path, _] = std::move(ev->Get()->GetArgs()); path = MakeLocalPath(path); @@ -169,7 +177,8 @@ void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribePathRequest::TPtr& ev) { } void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribeTableRequest::TPtr& ev) { - LOG_E("Handle " << ev->Get()->ToString()); + YDB_LOG_ERROR("Handle", + {"ev", ev->Get()->ToString()}); auto [path, settings] = std::move(ev->Get()->GetArgs()); @@ -182,6 +191,8 @@ void TLocalProxyActor::Handle(TEvYdbProxy::TEvDescribeTableRequest::TPtr& ev) { } STATEFN(TLocalProxyActor::StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix, + {"actorState", "StateWaitData"}); switch (ev->GetTypeRewrite()) { hFunc(TEvYdbProxy::TEvCreateTopicReaderRequest, Handle); hFunc(TEvYdbProxy::TEvAlterTopicRequest, Handle); diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.h b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.h index 0781b99bbe6..b0c4089aaef 100644 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.h +++ b/ydb/core/tx/replication/ydb_proxy/local_proxy/local_proxy.h @@ -33,7 +33,7 @@ private: private: const TString Database; - TString LogPrefix; + NActors::NStructuredLog::TStructuredMessage LogPrefix; }; } diff --git a/ydb/core/tx/replication/ydb_proxy/local_proxy/logging.h b/ydb/core/tx/replication/ydb_proxy/local_proxy/logging.h deleted file mode 100644 index 339a6597c23..00000000000 --- a/ydb/core/tx/replication/ydb_proxy/local_proxy/logging.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include <ydb/library/actors/core/log.h> -#include <ydb/library/services/services.pb.h> - -#define CLOG_T(ctx, stream) LOG_TRACE_S (ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) -#define CLOG_D(ctx, stream) LOG_DEBUG_S (ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) -#define CLOG_I(ctx, stream) LOG_INFO_S (ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) -#define CLOG_N(ctx, stream) LOG_NOTICE_S(ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) -#define CLOG_W(ctx, stream) LOG_WARN_S (ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) -#define CLOG_E(ctx, stream) LOG_ERROR_S (ctx, NKikimrServices::LOCAL_YDB_PROXY, LogPrefix << stream) - -#define LOG_T(stream) CLOG_T(*TlsActivationContext, stream) -#define LOG_D(stream) CLOG_D(*TlsActivationContext, stream) -#define LOG_I(stream) CLOG_I(*TlsActivationContext, stream) -#define LOG_N(stream) CLOG_N(*TlsActivationContext, stream) -#define LOG_W(stream) CLOG_W(*TlsActivationContext, stream) -#define LOG_E(stream) CLOG_E(*TlsActivationContext, stream) diff --git a/ydb/core/tx/scheme_board/backup.cpp b/ydb/core/tx/scheme_board/backup.cpp index c3a994bcb9e..2d3b88b40b7 100644 --- a/ydb/core/tx/scheme_board/backup.cpp +++ b/ydb/core/tx/scheme_board/backup.cpp @@ -20,6 +20,8 @@ #include <ranges> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::SCHEME_BOARD_BACKUP + namespace NKikimr::NSchemeBoard { using namespace NJson; @@ -67,8 +69,11 @@ public: return NKikimrServices::TActivity::SCHEME_BOARD_BACKUP_ACTOR; } - static constexpr TStringBuf LogPrefix() { - return "backup"sv; + NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TBackupActor"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", SelfId()}); } struct TPathDescriptionAggregate { @@ -123,6 +128,7 @@ public: } void Bootstrap() { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); try { OutputFile.ConstructInPlace(FilePath); } catch (...) { @@ -157,11 +163,10 @@ private: Send(MakeStateStorageProxyID(), new TEvStateStorage::TEvResolveSchemeBoard(path), 0, cookie); Schedule(DefaultTimeout, new TEvents::TEvWakeup(cookie)); - SBB_LOG_D("ProcessPaths" - << ", path: " << path - << ", cookie: " << cookie - << ", paths in progress: " << PathByCookie.size() - ); + YDB_LOG_DEBUG("ProcessPaths", + {"path", path}, + {"cookie", cookie}, + {"inProgress", PathByCookie.size()}); } SendProgressUpdate(); @@ -187,18 +192,22 @@ private: const ui64 cookie = ev->Cookie; auto it = PathByCookie.find(cookie); if (it == PathByCookie.end()) { - SBB_LOG_N("Unexpected cookie: " << cookie); + YDB_LOG_NOTICE("Unexpected cookie", + {"selfId", this->SelfId()}, + {"cookie", cookie}); return; } const TString& path = it->second; - SBB_LOG_D("Handle " << ev->Get()->ToString() << ", path: " << path); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"path", path}); const auto replicas = ev->Get()->GetPlainReplicas(); if (replicas.empty()) { - SBB_LOG_I("Empty replica list" - << ", path: " << path - ); + YDB_LOG_INFO("Empty replica list", + {"selfId", this->SelfId()}, + {"path", path}); EmptyReplicaList.emplace(path); return MarkPathCompleted(it); } @@ -232,12 +241,16 @@ private: const ui64 cookie = GetOriginalCookie(ev->Cookie); auto it = PathByCookie.find(cookie); if (it == PathByCookie.end()) { - SBB_LOG_D("Received description with inactive cookie: " << cookie); + YDB_LOG_DEBUG("Received description with inactive cookie", + {"selfId", this->SelfId()}, + {"cookie", cookie}); return; } const TString path = it->second; - SBB_LOG_D("Handle " << ev->Get()->ToString() << ", path: " << path); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"path", path}); TString jsonDescription = std::move(*ev->Get()->Record.MutableJson()); if (!jsonDescription.empty()) { @@ -251,13 +264,16 @@ private: const auto version = std::stoull(pathDescription["Self"]["PathVersion"].GetStringSafe()); auto aggregate = DescriptionsByCookie.find(cookie); if (aggregate == DescriptionsByCookie.end()) { - SBB_LOG_N("No description aggregate for cookie: " << cookie); + YDB_LOG_NOTICE("No description aggregate", + {"selfId", this->SelfId()}, + {"cookie", cookie}); return; } if (!aggregate->second.AddDescription(std::move(jsonDescription), version, GetReplicaGroupIndex(ev->Cookie))) { - SBB_LOG_N("Invalid replica group: " << GetReplicaGroupIndex(ev->Cookie) - << ", cookie: " << ev->Cookie - ); + YDB_LOG_NOTICE("Invalid replica group", + {"selfId", this->SelfId()}, + {"group", GetReplicaGroupIndex(ev->Cookie)}, + {"cookie", ev->Cookie}); return; } if (aggregate->second.IsMajorityReached()) { @@ -269,7 +285,8 @@ private: MarkPathCompleted(it); } } else { - SBB_LOG_I("No version in path description"); + YDB_LOG_INFO("No version in path description", + {"selfId", this->SelfId()}); } } else { AddChildrenToPending(pathDescription, path); @@ -280,9 +297,10 @@ private: } } } catch (const std::exception& e) { - SBB_LOG_I("Parsing error: " << e.what() - << ", path: " << path - ); + YDB_LOG_INFO("Parsing error", + {"selfId", this->SelfId()}, + {"error", e.what()}, + {"path", path}); } } @@ -306,9 +324,11 @@ private: void AddChildrenToPending(const TJsonValue& pathDescription, const TString& path) { if (pathDescription.Has("Children")) { const auto& children = pathDescription["Children"]; - SBB_LOG_D("Queue children: " << JoinSeq(", ", children.GetArraySafe() | std::views::transform([](const TJsonValue& child) { - return child["Name"].GetStringSafe().Quote(); - }))); + YDB_LOG_DEBUG("Queue children", + {"selfId", this->SelfId()}, + {"children", JoinSeq(", ", children.GetArraySafe() | std::views::transform([](const TJsonValue& child) { + return child["Name"].GetStringSafe().Quote(); + }))}); for (const auto& child : children.GetArraySafe()) { if (child.Has("Name")) { TString childPath = TStringBuilder() << path << "/" << child["Name"].GetStringSafe(); @@ -328,12 +348,13 @@ private: auto it = PathByCookie.find(cookie); if (it == PathByCookie.end()) { // assume already processed - SBB_LOG_D("Timeout with inactive cookie: " << cookie); + YDB_LOG_DEBUG("Timeout with inactive cookie", + {"selfId", this->SelfId()}, + {"cookie", cookie}); return; } - SBB_LOG_I("Timeout" - << ", path: " << it->second - ); + YDB_LOG_INFO("Timeout", + {"path", it->second}); Timeouts.emplace(it->second); MarkPathCompleted(it); if (RequireMajority) { @@ -345,28 +366,27 @@ private: const ui64 cookie = GetOriginalCookie(ev->Cookie); auto it = PathByCookie.find(cookie); if (it == PathByCookie.end()) { - SBB_LOG_N("Undelivered" - << ", unexpected cookie: " << cookie - ); + YDB_LOG_NOTICE("Undelivered", + {"selfId", this->SelfId()}, + {"cookie", cookie}); return; } - SBB_LOG_I("Undelivered" - << ", path: " << it->second - ); + YDB_LOG_INFO("Undelivered", + {"selfId", this->SelfId()}, + {"path", it->second}); Undelivered.emplace(it->second); MarkPathCompleted(it); } void SendProgressUpdate() { - SBB_LOG_D("SendProgressUpdate" - << ", paths in progress: " << PathByCookie.size() - << ", processed paths: " << ProcessedPaths - << ", total paths: " << TotalPaths - << ", pending paths: " << PendingPaths.size() - << ", timeouts: " << Timeouts.size() - << ", delivery problems: " << Undelivered.size() - << ", empty replica list: " << EmptyReplicaList.size() - ); + YDB_LOG_DEBUG("SendProgressUpdate", + {"inProgress", PathByCookie.size()}, + {"processedPaths", ProcessedPaths}, + {"totalPaths", TotalPaths}, + {"pendingPaths", PendingPaths.size()}, + {"timeouts", Timeouts.size()}, + {"deliveryProblems", Undelivered.size()}, + {"emptyReplicaList", EmptyReplicaList.size()}); Send(Parent, new TSchemeBoardMonEvents::TEvBackupProgress(TotalPaths, ProcessedPaths)); } @@ -398,6 +418,7 @@ private: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); switch (ev->GetTypeRewrite()) { hFunc(TEvStateStorage::TEvResolveReplicasList, Handle); hFunc(TSchemeBoardMonEvents::TEvDescribeResponse, Handle); @@ -433,8 +454,11 @@ public: return NKikimrServices::TActivity::SCHEME_BOARD_RESTORE_ACTOR; } - static constexpr TStringBuf LogPrefix() { - return "restore"sv; + NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TRestoreActor"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", SelfId()}); } TRestoreActor(const TString& filePath, ui64 schemeShardId, ui64 generation, const TActorId& parent) @@ -446,6 +470,7 @@ public: } void Bootstrap() { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); try { InputFile.ConstructInPlace(FilePath); } catch (...) { @@ -476,7 +501,9 @@ private: TotalPaths++; descriptions.emplace_back(pathId, std::move(description)); if (TotalPaths % 1'000 == 0) { - SBB_LOG_D("Reading file, parsed paths descriptions: " << TotalPaths); + YDB_LOG_DEBUG("Reading file, parsed paths", + {"selfId", this->SelfId()}, + {"descriptions", TotalPaths}); SendProgressUpdate(); } } @@ -488,9 +515,9 @@ private: } void Populate(std::vector<std::pair<TPathId, NSchemeBoard::TTwoPartDescription>>&& descriptions) { - SBB_LOG_D("Populate" - << ", total paths: " << TotalPaths - ); + YDB_LOG_DEBUG("Populate", + {"selfId", this->SelfId()}, + {"paths", TotalPaths}); std::sort(descriptions.begin(), descriptions.end()); auto paths = descriptions | std::views::keys; PathsToProcess = TVector<TPathId>(paths.begin(), paths.end()); @@ -507,9 +534,12 @@ private: } void Handle(TSchemeBoardMonEvents::TEvInfoResponse::TPtr& ev) { - SBB_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"selfId", this->SelfId()}, + {"ev", ev->Get()->ToString()}); if (ev->Sender != Populator || !ev->Get()->Record.HasPopulatorResponse()) { - SBB_LOG_N("Unexpected info response"); + YDB_LOG_NOTICE("Unexpected info response", + {"selfId", this->SelfId()}); return; } @@ -545,6 +575,7 @@ private: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); switch (ev->GetTypeRewrite()) { hFunc(TSchemeBoardMonEvents::TEvInfoResponse, Handle); cFunc(TEvents::TEvPoison::EventType, PassAway); diff --git a/ydb/core/tx/scheme_board/backup.h b/ydb/core/tx/scheme_board/backup.h index 7cfe216d6f5..cc6a3c9b069 100644 --- a/ydb/core/tx/scheme_board/backup.h +++ b/ydb/core/tx/scheme_board/backup.h @@ -11,13 +11,6 @@ struct TEvCommonResult; namespace NKikimr::NSchemeBoard { -#define SBB_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) -#define SBB_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) -#define SBB_LOG_I(stream) SB_LOG_I(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) -#define SBB_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) -#define SBB_LOG_W(stream) SB_LOG_W(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) -#define SBB_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_BACKUP, "[" << LogPrefix() << "]" << this->SelfId() << " " << stream) - struct TCommonProgress { enum class EStatus { Idle, diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index c014544c089..32d6b4ac7d2 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -42,14 +42,11 @@ #include <google/protobuf/util/json_util.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_PROXY_SCHEME_CACHE + namespace NKikimr { namespace NSchemeBoard { -#define SBC_LOG_T(stream) SB_LOG_T(TX_PROXY_SCHEME_CACHE, stream) -#define SBC_LOG_D(stream) SB_LOG_D(TX_PROXY_SCHEME_CACHE, stream) -#define SBC_LOG_N(stream) SB_LOG_N(TX_PROXY_SCHEME_CACHE, stream) -#define SBC_LOG_W(stream) SB_LOG_W(TX_PROXY_SCHEME_CACHE, stream) - using TEvNavigate = TEvTxProxySchemeCache::TEvNavigateKeySet; using TEvNavigateResult = TEvTxProxySchemeCache::TEvNavigateKeySetResult; using TNavigate = NSchemeCache::TSchemeCacheNavigate; @@ -263,10 +260,10 @@ namespace { } void SendResult() { - SBC_LOG_D("Send result" - << ": self# " << this->SelfId() - << ", recipient# " << Context->Sender - << ", result# " << Context->Request->ToString(*AppData()->TypeRegistry)); + YDB_LOG_DEBUG("Send result", + {"self", this->SelfId()}, + {"recipient", Context->Sender}, + {"result", Context->Request->ToString(*AppData()->TypeRegistry)}); this->Send(Context->Sender, new TEvResult(Context->Request.Release()), 0, Context->Cookie); this->PassAway(); @@ -302,10 +299,10 @@ namespace { ) { if (Context->ResolvedDomainInfo->DomainKey != entry.DomainInfo->DomainKey) { - SBC_LOG_W("Path does not belong to the specified domain" - << ": self# " << this->SelfId() - << ", domain# " << Context->ResolvedDomainInfo->DomainKey - << ", path's domain# " << entry.DomainInfo->DomainKey); + YDB_LOG_WARN("Path does not belong to the specified domain", + {"self", this->SelfId()}, + {"contextDomain", Context->ResolvedDomainInfo->DomainKey}, + {"entryDomain", entry.DomainInfo->DomainKey}); SetErrorAndClear(Context.Get(), entry, false); } @@ -319,10 +316,10 @@ namespace { const ui32 access = GetAccess(entry); if (!securityObject->CheckAccess(access, *token)) { - SBC_LOG_W("Access denied" - << ": self# " << this->SelfId() - << ", for# " << token->GetUserSID() - << ", access# " << NACLib::AccessRightsToString(access)); + YDB_LOG_WARN("Access denied", + {"self", this->SelfId()}, + {"sid", token->GetUserSID()}, + {"access", NACLib::AccessRightsToString(access)}); const auto hasDescribeAccess = securityObject->CheckAccess(NACLib::DescribeSchema, *token); SetErrorAndClear(Context.Get(), entry, hasDescribeAccess); @@ -557,11 +554,11 @@ namespace { private: template <typename TPath> TActorId CreateSubscriber(const TPath& path, const ui64 tabletId, const ui64 domainOwnerId) const { - SBC_LOG_T("Create subscriber" - << ": self# " << SelfId() - << ", path# " << path - << ", tabletId# " << tabletId - << ", domainOwnerId# " << domainOwnerId); + YDB_LOG_TRACE("Create subscriber", + {"self", SelfId()}, + {"path", path}, + {"tabletId", tabletId}, + {"domainOwnerId", domainOwnerId}); return Register(CreateSchemeBoardSubscriber(SelfId(), path, domainOwnerId)); } @@ -1915,11 +1912,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } void FillEntry(TNavigateContext* context, TNavigate::TEntry& entry, const TResponseProps& props = {}) const { - SBC_LOG_D("FillEntry for TNavigate" - << ": self# " << Owner->SelfId() - << ", cacheItem# " << ToString() - << ", entry# " << entry.ToString() - << ", props# " << props.ToString()); + YDB_LOG_DEBUG("FillEntry for TNavigate", + {"self", Owner->SelfId()}, + {"cacheItem", ToString()}, + {"entry", entry}, + {"props", props}); if (props.IsSync && props.Partial) { return SetError(context, entry, TNavigate::EStatus::LookupError); @@ -2145,11 +2142,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } void FillEntry(TResolveContext* context, TResolve::TEntry& entry, const TResponseProps& props = {}) const { - SBC_LOG_D("FillEntry for TResolve" - << ": self# " << Owner->SelfId() - << ", cacheItem# " << ToString() - << ", entry# " << entry.ToString() - << ", props# " << props.ToString()); + YDB_LOG_DEBUG("FillEntry for TResolve", + {"self", Owner->SelfId()}, + {"cacheItem", ToString()}, + {"entry", entry}, + {"props", props}); TKeyDesc& keyDesc = *entry.KeyDescription; @@ -2416,10 +2413,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { template <typename TPath> TSubscriber CreateSubscriber(const TPath& path, const ui64 domainOwnerId) const { - SBC_LOG_T("Create subscriber" - << ": self# " << SelfId() - << ", path# " << path - << ", domainOwnerId# " << domainOwnerId); + YDB_LOG_TRACE("Create subscriber", + {"self", SelfId()}, + {"path", path}, + {"domainOwnerId", domainOwnerId}); return TSubscriber( Register(CreateSchemeBoardSubscriber(SelfId(), path, domainOwnerId)), domainOwnerId, path @@ -2557,11 +2554,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { TCacheItem* byPath = Cache.FindPtr(notify.Path); TCacheItem* byPathId = Cache.FindPtr(notify.PathId); - SBC_LOG_D("ResolveCacheItem" - << ": self# " << SelfId() - << ", notify# " << notify.ToString() - << ", by path# " << (byPath ? byPath->ToString() : "nullptr") - << ", by pathId# " << (byPathId ? byPathId->ToString() : "nullptr")); + YDB_LOG_DEBUG("ResolveCacheItem", + {"self", SelfId()}, + {"notify", notify}, + {"path", (byPath ? byPath->ToString() : "nullptr")}, + {"pathId", (byPathId ? byPathId->ToString() : "nullptr")}); TCacheItem* commonResolve = ResolveCacheItemCommon(notify.PathId, notify.Path); if (commonResolve) { @@ -2579,20 +2576,22 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); } - SBC_LOG_D("ResolveCacheItemForNotify: subdomain case" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId - << ", byPath# " << (byPath ? byPath->ToString() : "nullptr") - << ", byPathId# " << (byPathId ? byPathId->ToString() : "nullptr")); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "subdomain"}, + {"path", notify.Path}, + {"pathId", notify.PathId}, + {"byPath", (byPath ? byPath->ToString() : "nullptr")}, + {"byPathId", (byPathId ? byPathId->ToString() : "nullptr")}); if (byPath->GetDomainId() != notifyDomainId && notifyDomainId) { - SBC_LOG_D("ResolveCacheItemForNotify: recreation domain case" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId - << ", byPath# " << (byPath ? byPath->ToString() : "nullptr") - << ", byPathId# " << (byPathId ? byPathId->ToString() : "nullptr")); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "recreation domain"}, + {"path", notify.Path}, + {"pathId", notify.PathId}, + {"byPath", (byPath ? byPath->ToString() : "nullptr")}, + {"byPathId", (byPathId ? byPathId->ToString() : "nullptr")}); if (PathIdLessThan(byPath->GetPathId(), notify.PathId)) { return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); @@ -2603,51 +2602,57 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { if (byPath->GetPathId() == notifyDomainId) { //Update from TSS, GSS->TSS if (byPath->GetAbandonedSchemeShardIds().contains(notify.PathId.OwnerId)) { - SBC_LOG_D("ResolveCacheItemForNotify: this is update from TSS, the update is ignored, present GSS reverted implicitly that TSS" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "update from TSS is ignored, present GSS reverted implicitly that TSS"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return byPathId; } - SBC_LOG_D("ResolveCacheItemForNotify: this is update from TSS, the update overrides GSS by path" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "update from TSS overrides GSS by path"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); } if (byPath->GetDomainId() == notify.PathId) { //Update from GSS, TSS->GSS if (abandonedSchemeShardIds.contains(byPath->GetPathId().OwnerId)) { //GSS reverts TSS - SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS, the update overrides TSS by path, GSS implicitly reverts that TSS" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "update from GSS overrides TSS by path, GSS implicitly reverts that TSS"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); } if (!notifyDomainId) { - SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS that removes TSS" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "update from GSS that removes TSS"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); } - SBC_LOG_D("ResolveCacheItemForNotify: this is update from GSS, the update is ignored, TSS is preferred" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "update from GSS is ignored, TSS is preferred"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return byPathId; } if (byPath->GetDomainId() == notifyDomainId && notifyDomainId) { - SBC_LOG_D("ResolveCacheItemForNotify: recreation migrated path case" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId - << ", byPath# " << (byPath ? byPath->ToString() : "nullptr") - << ", byPathId# " << (byPathId ? byPathId->ToString() : "nullptr")); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "recreation migrated path"}, + {"path", notify.Path}, + {"pathId", notify.PathId}, + {"byPath", (byPath ? byPath->ToString() : "nullptr")}, + {"byPathId", (byPathId ? byPathId->ToString() : "nullptr")}); if (PathIdLessThan(byPath->GetPathId(), notify.PathId)) { return SwapSubscriberAndUpsert(byPath, notify.PathId, notify.Path); @@ -2657,10 +2662,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } if (!notifyDomainId) { - SBC_LOG_D("ResolveCacheItemForNotify: path has gone, update only by pathId" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId); + YDB_LOG_DEBUG("ResolveCacheItemForNotify", + {"self", SelfId()}, + {"case", "path has gone, update only by pathId"}, + {"path", notify.Path}, + {"pathId", notify.PathId}); return byPathId; } @@ -2684,9 +2690,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { const TResponseProps response = TResponseProps::FromEvent(ev); auto& notify = *ev->Get(); - SBC_LOG_D("HandleNotify" - << ": self# " << SelfId() - << ", notify# " << notify.ToString()); + YDB_LOG_DEBUG("HandleNotify", + {"self", SelfId()}, + {"notify", notify}); if (notify.Path && notify.PathId) { Y_ABORT_UNLESS(!response.IsSync); @@ -2695,11 +2701,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { TCacheItem* cacheItem = ResolveCacheItemForNotify(notify); if (!cacheItem) { - SBC_LOG_W("HandleNotify doesn't find any cacheItem for Fill" - << ": self# " << SelfId() - << ", path# " << notify.Path - << ", pathId# " << notify.PathId - << ", isSync# " << response.IsSync); + YDB_LOG_WARN("HandleNotify doesn't find any cacheItem for Fill", + {"self", SelfId()}, + {"path", notify.Path}, + {"pathId", notify.PathId}, + {"isSync", response.IsSync}); return; } @@ -2843,9 +2849,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { // otherwise use original entry and remove fallback entry // - This approach allows to avoid adding new handlers to SchemeCache actor and changing ProcessInFlight behavior void Handle(TEvTxProxySchemeCache::TEvNavigateKeySet::TPtr& ev) { - SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvNavigateKeySet" - << ": self# " << SelfId() - << ", request# " << ev->Get()->Request->ToString(*AppData()->TypeRegistry)); + YDB_LOG_DEBUG("Handle TEvTxProxySchemeCache::TEvNavigateKeySet", + {"self", SelfId()}, + {"request", ev->Get()->Request->ToString(*AppData()->TypeRegistry)}); if (MaybeRunDbResolver(ev)) { return; @@ -2918,9 +2924,9 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } void Handle(TEvTxProxySchemeCache::TEvResolveKeySet::TPtr& ev) { - SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvResolveKeySet" - << ": self# " << SelfId() - << ", request# " << ev->Get()->Request->ToString(*AppData()->TypeRegistry)); + YDB_LOG_DEBUG("Handle TEvTxProxySchemeCache::TEvResolveKeySet", + {"self", SelfId()}, + {"request", ev->Get()->Request->ToString(*AppData()->TypeRegistry)}); if (MaybeRunDbResolver(ev)) { return; @@ -2947,8 +2953,8 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { } void Handle(TEvTxProxySchemeCache::TEvInvalidateTable::TPtr& ev) { - SBC_LOG_D("Handle TEvTxProxySchemeCache::TEvInvalidateTable" - << ": self# " << SelfId()); + YDB_LOG_DEBUG("Handle TEvTxProxySchemeCache::TEvInvalidateTable", + {"self", SelfId()}); Send(ev->Sender, new TEvTxProxySchemeCache::TEvInvalidateTableResult(ev->Get()->Sender), 0, ev->Cookie); } diff --git a/ydb/core/tx/scheme_board/helpers.h b/ydb/core/tx/scheme_board/helpers.h index 0adf8affda6..926035d8655 100644 --- a/ydb/core/tx/scheme_board/helpers.h +++ b/ydb/core/tx/scheme_board/helpers.h @@ -10,24 +10,6 @@ #include <util/generic/vector.h> #include <util/generic/set.h> -#if defined SB_LOG_T || \ - defined SB_LOG_D || \ - defined SB_LOG_I || \ - defined SB_LOG_N || \ - defined SB_LOG_W || \ - defined SB_LOG_E || \ - defined SB_LOG_CRIT -#error log macro redefinition -#endif - -#define SB_LOG_T(service, stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_D(service, stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_I(service, stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_N(service, stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_W(service, stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_E(service, stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) -#define SB_LOG_CRIT(service, stream) LOG_CRIT_S((TlsActivationContext->AsActorContext()), NKikimrServices::service, stream) - namespace NKikimr { namespace NSchemeBoard { diff --git a/ydb/core/tx/scheme_board/monitoring.cpp b/ydb/core/tx/scheme_board/monitoring.cpp index d4e8d741a73..f818111345a 100644 --- a/ydb/core/tx/scheme_board/monitoring.cpp +++ b/ydb/core/tx/scheme_board/monitoring.cpp @@ -32,6 +32,8 @@ #include <util/string/join.h> #include <util/string/split.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::SCHEME_BOARD_BACKUP + // additional html elements namespace NMonitoring { const char NavTag[] = "nav"; @@ -79,8 +81,15 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { static constexpr char ROOT[] = "scheme_board"; static constexpr TBackupLimits BackupLimits = TBackupLimits(); - static constexpr TStringBuf LogPrefix() { - return "monitoring"sv; + static constexpr auto ActorActivityType() { + return TActivity::SCHEME_BOARD_MONITORING_ACTOR; + } + + NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TMonitoring"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", SelfId()}); } using TActivity = NKikimrServices::TActivity; @@ -1757,10 +1766,11 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { BackupProgress = TBackupProgress(); BackupProgress.Status = TBackupProgress::EStatus::Starting; - SBB_LOG_I("Starting backup to file: " << filePath - << ", in-flight limit: " << inFlightLimit - << ", require majority: " << requireMajority - ); + YDB_LOG_INFO("Starting backup", + {"selfId", this->SelfId()}, + {"filePath", filePath}, + {"inFlightLimit", inFlightLimit}, + {"majority", requireMajority}); Register(CreateSchemeBoardBackuper(filePath, inFlightLimit, requireMajority, SelfId())); @@ -1811,10 +1821,11 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { RestoreProgress = TRestoreProgress(); RestoreProgress.Status = TRestoreProgress::EStatus::Starting; - SBB_LOG_I("Starting restore from " << filePath - << " for SchemeShard ID: " << schemeShardId - << " of generation: " << generation - ); + YDB_LOG_INFO("Starting restore", + {"selfId", this->SelfId()}, + {"filePath", filePath}, + {"schemeShardId", schemeShardId}, + {"generation", generation}); Register(CreateSchemeBoardRestorer(filePath, schemeShardId, generation, SelfId())); @@ -1840,7 +1851,9 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { template <typename TProgress, typename TEventPtr> void Handle(TProgress& progress, TEventPtr& ev) { const auto& msg = *ev->Get(); - SBB_LOG_D("Handle " << msg.ToString()); + YDB_LOG_DEBUG("Handle", + {"selfId", this->SelfId()}, + {"ev", msg}); progress = TProgress(msg); } @@ -1861,10 +1874,6 @@ class TMonitoring: public TActorBootstrapped<TMonitoring> { } public: - static constexpr auto ActorActivityType() { - return TActivity::SCHEME_BOARD_MONITORING_ACTOR; - } - void Bootstrap() { if (auto* mon = AppData()->Mon) { auto* actorsMonPage = mon->RegisterIndexPage("actors", "Actors"); @@ -1876,6 +1885,7 @@ public: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); switch (ev->GetTypeRewrite()) { hFunc(TSchemeBoardMonEvents::TEvRegister, Handle); hFunc(TSchemeBoardMonEvents::TEvUnregister, Handle); diff --git a/ydb/core/tx/scheme_board/populator.cpp b/ydb/core/tx/scheme_board/populator.cpp index 6085cfad95d..e2a84317b29 100644 --- a/ydb/core/tx/scheme_board/populator.cpp +++ b/ydb/core/tx/scheme_board/populator.cpp @@ -27,15 +27,11 @@ #include <util/generic/vector.h> #include <util/generic/algorithm.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::SCHEME_BOARD_POPULATOR + namespace NKikimr { namespace NSchemeBoard { -#define SBP_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_POPULATOR, "" << SelfId() << " " << stream) -#define SBP_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_POPULATOR, "" << SelfId() << " " << stream) -#define SBP_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_POPULATOR, "" << SelfId() << " " << stream) -#define SBP_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_POPULATOR, "" << SelfId() << " " << stream) -#define SBP_LOG_CRIT(stream) SB_LOG_CRIT(SCHEME_BOARD_POPULATOR, "" << SelfId() << " " << stream) - namespace { using TDelayedUpdates = TVector<THolder<IEventHandle>>; @@ -91,7 +87,9 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { if (msg->HasDescription()) { if (msg->Description.Status != NKikimrScheme::StatusSuccess) { - SBP_LOG_E("Ignore description: " << msg->Description.ToString()); + YDB_LOG_ERROR("Ignore description", + {"selfId", SelfId()}, + {"description", msg->Description}); } else { CurPathId = msg->Description.PathId; update->SetDescribeSchemeResultSerialized(std::move(msg->Description.DescribeSchemeResultSerialized)); @@ -99,9 +97,10 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { } if (msg->HasMigratedPath()) { - SBP_LOG_D("Ignore description of migrated path" - << ": owner# " << Owner - << ", localPathId# " << msg->MigratedPathId); + YDB_LOG_DEBUG("Ignore description of migrated path", + {"selfId", SelfId()}, + {"owner", Owner}, + {"localPathId", msg->MigratedPathId}); // this path should be described by another owner (tenant schemeshard) auto& migratedLocalPathIds = *update->Record.MutableMigratedLocalPathIds(); migratedLocalPathIds.SetBegin(msg->MigratedPathId); @@ -218,10 +217,11 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { template <typename TEvent, typename T> bool Check(TEvent& ev, T this_, T that, const TString& what) { if (this_ != that) { - SBP_LOG_E("Suspicious " << TypeName<TEvent>() - << ": sender# " << ev->Sender - << ", " << what << "# " << this_ - << ", other " << what << "# " << that); + YDB_LOG_ERROR(TStringBuilder() << "Suspicious " << what, + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"expected", this_}, + {"got", that}); return false; } @@ -239,8 +239,10 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { } void Handle(NInternalEvents::TEvHandshakeResponse::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); if (!CheckOwner(ev)) { return; @@ -248,20 +250,24 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { const ui64 generation = ev->Get()->Record.GetGeneration(); if (generation > Generation) { - SBP_LOG_CRIT("Keep calm"); + YDB_LOG_CRIT("Keep calm", + {"selfId", SelfId()}); Become(&TThis::StateCalm); } else { - SBP_LOG_N("Successful handshake" - << ": replica# " << ev->Sender); + YDB_LOG_NOTICE("Successful handshake", + {"selfId", SelfId()}, + {"replica", ev->Sender}); if (generation < Generation) { - SBP_LOG_N("Start full sync" - << ": replica# " << ev->Sender); + YDB_LOG_NOTICE("Start full sync", + {"selfId", SelfId()}, + {"replica", ev->Sender}); ProcessSync(); } else { - SBP_LOG_N("Resume sync" - << ": replica# " << ev->Sender - << ", fromPathId# " << LastAckedPathId.NextId()); + YDB_LOG_NOTICE("Resume sync", + {"selfId", SelfId()}, + {"replica", ev->Sender}, + {"fromPathId", LastAckedPathId.NextId()}); ResumeSync(LastAckedPathId.NextId()); } @@ -271,24 +277,30 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { } void Handle(NInternalEvents::TEvDescribeResult::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); ProcessSync(ev->Get()); } void Handle(NInternalEvents::TEvUpdate::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); EnqueueUpdate(ev, true); } void Handle(NSchemeshardEvents::TEvUpdateAck::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); if (!CheckOwner(ev) || !CheckGeneration(ev)) { return; @@ -305,8 +317,10 @@ class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> { } void Handle(NInternalEvents::TEvCommitResponse::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); if (!CheckOwner(ev) || !CheckGeneration(ev)) { return; @@ -539,16 +553,19 @@ class TPopulator: public TMonitorableActor<TPopulator> { } void Handle(NInternalEvents::TEvRequestDescribe::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); const TActorId replicaPopulator = ev->Sender; const TActorId replica = ev->Get()->Replica; if (ReplicaToReplicaPopulator[replica] != replicaPopulator) { - SBP_LOG_CRIT("Inconsistent replica populator" - << ": replica# " << replica - << ", replicaPopulator# " << replicaPopulator); + YDB_LOG_CRIT("Inconsistent replica populator", + {"selfId", SelfId()}, + {"replica", replica}, + {"replicaPopulator", replicaPopulator}); return; } @@ -634,8 +651,10 @@ class TPopulator: public TMonitorableActor<TPopulator> { } void Handle(NInternalEvents::TEvRequestUpdate::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); const TPathId pathId = ev->Get()->PathId; THolder<NInternalEvents::TEvUpdateBuilder> update; @@ -654,9 +673,11 @@ class TPopulator: public TMonitorableActor<TPopulator> { } void DelayUpdate(NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr& ev) { - SBP_LOG_D("DelayUpdate " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("DelayUpdate", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); DelayedUpdates.emplace_back(ev.Release()); } @@ -668,15 +689,17 @@ class TPopulator: public TMonitorableActor<TPopulator> { auto* msg = static_cast<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResultBuilder*>(ev->Get()); auto& record = msg->Record; - SBP_LOG_D("Handle TEvSchemeShard::TEvDescribeSchemeResult { " << record.ShortDebugString() << " }" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", event size# " << msg->GetCachedByteSize() - << ", preserialized size# " << msg->PreSerializedData.size() - ); + YDB_LOG_DEBUG("Handle TEvSchemeShard::TEvDescribeSchemeResult", + {"selfId", SelfId()}, + {"ev", record.ShortDebugString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"eventSize", msg->GetCachedByteSize()}, + {"preserializedSize", msg->PreSerializedData.size()}); if (!record.HasStatus()) { - SBP_LOG_E("Description without status"); + YDB_LOG_ERROR("Description without status", + {"selfId", SelfId()}); return; } @@ -684,20 +707,21 @@ class TPopulator: public TMonitorableActor<TPopulator> { const bool isDeletion = record.GetStatus() == NKikimrScheme::StatusPathDoesNotExist; const ui64 version = isDeletion ? Max<ui64>() : NSchemeBoard::GetPathVersion(record); - SBP_LOG_N("Update description" - << ": owner# " << Owner - << ", pathId# " << pathId - << ", cookie# " << ev->Cookie - << ", is deletion# " << (isDeletion ? "true" : "false") - << ", version: " << (isDeletion ? 0 : version) - ); + YDB_LOG_NOTICE("Update description", + {"selfId", SelfId()}, + {"owner", Owner}, + {"pathId", pathId}, + {"cookie", ev->Cookie}, + {"isDeletion", (isDeletion ? "true" : "false")}, + {"version", (isDeletion ? 0 : version)}); if (isDeletion) { if (!Descriptions.contains(pathId)) { - SBP_LOG_N("Immediate ack for deleted path" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", pathId# " << pathId); + YDB_LOG_NOTICE("Immediate ack for deleted path", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"pathId", pathId}); auto ack = MakeHolder<NSchemeshardEvents::TEvUpdateAck>(Owner, Generation, pathId, Max<ui64>()); Send(ev->Sender, std::move(ack), 0, ev->Cookie); @@ -753,22 +777,28 @@ class TPopulator: public TMonitorableActor<TPopulator> { void Handle(NSchemeshardEvents::TEvUpdateAck::TPtr& ev) { const auto& record = ev->Get()->Record; - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); auto it = UpdateAcks.find(ev->Cookie); if (it == UpdateAcks.end()) { - SBP_LOG_D("Ack for unknown update (already acked?)" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Ack for unknown update", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"reason", "already acked"}); return; } TActorId* ackedReplica = ReplicaToReplicaPopulatorBackMap.FindPtr(ev->Sender); if (ackedReplica == nullptr) { - SBP_LOG_D("Ack from unknown replica (config changed?)" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Ack from unknown replica", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"reason", "config changed"}); return; } const TPathId pathId = ev->Get()->GetPathId(); @@ -779,11 +809,12 @@ class TPopulator: public TMonitorableActor<TPopulator> { && pathIt->first.first == pathId && pathIt->first.second <= version) { if (CheckQuorum(pathIt->second, *ackedReplica)) { - SBP_LOG_N("Ack update" - << ": ack to# " << it->second.AckTo - << ", cookie# " << ev->Cookie - << ", pathId# " << pathId - << ", version# " << pathIt->first.second); + YDB_LOG_NOTICE("Ack update", + {"selfId", SelfId()}, + {"to", it->second.AckTo}, + {"cookie", ev->Cookie}, + {"pathId", pathId}, + {"version", pathIt->first.second}); auto ack = MakeHolder<NSchemeshardEvents::TEvUpdateAck>(Owner, Generation, pathId, pathIt->first.second); Send(it->second.AckTo, std::move(ack), 0, ev->Cookie); @@ -803,14 +834,17 @@ class TPopulator: public TMonitorableActor<TPopulator> { } void Handle(TEvStateStorage::TEvListSchemeBoardResult::TPtr& ev) { - SBP_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); const auto& info = ev->Get()->Info; if (!info) { Y_ABORT_UNLESS(!GroupInfo); - SBP_LOG_E("Publish on unconfigured SchemeBoard"); + YDB_LOG_ERROR("Publish on unconfigured SchemeBoard", + {"selfId", SelfId()}); Become(&TThis::StateCalm); return; } @@ -914,7 +948,8 @@ class TPopulator: public TMonitorableActor<TPopulator> { } void HandleUndelivered() { - SBP_LOG_E("Publish on unavailable SchemeBoard"); + YDB_LOG_ERROR("Publish on unavailable SchemeBoard", + {"selfId", SelfId()}); Become(&TThis::StateCalm); } diff --git a/ydb/core/tx/scheme_board/replica.cpp b/ydb/core/tx/scheme_board/replica.cpp index 24f53ffd506..bc755947ff2 100644 --- a/ydb/core/tx/scheme_board/replica.cpp +++ b/ydb/core/tx/scheme_board/replica.cpp @@ -27,15 +27,11 @@ #include <util/generic/variant.h> #include <util/string/builder.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::SCHEME_BOARD_REPLICA + namespace NKikimr { namespace NSchemeBoard { -#define SBR_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_REPLICA, "" << SelfId() << " " << stream) -#define SBR_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_REPLICA, "" << SelfId() << " " << stream) -#define SBR_LOG_I(stream) SB_LOG_I(SCHEME_BOARD_REPLICA, "" << SelfId() << " " << stream) -#define SBR_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_REPLICA, "" << SelfId() << " " << stream) -#define SBR_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_REPLICA, "" << SelfId() << " " << stream) - class TReplica: public TMonitorableActor<TReplica> { using TDescribeSchemeResult = NKikimrScheme::TEvDescribeSchemeResult; using TCapabilities = NKikimrSchemeBoard::TEvSubscribe::TCapabilities; @@ -313,9 +309,10 @@ public: << ": self# " << ToString() << ", other# " << other.ToString()); - SBR_LOG_T("Merge descriptions" - << ": self# " << ToLogString() - << ", other# " << other.ToLogString()); + YDB_LOG_TRACE("Merge descriptions", + {"selfId", SelfId()}, + {"self", ToLogString()}, + {"other", other.ToLogString()}); UntrackMemory(); other.UntrackMemory(); @@ -525,38 +522,40 @@ private: // register empty entry by path OR pathId template <typename TPath> TDescription& UpsertDescription(const TPath& path) { - SBR_LOG_I("Upsert description" - << ": path# " << path); + YDB_LOG_INFO("Upsert description", + {"selfId", SelfId()}, + {"path", path}); return Descriptions.Upsert(path, TDescription(this, path)); } // register empty entry by path AND pathId both TDescription& UpsertDescription(const TString& path, const TPathId& pathId) { - SBR_LOG_I("Upsert description" - << ": path# " << path - << ", pathId# " << pathId); + YDB_LOG_INFO("Upsert description", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}); return Descriptions.Upsert(path, pathId, TDescription(this, path, pathId)); } // upsert description only by pathId TDescription& UpsertDescriptionByPathId(const TString& path, const TPathId& pathId, TOpaquePathDescription&& pathDescription) { - SBR_LOG_I("Upsert description" - << ": pathId# " << pathId - << ", pathDescription# " << pathDescription.ToString() - ); + YDB_LOG_INFO("Upsert description", + {"selfId", SelfId()}, + {"pathId", pathId}, + {"pathDescription", pathDescription}); return Descriptions.Upsert(pathId, TDescription(this, path, pathId, std::move(pathDescription))); } // upsert description by path AND pathId both TDescription& UpsertDescription(const TString& path, const TPathId& pathId, TOpaquePathDescription&& pathDescription) { - SBR_LOG_I("Upsert description" - << ": path# " << path - << ", pathId# " << pathId - << ", pathDescription# " << pathDescription.ToString() - ); + YDB_LOG_INFO("Upsert description", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}, + {"pathDescription", pathDescription}); return Descriptions.Upsert(path, pathId, TDescription(this, path, pathId, std::move(pathDescription))); } @@ -579,9 +578,10 @@ private: auto path = desc->GetPath(); - SBR_LOG_I("Delete description" - << ": path# " << path - << ", pathId# " << pathId); + YDB_LOG_INFO("Delete description", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}); if (TDescription* descByPath = Descriptions.FindPtr(path)) { if (descByPath != desc && !descByPath->IsEmpty()) { @@ -591,12 +591,13 @@ private: auto domainId = desc->GetDomainId(); if (curDomainId == pathId) { // Deletion from GSS - SBR_LOG_N("Delete description by GSS" - << ": path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); + YDB_LOG_NOTICE("Delete description by GSS", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}, + {"domainId", domainId}, + {"curPathId", curPathId}, + {"curDomainId", curDomainId}); Descriptions.DeleteIndex(path); UpsertDescription(path, pathId); @@ -648,11 +649,12 @@ private: TDescription* desc = Descriptions.FindPtr(path); Y_ABORT_UNLESS(desc); - SBR_LOG_I("Subscribe" - << ": subscriber# " << subscriber - << ", path# " << path - << ", domainOwnerId# " << domainOwnerId - << ", capabilities# " << capabilities.ShortDebugString()); + YDB_LOG_INFO("Subscribe", + {"selfId", SelfId()}, + {"subscriber", subscriber}, + {"path", path}, + {"domainOwnerId", domainOwnerId}, + {"capabilities", capabilities.ShortDebugString()}); desc->Subscribe(subscriber, path, domainOwnerId, capabilities); @@ -666,9 +668,10 @@ private: TDescription* desc = Descriptions.FindPtr(path); Y_ABORT_UNLESS(desc); - SBR_LOG_I("Unsubscribe" - << ": subscriber# " << subscriber - << ", path# " << path); + YDB_LOG_INFO("Unsubscribe", + {"selfId", SelfId()}, + {"subscriber", subscriber}, + {"path", path}); desc->Unsubscribe(subscriber); Subscribers.erase(subscriber); @@ -756,8 +759,10 @@ private: } void Handle(NInternalEvents::TEvHandshakeRequest::TPtr& ev) { - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); const auto& record = ev->Get()->Record; const ui64 owner = record.GetOwner(); @@ -765,17 +770,19 @@ private: TPopulatorInfo& info = Populators[owner]; if (generation < info.PendingGeneration) { - SBR_LOG_E("Reject handshake from stale populator" - << ": sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation - << ", pending generation# " << info.PendingGeneration); + YDB_LOG_ERROR("Reject handshake from stale populator", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"owner", owner}, + {"generation", generation}, + {"pendingGeneration", info.PendingGeneration}); return; } - SBR_LOG_N("Successful handshake" - << ": owner# " << owner - << ", generation# " << generation); + YDB_LOG_NOTICE("Successful handshake", + {"selfId", SelfId()}, + {"owner", owner}, + {"generation", generation}); info.PendingGeneration = generation; info.PopulatorActor = ev->Sender; @@ -784,11 +791,12 @@ private: } void Handle(NInternalEvents::TEvUpdate::TPtr& ev) { - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", event size# " << ev->Get()->GetCachedByteSize() - ); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"size", ev->Get()->GetCachedByteSize()}); const TString& path = ev->Get()->GetPath(); const TPathId& pathId = ev->Get()->GetPathId(); @@ -800,18 +808,20 @@ private: const auto populatorIt = Populators.find(owner); if (populatorIt == Populators.end()) { - SBR_LOG_E("Reject update from unknown populator" - << ": sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation); + YDB_LOG_ERROR("Reject update from unknown populator", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"owner", owner}, + {"generation", generation}); return; } if (generation != populatorIt->second.PendingGeneration) { - SBR_LOG_E("Reject update from stale populator" - << ": sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation - << ", pending generation# " << populatorIt->second.PendingGeneration); + YDB_LOG_ERROR("Reject update from stale populator", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"owner", owner}, + {"generation", generation}, + {"pendingGeneration", populatorIt->second.PendingGeneration}); return; } @@ -825,10 +835,11 @@ private: return AckUpdate(ev); } - SBR_LOG_N("Update description" - << ": path# " << path - << ", pathId# " << pathId - << ", deletion# " << (record.GetIsDeletion() ? "true" : "false")); + YDB_LOG_NOTICE("Update description", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}, + {"deletion", (record.GetIsDeletion() ? "true" : "false")}); if (record.GetIsDeletion()) { SoftDeleteDescription(pathId, true); @@ -838,9 +849,10 @@ private: if (TDescription* desc = Descriptions.FindPtr(pathId)) { if (desc->IsExplicitlyDeleted()) { - SBR_LOG_N("Path was explicitly deleted, ignoring" - << ": path# " << path - << ", pathId# " << pathId); + YDB_LOG_NOTICE("Path was explicitly deleted, ignoring", + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}); return AckUpdate(ev); } @@ -885,12 +897,13 @@ private: const auto& domainId = pathDescription.SubdomainPathId; auto log = [&](const TString& message) { - SBR_LOG_N("" << message - << ": path# " << path - << ", pathId# " << pathId - << ", domainId# " << domainId - << ", curPathId# " << curPathId - << ", curDomainId# " << curDomainId); + YDB_LOG_NOTICE(message, + {"selfId", SelfId()}, + {"path", path}, + {"pathId", pathId}, + {"domainId", domainId}, + {"curPathId", curPathId}, + {"curDomainId", curDomainId}); }; if (curPathId == domainId) { // Update from TSS, GSS->TSS @@ -963,8 +976,10 @@ private: } void Handle(NInternalEvents::TEvCommitRequest::TPtr& ev) { - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); const auto& record = ev->Get()->Record; const ui64 owner = record.GetOwner(); @@ -972,26 +987,29 @@ private: auto it = Populators.find(owner); if (it == Populators.end()) { - SBR_LOG_E("Reject commit from unknown populator" - << ": sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation); + YDB_LOG_ERROR("Reject commit from unknown populator", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"owner", owner}, + {"generation", generation}); return; } TPopulatorInfo& info = it->second; if (generation != info.PendingGeneration) { - SBR_LOG_E("Reject commit from stale populator" - << ": sender# " << ev->Sender - << ", owner# " << owner - << ", generation# " << generation - << ", pending generation# " << info.PendingGeneration); + YDB_LOG_ERROR("Reject commit from stale populator", + {"selfId", SelfId()}, + {"sender", ev->Sender}, + {"owner", owner}, + {"generation", generation}, + {"pendingGeneration", info.PendingGeneration}); return; } - SBR_LOG_N("Commit generation" - << ": owner# " << owner - << ", generation# " << generation); + YDB_LOG_NOTICE("Commit generation", + {"selfId", SelfId()}, + {"owner", owner}, + {"generation", generation}); info.Generation = info.PendingGeneration; info.IsCommited = true; @@ -1003,19 +1021,23 @@ private: } void Handle(TEvPrivate::TEvSendStrongNotifications::TPtr& ev) { - SBR_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}); const auto owner = ev->Get()->Owner; if (!IsPopulatorCommited(owner)) { - SBR_LOG_N("Populator is not commited" - << ": owner# " << owner); + YDB_LOG_NOTICE("Populator is not commited", + {"selfId", SelfId()}, + {"owner", owner}); return; } auto itSubscribers = WaitStrongNotifications.find(owner); if (itSubscribers == WaitStrongNotifications.end()) { - SBR_LOG_E("Invalid owner" - << ": owner# " << owner); + YDB_LOG_ERROR("Invalid owner", + {"selfId", SelfId()}, + {"owner", owner}); return; } @@ -1066,8 +1088,10 @@ private: const ui64 domainOwnerId = record.GetDomainOwnerId(); const auto& capabilities = record.GetCapabilities(); - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); if (record.HasPath()) { SubscribeBy(ev->Sender, record.GetPath(), domainOwnerId, capabilities); @@ -1080,8 +1104,10 @@ private: void Handle(NInternalEvents::TEvUnsubscribe::TPtr& ev) { const auto& record = ev->Get()->Record; - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); if (record.HasPath()) { UnsubscribeBy(ev->Sender, record.GetPath()); @@ -1094,8 +1120,10 @@ private: void Handle(NInternalEvents::TEvNotifyAck::TPtr& ev) { const auto& record = ev->Get()->Record; - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); auto it = Subscribers.find(ev->Sender); if (it == Subscribers.end()) { @@ -1132,9 +1160,11 @@ private: void Handle(NInternalEvents::TEvSyncVersionRequest::TPtr& ev) { const auto& record = ev->Get()->Record; - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); auto it = Subscribers.find(ev->Sender); if (it == Subscribers.end()) { @@ -1237,7 +1267,9 @@ private: } void Handle(TEvStateStorage::TEvUpdateGroupConfig::TPtr ev) { - SBR_LOG_I("Handle " << ev->Get()->ToString()); + YDB_LOG_INFO("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}); Info = ev->Get()->SchemeBoardConfig; Y_ABORT_UNLESS(!ev->Get()->GroupConfig); @@ -1247,8 +1279,10 @@ private: void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { const ui32 nodeId = ev->Get()->NodeId; - SBR_LOG_D("Handle " << ev->Get()->ToString() - << ": nodeId# " << nodeId); + YDB_LOG_DEBUG("Handle", + {"selfId", SelfId()}, + {"ev", ev->Get()->ToString()}, + {"nodeId", nodeId}); auto it = Subscribers.lower_bound(TActorId(nodeId, 0, 0, 0)); while (it != Subscribers.end() && it->first.NodeId() == nodeId) { @@ -1267,7 +1301,8 @@ private: } void PassAway() override { - SBR_LOG_T("PassAway"); + YDB_LOG_TRACE("PassAway", + {"selfId", SelfId()}); for (const auto& [_, info] : Populators) { if (const auto& actorId = info.PopulatorActor) { Send(actorId, new TEvStateStorage::TEvReplicaShutdown()); @@ -1287,7 +1322,8 @@ public: } void Bootstrap() { - SBR_LOG_T("Bootstrap"); + YDB_LOG_TRACE("Bootstrap", + {"selfId", SelfId()}); TMonitorableActor::Bootstrap(); auto localNodeId = SelfId().NodeId(); auto whiteboardId = NNodeWhiteboard::MakeNodeWhiteboardServiceId(localNodeId); diff --git a/ydb/core/tx/scheme_board/subscriber.cpp b/ydb/core/tx/scheme_board/subscriber.cpp index facc6f5fc9a..80e428324f4 100644 --- a/ydb/core/tx/scheme_board/subscriber.cpp +++ b/ydb/core/tx/scheme_board/subscriber.cpp @@ -26,19 +26,14 @@ #include <util/string/cast.h> #include <util/generic/xrange.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::SCHEME_BOARD_SUBSCRIBER + namespace NKikimr { using EDeletionPolicy = ESchemeBoardSubscriberDeletionPolicy; namespace NSchemeBoard { -#define SBS_LOG_T(stream) SB_LOG_T(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) -#define SBS_LOG_D(stream) SB_LOG_D(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) -#define SBS_LOG_I(stream) SB_LOG_I(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) -#define SBS_LOG_N(stream) SB_LOG_N(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) -#define SBS_LOG_W(stream) SB_LOG_W(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) -#define SBS_LOG_E(stream) SB_LOG_E(SCHEME_BOARD_SUBSCRIBER, "[" << LogPrefix() << "]" << this->SelfId() << "[" << Path << "] " << stream) - namespace { template <typename T, typename U> @@ -355,14 +350,18 @@ class TReplicaSubscriber: public TMonitorableActor<TDerived> { void Handle(NInternalEvents::TEvNotify::TPtr& ev) { auto& record = *ev->Get()->MutableRecord(); - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"path", Path}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); this->Send(ev->Sender, new NInternalEvents::TEvNotifyAck(record.GetVersion())); if (!IsValidNotification(Path, record)) { - SBS_LOG_E("Suspicious " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_ERROR("Suspicious", + {"path", Path}, + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); return; } @@ -370,18 +369,20 @@ class TReplicaSubscriber: public TMonitorableActor<TDerived> { } void Handle(NInternalEvents::TEvSyncVersionRequest::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); CurrentSyncRequest = ev->Cookie; this->Send(Replica, ev->Release().Release(), IEventHandle::FlagTrackDelivery, ev->Cookie); } void Handle(NInternalEvents::TEvSyncVersionResponse::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); if (ev->Cookie != CurrentSyncRequest) { return; @@ -412,6 +413,7 @@ class TReplicaSubscriber: public TMonitorableActor<TDerived> { } void PassAway() override { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); if (Replica.NodeId() != this->SelfId().NodeId()) { this->Send(MakeInterconnectProxyId(Replica.NodeId()), new TEvents::TEvUnsubscribe()); } @@ -435,8 +437,12 @@ public: return NKikimrServices::TActivity::SCHEME_BOARD_REPLICA_SUBSCRIBER_ACTOR; } - static constexpr TStringBuf LogPrefix() { - return "replica"sv; + NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TReplicaSubscriber"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", this->SelfId()}, + {"path", Path}); } explicit TReplicaSubscriber( @@ -453,6 +459,7 @@ public: } void Bootstrap(const TActorContext&) { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); TMonitorableActor<TDerived>::Bootstrap(); this->Send(Replica, new NInternalEvents::TEvSubscribe(Path, DomainOwnerId), @@ -461,6 +468,9 @@ public: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorStateFn", "StateWork"}); + switch (ev->GetTypeRewrite()) { hFunc(NInternalEvents::TEvNotify, Handle); hFunc(NInternalEvents::TEvSyncVersionRequest, Handle); @@ -499,8 +509,8 @@ public: template <typename TPath, typename TDerived, typename TReplicaDerived> class TSubscriberProxy: public TMonitorableActor<TDerived> { void Sleep() { - SBS_LOG_T("Sleep" - << ": delay# " << Delay); + YDB_LOG_TRACE("Sleep", + {"delay", Delay}); ReplicaSubscriber = TActorId(); this->Send(Parent, new NInternalEvents::TEvNotifyBuilder(Path, true)); @@ -513,11 +523,10 @@ class TSubscriberProxy: public TMonitorableActor<TDerived> { return; } if (const auto& record = ev->Get()->GetRecord(); !ClusterStatesMatch(ClusterState, record)) { - SBS_LOG_D("Cluster state mismatch in replica notification" - << ": sender# " << ev->Sender - << ", subscriber cluster state# " << ClusterState - << ", replica cluster state# {" << record.GetClusterState().ShortDebugString() << "}" - ); + YDB_LOG_DEBUG("Cluster state mismatch in replica notification", + {"sender", ev->Sender}, + {"clusterState", ClusterState}, + {"state", record.GetClusterState().ShortDebugString()}); this->Send(ReplicaSubscriber, new TEvents::TEvPoisonPill()); return Sleep(); } @@ -532,18 +541,20 @@ class TSubscriberProxy: public TMonitorableActor<TDerived> { } void HandleSleep(NInternalEvents::TEvSyncVersionRequest::TPtr& ev) { - SBS_LOG_T("HandleSleep " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_TRACE("HandleSleep", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); this->Send(Parent, new NInternalEvents::TEvSyncVersionResponse(0), 0, ev->Cookie); } void Handle(NInternalEvents::TEvSyncVersionResponse::TPtr& ev) { - SBS_LOG_T("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", current sync request# " << CurrentSyncRequest); + YDB_LOG_TRACE("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"request", CurrentSyncRequest}); if (ev->Sender != ReplicaSubscriber || ev->Cookie != CurrentSyncRequest) { return; @@ -588,6 +599,7 @@ class TSubscriberProxy: public TMonitorableActor<TDerived> { } void PassAway() override { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); if (ReplicaSubscriber) { this->Send(ReplicaSubscriber, new TEvents::TEvPoisonPill()); } @@ -608,8 +620,12 @@ public: return NKikimrServices::TActivity::SCHEME_BOARD_SUBSCRIBER_PROXY_ACTOR; } - static constexpr TStringBuf LogPrefix() { - return "proxy"sv; + NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "TSubscriberProxy"}, + {"actorActivityType", ActorActivityType()}, + {"selfId", this->SelfId()}, + {"path", Path}); } explicit TSubscriberProxy( @@ -633,6 +649,7 @@ public: } void Bootstrap(const TActorContext&) { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); TMonitorableActor<TDerived>::Bootstrap(); ReplicaSubscriber = this->RegisterWithSameMailbox(new TReplicaDerived(this->SelfId(), Replica, Path, DomainOwnerId)); @@ -640,6 +657,8 @@ public: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorStateFn", "StateWork"}); switch (ev->GetTypeRewrite()) { hFunc(NInternalEvents::TEvNotify, Handle); hFunc(NInternalEvents::TEvSyncVersionRequest, Handle); @@ -653,6 +672,8 @@ public: } STFUNC(StateSleep) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorStateFn", "StateSleep"}); switch (ev->GetTypeRewrite()) { hFunc(NInternalEvents::TEvSyncVersionRequest, HandleSleep); @@ -772,8 +793,8 @@ class TSubscriber: public TMonitorableActor<TDerived> { if (const auto* groupIdx = ProxyToGroupMap.FindPtr(proxy)) { responsesByGroup[*groupIdx]++; } else { - SBS_LOG_N("Previously received response sender is currently unknown" - << ": sender# " << proxy); + YDB_LOG_NOTICE("Previously received response sender is currently unknown", + {"sender", proxy}); } } for (size_t groupIdx : xrange(ProxyGroups.size())) { @@ -789,8 +810,8 @@ class TSubscriber: public TMonitorableActor<TDerived> { } bool MaybeRunVersionSync() { - SBS_LOG_T("MaybeRunVersionSync" - << ": delayed sync request# " << DelayedSyncRequest); + YDB_LOG_TRACE("MaybeRunVersionSync", + {"request", DelayedSyncRequest}); if (!DelayedSyncRequest) { return false; } @@ -810,18 +831,21 @@ class TSubscriber: public TMonitorableActor<TDerived> { } void Handle(NInternalEvents::TEvNotify::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); if (!IsValidNotification(Path, ev->Get()->GetRecord())) { - SBS_LOG_E("Suspicious " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_ERROR("Suspicious", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); return; } if (!ProxyToGroupMap.contains(ev->Sender)) { - SBS_LOG_E("Unknown " << ev->Get()->ToString() - << ": sender# " << ev->Sender); + YDB_LOG_ERROR("Unknown", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}); return; } @@ -844,27 +868,29 @@ class TSubscriber: public TMonitorableActor<TDerived> { TString reason; if (!State) { - SBS_LOG_N("Set up state" - << ": owner# " << Owner - << ", state# " << newestState); + YDB_LOG_NOTICE("Set up state", + {"owner", Owner}, + {"state", newestState}); State = newestState; } else if (State->LessThan(newestState, reason)) { - SBS_LOG_N("" << reason - << ": owner# " << Owner - << ", state# " << *State - << ", new state# " << newestState); + YDB_LOG_NOTICE("New", + {"reason", reason}, + {"owner", Owner}, + {"state", *State}, + {"newestState", newestState}); State = newestState; } else if (!State->Deleted && newestState.Deleted && newestState.Strong) { - SBS_LOG_N("Path was deleted" - << ": owner# " << Owner - << ", state# " << *State - << ", new state# " << newestState); + YDB_LOG_NOTICE("Path was deleted new", + {"owner", Owner}, + {"state", *State}, + {"newestState", newestState}); State = newestState; } else { - SBS_LOG_I("" << reason - << ": owner# " << Owner - << ", state# " << *State - << ", other state# " << newestState); + YDB_LOG_INFO("Other", + {"reason", reason}, + {"owner", Owner}, + {"state", *State}, + {"newestState", newestState}); return; } @@ -872,37 +898,39 @@ class TSubscriber: public TMonitorableActor<TDerived> { if (!record.GetIsDeletion()) { NKikimrScheme::TEvDescribeSchemeResult proto = selectedNotify->GetDescribeSchemeResult(); - SBS_LOG_T("Send notification to the owner" - << ": owner# " << Owner - << ", proto# " << proto.ShortDebugString()); + YDB_LOG_TRACE("Send notification to the owner", + {"owner", Owner}, + {"proto", proto.ShortDebugString()}); this->Send(Owner, BuildNotify<TSchemeBoardEvents::TEvNotifyUpdate>(record, std::move(proto))); } else { - SBS_LOG_T("Send deletion notification to the owner" - << ": owner# " << Owner); + YDB_LOG_TRACE("Send deletion notification to the owner", + {"owner", Owner}); this->Send(Owner, BuildNotify<TSchemeBoardEvents::TEvNotifyDelete>(record, record.GetStrong())); } } void Handle(NInternalEvents::TEvSyncRequest::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); if (ev->Sender != Owner) { - SBS_LOG_W("Suspicious " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", owner# " << Owner); + YDB_LOG_WARN("Suspicious", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"owner", Owner}); return; } EnqueueSyncRequest(ev); if (PendingSync) { - SBS_LOG_T("Pending sync"); + YDB_LOG_TRACE("Pending sync"); return; } if (!IsMajorityReached()) { - SBS_LOG_T("Haven't reached the majority"); + YDB_LOG_TRACE("Haven't reached the majority"); return; } @@ -914,42 +942,42 @@ class TSubscriber: public TMonitorableActor<TDerived> { } void Handle(NInternalEvents::TEvSyncVersionResponse::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString() - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}, + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); if (ev->Cookie != CurrentSyncRequest) { - SBS_LOG_D("Sync cookie mismatch" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", current cookie# " << CurrentSyncRequest); + YDB_LOG_DEBUG("Sync cookie mismatch", + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"currentCookie", CurrentSyncRequest}); return; } auto it = PendingSync.find(ev->Sender); if (it == PendingSync.end()) { - SBS_LOG_D("Unexpected sync response" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_DEBUG("Unexpected sync response", + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); return; } if (!ProxyToGroupMap.contains(ev->Sender)) { - SBS_LOG_E("Sync sender is unknown" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie); + YDB_LOG_ERROR("Sync sender is unknown", + {"sender", ev->Sender}, + {"cookie", ev->Cookie}); return; } const auto& record = ev->Get()->Record; const bool clusterStatesMatch = ClusterStatesMatch(ClusterState, record); if (!clusterStatesMatch) { - SBS_LOG_I("Cluster State mismatch in sync version response" - << ": sender# " << ev->Sender - << ", cookie# " << ev->Cookie - << ", subscriber cluster state# " << ClusterState - << ", replica cluster state# {" << record.GetClusterState().ShortDebugString() << "}" - ); + YDB_LOG_INFO("Cluster State mismatch in sync version response", + {"sender", ev->Sender}, + {"cookie", ev->Cookie}, + {"clusterState", ClusterState}, + {"state", record.GetClusterState().ShortDebugString()}); } PendingSync.erase(it); @@ -962,8 +990,8 @@ class TSubscriber: public TMonitorableActor<TDerived> { for (const auto& [proxy, partial] : ReceivedSync) { const auto* groupIdx = ProxyToGroupMap.FindPtr(proxy); if (!groupIdx) { - SBS_LOG_N("Previously received sync sender is currently unknown" - << ": sender# " << proxy); + YDB_LOG_NOTICE("Previously received sync sender is currently unknown", + {"sender", proxy}); continue; } if (!partial) { @@ -977,13 +1005,13 @@ class TSubscriber: public TMonitorableActor<TDerived> { const ui32 size = ProxyGroups[groupIdx].Proxies.size(); const ui32 half = size / 2; if (!IsSyncFinished(successesByGroup[groupIdx], failuresByGroup[groupIdx], size, half)) { - SBS_LOG_D("Sync is in progress" - << ": cookie# " << ev->Cookie - << ", ring group# " << groupIdx - << ", size# " << size - << ", half# " << half - << ", successes# " << successesByGroup[groupIdx] - << ", failures# " << failuresByGroup[groupIdx]); + YDB_LOG_DEBUG("Sync is in progress", + {"cookie", ev->Cookie}, + {"group", groupIdx}, + {"size", size}, + {"half", half}, + {"successes", successesByGroup[groupIdx]}, + {"failures", failuresByGroup[groupIdx]}); return; } syncIsComplete &= successesByGroup[groupIdx] > half; @@ -996,14 +1024,16 @@ class TSubscriber: public TMonitorableActor<TDerived> { << ", failures# " << failuresByGroup[groupIdx] << ", partial# " << !syncIsComplete; if (syncIsComplete) { - SBS_LOG_D(finalMessage); + YDB_LOG_DEBUG("Dump logPrefix, SelfId, path, finalMessage", + {"finalMessage", finalMessage}); } else { - SBS_LOG_N(finalMessage); + YDB_LOG_NOTICE("Dump logPrefix, SelfId, path, finalMessage", + {"finalMessage", finalMessage}); } } if (!syncIsComplete) { - SBS_LOG_W("Sync is incomplete in one of the ring groups" - << ": cookie# " << ev->Cookie); + YDB_LOG_WARN("Sync is incomplete in one of the ring groups", + {"cookie", ev->Cookie}); } this->Send(Owner, new NInternalEvents::TEvSyncResponse(Path, !syncIsComplete), 0, ev->Cookie); @@ -1020,13 +1050,14 @@ class TSubscriber: public TMonitorableActor<TDerived> { } void Handle(TEvStateStorage::TEvResolveReplicasList::TPtr& ev) { - SBS_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"ev", ev->Get()->ToString()}); const auto& replicaGroups = ev->Get()->ReplicaGroups; if (replicaGroups.empty()) { Y_ABORT_UNLESS(ProxyGroups.empty()); - SBS_LOG_E("Subscribe on unconfigured SchemeBoard"); + YDB_LOG_ERROR("Subscribe on unconfigured SchemeBoard"); this->Become(&TDerived::StateCalm); return; } @@ -1038,7 +1069,8 @@ class TSubscriber: public TMonitorableActor<TDerived> { } if (CurrentSyncRequest) { - SBS_LOG_I("Delay current sync request: " << CurrentSyncRequest); + YDB_LOG_INFO("Delay current sync", + {"request", CurrentSyncRequest}); DelayedSyncRequest = Max(DelayedSyncRequest, CurrentSyncRequest); CurrentSyncRequest = 0; } @@ -1130,7 +1162,7 @@ class TSubscriber: public TMonitorableActor<TDerived> { } void HandleUndelivered() { - SBS_LOG_E("Subscribe on unavailable SchemeBoard"); + YDB_LOG_ERROR("Subscribe on unavailable SchemeBoard"); this->Become(&TDerived::StateCalm); } @@ -1158,8 +1190,13 @@ public: return NKikimrServices::TActivity::SCHEME_BOARD_SUBSCRIBER_ACTOR; } - static constexpr TStringBuf LogPrefix() { - return "main"sv; + ::NActors::NStructuredLog::TStructuredMessage GetLogContext() { + using namespace NActors::NStructuredLog; + return YDB_LOG_CREATE_MESSAGE( + {"selfId", this->SelfId()}, + {"actorName", "TSubscriber"}, + {"path", Path} + );; } explicit TSubscriber( @@ -1175,6 +1212,10 @@ public: } void Bootstrap(const TActorContext&) { + YDB_LOG_CREATE_CONTEXT( + GetLogContext(), + {"actorAction", "bootstrap"}, + ); TMonitorableActor<TDerived>::Bootstrap(); const TActorId proxy = MakeStateStorageProxyID(); @@ -1183,6 +1224,10 @@ public: } STATEFN(StateResolve) { + YDB_LOG_CREATE_CONTEXT( + GetLogContext(), + {"actorAction", "StateResolve"}, + ); switch (ev->GetTypeRewrite()) { hFunc(NInternalEvents::TEvSyncRequest, EnqueueSyncRequest); // from owner (cache) @@ -1196,6 +1241,10 @@ public: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT( + GetLogContext(), + {"actorAction", "StateWork"}, + ); switch (ev->GetTypeRewrite()) { hFunc(TEvStateStorage::TEvResolveReplicasList, Handle); @@ -1210,6 +1259,10 @@ public: } STATEFN(StateCalm) { + YDB_LOG_CREATE_CONTEXT( + GetLogContext(), + {"actorAction", "StateCalm"}, + ); switch (ev->GetTypeRewrite()) { hFunc(TSchemeBoardMonEvents::TEvInfoRequest, Handle); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp index feb502e970a..0e0f50ba5f4 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_backup.cpp @@ -192,6 +192,11 @@ struct TBackup { } static bool NeedToBill(const TPathId& pathId, TOperationContext& context) { + if (context.SS->ColumnTables.contains(pathId)) { + auto table = context.SS->ColumnTables.at(pathId); + return table->BackupSettings.GetNeedToBill(); + } + Y_ABORT_UNLESS(context.SS->Tables.contains(pathId)); auto table = context.SS->Tables.at(pathId); return table->BackupSettings.GetNeedToBill(); diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 8fff773c8be..240c0750a78 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -8010,7 +8010,7 @@ void TSchemeShard::Handle(TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, // Control op completed; finalize the tracked record. Execute(CreateTxFullBackupProgress(ui64(txId)), ctx); } - if (TxIdToSetColumnConstraintOperations.contains(txId)) { + if (TxIdToSetColumnConstraintOperations.contains(txId) || TxIdToDependentSetColumnConstraint.contains(txId)) { Execute(CreateTxReplyCompletedSetColumnConstraint(txId), ctx); executed = true; } diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index 1fa19209ba2..90ae47b49ad 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -1993,6 +1993,10 @@ public: THashMap<TString, std::shared_ptr<TSetColumnConstraintOperationInfo>> SetColumnConstraintOperationsByUid; TSet<std::pair<TInstant, TIndexBuildId>> SetColumnConstraintOperationsByTime; THashMap<TTxId, TIndexBuildId> TxIdToSetColumnConstraintOperations; + // txIds of concurrent operations (e.g. a backup CopyTable) that a + // SetColumnConstraint operation is currently waiting on before it can retry + // one of its own internal sub-transactions. + THashMap<TTxId, THashSet<TIndexBuildId>> TxIdToDependentSetColumnConstraint; void PersistCreateSetColumnConstraint(NIceDb::TNiceDb& db, const TSetColumnConstraintOperationInfo& indexInfo); void PersistSetColumnConstraintState(NIceDb::TNiceDb& db, const TSetColumnConstraintOperationInfo& indexInfo); diff --git a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp index 288caedd7c8..8e11a524455 100644 --- a/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_set_column_constraint__progress.cpp @@ -205,45 +205,109 @@ public: << ", status# " << NKikimrScheme::EStatus_Name(record.GetStatus())); NIceDb::TNiceDb db(txc.DB); - - auto replyOnCreation = [&] { + bool shouldForget = false; + auto replyOnCreation = [&]() -> void { auto statusCode = TranslateStatusCode(record.GetStatus()); if (statusCode != Ydb::StatusIds::SUCCESS) { - // TODO(flown4qqqq): persist issue - // TODO(flown4qqqq): forget operation on error - // TODO(flown4qqqq): EraseBuildInfo(operationInfo); + shouldForget = true; } ReplyOnCreation(operationInfo, statusCode); - return true; }; - if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::Locking) { - Y_ENSURE(txId == operationInfo.LockTxId); - operationInfo.LockTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintLockTxStatus(db, operationInfo); + // Most modifications of the main table are forbidden while SetColumnConstraint + // holds its lock, however copying the table (e.g. for a backup) is allowed to + // proceed concurrently (see TCopyTable::Propose: CheckLocks is skipped for + // IsBackup copies). While such a copy is in flight the table has + // PathStateCopying, and our own internal sub-transactions (which are gated by + // NotUnderOperation()) may transiently fail with StatusMultipleModifications. + // Just like build_index does, retry the very same proposal once the copy is + // done instead of failing the whole operation. + // Waiting for the conflicting operation's completion notification, not for + // an immediate self-retry - see the comment on shouldRetry() below for why. + bool waitingForDependency = false; - if (!replyOnCreation()) { + auto shouldRetry = [&]() { + if (record.GetStatus() != NKikimrScheme::StatusMultipleModifications) { return false; } - } else if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::LockingNullWrites) { - Y_ENSURE(txId == operationInfo.LockNullWritesTxId); - operationInfo.LockNullWritesTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintLockNullWritesTxStatus(db, operationInfo); - } else if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::Finishing) { - Y_ENSURE(txId == operationInfo.UnlockNullWritesTxId); - operationInfo.UnlockNullWritesTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintUnlockNullWritesTxStatus(db, operationInfo); - } else if (operationInfo.OperationState == TSetColumnConstraintOperationInfo::EOperationState::Unlocking) { - Y_ENSURE(txId == operationInfo.UnlockTxId); - operationInfo.UnlockTxStatus = record.GetStatus(); - Self->PersistSetColumnConstraintUnlockTxStatus(db, operationInfo); - } else { - Y_UNREACHABLE(); + + auto it = Self->PathsById.find(operationInfo.TablePathId); + if (it == Self->PathsById.end() || it->second->PathState != NKikimrSchemeOp::EPathStateCopying) { + return false; + } + + auto copyTxId = it->second->LastTxId; + LOG_I("TTxReplyModify : Waiting for txId " << copyTxId << " to retry SetColumnConstraint id# " << BuildId); + operationInfo.DependencyTxIds.insert(copyTxId); + Self->TxIdToDependentSetColumnConstraint[copyTxId].insert(BuildId); + Send(Self->SelfId(), MakeHolder<TEvSchemeShard::TEvNotifyTxCompletion>(ui64(copyTxId))); + return true; + }; + + switch (operationInfo.OperationState) { + case TSetColumnConstraintOperationInfo::EOperationState::Locking: { + Y_ENSURE(txId == operationInfo.LockTxId); + operationInfo.LockTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintLockTxStatus(db, operationInfo); + replyOnCreation(); + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::LockingNullWrites: { + Y_ENSURE(txId == operationInfo.LockNullWritesTxId); + if (shouldRetry()) { + operationInfo.LockNullWritesTxId = InvalidTxId; + Self->PersistSetColumnConstraintResetSubState(db, operationInfo); + waitingForDependency = true; + } else { + operationInfo.LockNullWritesTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintLockNullWritesTxStatus(db, operationInfo); + } + + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::Finishing: { + Y_ENSURE(txId == operationInfo.UnlockNullWritesTxId); + if (shouldRetry()) { + operationInfo.UnlockNullWritesTxId = InvalidTxId; + Self->PersistSetColumnConstraintResetSubState(db, operationInfo); + waitingForDependency = true; + } else { + operationInfo.UnlockNullWritesTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintUnlockNullWritesTxStatus(db, operationInfo); + } + + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::Unlocking: { + Y_ENSURE(txId == operationInfo.UnlockTxId); + if (shouldRetry()) { + operationInfo.UnlockTxId = InvalidTxId; + Self->PersistSetColumnConstraintResetSubState(db, operationInfo); + waitingForDependency = true; + } else { + operationInfo.UnlockTxStatus = record.GetStatus(); + Self->PersistSetColumnConstraintUnlockTxStatus(db, operationInfo); + } + + break; + } + case TSetColumnConstraintOperationInfo::EOperationState::Invalid: + case TSetColumnConstraintOperationInfo::EOperationState::Validating: + case TSetColumnConstraintOperationInfo::EOperationState::Done: { + Y_UNREACHABLE(); + } + } + + if (!waitingForDependency) { + Progress(BuildId); + } + + if (shouldForget) { + Self->ForgetSetColumnConstraint(db, operationInfo); } - Progress(BuildId); return true; } @@ -271,6 +335,23 @@ public: const auto txId = CompletedTxId; auto* operationIdPtr = Self->TxIdToSetColumnConstraintOperations.FindPtr(txId); + + if (Self->TxIdToDependentSetColumnConstraint.contains(txId)) { + THashSet<TIndexBuildId> deps = std::move(Self->TxIdToDependentSetColumnConstraint.at(txId)); + Self->TxIdToDependentSetColumnConstraint.erase(txId); + for (auto& dependentBuildId : deps) { + LOG_I("TTxReplyCompleted txId: " << txId << " : trying to resume dependent SetColumnConstraint " << dependentBuildId); + if (auto* dependentOperationInfoPtr = Self->SetColumnConstraintOperations.FindPtr(dependentBuildId)) { + auto& dependentOperationInfo = **dependentOperationInfoPtr; + dependentOperationInfo.DependencyTxIds.erase(txId); + Progress(dependentBuildId); + } + } + if (!operationIdPtr) { + return true; + } + } + if (!operationIdPtr) { LOG_I("TTxReplyCompleted: operation not found, txId# " << txId); return true; @@ -638,6 +719,11 @@ public: Y_ENSURE(operationInfoPtr); auto& operationInfo = *operationInfoPtr->get(); + if (!operationInfo.DependencyTxIds.empty()) { + LOG_N("TTxProgressSetColumnConstraint: " << BuildId << ": waiting for dependencies"); + return true; + } + LOG_D("TTxProgressSetColumnConstraint::DoExecute, id# " << BuildId << "; OperationState = " << ToString(operationInfo.OperationState) << "; IsCancelled = " << operationInfo.IsCancelled); diff --git a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp index ab4ab2f3995..404f5b0e760 100644 --- a/ydb/core/tx/schemeshard/ut_export/ut_export.cpp +++ b/ydb/core/tx/schemeshard/ut_export/ut_export.cpp @@ -54,7 +54,8 @@ namespace { void Run(TTestBasicRuntime& runtime, TTestEnv& env, const std::variant<TVector<TString>, TTablesWithAttrs>& tablesVar, const TString& request, Ydb::StatusIds::StatusCode expectedStatus = Ydb::StatusIds::SUCCESS, const TString& dbName = "/MyRoot", bool serverless = false, const TString& userSID = "", const TString& peerName = "", - const TVector<TString>& cdcStreams = {}, bool checkAutoDropping = false) { + const TVector<TString>& cdcStreams = {}, bool checkAutoDropping = false, + const TVector<TString>& columnTables = {}) { TTablesWithAttrs tables; @@ -149,6 +150,11 @@ namespace { env.TestWaitNotification(runtime, txId, schemeshardId); } + for (const auto& table : columnTables) { + TestCreateColumnTable(runtime, schemeshardId, ++txId, dbName, table); + env.TestWaitNotification(runtime, txId, schemeshardId); + } + for (const auto& cdcStream : cdcStreams) { TestCreateCdcStream(runtime, schemeshardId, ++txId, dbName, cdcStream); env.TestWaitNotification(runtime, txId, schemeshardId); @@ -5296,4 +5302,92 @@ CREATE EXTERNAL TABLE IF NOT EXISTS `ExternalTable` ( NLs::CheckColumnTableMultiColumnStatistics("s1", {"data"}, {NKikimrSchemeOp::EMultiColumnStatisticsType::COUNT_MIN_SKETCH}), }); } + + Y_UNIT_TEST(ShouldWriteBillRecordOnColumnTableServerlessDb) { + Env(); + Runtime().GetAppData().FeatureFlags.SetEnableColumnTablesBackup(true); + + TVector<TString> billRecords; + Runtime().SetObserverFunc([&billRecords](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() != NMetering::TEvMetering::EvWriteMeteringJson) { + return TTestActorRuntime::EEventAction::PROCESS; + } + + billRecords.push_back(ev->Get<NMetering::TEvMetering::TEvWriteMeteringJson>()->MeteringJson); + return TTestActorRuntime::EEventAction::PROCESS; + }); + + Run(Runtime(), Env(), TVector<TString>{}, Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/User/ColumnTable" + destination_prefix: "" + } + } + )", S3Port()), Ydb::StatusIds::SUCCESS, "/MyRoot/User", true, "", "", {}, false, { + R"( + Name: "ColumnTable" + ColumnShardCount: 1 + Schema { + Columns { Name: "timestamp" Type: "Timestamp" NotNull: true } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: "timestamp" + } + )" + }); + + if (billRecords.empty()) { + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&billRecords](IEventHandle&) -> bool { + return !billRecords.empty(); + }); + Runtime().DispatchEvents(opts); + } + + UNIT_ASSERT_VALUES_EQUAL(billRecords.size(), 1); + UNIT_ASSERT_STRING_CONTAINS(billRecords[0], "\"cloud_id\":\"CLOUD_ID_VAL\""); + UNIT_ASSERT_STRING_CONTAINS(billRecords[0], "\"folder_id\":\"FOLDER_ID_VAL\""); + UNIT_ASSERT_STRING_CONTAINS(billRecords[0], "\"resource_id\":\"DATABASE_ID_VAL\""); + UNIT_ASSERT_STRING_CONTAINS(billRecords[0], "\"schema\":\"ydb.serverless.requests.v1\""); + } + + Y_UNIT_TEST(ShouldNotWriteBillRecordOnColumnTableCommonDb) { + Env(); + Runtime().GetAppData().FeatureFlags.SetEnableColumnTablesBackup(true); + + TVector<TString> billRecords; + Runtime().SetObserverFunc([&billRecords](TAutoPtr<IEventHandle>& ev) { + if (ev->GetTypeRewrite() != NMetering::TEvMetering::EvWriteMeteringJson) { + return TTestActorRuntime::EEventAction::PROCESS; + } + + billRecords.push_back(ev->Get<NMetering::TEvMetering::TEvWriteMeteringJson>()->MeteringJson); + return TTestActorRuntime::EEventAction::PROCESS; + }); + + Run(Runtime(), Env(), TVector<TString>{}, Sprintf(R"( + ExportToS3Settings { + endpoint: "localhost:%d" + scheme: HTTP + items { + source_path: "/MyRoot/User/ColumnTable" + destination_prefix: "" + } + } + )", S3Port()), Ydb::StatusIds::SUCCESS, "/MyRoot/User", false, "", "", {}, false, { + R"( + Name: "ColumnTable" + ColumnShardCount: 1 + Schema { + Columns { Name: "timestamp" Type: "Timestamp" NotNull: true } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: "timestamp" + } + )" + }); + + UNIT_ASSERT(billRecords.empty()); + } } diff --git a/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp b/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp index a444ad78332..c73f6e485b6 100644 --- a/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp +++ b/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp @@ -2383,5 +2383,459 @@ Y_UNIT_TEST_SUITE(SetNotNullTest) { Cerr << "=== Successfully cancelled at stage: " << stage.StageName << " ===" << Endl; } } + + Y_UNIT_TEST(MoveTableFailsWhileValidating) { + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString movedTablePath = root + "/TableMoved"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + // Block validation requests to datashards so that the operation is stuck + // in the Validating stage (the table lock is already held at this point). + TBlockEvents<TEvDataShard::TEvValidateRowConditionRequest> validateBlocker(runtime); + + ui64 setConstraintTxId = ++txId; + AsyncSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + runtime.WaitFor("block validate request", [&]{ return validateBlocker.size() > 0; }); + + // While SetNotNull is stuck validating shards (table is locked), attempt to move + // the table. MoveTable must be rejected immediately at Propose (no TxState is even + // created for it), because CheckLocks() sees the path is locked by the SetNotNull + // operation and MoveTable's transaction carries no matching LockGuard. + ui64 moveTxId = ++txId; + TestMoveTable(runtime, moveTxId, tablePath, movedTablePath, + {NKikimrScheme::StatusMultipleModifications}); + + // The table must still be exactly where it was - move never actually started. + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathExist}); + TestDescribeResult(DescribePath(runtime, movedTablePath), {NLs::PathNotExist}); + + // Let SetNotNull finish normally. + validateBlocker.Stop(); + validateBlocker.Unblock(); + + env.TestWaitNotification(runtime, setConstraintTxId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + + // Now that the lock is released, the very same move must succeed. + TestMoveTable(runtime, ++txId, tablePath, movedTablePath); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathNotExist}); + TestDescribeResult(DescribePath(runtime, movedTablePath), {NLs::PathExist}); + } + + Y_UNIT_TEST(SetNotNullFailsWhileMoveTableInProgress) { + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString movedTablePath = root + "/TableMoved"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + // Keep MoveTable stuck in ConfigureParts by blocking the flat scheme tx + // it sends to the source table's datashards. By the time this event is + // observed, MoveTable's local Propose has already committed - the source + // path is already marked EPathStateMoving. + TBlockEvents<TEvDataShard::TEvProposeTransaction> moveProposeBlocker(runtime); + + ui64 moveTxId = ++txId; + AsyncMoveTable(runtime, moveTxId, tablePath, movedTablePath); + + runtime.WaitFor("block move propose to datashards", [&]{ return moveProposeBlocker.size() > 0; }); + + // The source path is now "under operation" (moving). SetNotNull's internal + // CreateLock sub-operation must be rejected right away with + // StatusMultipleModifications (translated to Ydb::StatusIds::OVERLOADED), + // so the whole SetColumnConstraint request fails and never gets to lock the + // table or validate anything. + auto response = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + Cerr << "SET COLUMN CONSTRAINT RESPONSE (while moving): " << response.ShortDebugString() << Endl; + + UNIT_ASSERT_VALUES_UNEQUAL_C( + response.GetStatus(), + Ydb::StatusIds::SUCCESS, + response.ShortDebugString()); + UNIT_ASSERT_VALUES_EQUAL_C( + response.GetStatus(), + Ydb::StatusIds::OVERLOADED, + response.ShortDebugString()); + + // Column must remain nullable - SetNotNull never actually ran. + TestCheckColumnsNotNull(runtime, tablePath, {{"value", false}}); + + // Let MoveTable finish normally. + moveProposeBlocker.Stop(); + moveProposeBlocker.Unblock(); + env.TestWaitNotification(runtime, moveTxId); + + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathNotExist}); + TestDescribeResult(DescribePath(runtime, movedTablePath), {NLs::PathExist}); + + // Now that the move is done, SetNotNull on the (now moved) table must succeed. + auto secondResponse = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + movedTablePath, + {"value"}); + + UNIT_ASSERT_VALUES_EQUAL_C( + secondResponse.GetStatus(), + Ydb::StatusIds::SUCCESS, + secondResponse.ShortDebugString()); + + env.TestWaitNotification(runtime, txId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, movedTablePath, {{"value", true}}); + } + + Y_UNIT_TEST(CopyTableNoBackupWhileValidating) { + /* + The test is based on the following idea. + + There are essentially two different ways CopyTable can be invoked: + + 1. As a standalone operation initiated by the user. + 2. Internally, as part of a Backup meta-operation. + + This test covers the first scenario. + + For a standalone CopyTable, it is fine to fail the request and tell the user that the table is currently locked + by another long-running operation. In that scenario, CopyTable is a single explicit operation that the user can simply retry. + This is also the case where CopyTable is expected to check whether the table is locked by other schema operations. + */ + + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString copyTablePath = root + "/TableBackup"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TBlockEvents<TEvDataShard::TEvValidateRowConditionRequest> validateBlocker(runtime); + + ui64 setConstraintTxId = ++txId; + AsyncSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + runtime.WaitFor("block validate request", [&]{ return validateBlocker.size() > 0; }); + + ui64 copyTx1 = ++txId; + TestConsistentCopyTables(runtime, copyTx1, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableBackup" + IsBackup: false + } + )", {NKikimrScheme::StatusMultipleModifications}); + env.TestWaitNotification(runtime, copyTx1); + + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathNotExist}); + + validateBlocker.Stop(); + validateBlocker.Unblock(); + + env.TestWaitNotification(runtime, setConstraintTxId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + + ui64 copyTx2 = ++txId; + TestConsistentCopyTables(runtime, copyTx2, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableBackup" + IsBackup: false + } + )"); + env.TestWaitNotification(runtime, copyTx2); + + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathExist}); + TestCheckColumnsNotNull(runtime, copyTablePath, {{"value", true}}); + } + + Y_UNIT_TEST(CopyTableWithBackupWhileValidating) { + /* + The test is based on the following idea. + + There are essentially two different ways CopyTable can be invoked: + + 1. As a standalone operation initiated by the user. + 2. Internally, as part of a Backup meta-operation. + + This test covers the second scenario. + + When CopyTable is invoked as part of Backup, it should not fail just because the table is locked by another + long-running operation. If ConsistentCopyTable gets blocked this way during a backup, it may lead to a large + number of retries. Each retry effectively means trying to copy all tables in the database again, which is a very + expensive and long-running operation. + + Therefore, the practical way to avoid such retries is to make the new long-running operation pause and wait + until the backup's CopyTable finishes. + */ + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString copyTablePath = root + "/TableBackup"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TBlockEvents<TEvDataShard::TEvValidateRowConditionRequest> validateBlocker(runtime); + + ui64 setConstraintTxId = ++txId; + AsyncSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + runtime.WaitFor("block validate request", [&]{ return validateBlocker.size() > 0; }); + + ui64 copyTx = ++txId; + TestConsistentCopyTables(runtime, copyTx, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableBackup" + IsBackup: true + } + )"); + env.TestWaitNotification(runtime, copyTx); + + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathExist}); + + validateBlocker.Stop(); + validateBlocker.Unblock(); + + env.TestWaitNotification(runtime, setConstraintTxId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + TestCheckColumnsNotNull(runtime, copyTablePath, {{"value", false}}); + + { + const auto describeResult = DescribePath(runtime, tablePath); + const auto& columns = describeResult.GetPathDescription().GetTable().GetColumns(); + for (const auto& column : columns) { + if (column.GetName() == "value") { + UNIT_ASSERT_VALUES_EQUAL(column.GetSetNotNullInProgress(), false); + } + } + } + } + + Y_UNIT_TEST_TWIN(SetNotNullFailsWhileCopyTableInProgress, IsBackup) { + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + TString root = "/MyRoot"; + + ui64 txId = 100; + TString tableName = "Table"; + TString tablePath = root + "/" + tableName; + TString copyTablePath = root + "/TableCopy"; + + TestCreateTable(runtime, ++txId, root, TStringBuilder() << R"( + Name: ")" << tableName << R"(" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + // Keep CopyTable stuck in ConfigureParts by blocking the flat scheme tx + // it sends to the source table's datashards. By the time this event is + // observed, CopyTable's local Propose has already committed - the source + // path is already marked EPathStateCopying. + TBlockEvents<TEvDataShard::TEvProposeTransaction> copyProposeBlocker(runtime); + + ui64 copyTxId = ++txId; + if (IsBackup) { + AsyncConsistentCopyTables(runtime, copyTxId, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableCopy" + IsBackup: true + } + )"); + } else { + AsyncConsistentCopyTables(runtime, copyTxId, root, R"( + CopyTableDescriptions { + SrcPath: "/MyRoot/Table" + DstPath: "/MyRoot/TableCopy" + IsBackup: false + } + )"); + } + + runtime.WaitFor("block copy propose to datashards", [&]{ return copyProposeBlocker.size() > 0; }); + + // The source path is now "under operation" (copying). SetNotNull's internal + // CreateLock sub-operation must be rejected right away with + // StatusMultipleModifications (translated to Ydb::StatusIds::OVERLOADED), + // so the whole SetColumnConstraint request fails and never gets to lock the + // table or validate anything. + auto response = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + Cerr << "SET COLUMN CONSTRAINT RESPONSE (while copying): " << response.ShortDebugString() << Endl; + + UNIT_ASSERT_VALUES_EQUAL_C( + response.GetStatus(), + Ydb::StatusIds::OVERLOADED, + response.ShortDebugString()); + + // Column must remain nullable - SetNotNull never actually ran. + TestCheckColumnsNotNull(runtime, tablePath, {{"value", false}}); + + // Let CopyTable finish normally. + copyProposeBlocker.Stop(); + copyProposeBlocker.Unblock(); + env.TestWaitNotification(runtime, copyTxId); + + TestDescribeResult(DescribePath(runtime, tablePath), {NLs::PathExist}); + TestDescribeResult(DescribePath(runtime, copyTablePath), {NLs::PathExist}); + + // Now that the copy is done, SetNotNull on the source table must succeed. + auto secondResponse = TestSetColumnConstraint( + runtime, ++txId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + UNIT_ASSERT_VALUES_EQUAL_C( + secondResponse.GetStatus(), + Ydb::StatusIds::SUCCESS, + secondResponse.ShortDebugString()); + + env.TestWaitNotification(runtime, txId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + } + + Y_UNIT_TEST(SetNotNullWithUsualSchemeOperation) { + // As an example of a 'usual' scheme operation, this test uses TRUNCATE TABLE. + + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + + ui64 txId = 100; + + TString root = "/MyRoot"; + TString tablePath = root + "/Table"; + + TestCreateTable(runtime, ++txId, root, R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + TBlockEvents<TEvDataShard::TEvProposeTransaction> truncateTableBlocker(runtime); + const ui64 truncateTxId = ++txId; + AsyncTruncateTable(runtime, truncateTxId, root, "Table", TTestTxConfig::SchemeShard); + runtime.WaitFor("block truncate propose to datashards", [&]{ return truncateTableBlocker.size() > 0; }); + + ui64 setConstraintTxId = ++txId; + auto response = TestSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + UNIT_ASSERT_VALUES_EQUAL_C( + response.GetStatus(), + Ydb::StatusIds::OVERLOADED, + response.ShortDebugString()); + + // Column must remain nullable - SetNotNull never actually ran. + TestCheckColumnsNotNull(runtime, tablePath, {{"value", false}}); + + truncateTableBlocker.Stop(); + truncateTableBlocker.Unblock(); + env.TestWaitNotification(runtime, truncateTxId); + } } // Y_UNIT_TEST_SUITE(SetNotNullTest) diff --git a/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make b/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make index 13dfda64b9e..9df7e2698c9 100644 --- a/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make +++ b/ydb/core/tx/schemeshard/ut_set_column_constraint/ya.make @@ -1,6 +1,8 @@ UNITTEST_FOR(ydb/core/tx/schemeshard) FORK_SUBTESTS() +SPLIT_FACTOR(60) +SIZE(MEDIUM) PEERDIR( library/cpp/getopt diff --git a/ydb/docs/en/core/integrations/gui/dbeaver-plugin.md b/ydb/docs/en/core/integrations/gui/dbeaver-plugin.md index 6a578cfe901..1ba4bab21e3 100644 --- a/ydb/docs/en/core/integrations/gui/dbeaver-plugin.md +++ b/ydb/docs/en/core/integrations/gui/dbeaver-plugin.md @@ -1,145 +1,180 @@ -# Connecting to {{ ydb-short-name }} Using the DBeaver Plugin +# Connecting to {{ ydb-short-name }} using the DBeaver plugin -[DBeaver](https://dbeaver.com) is a free cross-platform open-source database management tool that provides a visual interface for connecting to various databases and executing SQL queries. +[DBeaver](https://dbeaver.com) is a free, cross-platform, open-source database management tool that provides a visual interface for connecting to various databases and executing SQL queries. -[YDB DBeaver Plugin](https://github.com/ydb-platform/ydb-dbeaver-plugin) is a DBeaver extension with native support for {{ ydb-short-name }}. Unlike [connecting via a JDBC driver](dbeaver.md), the plugin provides a specialized interface for working with {{ ydb-short-name }} objects: a hierarchical navigator for tables, topics, views, external data sources, support for all authentication methods, a [YQL](../../concepts/glossary.md#yql) editor, query execution plan visualization, session and cluster monitoring, access control (ACL) management, and other features. +[YDB DBeaver Plugin](https://github.com/ydb-platform/ydb-dbeaver-plugin) is a DBeaver extension with native support for {{ ydb-short-name }}. Unlike [connecting via a JDBC driver](dbeaver.md), the plugin provides a specialized interface for working with {{ ydb-short-name }} objects: a hierarchical navigator for tables, topics, views, external data sources, support for all authentication methods, a [YQL](../../concepts/glossary.md#yql) editor, execution plan visualization, session and cluster monitoring, access control (ACL) management, and other features. -## Key Features of the Plugin {#features} +## Key features of the plugin {#features} -- Connecting to {{ ydb-name }} with all [authentication methods](../../security/authentication.md): anonymous, static, token-based, service account, metadata-based. +- Connecting to {{ ydb-name }} with all [authentication](../../security/authentication.md) methods: anonymous, static, token-based, service account, and metadata. - Hierarchical object navigator: tables, topics, external data sources, external tables, views. - System objects: [system views](../../dev/system-views.md) (`.sys`), [resource pools](../../concepts/glossary.md#resource-pool), and [resource pool classifiers](../../concepts/glossary.md#resource-pool-classifier). -- [YQL](../../concepts/glossary.md#yql) editor with syntax highlighting for over 150 keywords and built-in functions. -- Visualization of the query execution plan (`EXPLAIN` / `EXPLAIN ANALYZE`). -- Monitoring of active sessions via [`.sys/query_sessions`](../../dev/system-views.md#query-sessions). -- Cluster dashboard: CPU load, disk usage, memory usage, network traffic, node status (updated every 5 seconds). +- [YQL](../../concepts/glossary.md#yql) editor with syntax highlighting for 150+ keywords and built-in functions. +- Query execution plan visualization (`EXPLAIN` / `EXPLAIN ANALYZE`). +- Active session monitoring via [`.sys/query_sessions`](../../dev/system-views.md#query-sessions). +- Cluster dashboard: CPU load, disk usage, memory usage, network traffic, node status (updates every 5 seconds). - [Access control (ACL)](../../security/authorization.md#right) management: granting, revoking, and viewing permissions. -- Management of [streaming queries](../../concepts/glossary.md#streaming-query): viewing, modifying, starting, stopping. +- [Streaming query](../../concepts/glossary.md#streaming-query) management: viewing, modifying, starting, and stopping. - [Federated queries](../../concepts/query_execution/federated_query/index.md) via external data sources (S3, databases). -- [SQL Dialect Converter](../sql-dialect-converter.md) from other dialects (PostgreSQL, MySQL, ClickHouse, etc.) to YQL. -- Specialized editors for `JSON`, `JSONDOCUMENT`, `YSON` data types. +- [SQL query converter](../sql-translation/sql-dialect-converter.md) from other dialects (PostgreSQL, MySQL, ClickHouse, and others) to YQL. +- Specialized editors for `JSON`, `JSONDOCUMENT`, and `YSON` data types. ## Requirements {#requirements} The plugin requires DBeaver Community Edition version 24.x or later. -## Installing the Plugin {#installation} +## Installing the plugin {#installation} -The plugin is installed via DBeaver's extension installation mechanism from a URL repository, which ensures automatic updates. +The plugin can be installed in two ways: + +- **Via P2 repository URL (recommended)**: DBeaver downloads the plugin from cloud storage and remembers the source for subsequent automatic updates. +- **From a ZIP archive (GitHub Releases)**: download the archive and install via a local file. Suitable for networks where external Eclipse repositories are unavailable, or for reproducible installation of a specific version. Automatic updates are not supported with this method. + +### Installation via P2 repository URL {#installation-url} 1. Open DBeaver. In the top menu, select **Help → Install New Software...**. - {% cut "Screenshot" %} + {% cut "Screenshot" %} + +  -  + {% endcut %} +2. Click the **Add...** button to the right of the **Work with:** field. - {% endcut %} + {% cut "Screenshot" %} -1. Click the **Add...** button to the right of the **Work with:** field. +  - {% cut "Screenshot" %} + {% endcut %} +3. In the **Add Repository** window that opens, specify a repository name (e.g., `YDB Plugin`) and paste the following URL into the **Location** field: -  - {% endcut %} + ```text + https://storage.yandexcloud.net/ydb-dbeaver-plugin + ``` -1. In the **Add Repository** window that opens, specify a repository name (for example, `YDB Plugin`) and paste the following URL into the **Location** field: - ```text - https://storage.yandexcloud.net/ydb-dbeaver-plugin - ``` + Click **Add**. DBeaver will load the repository metadata. - Click **Add**. DBeaver will load the repository metadata. + {% cut "Screenshot" %} - {% cut "Screenshot" %} +  -  + {% endcut %} - {% endcut %} + {% note warning %} -1. The **DBeaver YDB Support** category will appear in the component list. Select it and click **Next >**. + During installation, DBeaver downloads not only the plugin itself but also its dependencies (OSGi components) from Eclipse project servers and other external repositories. If these resources are not accessible from your network, contact your network administrator or use [installation from a ZIP archive](#installation-zip). - {% cut "Screenshot" %} + {% endnote %} +4. A category **DBeaver YDB Support** will appear in the component list. Check it and click **Next >**. -  + {% cut "Screenshot" %} - {% endcut %} +  -1. On the **Install Details** screen, make sure both components (`org.jkiss.dbeaver.ext.ydb` and `org.jkiss.dbeaver.ext.ydb.ui`) are listed, then click **Next >**. + {% endcut %} +5. On the **Install Details** screen, make sure both components (`org.jkiss.dbeaver.ext.ydb` and `org.jkiss.dbeaver.ext.ydb.ui`) are in the list, and click **Next >**. - {% cut "Screenshot" %} + {% cut "Screenshot" %} -  +  - {% endcut %} + {% endcut %} +6. DBeaver may show a warning about unsigned content. This is expected behavior — the plugin's JAR files are not signed with a commercial certificate. Click **Install Anyway**. -1. DBeaver may display a warning about unsigned content. This is expected behavior — the plugin's JAR files are not signed with a commercial certificate. Click **Install Anyway**. + {% note info %} - {% note info %} + Eclipse, on which DBeaver is based, verifies JAR file signatures to confirm authenticity. This open-source plugin is distributed without a signature; the source code is available in the [repository](https://github.com/ydb-platform/ydb-dbeaver-plugin). - Eclipse, on which DBeaver is based, checks JAR file signatures to verify authenticity. This open-source plugin is distributed without a signature; the source code is available in the [repository](https://github.com/ydb-platform/ydb-dbeaver-plugin). + {% endnote %} +7. Review the license (Apache License 2.0), select **I accept the terms of the license agreements**, and click **Finish**. - {% endnote %} + {% cut "Screenshot" %} -1. Review the license (Apache License 2.0), select **I accept the terms of the license agreements**, and click **Finish**. +  - {% cut "Screenshot" %} + {% endcut %} +8. DBeaver will install the plugin and prompt you to restart. Click **Restart Now**. After restart, the plugin will be active. -  +### Installation from a ZIP archive (GitHub Releases) {#installation-zip} - {% endcut %} +1. Go to the [GitHub Releases page](https://github.com/ydb-platform/ydb-dbeaver-plugin/releases) and download the `ydb-dbeaver-plugin-*.zip` file of the required version. +2. Open DBeaver. In the top menu, select **Help → Install New Software...**. +3. Click the **Add...** button to the right of the **Work with:** field. +4. In the **Add Repository** window that opens, click **Archive...**, specify the path to the downloaded ZIP file, and click **Add**. +5. The **DBeaver YDB Support** category will appear in the component list. Check it. -1. DBeaver will install the plugin and prompt you to restart. Click **Restart Now**. After restarting, the plugin will become active. + {% note warning %} -## Creating a Connection to {{ ydb-name }} {#connection} + Before clicking **Next >**, uncheck the **Contact all update sites during install to find required software** checkbox at the bottom of the **Install New Software** window. If this checkbox is enabled, DBeaver contacts all registered P2 repositories (`dbeaver.io`, `eclipse.org`, and others) when calculating dependencies, and if they are unavailable or respond slowly, the installation dialog hangs at the **Calculating requirements and dependencies** step. The plugin only uses DBeaver components already installed locally, so contacting other repositories during installation is not required. + + {% endnote %} +6. Click **Next >**. +7. On the **Install Details** screen, make sure both components (`org.jkiss.dbeaver.ext.ydb` and `org.jkiss.dbeaver.ext.ydb.ui`) are present in the list, and click **Next >**. +8. DBeaver may show a warning about unsigned content. Click **Install Anyway**. +9. Review the license (Apache License 2.0), select **I accept the terms of the license agreements**, and click **Finish**. +10. DBeaver will install the plugin and prompt you to restart. Click **Restart Now**. After the restart, the plugin becomes active. + +{% note warning %} + +When installing from a ZIP archive, automatic updates are not supported: DBeaver does not know where to look for new versions. To update: + +1. Download the new archive from the [GitHub Releases](https://github.com/ydb-platform/ydb-dbeaver-plugin/releases) page. +2. Open **Help → Installation Information**, select the plugin, and click **Uninstall**. +3. Install the new version by following the [instructions for installing from a ZIP archive](#installation-zip). + +{% endnote %} + +## Creating a connection to {{ ydb-name }} {#connection} To create a connection to {{ ydb-name }}, follow these steps: 1. In the top menu, select **Database → New Database Connection** (or press `Ctrl+Shift+N`). -1. In the search field, type `YDB`. Select **YDB** from the list and click **Next**. -1. The {{ ydb-name }} connection setup page will open. Fill in the fields: +2. In the search field, enter `YDB`. Select **YDB** from the list and click **Next**. +3. The connection settings page for {{ ydb-name }} will open. Fill in the fields: - | Field | Description | Example | - |------|----------|--------| - | **Host** | Host of the [endpoint](../../concepts/connect.md#endpoint) for the {{ ydb-name }} cluster | `ydb.example.com` | - | **Port** | Port (default is `2135`) | `2135` | - | **Database** | Path to the [database](../../concepts/glossary.md#database) | `/Root/database` | - | **Monitoring URL** | URL of the [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md) with the database path, used for the dashboard (optional) | `http://ydb.example.com:8765/monitoring/tenant?name=%2FRoot%2Fdatabase` | - | **Use secure connection** | Use a secure connection (`grpcs://`) | ☑ | - | **Enable autocomplete API** | Autocomplete via {{ ydb-short-name }} API | ☑ | + | Field | Description | Example | + | --- | --- | --- | + | **Host** | Host of the {{ ydb-name }} cluster [endpoint](../../concepts/connect.md#endpoint) | `ydb.example.com` | + | **Port** | Port (default `2135`) | `2135` | + | **Database** | Path to the [database](../../concepts/glossary.md#database) | `/Root/database` | + | **Monitoring URL** | URL of the [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md) with the database path, used for the dashboard (optional) | `http://ydb.example.com:8765/monitoring/tenant?name=%2FRoot%2Fdatabase` | + | **Use secure connection** | Use a secure connection (`grpcs://`) | ☑ | + | **Enable autocomplete API** | Autocomplete via {{ ydb-short-name }} API | ☑ | +4. Select the authentication method from the **Auth type** drop-down list (see [Authentication methods](#auth-methods)). +5. Click the **Test Connection** button to verify the settings. If the connection is successful, a dialog with the connection time in milliseconds will appear. +6. Click the **Finish** button. The connection will appear in the **Database Navigator** panel. -1. Select an authentication method from the **Auth type** dropdown list (see [Authentication Methods](#auth-methods)). -1. Click the **Test Connection** button to verify the settings. If the connection is successful, a dialog will appear with the connection time in milliseconds. -1. Click the **Finish** button. The connection will appear in the **Database Navigator** panel. +## Authentication methods {#auth-methods} -## Authentication Methods {#auth-methods} - -The plugin supports all [authentication methods](../../security/authentication.md) available in {{ ydb-short-name }}. The method is selected from the **Auth type** dropdown list on the connection setup page. +The plugin supports all [authentication](../../security/authentication.md) methods available in {{ ydb-short-name }}. The method is selected from the **Auth type** drop-down list on the connection settings page. ### Anonymous {#auth-anonymous} -Connection without credentials. Used for local or test installations of {{ ydb-short-name }}. No additional fields need to be filled out. +Connection without credentials. Used for local or test installations of {{ ydb-short-name }}. No additional fields need to be filled in. -### Static (Username and Password) {#auth-static} +### Static (login and password) {#auth-static} -Authentication using a username and password. Enter the username in the **User** field and the password in the **Password** field. Used if {{ ydb-short-name }} server has [username and password authentication](../../security/authentication.md#static-credentials) enabled. +Authentication by login and password. Enter the username in the **User** field and the password in the **Password** field. on the {{ ydb-short-name }} server. Used if [login and password authentication](../../security/authentication.md#static-credentials) is enabled {% note info %} -In managed installations of {{ ydb-name }}, username and password authentication is disabled: managed services use the centralized access control system of the cloud platform ([IAM](https://yandex.cloud/ru/docs/iam/)). +In managed installations of {{ ydb-name }}, login and password authentication is disabled: managed services use the cloud platform's centralized access control system ([IAM](https://yandex.cloud/en/docs/iam/)). {% endnote %} ### Token {#auth-token} -Authentication using an [IAM](https://yandex.cloud/ru/docs/iam/concepts/authorization/iam-token) or [OAuth token](https://yandex.cloud/ru/docs/iam/concepts/authorization/oauth-token). Enter the token in the **Token** field. The token is passed in the header of each request. +Authentication by [IAM](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token) or [OAuth token](https://yandex.cloud/en/docs/iam/concepts/authorization/oauth-token). Enter the token in the **Token** field. The token is passed in the header of each request. ### Service Account {#auth-service-account} -Authentication using the key of a [Yandex Cloud service account](https://yandex.cloud/ru/docs/iam/concepts/users/service-accounts). Specify the path to the JSON file with the key in the **SA Key File** field (use the **...** button to select the file). For more information on how to create an authorized key, see the [Yandex Cloud documentation](https://yandex.cloud/ru/docs/iam/operations/authentication/manage-authorized-keys). +Authentication by a [service account](https://yandex.cloud/en/docs/iam/concepts/users/service-accounts) key of Yandex Cloud. Specify the path to the JSON file with the key in the **SA Key File** field (use the **...** button to select the file). For more information on how to create an authorized key, see the [Yandex Cloud documentation](https://yandex.cloud/en/docs/iam/operations/authentication/manage-authorized-keys). Key file format: + ```json { "id": "aje...", @@ -148,15 +183,16 @@ Key file format: } ``` + ### Metadata {#auth-metadata} -Authentication via the [Yandex Cloud metadata service](https://yandex.cloud/ru/docs/compute/operations/vm-metadata/get-vm-metadata). The plugin obtains an IAM token from the virtual machine's metadata service. Used only when DBeaver is running on a Yandex Cloud virtual machine. +Authentication via [Yandex Cloud Metadata Service](https://yandex.cloud/en/docs/compute/operations/vm-metadata/get-vm-metadata). The plugin obtains an IAM token from the virtual machine metadata service. It is used only when DBeaver is running on a Yandex Cloud virtual machine. -## Object Navigator {#object-navigator} +## Object navigator {#object-navigator} -After connecting, the {{ ydb-short-name }} object hierarchy is displayed in the **Database Navigator** panel. The root node is the connection, and inside it is the database path, which contains the following folders: +After connection, the **Database Navigator** panel displays the {{ ydb-short-name }} object hierarchy. The root node is the connection, inside it is the database path, which contains the following folders: -- **Tables** — tables organized by subdirectories according to the path in {{ ydb-short-name }} (for example, a table with the path `folder1/subfolder/mytable` will be nested under `folder1 → subfolder`). +- **Tables** — tables organized into subdirectories according to the path in {{ ydb-short-name }} (for example, a table at path `folder1/subfolder/mytable` will be nested in `folder1 → subfolder`). - **Topics** — [topics](../../concepts/datamodel/topic.md). - **Views** — [views](../../concepts/datamodel/view.md). - **External Data Sources** — [external data sources](../../concepts/glossary.md#external-data-source). @@ -164,95 +200,98 @@ After connecting, the {{ ydb-short-name }} object hierarchy is displayed in the - **System Views (.sys)** — [system views](../../dev/system-views.md), such as `partition_stats`, `query_sessions`. - **Resource Pools** — [resource pools](../../concepts/glossary.md#resource-pool). -## Working with the Plugin {#capabilities} +## Working with the plugin {#capabilities} -### YQL Editor {#yql-editor} +### YQL editor {#yql-editor} -Open the **SQL Editor** (`F3` or double-click on the connection). The editor supports: +Open the **SQL Editor** (`F3` or double-click the connection). The editor supports: -- Syntax highlighting for [YQL](../../yql/reference/index.md): keywords (`UPSERT`, `REPLACE`, `EVALUATE`, `PRAGMA`, `WINDOW`, and 145+ others), data types, built-in functions. -- Autocompletion of table, column, and function names. -- Query execution: `Ctrl+Enter` for the current query, `Ctrl+Shift+Enter` for the entire script. +- Syntax highlighting for [YQL](../../yql/reference/index.md): keywords (`UPSERT`, `REPLACE`, `EVALUATE`, `PRAGMA`, `WINDOW` and 145+ others), data types, built-in functions. +- Auto-completion of table names, columns, and functions. +- Query execution: `Ctrl+Enter` — current query, `Ctrl+Shift+Enter` — entire script. Example YQL query: + ```yql UPSERT INTO `users` (id, name, created_at) VALUES (1, "Alice", CurrentUtcDatetime()); ``` -### EXPLAIN and Execution Plan {#explain} + +### EXPLAIN and execution plan {#explain} Click **Explain** (or `Ctrl+Shift+E`) to get the [query execution plan](../../dev/query-execution-optimization/query-plans-optimization.md). The plugin displays: -- **Text plan** — a tree of operations in text form. -- **Diagram** — a graphical representation as a DAG. -- **SVG plan** — an interactive visualization. +- **Text plan** — operation tree in text form. +- **Diagram** — graphical representation as a DAG. +- **SVG plan** — interactive visualization. -`EXPLAIN ANALYZE` additionally shows execution statistics (number of rows, execution time). +`EXPLAIN ANALYZE` additionally shows execution statistics (row count, execution time). -### Session Manager {#session-manager} +### Session manager {#session-manager} -Right-click on the connection and select **Manage Sessions**, or use the **Database → Manage Sessions** menu item. The opened view displays all active sessions with the current query, state, and duration (data from the system view [`.sys/query_sessions`](../../dev/system-views.md#query-sessions)). The **Hide Idle** checkbox hides sessions without an active query. +Select the menu item **Tools → Sessions Manager**, or right-click the connection and select the corresponding item. The opened view displays all active sessions with the current query, state, and duration (data from the system view [`.sys/query_sessions`](../../dev/system-views.md#query-sessions)). The **Hide Idle** checkbox hides sessions without an active query. -### Cluster Dashboard {#cluster-dashboard} +### Cluster dashboard {#cluster-dashboard} -Open the **Dashboard** tab in the connection editor (requires the **Monitoring URL** field to be filled in during connection setup). +Open the **Dashboard** tab in the connection editor (requires the **Monitoring URL** field to be filled in when configuring the connection). {% note warning %} -The dashboard is only available when working with self-hosted installations of {{ ydb-short-name }} where access to the [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md) is available. In Yandex Cloud Managed Service for {{ ydb-short-name }}, Embedded UI is not published, so dashboard data is unavailable — use [cloud platform tools](https://yandex.cloud/ru/docs/ydb/operations/monitoring) for monitoring. +The dashboard is only available when working with self-hosted {{ ydb-short-name }} installations that have access to [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md). In Yandex Cloud Managed Service for {{ ydb-short-name }}, the Embedded UI is not published, so dashboard data is unavailable — use [cloud platform tools](https://yandex.cloud/en/docs/ydb/operations/monitoring) for monitoring. {% endnote %} -The dashboard displays real-time data (updated every 5 seconds): +The dashboard displays in real time (updates every 5 seconds): -- CPU load by nodes. +- CPU load by node. - Disk space usage. - Memory usage. - Network traffic. -- Number of executing queries. -- Status of cluster nodes. +- Number of running queries. +- Cluster node status. -### Streaming Queries {#streaming-queries} +### Streaming queries {#streaming-queries} -In the navigator, expand the **Streaming Queries** folder. For each query, you can: +In the navigator, expand the **Streaming Queries** folder. For each query, the following are available: -- View the original YQL. +- View the source YQL. - View errors (issues). - View the execution plan. -- Perform actions: **Start**, **Stop**, **Alter**. +- Actions: **Start**, **Stop**, **Alter**. -### SQL Dialect Converter {#convert-dialect} +### SQL dialect converter {#convert-dialect} -The plugin allows you to convert an SQL query written in another dialect (PostgreSQL, MySQL, ClickHouse, etc.) to YQL. The converter is available on the **Convert Dialect** tab in the connection editor. +The plugin allows you to convert an SQL query written in another dialect (PostgreSQL, MySQL, ClickHouse, and others) to YQL. The converter is available in the context menu **Tools → Convert Dialect** in the connection editor. To convert a query: -1. In the **Source Dialect** dropdown list, select the source SQL dialect. The list of dialects is requested from the plugin's external service when the tab is first opened. -1. Paste the source SQL code into the **Input SQL** field. -1. Click **Convert**. The result will appear in the lower field. -1. Click **Copy** to copy the result to the clipboard. +1. In the **Source Dialect** dropdown, select the source SQL dialect. The list of dialects is fetched from the plugin's external service when the tab is first opened. +2. Paste the source SQL code into the **Input SQL** field. +3. Click **Convert**. The result appears in the lower field. +4. Click **Copy** to copy the result to the clipboard. -For more information on how the converter works, supported dialects, and limitations, see the [SQL Dialect Converter to YQL](../sql-dialect-converter.md) article. +For more details on how the converter works, supported dialects, and limitations, see the article [SQL dialect converter to YQL](../sql-translation/sql-dialect-converter.md). {% note warning %} -The plugin sends the original query to an external HTTPS service for conversion. Do not use the converter for queries containing confidential data. +To perform the conversion, the plugin sends the original query to an external HTTPS service. The converter does not work without internet access. Do not use the converter for queries containing confidential data. {% endnote %} -### Creating Objects {#create-objects} +### Creating objects {#create-objects} -Right-click on a folder or object and select **Create New**: +Right-click a folder or object and select **Create New**: - **Create Table** — create a new table. - **Create Topic** — create a new topic. -- **Create Resource Pool** — create a resource pool. -## Updating the Plugin {#updates} +## Plugin update {#updates} + +Automatic updates are only supported when [installing via the P2 repository URL](#installation-url). DBeaver remembers the source (repository URL) and when a new version is published, it offers an update in one of two ways: -DBeaver uses the Eclipse P2 mechanism to detect and install updates. When you install the plugin, DBeaver remembers the source — the repository URL. When a new version is published, DBeaver compares the installed version with the version in the repository and offers an update in one of two ways: +1. Automatically on the next DBeaver launch (if update checking is enabled in **Window → Preferences → Install/Update → Automatic Updates**). +2. Manually via **Help → Check for Updates**: select the available update and follow the same steps as during the initial installation (license → unsigned content warning → restart). -1. Automatically on the next DBeaver startup (if update checks are enabled in **Window → Preferences → Install/Update → Automatic Updates**). -1. Manually via **Help → Check for Updates**: select the available update and follow the same steps as during the initial installation (license → warning about unsigned content → restart). +When [installing from a ZIP archive](#installation-zip), the update is performed manually: download the new archive from the [GitHub Releases](https://github.com/ydb-platform/ydb-dbeaver-plugin/releases) page, open **Help → Installation Information**, select the plugin, click **Uninstall**, then install the new version again. diff --git a/ydb/docs/en/core/integrations/gui/toc-ide.yaml b/ydb/docs/en/core/integrations/gui/toc-ide.yaml index 9a7aba07bc7..c8e6947778c 100644 --- a/ydb/docs/en/core/integrations/gui/toc-ide.yaml +++ b/ydb/docs/en/core/integrations/gui/toc-ide.yaml @@ -3,6 +3,8 @@ items: href: dbeaver.md - name: DBeaver Plugin href: dbeaver-plugin.md +- name: VS Code + href: vscode-plugin.md - name: DataGrip href: datagrip.md - name: Jupyter Notebook diff --git a/ydb/docs/en/core/integrations/gui/vscode-plugin.md b/ydb/docs/en/core/integrations/gui/vscode-plugin.md index 454f6dc9937..7bc93119669 100644 --- a/ydb/docs/en/core/integrations/gui/vscode-plugin.md +++ b/ydb/docs/en/core/integrations/gui/vscode-plugin.md @@ -1,110 +1,128 @@ # Connecting to {{ ydb-short-name }} using the VS Code plugin -[Visual Studio Code](https://code.visualstudio.com) is a free cross-platform open-source code editor that supports a wide ecosystem of extensions for working with databases, language servers, and development tools. +[Visual Studio Code](https://code.visualstudio.com) is a free, cross-platform, open-source code editor that supports a wide ecosystem of extensions for working with databases, language servers, and development tools. -[YDB for VS Code](https://github.com/ydb-platform/ydb-vscode-plugin) is a VS Code extension with native support for {{ ydb-short-name }}. The plugin provides a specialized interface for working with {{ ydb-short-name }} objects: a hierarchical navigator for tables, topics, views, external data sources, support for all authentication methods, a [YQL](../../concepts/glossary.md#yql) editor with syntax highlighting, visualization of execution plans, monitoring of sessions and the cluster, access control management (ACL), a built-in [MCP server](https://modelcontextprotocol.io/) for AI assistants, and other features. -## Key Features of the Plugin {#features} +[YDB for VS Code](https://github.com/ydb-platform/ydb-vscode-plugin) is a VS Code extension with native support for {{ ydb-short-name }}. The plugin provides a specialized interface for working with {{ ydb-short-name }} objects: a hierarchical navigator for tables, topics, views, external data sources, support for all authentication methods, a [YQL](../../concepts/glossary.md#yql) editor with syntax highlighting, visualization of execution plans, session and cluster monitoring, access control management (ACL), a built-in [MCP server](https://modelcontextprotocol.io/) for AI assistants, and other features. -- Connecting to {{ ydb-name }} with all [authentication methods](../../security/authentication.md): anonymous, static, token-based, service account, metadata-based. -- Hierarchical object navigator: tables ([row-oriented](../../concepts/glossary.md#row-oriented-table) and [column-oriented](../../concepts/glossary.md#column-oriented-table)), [topics](../../concepts/datamodel/topic.md), [views](../../concepts/datamodel/view.md), [external data sources](../../concepts/glossary.md#external-data-source), [external tables](../../concepts/glossary.md#external-table), [transfers](../../concepts/transfer.md), [streaming queries](../../concepts/glossary.md#streaming-query). +## Key features of the plugin {#features} + +- Connecting to {{ ydb-name }} with all [authentication](../../security/authentication.md) methods: anonymous, static, token-based, service account, and metadata. +- Hierarchical object navigator: tables ([row](../../concepts/glossary.md#row-oriented-table) and [column](../../concepts/glossary.md#column-oriented-table)), [topics](../../concepts/datamodel/topic.md), [views](../../concepts/datamodel/view.md), [external data sources](../../concepts/glossary.md#external-data-source), [external tables](../../concepts/glossary.md#external-table), [transfers](../../concepts/transfer.md), [streaming queries](../../concepts/glossary.md#streaming-query). - System objects: [system views](../../dev/system-views.md) (`.sys`), [resource pools](../../concepts/glossary.md#resource-pool). -- YQL editor with syntax highlighting, table and column autocompletion. -- Query execution and results visualization: table, JSON, chart. +- [YQL](../../concepts/glossary.md#yql) editor with syntax highlighting, autocompletion of tables and columns. +- Query execution and result visualization: table, JSON, diagram. - Visualization of the [query execution plan](../../dev/query-execution-optimization/query-plans-optimization.md) (`EXPLAIN`). -- Monitoring active sessions via [`.sys/query_sessions`](../../dev/system-views.md#query-sessions). -- Cluster dashboard based on [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md): CPU load, memory usage, network traffic (updated every 10 seconds). -- Managing [access rights (ACL)](../../security/authorization.md#right): viewing permissions for database objects. -- Generating DDL scripts ([`CREATE`](../../yql/reference/syntax/create_table/index.md)) for any database object. -- Managing [streaming queries](../../concepts/glossary.md#streaming-query): viewing, starting, stopping. -- [SQL query converter](../sql-dialect-converter.md) from other dialects (PostgreSQL, MySQL, ClickHouse, etc.) to YQL. +- Active session monitoring via [`.sys/query_sessions`](../../dev/system-views.md#query-sessions). +- Cluster dashboard based on [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md): CPU load, memory usage, network traffic (updates every 10 seconds). +- Management of [access rights (ACL)](../../security/authorization.md#right): viewing permissions on database objects. +- Generation of DDL scripts ( [`CREATE`](../../yql/reference/syntax/create_table/index.md)) for any database object. +- Management of [streaming queries](../../concepts/glossary.md#streaming-query): viewing, starting, stopping. +- [SQL query converter](../sql-translation/sql-dialect-converter.md) from other dialects (PostgreSQL, MySQL, ClickHouse, and others) to YQL. - Built-in [MCP server](#mcp) — direct access to databases from AI assistants (Claude Code and others). -- [Semantic search for YQL documentation](#rag) (RAG) for AI-assisted query writing. +- [Semantic search in YQL documentation](#rag) (RAG) for AI-assisted query writing. + ## Requirements {#requirements} -The plugin requires Visual Studio Code version 1.75.0 or later. -## Plugin installation {#installation} +The plugin requires Visual Studio Code version 1.75.0 or newer. + +## Installing the plugin {#installation} The plugin can be installed from the VS Code Marketplace or from a `.vsix` file on the GitHub Releases page. -### Installing from the VS Code Marketplace {#install-marketplace} + +### Installation from VS Code Marketplace {#install-marketplace} 1. Open the Extensions panel in VS Code (`Ctrl+Shift+X` / `Cmd+Shift+X`). -1. In the search bar, enter `YDB for VS Code` and select the extension from the publisher `ydb-tech` ([direct link](https://marketplace.visualstudio.com/items?itemName=ydb-tech.ydb-vscode-plugin)). -1. Click **Install**. +2. In the search bar, enter `YDB for VS Code` and select the extension from publisher `ydb-tech` ( [direct link](https://marketplace.visualstudio.com/items?itemName=ydb-tech.ydb-vscode-plugin)). +3. Click **Install**. Alternatively, you can install the plugin from the Marketplace with a single command in the terminal: + + ```bash code --install-extension ydb-tech.ydb-vscode-plugin ``` + + ### Installation from a VSIX file {#install-vsix} This method is suitable if you need a specific version or do not have access to the Marketplace. 1. Go to the [GitHub Releases page](https://github.com/ydb-platform/ydb-vscode-plugin/releases) and download the `ydb-vscode-plugin-*.vsix` file of the required version. +2. Install the extension using one of the following methods: + + - **Via terminal:** -1. Install the extension using one of the methods: - - **Via the terminal:** - ```bash - code --install-extension ydb-vscode-plugin-X.X.X.vsix - ``` -Where `X.X.X` is the number of the downloaded version. + ```bash + code --install-extension ydb-vscode-plugin-X.X.X.vsix + ``` -- **Via the VS Code interface:** - 1. Open the extensions panel (`Ctrl+Shift+X` / `Cmd+Shift+X`). - 1. Click `...` (three dots) in the top right corner of the panel. - 1. Select **Install from VSIX...**. - 1. Specify the path to the downloaded `.vsix` file. + Where `X.X.X` — the number of the downloaded version. + - **Via VS Code interface:** + + 1. Open the Extensions panel (`Ctrl+Shift+X` / `Cmd+Shift+X`). + 2. Click `...` (three dots) in the top-right corner of the panel. + 3. Select **Install from VSIX...**. + 4. Specify the path to the downloaded `.vsix` file. + +After installation by any method, restart VS Code. The **YDB** icon will appear in the Activity Bar. -After installing using any of the methods, restart VS Code. The **YDB** icon will appear in the Activity Bar. ## Creating a connection to {{ ydb-name }} {#connection} 1. Click the **YDB** icon in the Activity Bar on the left. -1. In the **Connections** panel, click the **Add Connection** button (the `+` icon). -1. The connection creation form will open. Fill in the fields: -| Field | Description | Example | -|------|----------|--------| -| **Connection Name** | Arbitrary connection name | `my-ydb` | -| **Host** | Host of the [endpoint](../../concepts/connect.md#endpoint) of the {{ ydb-name }} cluster | `ydb.example.com` | -| **Port** | Port (default `2135`) | `2135` | -| **Database** | Path to the [database](../../concepts/glossary.md#database) | `/Root/database` | -| **Monitoring URL** | URL of [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md), used for the dashboard (filled in automatically based on the host, can be overridden) | `http://ydb.example.com:8765` | -| **Secure connection (grpcs)** | Use a secure connection (`grpcs://`) | ☑ | -| **Use RAG** | Enable [YQL documentation search](#rag) for this connection | ☑ | -1. If necessary, specify the path to the custom CA certificate (PEM) in the **CA Certificate File** field for connections with non-standard TLS. If the field is left empty, the built-in Yandex Cloud certificate is used. - -1. Select an authentication method from the **Auth type** dropdown list (see [Authentication methods](#auth-methods)). +2. In the **Connections** panel, click the **Add Connection** button (`+` icon). +3. The connection creation form will open. Fill in the fields: -1. Click **Test Connection** to check the settings. If the connection is successful, a success message will appear. + | Field | Description | Example | + | --- | --- | --- | + | **Connection Name** | Arbitrary connection name | `my-ydb` | + | **Host** | Endpoint of the [endpoint](../../concepts/connect.md#endpoint) of the {{ ydb-name }} cluster | `ydb.example.com` | + | **Port** | Port (default `2135`) | `2135` | + | **Database** | Path to the [database](../../concepts/glossary.md#database) | `/Root/database` | + | **Monitoring URL** | URL of [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md), used for the dashboard (filled in automatically based on the host, can be overridden) | `http://ydb.example.com:8765` | + | **Secure connection (grpcs)** | Use a secure connection (`grpcs://`) | ☑ | + | **Use RAG** | Enable [search in YQL documentation](#rag) for this connection | ☑ | +4. If necessary, specify the path to a custom CA certificate (PEM) in the **CA Certificate File** field — for connections with non-standard TLS. If the field is left empty, the built-in Yandex Cloud certificate is used. +5. Select the authentication method from the **Auth type** drop-down list (see [Authentication methods](#auth-methods)). +6. Click **Test Connection** to verify the settings. If the connection is successful, a success message will appear. +7. Click **Save**. The connection will appear in the **Connections** panel. -1. Click **Save**. The connection will appear in the **Connections** panel. ## Authentication methods {#auth-methods} -The plugin supports all [authentication methods](../../security/authentication.md) available in {{ ydb-short-name }}. The method is selected in the **Auth type** dropdown list on the connection creation form. +The plugin supports all [authentication](../../security/authentication.md) methods available in {{ ydb-short-name }}. The method is selected in the **Auth type** dropdown on the connection creation form. + ### Anonymous {#auth-anonymous} Connection without credentials. Used for local or test installations of {{ ydb-short-name }}. No additional fields need to be filled in. + ### Static Credentials (login and password) {#auth-static} -Authentication using login and password. Specify the username in the **Username** field and the password in the **Password** field. Used if {{ ydb-short-name }} server has [login and password authentication enabled](../../security/authentication.md#static-credentials). +Authentication by login and password. Specify the username in the **Username** field and the password in the **Password** field. Used if [login and password authentication](../../security/authentication.md#static-credentials) is enabled on the {{ ydb-short-name }} server. + {% note info %} -In managed installations of {{ ydb-name }}, authentication via login and password is disabled: managed services use the centralized access management system of the cloud platform ([IAM](https://yandex.cloud/en/docs/iam/)). +In managed installations of {{ ydb-name }}, login and password authentication is disabled: managed services use the cloud platform's centralized access control system ( [IAM](https://yandex.cloud/en/docs/iam/)). {% endnote %} + ### Access Token {#auth-token} -Authentication using [IAM-](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token) or [OAuth token](https://yandex.cloud/en/docs/iam/concepts/authorization/oauth-token). Enter the token in the **Token** field. The token is passed in the header of each request. +Authentication by [IAM](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token) or [OAuth token](https://yandex.cloud/en/docs/iam/concepts/authorization/oauth-token). Enter the token in the **Token** field. The token is passed in the header of each request. + {% note warning %} -The IAM token has a limited [lifespan — no more than 12 hours](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token#lifetime) — after which it must be obtained again and updated in the connection settings. For long-lived connections, use authentication via a [service account](#auth-service-account) or [metadata service](#auth-metadata). +The IAM token has a limited [lifetime — no more than 12 hours](https://yandex.cloud/en/docs/iam/concepts/authorization/iam-token#lifetime), after which it must be obtained again and updated in the connection settings. For long-lived connections, use authentication via a [service account](#auth-service-account) or [metadata service](#auth-metadata). {% endnote %} + ### Service Account Key File {#auth-service-account} -Authentication using the [service account](https://yandex.cloud/en/docs/iam/concepts/users/service-accounts) key of Yandex Cloud. Specify the path to the JSON file with the key in the **Service Account Key File** field (use the **Browse** button to select the file). For more information on how to create an authorized key, see the [Yandex Cloud documentation](https://yandex.cloud/en/docs/iam/operations/authentication/manage-authorized-keys). +Authentication by a [service account](https://yandex.cloud/en/docs/iam/concepts/users/service-accounts) key of Yandex Cloud. Specify the path to the JSON file with the key in the **Service Account Key File** field (use the **Browse** button to select the file). For more details on how to create an authorized key, see the [Yandex Cloud documentation](https://yandex.cloud/en/docs/iam/operations/authentication/manage-authorized-keys). + +Key file format: + -The format of the key file: ```json { "id": "aje...", @@ -112,14 +130,17 @@ The format of the key file: "private_key": "-----BEGIN RSA PRIVATE KEY-----\n..." } ``` + + ### Metadata Service {#auth-metadata} -Authentication via the [Yandex Cloud metadata service](https://yandex.cloud/en/docs/compute/operations/vm-metadata/get-vm-metadata). The plugin receives an IAM token from the virtual machine's metadata service. It is used only when VS Code is running on a Yandex Cloud virtual machine. -## Object Navigator {#object-navigator} +Authentication via [Yandex Cloud metadata service](https://yandex.cloud/en/docs/compute/operations/vm-metadata/get-vm-metadata). The plugin obtains an IAM token from the virtual machine's metadata service. Used only when VS Code is running on a Yandex Cloud virtual machine. + +## Object navigator {#object-navigator} -After connecting, click on the connection in the **Connections** panel — the **Navigator** panel will open with the hierarchy of {{ ydb-short-name }} objects. The navigator contains the following sections: +After connecting, click the connection in the **Connections** panel — the **Navigator** panel opens with a hierarchy of {{ ydb-short-name }} objects. The navigator contains the following sections: -- **Tables** — tables (string and columnar), organized into subdirectories according to the path in {{ ydb-short-name }}. +- **Tables** — tables (row and column), organized by subdirectories according to the path in {{ ydb-short-name }}. - **System Views** — [system views](../../dev/system-views.md) (`.sys`), such as `query_sessions`, `partition_stats`. - **Views** — [views](../../concepts/datamodel/view.md). - **Topics** — [topics](../../concepts/datamodel/topic.md). @@ -129,80 +150,97 @@ After connecting, click on the connection in the **Connections** panel — the * - **Transfers** — data transfers. - **Streaming Queries** — [streaming queries](../../concepts/glossary.md#streaming-query). -Right-clicking on any object in the navigator opens a context menu with available actions. +Right-clicking any object in the navigator opens a context menu with available actions. + ## Working with the plugin {#capabilities} -### Query Workspace {#query-workspace} -Open the query workspace using `Ctrl+Shift+Q` (`Cmd+Shift+Q` on macOS) or click **Open Query Workspace** in the connection context menu. In the workspace, you can write and execute YQL queries, view history and results. +### Query workspace {#query-workspace} + +Open the query workspace via `Ctrl+Shift+Q` (`Cmd+Shift+Q` on macOS) or click **Open Query Workspace** in the connection context menu. In the workspace, you can write and execute YQL queries, view history and results. -To quickly open the editor with a pre-filled query, right-click on a table or view in the navigator and select: +To quickly open the editor with a pre-filled query, right-click a table or view in the navigator and select: - **Show Preview** — `SELECT` of the first 100 rows. - **Make Query** — `SELECT` with the object name. -### YQL Editor {#yql-editor} + +### YQL editor {#yql-editor} The editor supports: - Syntax highlighting for [YQL](../../yql/reference/index.md): keywords, data types, built-in functions. -- Autocompletion of table and column names (based on the connected database schema). +- Auto-completion of table and column names (based on the schema of the connected database). - Query execution: `Ctrl+Enter` (`Cmd+Enter` on macOS). -Example of a YQL query: +Example YQL query: + + ```yql UPSERT INTO `users` (id, name, created_at) VALUES (1, "Alice", CurrentUtcDatetime()); ``` -The execution results are displayed in the **Results** panel as a table, JSON, or chart (switch between them using tabs). -### EXPLAIN and query execution plan {#explain} + + +Execution results are displayed in the **Results** panel as a table, JSON, or diagram (switch via tabs). + +### EXPLAIN and execution plan {#explain} Select **Explain YQL Query** in the editor context menu or in the command palette (`Ctrl+Shift+P`) to get the [query execution plan](../../dev/query-execution-optimization/query-plans-optimization.md). The plugin displays the plan's operation tree in text form. -### Session Manager {#session-manager} + +### Session manager {#session-manager} The **Sessions** panel (Activity Bar → YDB) displays all active sessions with the current query, state, and duration (data from the system view [`.sys/query_sessions`](../../dev/system-views.md#query-sessions)). The **Toggle Hide Idle** button hides sessions without an active query. -### Cluster Dashboard {#cluster-dashboard} + +### Cluster dashboard {#cluster-dashboard} The **Database Load** panel (Activity Bar → YDB) displays real-time cluster load (updated every 10 seconds): -- CPU usage (percentage and number of cores). -- Memory usage (percentage and volume). +- CPU load (% and number of cores). +- Memory usage (% and amount). - Network traffic. + {% note warning %} -The dashboard is available only when working with self-hosted installations of {{ ydb-short-name }} that have access to [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md). In Yandex Cloud Managed Service for {{ ydb-short-name }}, Embedded UI is not published, so dashboard data is unavailable — use [cloud platform tools](https://yandex.cloud/en/docs/ydb/operations/monitoring) for monitoring. +The dashboard is only available when working with self-hosted {{ ydb-short-name }} installations that have access to [{{ ydb-short-name }} Embedded UI](../../reference/embedded-ui/index.md). In Yandex Cloud Managed Service for {{ ydb-short-name }}, the Embedded UI is not published, so dashboard data is unavailable — use [cloud platform tools](https://yandex.cloud/en/docs/ydb/operations/monitoring) for monitoring. {% endnote %} -### Streaming Queries {#streaming-queries} -In the navigator, expand the **Streaming Queries** section. For each query, the following is available: +### Streaming queries {#streaming-queries} + +In the navigator, expand the **Streaming Queries** section. For each query, the following are available: -- Viewing the original YQL (**View Source**). -- Starting and stopping (**Start** / **Stop**). -### SQL Dialect Converter {#convert-dialect} +- View the source YQL (**View Source**). +- Start and stop (**Start** / **Stop**). -The plugin allows you to convert an SQL query written in another dialect (PostgreSQL, MySQL, ClickHouse, and others) to YQL. The converter is available in the **Convert Dialect** panel on the Activity Bar. +### SQL dialect converter {#convert-dialect} + +The plugin allows converting an SQL query written in another dialect (PostgreSQL, MySQL, ClickHouse, and others) to YQL. The converter is available in the **Convert Dialect** panel on the Activity Bar. To convert a query: -1. Select the source SQL dialect in the **Source Dialect** dropdown list. -1. Paste the source SQL code into the **Input SQL** field. -1. Click **Convert**. The result will appear in the lower field. -1. Copy the result for use in the YQL editor. +1. In the **Source Dialect** dropdown, select the source SQL dialect. +2. Paste the source SQL code into the **Input SQL** field. +3. Click **Convert**. The result appears in the lower field. +4. Copy the result to use in the YQL editor. + +For more details on how the converter works, supported dialects, and limitations, see the article [SQL dialect converter to YQL](../sql-translation/sql-dialect-converter.md). -For more information about how the converter works, the supported dialects, and the limitations, see the article [SQL Dialect Converter to YQL](../sql-dialect-converter.md). {% note warning %} -To convert, the plugin sends the original query to an external HTTPS service. The converter does not work without internet access. Do not use the converter for queries containing confidential data. +To perform the conversion, the plugin sends the original query to an external HTTPS service. The converter does not work without internet access. Do not use the converter for queries containing confidential data. {% endnote %} -### Viewing Permissions {#permissions} -Right-click on an object in the navigator and select **View Permissions** to view the [access rights (ACL)](../../security/authorization.md#right) assigned to this object. -### DDL Generation {#ddl} +### View permissions {#permissions} + +Right-click an object in the navigator and select **View Permissions** to view the [access rights (ACL)](../../security/authorization.md#right) assigned to this object. + +### DDL generation {#ddl} + +Right-click a table, topic, or other object in the navigator and select **Create DDL** to get the `CREATE` script of the object. -Right-click on a table, topic, or another object in the navigator and select **Create DDL** to get the `CREATE` script for the object. ### Creating objects {#create-objects} -Right-click on the corresponding folder in the navigator to create a new object: +Right-click the appropriate folder in the navigator to create a new object: - **New Row Table** / **New Column Table** — create a row or column table. - **New Topic** — create a topic. @@ -211,100 +249,130 @@ Right-click on the corresponding folder in the navigator to create a new object: - **New CSV/JSON/Parquet External Table** — create an external table. - **New Transfer** — create a transfer. - **New Streaming Query** — create a streaming query. + ## Integration with AI assistants (MCP) {#mcp} -The plugin launches a built-in [MCP server](https://modelcontextprotocol.io/) (Model Context Protocol), which allows AI assistants (Claude Code and others) to directly execute queries to {{ ydb-short-name }} databases configured in the plugin. -### Configuring the port {#mcp-port} +The plugin runs a built-in [MCP server](https://modelcontextprotocol.io/) (Model Context Protocol), which allows AI assistants (Claude Code and others) to directly execute queries against {{ ydb-short-name }} databases configured in the plugin. + +### Port configuration {#mcp-port} + +The server runs on port **3333** (localhost only) by default. You can change the port in VS Code settings: + -The server starts on port **3333** (localhost only) by default. You can change the port in VS Code settings: ```json { "ydb.mcpPort": 3333 } ``` -If the port is already in use, the extension will display a warning and continue working without MCP. + + +If the port is already in use, the extension will show a warning and continue working without MCP. + ### Connecting Claude Code {#mcp-claude} -1. Make sure that the YDB extension is running in VS Code and at least one connection is added in the **Connections** panel (see [Creating a connection](#connection)). +1. Make sure the YDB extension is running in VS Code and at least one connection is added in the **Connections** panel (see [Creating a connection](#connection)). +2. Register the MCP server in Claude Code globally for the current user: + + + ```bash + claude mcp add --scope user --transport sse ydb http://localhost:3333/sse + ``` -1. Register the MCP server in Claude Code globally for the current user: - ```bash - claude mcp add --scope user --transport sse ydb http://localhost:3333/sse - ``` -The `--scope user` flag saves the configuration globally — the server will be accessible from any directory where Claude Code is launched. Without this flag, the `local` scope is used (by default), and registration will have to be repeated for each project. -1. Check the connection: - ```bash - claude mcp list - ``` + The `--scope user` flag saves the configuration globally — the server will be accessible from any directory where Claude Code is launched. Without this flag, the [scope `local`](https://docs.claude.com/en/docs/claude-code/mcp#mcp-installation-scopes) (default) is used, and registration will have to be repeated for each project. +3. Check the connection: + + + ```bash + claude mcp list + ``` + ### Available MCP tools {#mcp-tools} + | Tool | Parameters | Description | -|------------|-----------|----------| +| --- | --- | --- | | `ydb_list_connections` | — | List of all connections configured in the plugin | | `ydb_query` | `connection`, `sql` | Execute a YQL query | -| `ydb_describe_table` | `connection`, `path` | Get the table schema (columns, primary key) | +| `ydb_describe_table` | `connection`, `path` | Get table schema (columns, primary key) | | `ydb_list_directory` | `connection`, `path?` | List of objects in the database directory | | `ydb_list_all` | `connection`, `path?`, `limit?`, `offset?` | Recursive list of all objects | | `ydb_yql_help` | `query`, `connection?` | Search YQL documentation (requires [RAG](#rag) enabled) | -The `connection` parameter is the name of the connection as it appears in the **Connections** panel. -## YQL documentation search (RAG) {#rag} -The plugin supports semantic and keyword search of YQL documentation using the built-in RAG index (Retrieval-Augmented Generation). This allows AI assistants (via the `ydb_yql_help` tool) to find relevant fragments of the syntax reference when writing queries. +The `connection` parameter is the connection name as it appears in the **Connections** panel. + +## Search YQL documentation (RAG) {#rag} + +The plugin supports semantic and keyword search over YQL documentation using a built-in RAG index (Retrieval-Augmented Generation). This allows AI assistants (via the `ydb_yql_help` tool) to find relevant fragments of the syntax reference when writing queries. + ### Enabling RAG {#rag-enable} 1. When creating or editing a connection, set the **Use RAG** flag. -1. The plugin will automatically detect the {{ ydb-short-name }} version and load the corresponding index from the cloud on the first connection. +2. The plugin will automatically detect the {{ ydb-short-name }} version and download the corresponding index from the cloud on first connection. + ### Search modes {#rag-modes} - **Keyword search** — works without additional dependencies, used by default. -- **Semantic search (vector)** — requires locally running [Ollama](https://ollama.com) with the embedding model [`nomic-embed-text`](https://ollama.com/library/nomic-embed-text). See [Installing Ollama and the embedding model](#install-ollama). +- **Semantic search (vector)** — requires a locally running [Ollama](https://ollama.com) with the [`nomic-embed-text`](https://ollama.com/library/nomic-embed-text) embedding model. See [Installing Ollama and the embedding model](#install-ollama). + +The RAG status (Running / Not running) and Ollama status are displayed directly in the connection form. -The RAG status (Running / Not running) and the Ollama status are displayed directly in the connection form. ### Installing Ollama and the embedding model {#install-ollama} -[Ollama](https://ollama.com) is a local server for running language models and embedding models. The plugin accesses it via HTTP to convert documentation texts and search queries into vectors for semantic search. +[Ollama](https://ollama.com) is a local server for running language models and embedding models. The plugin communicates with it over HTTP to convert documentation texts and search queries into vectors for semantic search. 1. Install Ollama following the instructions on the [official download page](https://ollama.com/download) (macOS, Windows, and Linux are supported). +2. Make sure the Ollama service is running and accessible at `http://localhost:11434`: + + + ```bash + curl http://localhost:11434/api/tags + ``` + + + A JSON response with a list of models means the service is running. +3. Download the `nomic-embed-text` embedding model (~270 MB): -1. Make sure that the Ollama service is running and accessible at `http://localhost:11434`: - ```bash - curl http://localhost:11434/api/tags - ``` -A response in JSON format with a list of models means that the service is working. -1. Download the `nomic-embed-text` embedding model (~270 MB): - ```bash - ollama pull nomic-embed-text - ``` -1. Make sure the model is available: - ```bash - ollama list - ``` -The output should include the line `nomic-embed-text`. + ```bash + ollama pull nomic-embed-text + ``` + +4. Make sure the model is available: + + + ```bash + ollama list + ``` + + + The output should contain the line `nomic-embed-text`. +5. Open the connection form in the plugin and make sure the Ollama status is displayed as **Running**. If the status is **Not running**, check that the Ollama service is running and accessible at `http://localhost:11434`. -1. Open the connection form in the plugin and make sure that the Ollama status is displayed as **Running**. If the status is **Not running**, check that the Ollama service is running and available at `http://localhost:11434`. {% note info %} -The Ollama URL (`http://localhost:11434`) and model name (`nomic-embed-text`) are already set in the plugin by default — you only need to override the `ydb.ragOllamaUrl` and `ydb.ragOllamaModel` settings in case of a non-standard installation. +The Ollama URL (`http://localhost:11434`) and model name (`nomic-embed-text`) are already set by default in the plugin — you only need to override the `ydb.ragOllamaUrl` and `ydb.ragOllamaModel` settings for a non-standard installation. -It is not possible to use another embedding model: the indices published by the plugin in the cloud are based on `nomic-embed-text`, and if you change the model, the vectors of the query and documentation will become incompatible. +You cannot use a different embedding model: the indexes published by the plugin in the cloud are built specifically on `nomic-embed-text`, and changing the model will make the query and documentation vectors incompatible. {% endnote %} + ## Plugin update {#updates} The update method depends on how the plugin was installed. -### Updating from the VS Code Marketplace {#update-marketplace} + +### Updating from VS Code Marketplace {#update-marketplace} If the plugin is installed from the [Marketplace](#install-marketplace), VS Code updates it automatically. To update the plugin manually: 1. Open the Extensions panel (`Ctrl+Shift+X` / `Cmd+Shift+X`). -1. Enter `YDB for VS Code` in the search bar and open the installed extension. -1. If a new version is available, the **Update** button will appear — click it. +2. In the search bar, enter `YDB for VS Code` and open the installed extension. +3. If a new version is available, the **Update** button will appear — click it. + ### Updating from a VSIX file {#update-vsix} If the plugin is installed from a [VSIX file](#install-vsix), automatic updates are not available — VS Code does not track updates for manually installed extensions. -1. Download a new `.vsix` file from the [GitHub Releases](https://github.com/ydb-platform/ydb-vscode-plugin/releases) page. -1. Install it over the old version using the same method as during the initial installation — VS Code will automatically replace the previous version. +1. Download the new `.vsix` file from the [GitHub Releases](https://github.com/ydb-platform/ydb-vscode-plugin/releases) page. +2. Install it over the old version using the same method as the initial installation — VS Code will automatically replace the previous version. diff --git a/ydb/docs/en/core/integrations/index.md b/ydb/docs/en/core/integrations/index.md index 3e253291b7b..ecb78b24a18 100644 --- a/ydb/docs/en/core/integrations/index.md +++ b/ydb/docs/en/core/integrations/index.md @@ -4,7 +4,7 @@ This section provides basic information about {{ ydb-name }} integrations with t {% note info %} -In addition to its own native protocol, {{ ydb-name }} has a compatibility layer that allows external systems to connect to databases via [PostgreSQL](../postgresql/intro.md) or [Apache Kafka](../reference/kafka-api/index.md) network protocols. Thanks to the compatibility layer, many tools designed to work with these systems can also interact with {{ ydb-name }}. The compatibility level of each specific application should be clarified separately. +In addition to its own native protocol, {{ ydb-name }} has a compatibility layer that allows external systems to connect to databases via [PostgreSQL](../postgresql/intro.md) or [Apache Kafka](../reference/kafka-api/index.md) network protocols. Thanks to the compatibility layer, many tools designed to work with these systems can also interact with {{ ydb-name }}. The compatibility level of each specific application must be clarified separately. {% endnote %} @@ -44,10 +44,6 @@ In addition to its own native protocol, {{ ydb-name }} has a compatibility layer {% include notitle [Contents](orm/_includes/toc-table.md) %} -## Spring {#spring} - -{% include notitle [Contents](spring/_includes/toc-table.md) %} - ## Vector search {#vectorsearch} {% include notitle [Contents](vectorsearch/_includes/toc-table.md) %} diff --git a/ydb/docs/en/core/integrations/sql-translation/_includes/toc-table.md b/ydb/docs/en/core/integrations/sql-translation/_includes/toc-table.md index b7fd39554e5..6dddf148bd5 100644 --- a/ydb/docs/en/core/integrations/sql-translation/_includes/toc-table.md +++ b/ydb/docs/en/core/integrations/sql-translation/_includes/toc-table.md @@ -1,6 +1,6 @@ # SQL dialect conversion -| Tool | Instruction | +| Tool | Instructions | | --- | --- | -| Plugin [ydb-sqlglot-plugin](https://github.com/ydb-platform/ydb-sqlglot-plugin) for [SQLGlot](https://github.com/tobymao/sqlglot) | Instructions | -| Converter of SQL dialects to YQL | Instructions | +| Plugin [ydb-sqlglot-plugin](https://github.com/ydb-platform/ydb-sqlglot-plugin) for [SQLGlot](https://github.com/tobymao/sqlglot) | [Instructions](../sqlglot.md) | +| SQL dialect converter to YQL | [Instructions](../sql-dialect-converter.md) | diff --git a/ydb/docs/en/core/integrations/sql-translation/index.md b/ydb/docs/en/core/integrations/sql-translation/index.md new file mode 100644 index 00000000000..27ac349138e --- /dev/null +++ b/ydb/docs/en/core/integrations/sql-translation/index.md @@ -0,0 +1,3 @@ +# SQL dialect conversion + +{% include notitle [Contents](_includes/toc-table.md) %} diff --git a/ydb/docs/en/core/integrations/sql-translation/sql-dialect-converter.md b/ydb/docs/en/core/integrations/sql-translation/sql-dialect-converter.md new file mode 100644 index 00000000000..ee54fa46f3d --- /dev/null +++ b/ydb/docs/en/core/integrations/sql-translation/sql-dialect-converter.md @@ -0,0 +1,78 @@ +# SQL dialect converter to YQL + +The SQL dialect converter is a service that converts SQL queries written in other dialects (PostgreSQL, MySQL, ClickHouse, and others) into [YQL](../../yql/reference/index.md) syntax. The service is designed to simplify migration of existing queries to {{ ydb-name }} and lower the entry barrier for developers already familiar with other DBMSs. + +The converter is available as a standalone web application at [ydb-dialect-converter.website.yandexcloud.net](https://ydb-dialect-converter.website.yandexcloud.net/), and is also built into the [DBeaver plugin for {{ ydb-short-name }}](../gui/dbeaver-plugin.md#convert-dialect) and the [VS Code plugin for {{ ydb-short-name }}](../gui/vscode-plugin.md). The user pastes a query in the source dialect, selects the dialect from the list, and gets a version of the same query in YQL. + +## How to use {#usage} + +1. Open the [converter web interface](https://ydb-dialect-converter.website.yandexcloud.net/). +2. Select the source SQL dialect from the drop-down list. +3. Paste the query into the **Input SQL query** field. If necessary, use the ready-made examples (`CTE Example`, `Create Table Example`, and others). +4. Click **Convert** — the YQL result will appear in the **Conversion result (YDB)** field. + +## How it works {#principles} + +The conversion is performed in three stages: + +1. **Parsing the source SQL.** The source query is parsed by the specific dialect parser into an abstract syntax tree (AST) — a structure that describes the query's meaning independently of syntax. +2. **AST transformation.** Constructs specific to the source dialect are replaced with their YQL equivalents. For example, type casting functions, date and string handling, `LIMIT`/`OFFSET` operators, and `WITH` constructs are adapted to YQL rules. +3. **YQL generation.** A textual representation of the query in YQL is generated from the modified AST. + +The stages are implemented using the [SQLGlot](https://github.com/tobymao/sqlglot) library and the [{{ ydb-short-name }} plugin for it](sqlglot.md), which adds a separate {{ ydb-short-name }} dialect to SQLGlot. SQLGlot is an open-source SQL parser and transpiler that supports over twenty dialects. It serves both as the parser for the source query and as the YQL generator. + +If you need not a one-time conversion in the GUI but to embed the transformation into your own code or data processing pipeline, use the plugin directly — see the [SQLGlot](sqlglot.md) article. + +{% note warning %} + +The source query is sent to an external HTTPS service. Do not send queries containing confidential data (personal data, trade secrets, identifiers of real objects in production) to the converter. + +{% endnote %} + +## Supported dialects {#dialects} + +The list of source dialects is determined by the set supported by SQLGlot. Among the main ones: + +- PostgreSQL +- MySQL +- ClickHouse +- Microsoft SQL Server (T-SQL) +- Oracle +- Snowflake +- BigQuery +- Presto / Trino +- Spark SQL / Databricks +- SQLite + +The current full list of dialects can be obtained via the converter API or viewed in the [SQLGlot source code](https://github.com/tobymao/sqlglot/tree/main/sqlglot/dialects). The target dialect is always YQL. + +## Supported constructs {#supported-constructs} + +The converter covers typical constructs of analytical and OLTP queries: + +- SELECT operators: `SELECT`, `JOIN` (all types), `WHERE`, `GROUP BY`, `HAVING`, `ORDER BY`, `LIMIT`, `OFFSET`. +- Window functions (`OVER`, `PARTITION BY`, `ROWS BETWEEN`). +- Subqueries and CTE (`WITH`). +- Type casting (`CAST`, `::`), arithmetic and logical operations. +- Functions for strings, numbers, dates, and time — mapped to the nearest equivalent in YQL. +- Aggregate functions (`COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, and others). +- DML operations: `INSERT`, `UPDATE`, `DELETE` (taking into account [transaction features of {{ ydb-short-name }}](../../concepts/transactions.md)). + +## Limitations {#limitations} + +Full automatic conversion is not always possible, since YQL and other SQL dialects differ in data models and operation semantics. In particular: + +- **Dialect-specific functions.** Functions that have no direct equivalent in YQL (for example, PostgreSQL arrays, `JSON_EXTRACT` MySQL with special path syntax) are translated to approximate equivalents or left for manual refinement. +- **Stored procedures and triggers.** Not supported, as {{ ydb-short-name }} uses a different model for executing code in the database. +- **Dialect-specific DDL.** `CREATE TABLE` for complex types may require manual adjustment considering [columnar and row-based tables](../../concepts/datamodel/table.md) in {{ ydb-short-name }}. +- **Optimizer hints.** Ignored: the {{ ydb-short-name }} optimizer has its own control mechanisms. + +The conversion result should be considered a draft: complex queries require manual review and adaptation before execution. + +## See also {#see-also} + +- [SQL dialect converter web interface](https://ydb-dialect-converter.website.yandexcloud.net/) +- [{#T}](sqlglot.md) +- [{#T}](../gui/dbeaver-plugin.md) +- [YQL Reference](../../yql/reference/index.md) +- [SQLGlot](https://github.com/tobymao/sqlglot) diff --git a/ydb/docs/en/core/integrations/sql-translation/sqlglot.md b/ydb/docs/en/core/integrations/sql-translation/sqlglot.md new file mode 100644 index 00000000000..8772c2cff6d --- /dev/null +++ b/ydb/docs/en/core/integrations/sql-translation/sqlglot.md @@ -0,0 +1,173 @@ +# SQLGlot + +[SQLGlot](https://github.com/tobymao/sqlglot) is a pure Python SQL parser, transpiler, optimizer, and formatter that supports over twenty dialects (PostgreSQL, MySQL, ClickHouse, BigQuery, Snowflake, Spark SQL, and others). SQLGlot parses a query into an abstract syntax tree (AST), which can be analyzed and transformed programmatically, and then generate back to SQL in any of the supported dialects. + +[ydb-sqlglot-plugin plugin](https://github.com/ydb-platform/ydb-sqlglot-plugin) adds the {{ ydb-short-name }} dialect to SQLGlot. After installation, SQLGlot can both parse [YQL](../../yql/reference/index.md) queries and generate YQL from queries written in other dialects. The conversion is bidirectional: any supported dialect ↔ YQL. + +Unlike the ready-made [SQL dialect converter](sql-dialect-converter.md), which operates as an external service and is built into graphical tools, the plugin is a library that you integrate into your own Python code. This provides two important advantages: + +- **Local processing.** Queries never leave your machine — no data is sent to an external HTTPS service. Suitable for working with confidential queries. +- **Programmatic access to AST.** In addition to transpilation, query parsing, column lineage analysis, optimization, and formatting are available — everything SQLGlot can do. + +## Features {#features} + +- Transpilation of queries from other dialects to YQL and back. +- Parsing a YQL query into AST for programmatic analysis and modification. +- Column lineage analysis (column lineage) — tracking which tables and columns each resulting column originates from. +- Query optimization and formatting using SQLGlot. +- Support for YQL-specific constructs: named expressions (`$variable`), module functions (`DateTime::GetYear()`), `FLATTEN`, lambda expressions, container types, and others. + +{% note info %} + +The full list of supported features and current limitations is provided in the [plugin repository README](https://github.com/ydb-platform/ydb-sqlglot-plugin). + +{% endnote %} + +## Installation {#install} + +The plugin is published on PyPI under the name `ydb-sqlglot-plugin`: + + +```bash +pip install ydb-sqlglot-plugin +``` + + +Requirements: + +- Python 3.9 or newer. +- SQLGlot version 28.6.0 or newer (installed automatically as a dependency). + +The plugin is distributed under the Apache 2.0 license. + +## Quick start {#quickstart} + +After installation, the {{ ydb-short-name }} dialect is available in SQLGlot automatically — no additional imports are needed. It has two equivalent names, `ydb` and `yql`; either of them can be specified in the `read` and `write` arguments. + +Converting a query from MySQL to YQL: + + +```python +import sqlglot + +result = sqlglot.transpile( + "SELECT * FROM users WHERE id = 1", + read="mysql", + write="ydb", +)[0] + +print(result) +# SELECT * FROM `users` WHERE id = 1 +``` + + +Reverse transformation — from YQL to PostgreSQL. The YQL named expression (`$t = (...)`) turns into a CTE (`WITH ... AS`): + + +```python +import sqlglot + +result = sqlglot.transpile( + "$t = (SELECT id FROM users); SELECT * FROM $t AS t", + read="ydb", + write="postgres", +)[0] + +print(result) +# WITH t AS (SELECT id FROM users) SELECT * FROM t AS t +``` + + +{% note tip %} + +The `sqlglot.transpile()` function returns a list of strings — one for each statement in the source text. If the query has one statement, take the first element (`[0]`). + +{% endnote %} + +## Usage examples {#usage} + +### Migrating queries from other DBMS {#migration} + +When migrating to {{ ydb-short-name }}, the plugin automatically converts the original dialect constructs to YQL rules. For example, names with schema references (`schema.table`) are transformed into a [{{ ydb-short-name }} path](../../concepts/datamodel/dir.md) with backticks: + + +```python +import sqlglot + +print(sqlglot.transpile("SELECT * FROM analytics.events", read="postgres", write="ydb")[0]) +# SELECT * FROM `analytics/events` +``` + + +Correlated subqueries not directly supported in {{ ydb-short-name }} are, where possible, rewritten into `JOIN`, and `WITH` constructs into named YQL expressions. + +### Parsing a query into AST {#parse} + +If you need not only transpilation but also query structure analysis, parse it into a tree using `sqlglot.parse_one()`: + + +```python +import sqlglot +from sqlglot import exp + +tree = sqlglot.parse_one("SELECT id, name FROM `users` WHERE age > 18", dialect="ydb") + +# List all tables accessed by the query +for table in tree.find_all(exp.Table): + print(table.name) +# users + +# Generate the query back — in any dialect +print(tree.sql(dialect="clickhouse")) +``` + + +### Column lineage analysis {#lineage} + +Since YQL is parsed into the standard SQLGlot AST, the built-in SQLGlot column lineage analysis works for YQL queries. This allows you to trace which source tables and columns each resulting column comes from: + + +```python +from sqlglot.lineage import lineage + +node = lineage( + "total", + "SELECT SUM(amount) AS total FROM orders", + dialect="ydb", +) + +print(node.name) +# total +``` + + +Lineage analysis is useful when building data documentation tools, assessing the impact of schema changes (impact analysis), and auditing queries. + +## Function and type mapping {#mappings} + +The plugin contains mapping tables between standard SQL constructs and their counterparts in {{ ydb-short-name }}. + +Data types are mapped to {{ ydb-short-name }} equivalents: + +| Type in the source dialect | {{ ydb-short-name }} | +| --- | --- | +| `TINYINT` | `Int8` | +| `INT` | `Int32` | +| `BIGINT` | `Int64` | +| `VARCHAR`, `TEXT` | `Utf8` | +| `TIMESTAMP` | `Timestamp` | + +Functions are matched by value family: + +- **Date and time:** `DATE_TRUNC`, `EXTRACT`, interval operations. +- **Rows:** `CONCAT`, `UPPER`, `LOWER`, `LENGTH`, and others. +- **Collections:** `ARRAY`, `ARRAY_FILTER`, `UNNEST` → `FLATTEN BY`. +- **Conditional and numeric:** `NULLIF`, `ROUND`, `COUNT()`. +- **JSON:** `JSON_VALUE`, `JSON_QUERY` with support for related constructs. + +## See also {#see-also} + +- [{#T}](sql-dialect-converter.md) +- [ydb-sqlglot-plugin on GitHub](https://github.com/ydb-platform/ydb-sqlglot-plugin) +- [SQLGlot](https://github.com/tobymao/sqlglot) +- [YQL Reference](../../yql/reference/index.md) diff --git a/ydb/docs/en/core/integrations/sql-translation/toc-sql-translation.yaml b/ydb/docs/en/core/integrations/sql-translation/toc-sql-translation.yaml new file mode 100644 index 00000000000..4974c8b8eb5 --- /dev/null +++ b/ydb/docs/en/core/integrations/sql-translation/toc-sql-translation.yaml @@ -0,0 +1,5 @@ +items: +- name: SQLGlot + href: sqlglot.md +- name: SQL dialect converter to YQL + href: sql-dialect-converter.md diff --git a/ydb/docs/en/core/integrations/toc_i.yaml b/ydb/docs/en/core/integrations/toc_i.yaml index 10ed0517d1e..02258884ee5 100644 --- a/ydb/docs/en/core/integrations/toc_i.yaml +++ b/ydb/docs/en/core/integrations/toc_i.yaml @@ -39,15 +39,25 @@ items: include: mode: link path: orm/toc-orm.yaml +- name: Vector search + href: vectorsearch/index.md + include: + mode: link + path: vectorsearch/toc-vectorsearch.yaml +- name: SQL dialect conversion + href: sql-translation/index.md + include: + mode: link + path: sql-translation/toc-sql-translation.yaml - name: Spring href: spring/index.md include: mode: link path: spring/toc-spring.yaml -- name: Vector search - href: vectorsearch/index.md +- name: Spring + href: spring/index.md include: mode: link - path: vectorsearch/toc-vectorsearch.yaml + path: spring/toc-spring.yaml - name: SQL Dialect Converter to YQL href: sql-dialect-converter.md diff --git a/ydb/library/actors/core/servicemap.h b/ydb/library/actors/core/servicemap.h index d72e50cae5f..209d8cf13e3 100644 --- a/ydb/library/actors/core/servicemap.h +++ b/ydb/library/actors/core/servicemap.h @@ -2,23 +2,60 @@ #include "defs.h" +#include <util/generic/algorithm.h> + +#include <array> +#include <atomic> +#include <cstring> +#include <type_traits> + namespace NActors { // wait-free one writer multi reader hash-tree for service mapping purposes // on fast updates on same key - could lead to false-negatives, we don't care as such cases are broken from service-map app logic template <typename TKey, typename TValue, typename THash, ui64 BaseSize = 256 * 1024, ui64 ExtCount = 4, ui64 ExtBranching = 4> class TServiceMap : TNonCopyable { + // CounterIn/CounterOut validate the whole snapshot, but every part still + // has to be accessed atomically to make a discarded torn read well-defined. + template <typename T> + class TAtomicStorage : TNonCopyable { + static_assert(std::is_trivially_copyable_v<T>); + static_assert(sizeof(T) % sizeof(ui64) == 0); + + static constexpr size_t PartCount = sizeof(T) / sizeof(ui64); + std::array<std::atomic<ui64>, PartCount> Parts{}; + + public: + T Load() const noexcept { + std::array<ui64, PartCount> parts; + for (size_t i = 0; i != PartCount; ++i) { + parts[i] = Parts[i].load(std::memory_order_relaxed); + } + + T value; + std::memcpy(&value, parts.data(), sizeof(value)); + return value; + } + + void Store(const T& value) noexcept { + std::array<ui64, PartCount> parts; + std::memcpy(parts.data(), &value, sizeof(value)); + + for (size_t i = 0; i != PartCount; ++i) { + Parts[i].store(parts[i], std::memory_order_relaxed); + } + } + }; + struct TEntry : TNonCopyable { ui32 CounterIn; ui32 CounterOut; - TKey Key; - TValue Value; + TAtomicStorage<TKey> Key; + TAtomicStorage<TValue> Value; TEntry() : CounterIn(0) , CounterOut(0) - , Key() - , Value() { } }; @@ -39,8 +76,8 @@ namespace NActors { for (ui32 i = 0; i != ExtCount; ++i) { const TEntry& entry = branch->Entries[i]; const ui32 counterIn = AtomicLoad(&entry.CounterIn); - if (counterIn != 0 && entry.Key == key) { - ret = entry.Value; + if (counterIn != 0 && entry.Key.Load() == key) { + ret = entry.Value.Load(); const ui32 counterOut = AtomicLoad(&entry.CounterOut); if (counterOut == counterIn) return true; @@ -64,7 +101,7 @@ namespace NActors { return; } } else { - if (entry.Key == key) { + if (entry.Key.Load() == key) { oldEntry = &entry; if (!zeroEntry || *zeroEntry) return; @@ -131,12 +168,12 @@ namespace NActors { // now we got both entries, first - push new one const ui32 counter = AtomicUi32Increment(&Counter); AtomicStore(&zeroEntry->CounterOut, counter); - zeroEntry->Key = key; - zeroEntry->Value = value; + zeroEntry->Key.Store(key); + zeroEntry->Value.Store(value); AtomicStore(&zeroEntry->CounterIn, counter); if (oldEntry != nullptr) { - const TValue ret = oldEntry->Value; + const TValue ret = oldEntry->Value.Load(); AtomicStore<ui32>(&oldEntry->CounterOut, 0); AtomicStore<ui32>(&oldEntry->CounterIn, 0); return ret; diff --git a/ydb/library/actors/core/ut/servicemap_ut.cpp b/ydb/library/actors/core/ut/servicemap_ut.cpp new file mode 100644 index 00000000000..5f32b68ffef --- /dev/null +++ b/ydb/library/actors/core/ut/servicemap_ut.cpp @@ -0,0 +1,57 @@ +#include "../actorid.h" +#include "../servicemap.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <atomic> +#include <latch> +#include <thread> +#include <vector> + +namespace NActors { + +Y_UNIT_TEST_SUITE(ServiceMap) { + + Y_UNIT_TEST(ConcurrentUpdateAndFind) { + constexpr size_t readerCount = 4; + constexpr size_t updateCount = 1'000'000; + + TServiceMap<TActorId, TActorId, TActorId::THash, 1, 2, 2> map; + const TActorId serviceId(1, TStringBuf("service")); + const TActorId actorId1(1, 0, 1, 0); + const TActorId actorId2(1, 0, 2, 0); + map.Update(serviceId, actorId1); + + std::atomic<bool> stop = false; + std::atomic<size_t> invalidReads = 0; + std::latch start(readerCount + 1); + std::vector<std::thread> readers; + readers.reserve(readerCount); + + for (size_t i = 0; i != readerCount; ++i) { + readers.emplace_back([&] { + start.arrive_and_wait(); + while (!stop.load(std::memory_order_relaxed)) { + const TActorId actorId = map.Find(serviceId); + if (actorId && actorId != actorId1 && actorId != actorId2) { + invalidReads.fetch_add(1, std::memory_order_relaxed); + } + } + }); + } + + start.arrive_and_wait(); + for (size_t i = 0; i != updateCount; ++i) { + map.Update(serviceId, i % 2 ? actorId1 : actorId2); + } + stop.store(true, std::memory_order_relaxed); + + for (std::thread& reader : readers) { + reader.join(); + } + + UNIT_ASSERT_VALUES_EQUAL(invalidReads.load(), 0); + } +} + +} // namespace NActors diff --git a/ydb/library/actors/core/ut/ya.make b/ydb/library/actors/core/ut/ya.make index 6c18ed3f37a..c68796ac696 100644 --- a/ydb/library/actors/core/ut/ya.make +++ b/ydb/library/actors/core/ut/ya.make @@ -38,6 +38,7 @@ SRCS( mon_ut.cpp scheduler_actor_ut.cpp mailbox_lockfree_ut.cpp + servicemap_ut.cpp struct_log_ut.cpp ) diff --git a/ydb/library/actors/interconnect/load.cpp b/ydb/library/actors/interconnect/load.cpp index 1f2e2db20c1..727ae35f1f3 100644 --- a/ydb/library/actors/interconnect/load.cpp +++ b/ydb/library/actors/interconnect/load.cpp @@ -364,12 +364,13 @@ namespace NInterconnect { msg << " Throughput# "; const TDuration duration = now - ThroughputFirstSample; + const ui64 durationUs = duration.MicroSeconds(); const ui64 traffic = *Traffic; msg << "{window# " << duration << " bytes# " << ThroughputBytes << " samples# " << ThroughputSamples - << " b/s# " << ui64(ThroughputBytes * 1000000 / duration.MicroSeconds()) - << " common# " << ui64((traffic - TrafficAtBegin) * 1000000 / duration.MicroSeconds()) + << " b/s# " << (durationUs ? ui64(ThroughputBytes * 1000000 / durationUs) : ui64(0)) + << " common# " << (durationUs ? ui64((traffic - TrafficAtBegin) * 1000000 / durationUs) : ui64(0)) << "}"; if (RunFirstSample == TMonotonic::Zero()) { @@ -478,6 +479,7 @@ namespace NInterconnect { TString RenderHTML(bool finished, const TActorContext& ctx) { const auto duration = ctx.Monotonic() - ThroughputFirstSample; + const ui64 durationUs = duration.MicroSeconds(); TStringStream msg; @@ -498,8 +500,8 @@ namespace NInterconnect { TABLED() { str << duration; } TABLED() { str << ThroughputBytes; } TABLED() { str << ThroughputSamples; } - TABLED() { str << ui64(ThroughputBytes * 1000000 / duration.MicroSeconds()); } - TABLED() { str << ui64((*Traffic - TrafficAtBegin) * 1000000 / duration.MicroSeconds()); } + TABLED() { str << (durationUs ? ui64(ThroughputBytes * 1000000 / durationUs) : ui64(0)); } + TABLED() { str << (durationUs ? ui64((*Traffic - TrafficAtBegin) * 1000000 / durationUs) : ui64(0)); } } } } diff --git a/ydb/library/yql/dq/actors/spilling/spilling_file.cpp b/ydb/library/yql/dq/actors/spilling/spilling_file.cpp index 055dd2d346c..bab5517834e 100644 --- a/ydb/library/yql/dq/actors/spilling/spilling_file.cpp +++ b/ydb/library/yql/dq/actors/spilling/spilling_file.cpp @@ -205,7 +205,7 @@ private: ui32 NodeId; TString SpillingSessionId; - TEvRemoveOldTmp(TFsPath tmpRoot, ui32 nodeId, TString spillingSessionId) + TEvRemoveOldTmp(TFsPath tmpRoot, ui32 nodeId, TString spillingSessionId) : TmpRoot(std::move(tmpRoot)), NodeId(nodeId), SpillingSessionId(std::move(spillingSessionId)) {} }; }; @@ -243,7 +243,7 @@ public: Become(&TDqLocalFileSpillingService::BrokenState); return; } - + Send(SelfId(), MakeHolder<TEvPrivate::TEvRemoveOldTmp>(rootToRemoveOldTmp, nodeId, sessionId)); Become(&TDqLocalFileSpillingService::WorkState); @@ -483,7 +483,7 @@ private: LOG_E(error); Send(ev->Sender, new TEvDqSpilling::TEvError(error)); - } + } } void HandleWork(TEvPrivate::TEvWriteFileResponse::TPtr& ev) { @@ -536,6 +536,7 @@ private: Send(msg.Client, new TEvDqSpilling::TEvError(*fd.Error)); fd.Ops.clear(); + fd.HasActiveOp = false; CloseFile(it, fd.Error); return; } @@ -630,7 +631,7 @@ private: LOG_E(error); Send(ev->Sender, new TEvDqSpilling::TEvError(error)); - } + } } void HandleWork(TEvPrivate::TEvReadFileResponse::TPtr& ev) { @@ -684,6 +685,7 @@ private: Send(msg.Client, new TEvDqSpilling::TEvError(*fd.Error)); fd.Ops.clear(); + fd.HasActiveOp = false; CloseFile(it, fd.Error); return; } @@ -792,7 +794,7 @@ private: LOG_I("[RemoveOldTmp] removing at root: " << root); - const auto isDirOldTmp = [&nodePrefix, &nodeIdString, &sessionId](const TString& dirName) -> bool { + const auto isDirOldTmp = [&nodePrefix, &nodeIdString, &sessionId](const TString& dirName) -> bool { // dirName: node_<nodeId>_<sessionId> TVector<TString> parts; StringSplitter(dirName).Split('_').Limit(3).Collect(&parts); @@ -805,13 +807,13 @@ private: try { TDirIterator iter(root, TDirIterator::TOptions().SetMaxLevel(1)); - + TVector<TString> oldTmps; for (const auto& dirEntry : iter) { if (dirEntry.fts_info == FTS_DP) { continue; } - + const auto dirName = dirEntry.fts_name; if (isDirOldTmp(dirName)) { LOG_D("[RemoveOldTmp] found old temporary at " << (root / dirName)); diff --git a/ydb/library/yql/dq/actors/spilling/spilling_file_ut.cpp b/ydb/library/yql/dq/actors/spilling/spilling_file_ut.cpp index ae7cfe55621..71ee1269fe5 100644 --- a/ydb/library/yql/dq/actors/spilling/spilling_file_ut.cpp +++ b/ydb/library/yql/dq/actors/spilling/spilling_file_ut.cpp @@ -496,6 +496,103 @@ Y_UNIT_TEST_SUITE(DqSpillingFileTests) { UNIT_ASSERT_C(err.Contains("No such file or directory"), err); UNIT_ASSERT_C(err.Contains(expected), err); } + + // The read error must trigger a proper cleanup: the file has to be closed and its + // descriptor released. If the close operation is not dispatched (e.g. because the + // active-op flag was left set), EvCloseFileResponse never arrives and the file stays + // stuck in the active list, keeping the file descriptor counter above zero. + { + std::atomic<bool> closed = false; + runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& event) { + if (event->GetRecipientRewrite() == spillingSvc) { + if (event->GetTypeRewrite() == 2146435074 /* EvCloseFileResponse */) { + closed = true; + } + } + return TTestActorRuntimeBase::EEventAction::PROCESS; + }); + + TDispatchOptions options; + options.CustomFinalCondition = [&]() { + return (bool) closed; + }; + runtime.DispatchEvents(options, TDuration::Seconds(1)); + UNIT_ASSERT_C(closed, "file was not closed after read error"); + } + + { + THttpRequest httpReq(HTTP_METHOD_GET); + NMonitoring::TMonService2HttpRequest monReq(nullptr, &httpReq, nullptr, nullptr, "", nullptr); + + runtime.Send(new IEventHandle(spillingSvc, tester, new NMon::TEvHttpInfo(monReq))); + auto resp = runtime.GrabEdgeEvent<NMon::TEvHttpInfoRes>(tester, TDuration::Seconds(1)); + UNIT_ASSERT(((NMon::TEvHttpInfoRes*) resp->Get())->Answer.Contains("Used file descriptors (compute): 0")); + } + } + + Y_UNIT_TEST(WriteError) { + TTestActorRuntime runtime; + runtime.Initialize(); + + auto spillingSvc = runtime.StartSpillingService(1000, 100, 25); + auto tester = runtime.AllocateEdgeActor(); + auto spillingActor = runtime.StartSpillingActor(tester); + + runtime.WaitBootstrap(); + + auto nodePath = TFsPath("node_" + std::to_string(spillingSvc.NodeId()) + "_" + runtime.GetSpillingSessionId()); + const TFsPath spillingDir = runtime.GetSpillingRoot() / nodePath; + const TFsPath firstFile = spillingDir / "1_test_0"; + const TFsPath secondFile = spillingDir / "1_test_1"; + + // Write the first blob. Because MaxFilePartSize is small and RemoveBlobsAfterRead is on, + // this blob lands in its own file part "1_test_0". + { + auto ev = new TEvDqSpilling::TEvWrite(0, CreateRope(20, 'a')); + runtime.Send(new IEventHandle(spillingActor, tester, ev)); + + auto resp = runtime.GrabEdgeEvent<TEvDqSpilling::TEvWriteResult>(tester); + UNIT_ASSERT_VALUES_EQUAL(0, resp->Get()->BlobId); + } + UNIT_ASSERT(NFs::Exists(firstFile.GetPath())); + + // File part names are predictable, so occupy the name of the would-be next file part + // ("1_test_1") with a directory. Opening it for writing will fail and produce an IO error. + secondFile.MkDirs(); + + // Write the second blob. It requires a fresh file part ("1_test_1"), whose creation now + // fails with an IO error. + { + auto ev = new TEvDqSpilling::TEvWrite(1, CreateRope(20, 'b')); + runtime.Send(new IEventHandle(spillingActor, tester, ev)); + + auto resp = runtime.GrabEdgeEvent<TEvDqSpilling::TEvError>(tester); + UNIT_ASSERT_C(resp->Get()->Message.Contains("1_test_1"), resp->Get()->Message); + } + + // The write error must trigger cleanup of the whole file, including the already-written + // "1_test_0" part. Wait for the close operation to complete. + { + std::atomic<bool> closed = false; + runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& event) { + if (event->GetRecipientRewrite() == spillingSvc) { + if (event->GetTypeRewrite() == 2146435074 /* EvCloseFileResponse */) { + closed = true; + } + } + return TTestActorRuntimeBase::EEventAction::PROCESS; + }); + + TDispatchOptions options; + options.CustomFinalCondition = [&]() { + return (bool) closed; + }; + runtime.DispatchEvents(options, TDuration::Seconds(1)); + UNIT_ASSERT_C(closed, "file was not closed after write error"); + } + + // The successfully written part must have been removed from disk during cleanup. + UNIT_ASSERT_C(!NFs::Exists(firstFile.GetPath()), "partially written file was not cleaned up"); } Y_UNIT_TEST(ThreadPoolQueueOverflow) { diff --git a/ydb/library/yql/dq/runtime/dq_channel_service.cpp b/ydb/library/yql/dq/runtime/dq_channel_service.cpp index 4bb02dcb1fe..6446cd1d05a 100644 --- a/ydb/library/yql/dq/runtime/dq_channel_service.cpp +++ b/ydb/library/yql/dq/runtime/dq_channel_service.cpp @@ -1042,6 +1042,7 @@ void TNodeState::PushDataChunk(TDataChunk&& data, std::shared_ptr<TOutputDescrip item->Leading = descriptor->Leading.exchange(false); Queue.push_back(item); SendMessage(item); + SendCount++; InflightBytes += bytes; *OutputBufferInflightBytes += bytes; (*OutputBufferInflightMessages)++; @@ -1080,7 +1081,7 @@ void TNodeState::PushDataChunk(TDataChunk&& data, std::shared_ptr<TOutputDescrip } void TNodeState::SendMessage(std::shared_ptr<TOutputItem> item) { - Y_ENSURE(PeerActorId); + Y_ENSURE(InputNodeActorId); auto ev = MakeHolder<TEvDqCompute::TEvChannelDataV2>(); ev->Record.SetGenMajor(GenMajor); @@ -1132,20 +1133,20 @@ void TNodeState::SendMessage(std::shared_ptr<TOutputItem> item) { FailureDoubleSend.store(failCount - 1); auto ev2 = MakeHolder<TEvDqCompute::TEvChannelDataV2>(); ev2->Record = ev->Record; - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, ev2.Release(), flags, item->SeqNo)); + ActorSystem->Send(new NActors::IEventHandle(InputNodeActorId, NodeActorId, ev2.Release(), flags, item->SeqNo)); } #endif - LOG_T(LogPrefix << "SEND MSG, G=" << GenMajor << '.' << GenMinor << ", SeqNo=" << item->SeqNo - << ", ChannelSeqNo=" << item->ChannelSeqNo - << ", ChannelId=" << item->Descriptor->Info.ChannelId << ", Leading=" << item->Leading << ", Bytes=" << item->Data.Bytes); - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, ev.Release(), flags, item->SeqNo)); + LOG_T(LogPrefix << "SEND DATA, G=" << GenMajor << '.' << GenMinor << ", SeqNo=" << item->SeqNo + << ", ChannelSeqNo=" << item->ChannelSeqNo << ", ChannelId=" << item->Descriptor->Info.ChannelId + << ", Leading=" << item->Leading << ", Finished=" << item->Data.Finished << ", Bytes=" << item->Data.Bytes); + ActorSystem->Send(new NActors::IEventHandle(InputNodeActorId, NodeActorId, ev.Release(), flags, item->SeqNo)); #if !defined(NDEBUG) } #endif item->State.store(TOutputItem::EState::Sent); } -void TNodeState::FailInputs(const NActors::TActorId& peerActorId, ui64 peerGenMajor) { +void TNodeState::FailInputs(const NActors::TActorId& outputNodeActorId, ui64 outputNodeGenMajor) { if (InputDescriptors.empty()) { return; } @@ -1153,11 +1154,11 @@ void TNodeState::FailInputs(const NActors::TActorId& peerActorId, ui64 peerGenMa std::vector<TChannelInfo> failedBuffers; for (auto& [info, descriptor] : InputDescriptors) { - if (!descriptor->IsFinished() && descriptor->PeerGenMajor) { - if (descriptor->PeerActorId != peerActorId || descriptor->PeerGenMajor != peerGenMajor) { + if (!descriptor->IsFinished() && descriptor->OutputNodeGenMajor) { + if (descriptor->OutputNodeActorId != outputNodeActorId || descriptor->OutputNodeGenMajor != outputNodeGenMajor) { descriptor->AbortChannel( - TStringBuilder() << "PeerActorId=" << descriptor->PeerActorId << ", PeerGenMajor=" << descriptor->PeerGenMajor - << " DO NOT MATCH peerActorId=" << peerActorId << ", peerGenMajor=" << peerGenMajor + TStringBuilder() << "OutputNodeActorId=" << descriptor->OutputNodeActorId << ", OutputNodeGenMajor=" << descriptor->OutputNodeGenMajor + << " DO NOT MATCH outputNodeActorId=" << outputNodeActorId << ", outputNodeGenMajor=" << outputNodeGenMajor << ", Session=" << LogPrefix << ", Log=" << GetReconciliationLog() ); failedBuffers.push_back(info); @@ -1207,14 +1208,14 @@ void TNodeState::SendAck(THolder<TEvDqCompute::TEvChannelAckV2>& evAck, ui64 coo flags |= NActors::IEventHandle::FlagSubscribeOnSession; } - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, evAck.Release(), flags, cookie)); + ActorSystem->Send(new NActors::IEventHandle(OutputNodeActorId, NodeActorId, evAck.Release(), flags, cookie)); } void TNodeState::SendAckWithError(ui64 cookie, const TString& message) { auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - evAck->Record.SetGenMajor(PeerGenMajor.load()); - evAck->Record.SetGenMinor(PeerGenMinor.load()); + evAck->Record.SetGenMajor(OutputNodeGenMajor.load()); + evAck->Record.SetGenMinor(OutputNodeGenMinor.load()); evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::ERROR); evAck->Record.SetSeqNo(ConfirmedSeqNo); evAck->Record.SetMessage(message); @@ -1245,12 +1246,12 @@ void TNodeState::HandleChannelData(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { return; } - if (descriptor->PeerGenMajor) { - if (descriptor->PeerActorId != PeerActorId || descriptor->PeerGenMajor != PeerGenMajor.load()) { + if (descriptor->OutputNodeGenMajor) { + if (descriptor->OutputNodeActorId != OutputNodeActorId || descriptor->OutputNodeGenMajor != OutputNodeGenMajor.load()) { descriptor->Terminate(); - TString errorMessage = TStringBuilder() << "MISMATCH G=" << descriptor->PeerGenMajor - << ", Peer=" << descriptor->PeerGenMajor << " vs G=" << PeerGenMajor.load() - << ", Peer=" << PeerActorId + TString errorMessage = TStringBuilder() << "MISMATCH ID G=" << descriptor->OutputNodeGenMajor + << ", Peer=" << descriptor->OutputNodeGenMajor << " vs G=" << OutputNodeGenMajor.load() + << ", Peer=" << OutputNodeActorId << ", ChannelId=" << info.ChannelId << ", OA=" << info.OutputActorId << ", IA=" << info.InputActorId << ", Bytes=" << record.GetBytes() << ", L=" << record.GetLeading() @@ -1264,8 +1265,8 @@ void TNodeState::HandleChannelData(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { return; } } else { - descriptor->PeerActorId = PeerActorId; - descriptor->PeerGenMajor = PeerGenMajor.load(); + descriptor->OutputNodeActorId = OutputNodeActorId; + descriptor->OutputNodeGenMajor = OutputNodeGenMajor.load(); } TDataChunk data(TChunkedBuffer(), record.GetRows(), record.GetTransportVersion(), @@ -1322,8 +1323,8 @@ void TNodeState::HandleChannelData(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - evAck->Record.SetGenMajor(PeerGenMajor.load()); - evAck->Record.SetGenMinor(PeerGenMinor.load()); + evAck->Record.SetGenMajor(OutputNodeGenMajor.load()); + evAck->Record.SetGenMinor(OutputNodeGenMinor.load()); evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::OK); evAck->Record.SetSeqNo(ConfirmedSeqNo); @@ -1346,25 +1347,18 @@ void TNodeState::HandleDisconnected(NActors::TEvInterconnect::TEvNodeDisconnecte void TNodeState::HandleUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { - if (ev->Get()->Reason == NActors::TEvents::TEvUndelivered::ReasonActorUnknown) { - std::lock_guard lock(Mutex); - if (Reconciliation.load() == 0) { // ignore errors in recovery - if (ev->Get()->SourceType == TEvDqCompute::TEvChannelUpdateV2::EventType) { - LOG_D(LogPrefix << "UNDELIVERED/UNKNOWN UPDATE, PeerActorId " << PeerActorId << ", Sender=" << ev->Sender); - } else { - LOG_W(LogPrefix << "UNDELIVERED/UNKNOWN, PeerActorId " << PeerActorId << ", Sender=" << ev->Sender); - PeerActorId = NActors::TActorId{}; - StartReconciliation(true, 'U'); - } - } - return; - } - switch (ev->Get()->SourceType) { case TEvDqCompute::TEvChannelDataV2::EventType: { - LOG_W(LogPrefix << "UNDELIVERED/OTHER"); std::lock_guard lock(Mutex); - StartReconciliation(false, 'O'); + if (ev->Get()->Reason == NActors::TEvents::TEvUndelivered::ReasonActorUnknown) { + if (Reconciliation.load() == 0) { // ignore errors in recovery + LOG_W(LogPrefix << "UNDELIVERED/UNKNOWN, InputNodeActorId " << InputNodeActorId << ", Sender=" << ev->Sender); + StartReconciliation(true, 'U'); + } + } else { + LOG_W(LogPrefix << "UNDELIVERED/OTHER"); + StartReconciliation(false, 'O'); + } break; } case TEvDqCompute::TEvChannelAckV2::EventType: { @@ -1380,45 +1374,32 @@ void TNodeState::HandleUndelivered(NActors::TEvents::TEvUndelivered::TPtr& ev) { } } -void TNodeState::ConnectSession(NActors::TActorId& sender, ui64 genMajor, ui64 genMinor, ui64 seqNo) { - std::lock_guard lock(Mutex); - if (!Connected) { - PeerActorId = sender; - PeerGenMajor.store(genMajor); - PeerGenMinor.store(genMinor); - ConfirmedSeqNo = seqNo; - Connected = true; - LOG_D(LogPrefix << "CONNECTED, PeerActorId=" << sender << ", PG=" << genMajor << '.' << genMinor); +void TNodeState::ConnectSession(NActors::TActorId& sender, ui64 genMajor, ui64 genMinor) { + if (OutputNodeActorId == sender && OutputNodeGenMajor.load() == genMajor) { + LOG_D(LogPrefix << "RECONNECTED, OutputNodeActorId=" << sender << ", PG=" << genMajor << '.' << genMinor); } else { - if (PeerActorId == sender && PeerGenMajor.load() == genMajor) { - LOG_D(LogPrefix << "RECONNECTED, PeerActorId=" << sender << ", PG=" << genMajor << '.' << genMinor); - } else { - PeerActorId = sender; - PeerGenMajor.store(genMajor); - LOG_W(LogPrefix << "RECONNECTED, PeerActorId=" << sender << ", PG=" << genMajor << '.' << genMinor); - } - PeerGenMinor.store(genMinor); - ConfirmedSeqNo = seqNo; - FailInputs(PeerActorId, PeerGenMajor.load()); - } - for (auto& [_, descriptor] : InputDescriptors) { - if (descriptor->EarlyFinished.load() || descriptor->PushStats.Bytes.load()) { - SendUpdateProgress(descriptor); - } + OutputNodeActorId = sender; + OutputNodeGenMajor.store(genMajor); + ConfirmedSeqNo = 0; + LOG_W(LogPrefix << "RECONNECTED, OutputNodeActorId=" << sender << ", PG=" << genMajor << '.' << genMinor); } + OutputNodeGenMinor.store(genMinor); + FailInputs(OutputNodeActorId, OutputNodeGenMajor.load()); } void TNodeState::HandleDiscovery(TEvDqCompute::TEvChannelDiscoveryV2::TPtr& ev) { LastPeerActivity.store(TInstant::Now()); auto& record = ev->Get()->Record; - ConnectSession(ev->Sender, record.GetGenMajor(), record.GetGenMinor(), record.GetSeqNo()); + + std::lock_guard lock(Mutex); + ConnectSession(ev->Sender, record.GetGenMajor(), record.GetGenMinor()); auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - evAck->Record.SetGenMajor(PeerGenMajor.load()); - evAck->Record.SetGenMinor(PeerGenMinor.load()); - evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::OK); + evAck->Record.SetGenMajor(OutputNodeGenMajor.load()); + evAck->Record.SetGenMinor(OutputNodeGenMinor.load()); + evAck->Record.SetStatus(record.GetSeqNo() <= ConfirmedSeqNo ? NYql::NDqProto::TEvChannelAckV2::OK : NYql::NDqProto::TEvChannelAckV2::RESEND); evAck->Record.SetSeqNo(ConfirmedSeqNo); ui32 flags = NActors::IEventHandle::FlagTrackDelivery; @@ -1426,67 +1407,46 @@ void TNodeState::HandleDiscovery(TEvDqCompute::TEvChannelDiscoveryV2::TPtr& ev) flags |= NActors::IEventHandle::FlagSubscribeOnSession; } - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, evAck.Release(), flags, ev->Cookie)); + ActorSystem->Send(new NActors::IEventHandle(OutputNodeActorId, NodeActorId, evAck.Release(), flags, ev->Cookie)); + + for (auto& [_, descriptor] : InputDescriptors) { + if (descriptor->EarlyFinished.load() || descriptor->PushStats.Bytes.load()) { + SendUpdateProgress(descriptor); + } + } } void TNodeState::HandleData(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { - LastPeerActivity.store(TInstant::Now()); auto& record = ev->Get()->Record; - LOG_T(LogPrefix << "RECV MSG, PG=" << PeerGenMajor.load() << '.' << PeerGenMinor.load() << ", SeqNo=" << record.GetSeqNo() - << ", ChannelSeqNo=" << record.GetChannelSeqNo() - << ", ChannelId=" << record.GetChannelId() << ", Bytes=" << record.GetBytes()); - - auto prevPeerGenMajor = PeerGenMajor.exchange(record.GetGenMajor()); - if (PeerActorId != ev->Sender || (prevPeerGenMajor && prevPeerGenMajor != record.GetGenMajor())) { - auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - - auto reconciliationLog = GetReconciliationLog(); - evAck->Record.SetGenMajor(record.GetGenMajor()); - evAck->Record.SetGenMinor(record.GetGenMinor()); - evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::FAIL); - evAck->Record.SetSeqNo(ConfirmedSeqNo); - evAck->Record.SetMessage(reconciliationLog); - LOG_E(LogPrefix << "FAIL, PeerActorId=" << PeerActorId << ", ev->Sender" << ev->Sender - << ", PeerGenMajor=" << prevPeerGenMajor << ", record.GetGenMajor()=" << record.GetGenMajor() - << ", Log=" << reconciliationLog); - - ui32 flags = NActors::IEventHandle::FlagTrackDelivery; - if (!Subscribed.exchange(true)) { - flags |= NActors::IEventHandle::FlagSubscribeOnSession; - } + auto genMajor = record.GetGenMajor(); + auto genMinor = record.GetGenMinor(); + auto seqNo = record.GetSeqNo(); - ActorSystem->Send(new NActors::IEventHandle(ev->Sender, NodeActorId, evAck.Release(), flags, ev->Cookie)); - return; + if (OutputNodeActorId != ev->Sender || OutputNodeGenMajor.load() != genMajor || OutputNodeGenMinor.load() != genMinor) { + LOG_D(LogPrefix << "OBSOLETE DATA, OutputNodeActorId=" << OutputNodeActorId << ", G=" << OutputNodeGenMajor.load() << '.' << OutputNodeGenMinor.load() + << " vs Sender=" << ev->Sender << ", G=" << genMajor << '.' << genMinor + << ", SeqNo=" << seqNo); + } else { + LOG_T(LogPrefix << "RECV DATA, G=" << genMajor << '.' << genMinor << ", SeqNo=" << seqNo + << ", ChannelSeqNo=" << record.GetChannelSeqNo() << ", ChannelId=" << record.GetChannelId() + << ", Leading=" << record.GetLeading() << ", Finished=" << record.GetFinished() + << ", Bytes=" << record.GetBytes()); } - PeerGenMinor.store(std::max<ui64>(record.GetGenMinor(), PeerGenMinor.load())); - - auto seqNo = record.GetSeqNo(); - if (seqNo <= ConfirmedSeqNo) { LOG_W(LogPrefix << "DATA/IGNORED, SeqNo=" << seqNo << ", ConfirmedSeqNo=" << ConfirmedSeqNo); return; } - switch (seqNo - ConfirmedSeqNo) { - case 1: { - break; - } - case 2: { - // allow 1 out of order message - LOG_W(LogPrefix << "DATA/OOO, SeqNo=" << seqNo << ", ConfirmedSeqNo=" << ConfirmedSeqNo); - OutOfOrderMessage = ev.Release(); - return; - } - default: { - if (!ResendAsked.exchange(true)) { - LOG_W(LogPrefix << "DATA/RESEND, SeqNo=" << seqNo << ", ConfirmedSeqNo=" << ConfirmedSeqNo); - } + if (seqNo - ConfirmedSeqNo > 1) { + if (!ResendAsked.exchange(true)) { + LOG_W(LogPrefix << "DATA/RESEND, SeqNo=" << seqNo << ", ConfirmedSeqNo=" << ConfirmedSeqNo); + auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - evAck->Record.SetGenMajor(PeerGenMajor.load()); - evAck->Record.SetGenMinor(PeerGenMinor.load()); + evAck->Record.SetGenMajor(OutputNodeGenMajor.load()); + evAck->Record.SetGenMinor(OutputNodeGenMinor.load()); evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::RESEND); evAck->Record.SetSeqNo(ConfirmedSeqNo + 1); @@ -1495,27 +1455,15 @@ void TNodeState::HandleData(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { flags |= NActors::IEventHandle::FlagSubscribeOnSession; } - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, evAck.Release(), flags, ev->Cookie)); - return; + ActorSystem->Send(new NActors::IEventHandle(OutputNodeActorId, NodeActorId, evAck.Release(), flags, ev->Cookie)); } + return; } // happy path - ResendAsked.store(false); ConfirmedSeqNo++; HandleChannelData(ev); - - if (OutOfOrderMessage) { - auto& record = OutOfOrderMessage->Get()->Record; - - if (record.GetSeqNo() == ConfirmedSeqNo + 1) { - ConfirmedSeqNo++; - HandleChannelData(OutOfOrderMessage); - } - - OutOfOrderMessage.Reset(); - } } void TNodeState::SendFromWaiters(ui64 deltaBytes) { @@ -1540,6 +1488,7 @@ void TNodeState::SendFromWaiters(ui64 deltaBytes) { (*OutputBufferWaiterCount)--; /* +now may need to send very last msg from terminated descriptor if (WaitersQueue.top()->IsTerminatedOrAborted()) { auto waitQueueBytes = WaitersQueue.top()->WaitQueueBytes.load(); @@ -1599,6 +1548,7 @@ void TNodeState::SendFromWaiters(ui64 deltaBytes) { item->Leading = waiter->Leading.exchange(false); Queue.push_back(item); SendMessage(item); + SendCount++; inflightBytes += bytes; InflightBytes += bytes; *OutputBufferInflightBytes += bytes; @@ -1645,7 +1595,7 @@ void TNodeState::HandleAck(TEvDqCompute::TEvChannelAckV2::TPtr& ev) { auto genMajor = record.GetGenMajor(); auto genMinor = record.GetGenMinor(); if (GenMajor != genMajor || GenMinor != genMinor) { - LOG_W(LogPrefix << "ACK/IGNORED, G=" << GenMajor << '.' << GenMinor << ", ack.G=" << genMajor << '.' << genMinor); + LOG_W(LogPrefix << "ACK/IGNORED, G=" << GenMajor << '.' << GenMinor << ", ack.G=" << genMajor << '.' << genMinor << ", Sender=" << ev->Sender); return; } @@ -1658,9 +1608,9 @@ void TNodeState::HandleAck(TEvDqCompute::TEvChannelAckV2::TPtr& ev) { return; } - if (PeerActorId == NActors::TActorId{}) { - PeerActorId = ev->Sender; - LOG_D(LogPrefix << "PEER/ACK, PeerActorId=" << ev->Sender); + if (InputNodeActorId == NActors::TActorId{}) { + InputNodeActorId = ev->Sender; + LOG_D(LogPrefix << "PEER/ACK, InputNodeActorId=" << ev->Sender); } if (seqNo > SeqNo) { @@ -1701,11 +1651,8 @@ void TNodeState::HandleAck(TEvDqCompute::TEvChannelAckV2::TPtr& ev) { } } else { if (status == NYql::NDqProto::TEvChannelAckV2::RESEND) { - // if we're reconcilating, ignore next RESENDs - if (record.GetGenMinor() == GenMinor) { - LOG_W(LogPrefix << "SEQ/RESEND, SeqNo=" << seqNo); - StartReconciliation(false, 'R'); - } + LOG_W(LogPrefix << "SEQ/RESEND, SeqNo=" << seqNo); + StartReconciliation(false, 'R'); return; } @@ -1735,10 +1682,12 @@ void TNodeState::HandleAck(TEvDqCompute::TEvChannelAckV2::TPtr& ev) { if (Reconciliation.exchange(0) > 0) { ReconciliationCount = 0; + ReconSent.store(TInstant::Zero()); LOG_I(LogPrefix << "RECONCILED, Q=" << (Queue.empty() ? "E" : ToString(Queue.front()->SeqNo)) << ':' << SeqNo << ", WQ=" << WaitersQueueSize.load() << ", InflightBytes=" << InflightBytes.load() << '-' << deltaBytes); if (!Queue.empty()) { for (auto item : Queue) { SendMessage(item); + ResendCount++; item->Descriptor->CheckGenMajor(GenMajor, TStringBuilder() << "Abort by Repeat from SeqNo=" << Queue.front()->SeqNo << ", item->SeqNo=" << item->SeqNo); } } @@ -1817,8 +1766,8 @@ void TNodeState::SendUpdateProgress(std::shared_ptr<TInputDescriptor>& descripto auto evUpdate = MakeHolder<TEvDqCompute::TEvChannelUpdateV2>(); - evUpdate->Record.SetGenMajor(PeerGenMajor.load()); - evUpdate->Record.SetGenMinor(PeerGenMinor.load()); + evUpdate->Record.SetGenMajor(OutputNodeGenMajor.load()); + evUpdate->Record.SetGenMinor(OutputNodeGenMinor.load()); // evUpdate->Record.SetSeqNo(ConfirmedSeqNo); NActors::ActorIdToProto(descriptor->Info.OutputActorId, evUpdate->Record.MutableSrcActorId()); @@ -1839,7 +1788,7 @@ void TNodeState::SendUpdateProgress(std::shared_ptr<TInputDescriptor>& descripto << ", EarlyFinished=" << descriptor->EarlyFinished.load() << ", PopBytes=" << descriptor->PopStats.Bytes.load() << ", Finishing=" << descriptor->Finishing.load()); - ActorSystem->Send(new NActors::IEventHandle(PeerActorId, NodeActorId, evUpdate.Release(), flags)); + ActorSystem->Send(new NActors::IEventHandle(OutputNodeActorId, NodeActorId, evUpdate.Release(), flags)); } std::shared_ptr<TOutputDescriptor> TNodeState::GetOrCreateOutputDescriptor(const TChannelFullInfo& info, IMemoryQuotaManager::TPtr quotaManager, bool bound, bool leading) { @@ -2035,10 +1984,12 @@ void TNodeState::HandleReconciliation(TEvPrivate::TEvReconciliation::TPtr& ev) { void TNodeState::StartReconciliation(bool major, char logSymbol) { if (Reconciliation.load() == 0 || (major && (GenMinor > 1))) { + ReconCount++; if (major) { GenMajor++; GenMinor = 1; SeqNo = 0; + InputNodeActorId = NActors::TActorId{}; } else { GenMinor++; } @@ -2087,70 +2038,70 @@ void TNodeState::DoReconciliation(char logSymbol) { << ", Q=" << (Queue.empty() ? "E" : ToString(Queue.front()->SeqNo)) << ':' << SeqNo << ", WQ=" << WaitersQueueSize.load() << ", Log=" << reconciliationLog); } else { - LOG_D(LogPrefix << "RECONCILIATION, G" << GenMajor << '.' << GenMinor + LOG_D(LogPrefix << "RECONCILIATION, G=" << GenMajor << '.' << GenMinor << ", Q=" << (Queue.empty() ? "E" : ToString(Queue.front()->SeqNo)) << ':' << SeqNo << ", WQ=" << WaitersQueueSize.load() << ", Log=" << reconciliationLog); } ui32 delta = 0; - std::deque<std::shared_ptr<TOutputItem>> RebuiltQueue; - - auto seqNo = SeqNo; + if (GenMinor == 1) { // => major reconciliation + std::deque<std::shared_ptr<TOutputItem>> RebuiltQueue; + while (!Queue.empty()) { - while (!Queue.empty()) { + auto& item = Queue.front(); - auto& item = Queue.front(); + if (item->Leading) { + auto prevGenMajor = item->Descriptor->GenMajor.exchange(GenMajor); + if (prevGenMajor != GenMajor) { + LOG_W(LogPrefix << "CHANGE OD G=" << prevGenMajor << " to " << GenMajor << " by RECONCILIATION" + << " Channel: " << item->Descriptor->Info.ChannelId + << ", SrcStageId: " << item->Descriptor->Info.SrcStageId + << ", DstStageId: " << item->Descriptor->Info.DstStageId + << ", Log=" << reconciliationLog); + } + } - if (item->Leading) { - auto prevGenMajor = item->Descriptor->GenMajor.exchange(GenMajor); - if (prevGenMajor != GenMajor) { - LOG_W(LogPrefix << "CHANGE OG G=" << prevGenMajor << " to " << GenMajor << " by RECONCILIATION" - << " Channel: " << item->Descriptor->Info.ChannelId - << ", SrcStageId: " << item->Descriptor->Info.SrcStageId - << ", DstStageId: " << item->Descriptor->Info.DstStageId - << ", Log=" << reconciliationLog); + if (item->Descriptor->CheckGenMajor(GenMajor, TStringBuilder() << "Abort by Reconciliation, Log=" << reconciliationLog)) { + item->SeqNo = ++SeqNo; + RebuiltQueue.push_back(std::move(item)); + } else { + delta += item->Data.Bytes; } - } - if (item->Descriptor->CheckGenMajor(GenMajor, TStringBuilder() << "Abort by Reconciliation, Log=" << reconciliationLog)) { - item->SeqNo = ++SeqNo; - RebuiltQueue.push_back(std::move(item)); - } else { - delta += item->Data.Bytes; + Queue.pop_front(); } - - Queue.pop_front(); + Queue.swap(RebuiltQueue); } - Queue.swap(RebuiltQueue); InflightBytes -= delta; - SendDiscovery(MakeChannelServiceActorID(NodeId), seqNo); + SendDiscovery(); + ReconSent.store(TInstant::Now()); ActorSystem->Schedule(reconciliationTimeout, new NActors::IEventHandle(NodeActorId, NodeActorId, new TEvPrivate::TEvReconciliation(GenMajor, GenMinor, ReconciliationCount))); } -void TNodeState::SendDiscovery(NActors::TActorId actorId, ui64 seqNo) { +void TNodeState::SendDiscovery() { auto evDiscovery = MakeHolder<TEvDqCompute::TEvChannelDiscoveryV2>(); evDiscovery->Record.SetGenMajor(GenMajor); evDiscovery->Record.SetGenMinor(GenMinor); - evDiscovery->Record.SetSeqNo(seqNo); + evDiscovery->Record.SetSeqNo(SeqNo); ui32 flags = NActors::IEventHandle::FlagTrackDelivery; if (!Subscribed.exchange(true)) { flags |= NActors::IEventHandle::FlagSubscribeOnSession; } - ActorSystem->Send(new NActors::IEventHandle(actorId, NodeActorId, evDiscovery.Release(), flags)); + ActorSystem->Send(new NActors::IEventHandle(MakeChannelServiceActorID(NodeId), NodeActorId, evDiscovery.Release(), flags)); } TString TNodeState::GetDebugInfo() { std::lock_guard lock(Mutex); TStringBuilder builder; - builder << "TNodeState, NodeId=" << NodeActorId.NodeId() << ", Peer NodeId=" << PeerActorId.NodeId() + builder << "TNodeState, NodeId=" << NodeActorId.NodeId() << ", Peer NodeId=" << NodeId << ", SeqNo=" << SeqNo << ", ConfirmedSeqNo=" << ConfirmedSeqNo << ", InflightBytes=" << InflightBytes.load() << ", Reconciliation=" << Reconciliation.load() << Endl; @@ -2175,7 +2126,7 @@ void TDebugNodeState::HandleNullMode(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { auto& record = ev->Get()->Record; - PeerGenMinor.store(std::max<ui64>(record.GetGenMinor(), PeerGenMinor.load())); + OutputNodeGenMinor.store(record.GetGenMinor()); auto seqNo = record.GetSeqNo(); @@ -2196,8 +2147,8 @@ void TDebugNodeState::HandleNullMode(TEvDqCompute::TEvChannelDataV2::TPtr& ev) { auto evAck = MakeHolder<TEvDqCompute::TEvChannelAckV2>(); - evAck->Record.SetGenMajor(PeerGenMajor.load()); - evAck->Record.SetGenMinor(PeerGenMinor.load()); + evAck->Record.SetGenMajor(OutputNodeGenMajor.load()); + evAck->Record.SetGenMinor(OutputNodeGenMinor.load()); evAck->Record.SetStatus(NYql::NDqProto::TEvChannelAckV2::OK); evAck->Record.SetSeqNo(ConfirmedSeqNo); @@ -2396,6 +2347,7 @@ IDqInputChannel::TPtr TDqChannelService::GetInputChannel(const TDqChannelSetting } void TDqChannelService::NotifyCleanup() { + LOG_T("CLEANUP"); ActorSystem->Schedule(Limits.CleanupPeriod, new NActors::IEventHandle(ServiceActorId, ServiceActorId, new TEvPrivate::TEvCleanup())); std::lock_guard lock(Mutex); for (auto& [_, nodeState] : NodeStates) { @@ -2611,18 +2563,22 @@ void TChannelServiceActor::Handle(NActors::NMon::TEvHttpInfo::TPtr& ev) { TABLEH_ATTRS({{"title", "FailureReconciliation"}}) {str << "FR";} #endif TABLEH_ATTRS({{"title", "FailureDestroy"}}) {str << "F~";} - TABLEH_ATTRS({{"title", "GenMajor"}}) {str << "GM";} - TABLEH_ATTRS({{"title", "GenMinor"}}) {str << "gm";} + TABLEH_ATTRS({{"title", "Gen"}}) {str << "G";} + TABLEH_ATTRS({{"title", "Queue.front()->SeqNo"}}) {str << "f/SeqNo";} TABLEH() {str << "SeqNo";} - TABLEH_ATTRS({{"title", "Queue.front()->SeqNo"}}) {str << "front()";} TABLEH_ATTRS({{"title", "InflightBytes"}}) {str << "InflightB";} + TABLEH_ATTRS({{"title", "SendCount"}}) {str << "Send";} + TABLEH_ATTRS({{"title", "ResendCount"}}) {str << "Resend";} + TABLEH_ATTRS({{"title", "ReconCount"}}) {str << "Recon";} + TABLEH() {str << "ReconSent";} TABLEH_ATTRS({{"title", "WaitersQueueSize"}}) {str << "W/Queue";} TABLEH_ATTRS({{"title", "WaitersMessages"}}) {str << "W/Msg";} - TABLEH_ATTRS({{"title", "PeerGenMajor"}}) {str << "PM";} - TABLEH_ATTRS({{"title", "PeerGenMinor"}}) {str << "pm";} + TABLEH_ATTRS({{"title", "OutputNodeGen"}}) {str << "OG";} TABLEH_ATTRS({{"title", "ConfirmedSeqNo"}}) {str << "C/SeqNo";} - TABLEH() {str << "PeerActorId";} + TABLEH() {str << "OutputNodeActorId";} + TABLEH() {str << "InputNodeActorId";} TABLEH() {str << "NodeActorId";} + TABLEH() {str << "ReconciliationLog";} } } TABLEBODY() { @@ -2656,22 +2612,26 @@ void TChannelServiceActor::Handle(NActors::NMon::TEvHttpInfo::TPtr& ev) { str << "X"; } } - TABLED() {str << state->GenMajor;} - TABLED() {str << state->GenMinor;} - TABLED() {str << state->SeqNo;} + TABLED() {str << state->GenMajor << '.' << state->GenMinor;} TABLED() { if (!state->Queue.empty()) { str << state->Queue.front()->SeqNo; } } + TABLED() {str << state->SeqNo;} TABLED() {str << state->InflightBytes.load();} + TABLED() {str << state->SendCount.load();} + TABLED() {str << state->ResendCount.load();} + TABLED() {str << state->ReconCount.load();} + TABLED() {str << state->ReconSent.load();} TABLED() {str << state->WaitersQueue.size();} TABLED() {str << state->WaiterMessages.load();} - TABLED() {str << state->PeerGenMajor.load();} - TABLED() {str << state->PeerGenMinor.load();} + TABLED() {str << state->OutputNodeGenMajor.load() << '.' << state->OutputNodeGenMinor.load();} TABLED() {str << state->ConfirmedSeqNo;} - TABLED() {str << state->PeerActorId;} + TABLED() {str << state->OutputNodeActorId;} + TABLED() {str << state->InputNodeActorId;} TABLED() {str << state->NodeActorId;} + TABLED() {str << state->GetReconciliationLog();} } } } diff --git a/ydb/library/yql/dq/runtime/dq_channel_service_impl.h b/ydb/library/yql/dq/runtime/dq_channel_service_impl.h index 032f9412e1b..b1e43f6194d 100644 --- a/ydb/library/yql/dq/runtime/dq_channel_service_impl.h +++ b/ydb/library/yql/dq/runtime/dq_channel_service_impl.h @@ -439,8 +439,8 @@ public: TChannelFullInfo Info; NActors::TActorSystem* ActorSystem; - ui64 PeerGenMajor = 0; - NActors::TActorId PeerActorId; + ui64 OutputNodeGenMajor = 0; + NActors::TActorId OutputNodeActorId; bool IsBound = false; TDqThreadSafeStats PushStats; TDqThreadSafeStats PopStats; @@ -586,13 +586,13 @@ public: void TerminateOutputDescriptor(const std::shared_ptr<TOutputDescriptor>& descriptor); void TerminateInputDescriptor(const std::shared_ptr<TInputDescriptor>& descriptor); void HandleCleanup(); - void FailInputs(const NActors::TActorId& peerActorId, ui64 peerGenMajor); + void FailInputs(const NActors::TActorId& outputNodeActorId, ui64 outputNodeGenMajor); void FailOutputs(const NActors::TActorId& peerActorId, ui64 peerGenMajor); void SendAck(THolder<TEvDqCompute::TEvChannelAckV2>& evAck, ui64 cookie); void SendAckWithError(ui64 cookie, const TString& message); void HandleChannelData(TEvDqCompute::TEvChannelDataV2::TPtr& ev); void SendFromWaiters(ui64 deltaBytes); - void ConnectSession(NActors::TActorId& sender, ui64 genMajor, ui64 genMinor, ui64 seqNo); + void ConnectSession(NActors::TActorId& sender, ui64 genMajor, ui64 genMinor); virtual TString GetDebugInfo(); void UpdateProgress(std::shared_ptr<TInputDescriptor>& descriptor); void SendUpdateProgress(std::shared_ptr<TInputDescriptor>& descriptor); @@ -602,7 +602,7 @@ public: void DoReconciliation(char logSymbol); void AddReconciliationLog(char logSymbol); TString GetReconciliationLog(); - void SendDiscovery(NActors::TActorId actorId, ui64 seqNo); + void SendDiscovery(); NActors::TActorId NodeActorId; TString LogPrefix; @@ -615,7 +615,6 @@ public: mutable std::unordered_map<TChannelInfo, std::shared_ptr<TInputDescriptor>> InputDescriptors; mutable std::queue<std::pair<TChannelInfo, TInstant>> UnboundInputs; mutable std::queue<std::pair<TChannelInfo, TInstant>> UnboundOutputs; - bool Connected = false; std::weak_ptr<TNodeState> Self; // Sender ui64 GenMajor = 1; @@ -623,12 +622,12 @@ public: ui64 ReconciliationCount = 0; ui64 SeqNo = 0; std::atomic<ui64> InflightBytes = 0; + NActors::TActorId InputNodeActorId; // Receiver - NActors::TActorId PeerActorId; - std::atomic<ui64> PeerGenMajor = 0; - std::atomic<ui64> PeerGenMinor = 0; + NActors::TActorId OutputNodeActorId; + std::atomic<ui64> OutputNodeGenMajor = 0; + std::atomic<ui64> OutputNodeGenMinor = 0; ui64 ConfirmedSeqNo = 0; - TEvDqCompute::TEvChannelDataV2::TPtr OutOfOrderMessage; // ... const TDqChannelLimits Limits; const ui64 MaxInflightMessages = 8192; @@ -660,6 +659,10 @@ public: std::atomic<bool> ResendAsked = false; std::deque<char> ReconciliationLog; TChannelInfo LastLostInfo = TChannelInfo(0, NActors::TActorId{}, NActors::TActorId{}); + std::atomic<ui64> SendCount = 0; + std::atomic<ui64> ResendCount = 0; + std::atomic<ui64> ReconCount = 0; + std::atomic<TInstant> ReconSent; }; class TDebugNodeState : public TNodeState { diff --git a/ydb/library/yql/providers/dq/actors/actor_helpers.h b/ydb/library/yql/providers/dq/actors/actor_helpers.h index 67cdbbd2460..69eaee8cce7 100644 --- a/ydb/library/yql/providers/dq/actors/actor_helpers.h +++ b/ydb/library/yql/providers/dq/actors/actor_helpers.h @@ -10,6 +10,13 @@ namespace NYql { +enum class EActorFutureCallbackFailureReason { + None, + Undelivered, + NodeDisconnected, + Timeout, +}; + enum EExecutorPoolType { Main, FullResultWriter, @@ -25,17 +32,27 @@ struct TRichActorFutureCallback : public TRichActor<TRichActorFutureCallback<Eve static constexpr char ActorName[] = "YQL_DQ_ACTOR_FUTURE_CALLBACK"; - TRichActorFutureCallback(TCallback&& callback, TFailure&& failure, TDuration timeout) + TRichActorFutureCallback( + TCallback&& callback, + TFailure&& failure, + TDuration timeout, + TString context = {}, + NActors::TActorId targetActorId = {}) : TBase(&TRichActorFutureCallback::StateWaitForEvent) , Callback(std::move(callback)) , Failure(std::move(failure)) , Timeout(timeout) + , Context(std::move(context)) + , TargetActorId(targetActorId) { } private: const TCallback Callback; const TFailure Failure; const TDuration Timeout; + const TString Context; + const NActors::TActorId TargetActorId; + EActorFutureCallbackFailureReason FailureReason = EActorFutureCallbackFailureReason::None; bool TimerStarted = false; NActors::TSchedulerCookieHolder TimerCookieHolder; @@ -47,11 +64,26 @@ private: }) hFunc(NActors::TEvInterconnect::TEvNodeDisconnected, [this] (NActors::TEvInterconnect::TEvNodeDisconnected::TPtr& ev) mutable { this->Unsubscribe(ev->Get()->NodeId); + FailureReason = EActorFutureCallbackFailureReason::NodeDisconnected; + YQL_CLOG(DEBUG, ProviderDq) << "ActorFutureCallback failed: interconnect node disconnected" + << " context=" << Context + << " disconnectedNodeId=" << ev->Get()->NodeId + << " targetActorId=" << TargetActorId + << " callbackActorId=" << this->SelfId(); TimerStarted = true; OnFailure(); }) hFunc(NActors::TEvents::TEvUndelivered, [this] (NActors::TEvents::TEvUndelivered::TPtr& ev) mutable { + const auto* undelivered = ev->Get(); this->Unsubscribe(ev->Sender.NodeId()); + FailureReason = EActorFutureCallbackFailureReason::Undelivered; + YQL_CLOG(DEBUG, ProviderDq) << "ActorFutureCallback failed: event undelivered (fast failure, not a timeout)" + << " context=" << Context + << " undeliveredReason=" << static_cast<int>(undelivered->Reason) + << " sourceEventType=" << undelivered->SourceType + << " unsure=" << undelivered->Unsure + << " targetActorId=" << TargetActorId + << " callbackActorId=" << this->SelfId(); TimerStarted = true; OnFailure(); }) @@ -69,6 +101,14 @@ private: void OnFailure() { if (TimerStarted) { + if (FailureReason == EActorFutureCallbackFailureReason::None) { + FailureReason = EActorFutureCallbackFailureReason::Timeout; + YQL_CLOG(DEBUG, ProviderDq) << "ActorFutureCallback failed: response timeout" + << " context=" << Context + << " timeout=" << Timeout + << " targetActorId=" << TargetActorId + << " callbackActorId=" << this->SelfId(); + } Failure(); this->PassAway(); } else { diff --git a/ydb/library/yql/providers/dq/actors/resource_allocator.cpp b/ydb/library/yql/providers/dq/actors/resource_allocator.cpp index 8c143ec3dad..95c31712355 100644 --- a/ydb/library/yql/providers/dq/actors/resource_allocator.cpp +++ b/ydb/library/yql/providers/dq/actors/resource_allocator.cpp @@ -85,7 +85,10 @@ private: Fail((ui64)-1, "GWM Disconnected"); }) hFunc(TEvents::TEvUndelivered, [this](TEvents::TEvUndelivered::TPtr& ev) { - Fail(ev->Cookie, "Undelivered"); + Fail(ev->Cookie, TStringBuilder() + << "Undelivered: sender: " << ev->Sender + << ", reason: " << ev->Get()->Reason + << ", source type: " << ev->Get()->SourceType); }) }) @@ -299,7 +302,8 @@ private: QueryStat.GetCounterName("Actor", {{"ClusterName", maybeRequestInfo->second.ClusterName}}, "CreateFailTimeUs"), delta); } - TString message = "Disconnected from worker: `" + workerInfo + "', reason: " + reason; + TString message = TStringBuilder() + << "Disconnected from worker: `" << workerInfo << "`, reason: " << reason; YQL_CLOG(ERROR, ProviderDq) << message; auto response = MakeHolder<TEvAllocateWorkersResponse>(message, NYql::NDqProto::StatusIds::UNAVAILABLE); QueryStat.FlushCounters(response->Record); diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp index 6216abea8ad..6f3b24cddca 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp @@ -979,7 +979,8 @@ private: return true; }); IDqGateway::TDqProgressWriter progressWriter = MakeDqProgressWriter(publicIds); - bool enableFullResultWrite = settings->EnableFullResultWrite.Get().GetOrElse(false); + const bool initialFullResultWrite = settings->EnableFullResultWrite.Get().GetOrElse(false); + bool enableFullResultWrite = initialFullResultWrite; if (enableFullResultWrite) { const auto type = result.Input().Ref().GetTypeAnn(); const auto integration = GetDqIntegrationForFullResTable(State); @@ -991,6 +992,11 @@ private: && integration->PrepareFullResultTableParams(result.Ref(), ctx, graphParams, secureParams, TColumnOrder(columns)); settings->EnableFullResultWrite = enableFullResultWrite; } + YQL_CLOG(INFO, ProviderDq) << "FullResultWrite decision: initial=" << initialFullResultWrite + << " effective=" << enableFullResultWrite + << " discard=" << fillSettings.Discard + << " hasDqGateway=" << (bool)State->DqGateway + << " hasYtFullResultParam=" << graphParams.contains("yt.full_result_table"); TString lambda; bool untrustedUdfFlag; @@ -1086,10 +1092,13 @@ private: } return WrapFutureCallback<false>(future, [localRun, startTime, type, rowSpec, skiffType, fillSettings, level, settings, enableFullResultWrite, columns, graphParams, state = State, skiffConverter = SkiffConverter](const IDqGateway::TResult& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { - YQL_CLOG(DEBUG, ProviderDq) << state->SessionId << " WrapFutureCallback"; - auto duration = TInstant::Now() - startTime; - YQL_CLOG(INFO, ProviderDq) << "Execution Result complete, duration: " << duration; + YQL_CLOG(INFO, ProviderDq) << state->SessionId << " Execution Result complete, duration: " << duration + << " localRun=" << localRun + << " enableFullResultWrite=" << enableFullResultWrite + << " truncated=" << res.Truncated + << " rowsCount=" << res.RowsCount + << " dataBytes=" << res.Data.size(); if (state->Metrics) { state->Metrics->SetCounter("dq", "TotalExecutionTime", duration.MilliSeconds()); state->Metrics->SetCounter( diff --git a/ydb/library/yql/utils/actors/http_sender_actor_ut.cpp b/ydb/library/yql/utils/actors/http_sender_actor_ut.cpp index 23e559c28b9..bfbc2f79713 100644 --- a/ydb/library/yql/utils/actors/http_sender_actor_ut.cpp +++ b/ydb/library/yql/utils/actors/http_sender_actor_ut.cpp @@ -1,6 +1,5 @@ #include "ydb/core/testlib/basics/appdata.h" -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/ydb/util.h> #include <ydb/core/fq/libs/ydb/ydb.h> diff --git a/ydb/public/sdk/cpp/src/library/kafka/kafka_messages_int.h b/ydb/public/sdk/cpp/src/library/kafka/kafka_messages_int.h index 99469c65fa1..89543bf4b54 100644 --- a/ydb/public/sdk/cpp/src/library/kafka/kafka_messages_int.h +++ b/ydb/public/sdk/cpp/src/library/kafka/kafka_messages_int.h @@ -640,7 +640,7 @@ public: // template<typename Meta> inline void Write(TWriteCollector& collector, TKafkaWritable& writable, TKafkaInt16 version, const typename Meta::Type& value) { - if (VersionCheck<Meta::PresentVersions.Min, Meta::PresentVersions.Max>(version)) { + if (VersionCheck<Meta::PresentVersions.Min, Meta::PresentVersions.Max>(version)) { if (VersionCheck<Meta::TaggedVersions.Min, Meta::TaggedVersions.Max>(version)) { if (!IsDefaultValue<Meta>(value)) { ++collector.NumTaggedFields; @@ -653,7 +653,7 @@ inline void Write(TWriteCollector& collector, TKafkaWritable& writable, TKafkaIn template<typename Meta> inline void Read(TKafkaReadable& readable, TKafkaInt16 version, typename Meta::Type& value) { - if (!VersionNone<Meta::TaggedVersions.Min, Meta::TaggedVersions.Max>() + if (!VersionNone<Meta::TaggedVersions.Min, Meta::TaggedVersions.Max>() && VersionCheck<Meta::TaggedVersions.Min, Meta::TaggedVersions.Max>(version)) { return; } else { diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp index a3555725f02..ba828b2907d 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_clusters_updater_actor.cpp @@ -5,6 +5,8 @@ #include <ydb/library/mkql_proto/protos/minikql.pb.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/result/result.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PQ_WRITE_PROXY + namespace NKikimr { namespace NGRpcProxy { @@ -78,7 +80,8 @@ void TClustersUpdater::Handle(NKqp::TEvKqp::TEvQueryResponse::TPtr &ev, const TA } ctx.Schedule(TDuration::Seconds(AppData(ctx)->PQConfig.GetClustersUpdateTimeoutSec()), new TEvPQClustersUpdater::TEvUpdateClusters()); } else { - LOG_ERROR_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "can't update clusters " << record); + YDB_LOG_ERROR_CTX(ctx, "Can't update clusters", + {"record", record}); ctx.Schedule(TDuration::Seconds(CLUSTERS_UPDATER_TIMEOUT_ON_ERROR), new TEvPQClustersUpdater::TEvUpdateClusters()); } } diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp index 54e10ae2006..0c20b05f00f 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_read_actor.cpp @@ -24,6 +24,8 @@ #include <algorithm> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PQ_READ_PROXY + using namespace NActors; using namespace NKikimrClient; @@ -39,8 +41,8 @@ using namespace NSchemeCache; #ifdef PQ_LOG_PREFIX #undef PQ_LOG_PREFIX #endif -#define PQ_LOG_PREFIX "session cookie " << Cookie << " client " << InternalClientId << " session " << Session +#define PQ_LOG_PREFIX (TStringBuilder() << "session cookie " << Cookie << " client " << InternalClientId << " session " << Session) //11 tries = 10,23 seconds, then each try for 5 seconds , so 21 retries will take near 1 min static const NTabletPipe::TClientRetryPolicy RetryPolicyForPipes = { @@ -352,7 +354,8 @@ void TReadSessionActor::Die(const TActorContext& ctx) { if (t.second->PipeClient) NTabletPipe::CloseClient(ctx, t.second->PipeClient); } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " is DEAD"); + YDB_LOG_INFO_CTX(ctx, "Is DEAD", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); if (SessionsActive) { --(*SessionsActive); @@ -408,7 +411,9 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvCommit::TPtr& ev, const TActorCont CloseSession(TStringBuilder() << "double commit of cookie " << readId, NPersQueue::NErrorCode::BAD_REQUEST, ctx); return; } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " commit request from client for " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Commit request from client", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"readId", readId}); MakeCommit(ctx); } @@ -427,7 +432,10 @@ void TReadSessionActor::MakeCommit(const TActorContext& ctx) { if (readId == ReadIdCommitted) return; NextCommits.erase(NextCommits.begin(), it); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " commit request from " << ReadIdCommitted + 1 << " to " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Commit request", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"readIdCommittedPlus1", ReadIdCommitted + 1}, + {"readId", readId}); auto& commit = Commits[readId]; commit.StartReadId = ReadIdCommitted + 1; @@ -486,8 +494,11 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvGetStatus::TPtr& ev, const TActorC if (it == Partitions.end() || it->second.Releasing || it->second.LockGeneration != ev->Get()->Generation) { //do nothing - already released partition - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got NOTACTUAL get status request from client for " << ev->Get()->Topic - << ":" << ev->Get()->Partition << " generation " << ev->Get()->Generation); + YDB_LOG_WARN_CTX(ctx, "Got NOTACTUAL get status request from client for generation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", ev->Get()->Topic}, + {"partition", ev->Get()->Partition}, + {"generation", ev->Get()->Generation}); return; } @@ -514,13 +525,22 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvLocked::TPtr& ev, const TActorCont if (it == Partitions.end() || it->second.Releasing || it->second.LockGeneration != ev->Get()->Generation) { //do nothing - already released partition - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got NOTACTUAL lock from client for " << topic - << ":" << ev->Get()->Partition << " at offset " << ev->Get()->ReadOffset << " generation " << ev->Get()->Generation); + YDB_LOG_WARN_CTX(ctx, "Got NOTACTUAL lock from client for at offset generation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", topic}, + {"partition", ev->Get()->Partition}, + {"readOffset", ev->Get()->ReadOffset}, + {"generation", ev->Get()->Generation}); return; } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got lock from client for " << ev->Get()->Topic - << ":" << ev->Get()->Partition << " at readOffset " << ev->Get()->ReadOffset << " commitOffset " << ev->Get()->CommitOffset << " generation " << ev->Get()->Generation); + YDB_LOG_INFO_CTX(ctx, "Got lock from client for at readOffset commitOffset generation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", ev->Get()->Topic}, + {"partition", ev->Get()->Partition}, + {"readOffset", ev->Get()->ReadOffset}, + {"commitOffset", ev->Get()->CommitOffset}, + {"generation", ev->Get()->Generation}); //proxy request to partition - allow initing //TODO: add here VerifyReadOffset too and check it against Committed position @@ -582,8 +602,10 @@ void TReadSessionActor::AnswerForCommitsIfCan(const TActorContext& ctx) { for (ui64 i = it->second.StartReadId; i <= readId; ++i){ result.MutableCommit()->AddCookie(i); } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " replying for commits from " << it->second.StartReadId - << " to " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Replying for commits", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"startReadId", it->second.StartReadId}, + {"readId", readId}); ui64 diff = result.ByteSize(); BytesInflight_ += diff; if (BytesInflight) (*BytesInflight) += diff; @@ -706,7 +728,10 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo topicsToResolve.insert(t); } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " init: " << event->Request << " from " << PeerName); + YDB_LOG_INFO_CTX(ctx, "From", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"init", event->Request}, + {"peerName", PeerName}); ClientsideLocksAllowed = init.GetClientsideLocksAllowed(); BalanceRightNow = init.GetBalancePartitionRightNow() || CommitsDisabled; @@ -716,7 +741,9 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadInit::TPtr& ev, const TActorCo } if (Auth.GetCredentialsCase() == NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) { - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, "session without AuthInfo : " << ExternalClientId << " from " << PeerName); + YDB_LOG_WARN_CTX(ctx, "Session without AuthInfo", + {"externalClientId", ExternalClientId}, + {"peerName", PeerName}); if (SessionsWithoutAuth) { ++(*SessionsWithoutAuth); } @@ -811,8 +838,10 @@ void TReadSessionActor::CreateInitAndAuthActor(const TActorContext& ctx) { void TReadSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& ev, const TActorContext& ctx) { TString ticket = ev->Get()->Ticket; TString maskedTicket = ticket.size() > 5 ? (ticket.substr(0, 5) + "***" + ticket.substr(ticket.size() - 5)) : "***"; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, "CheckACL ticket " << maskedTicket << " got result from TICKET_PARSER response: error: " << ev->Get()->Error << " user: " - << (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")); + YDB_LOG_DEBUG_CTX(ctx, "CheckACL ticket got result from TICKET_PARSER response", + {"maskedTicket", maskedTicket}, + {"error", ev->Get()->Error}, + {"user", (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")}); if (ev->Get()->HasError()) { if (AppData()->EnforceUserTokenRequirement || AppData()->EnforceUserTokenCheckRequirement) { @@ -827,7 +856,9 @@ void TReadSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& void TReadSessionActor::RegisterSession(const TActorId& pipe, const TString& topic, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " register session to " << topic); + YDB_LOG_INFO_CTX(ctx, "Register session", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", topic}); THolder<TEvPersQueue::TEvRegisterReadSession> request; request.Reset(new TEvPersQueue::TEvRegisterReadSession); auto& req = request->Record; @@ -940,7 +971,10 @@ void TReadSessionActor::Handle(V1::TEvPQProxy::TEvAuthResultOk::TPtr& ev, const LastACLCheckTimestamp = ctx.Now(); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " auth ok, got " << ev->Get()->TopicAndTablets.size() << " topics, init done " << InitDone); + YDB_LOG_DEBUG_CTX(ctx, "Auth ok, got topics, init done", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topicAndTablets", ev->Get()->TopicAndTablets.size()}, + {"initDone", InitDone}); AuthInitActor = TActorId(); AuthInflight = false; @@ -1046,10 +1080,9 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(path)); if (converterIter.IsEnd()) { - LOG_ALERT_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " ignored ev lock for event = " << record.ShortDebugString() << " path not recognized" - ); + YDB_LOG_ALERT_CTX(ctx, "Ignored ev lock for event path not recognized", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"record", record.ShortDebugString()}); CloseSession( TStringBuilder() << "Internal server error, cannot parse lock event: " << record.ShortDebugString() << ", reason: topic not found", NPersQueue::NErrorCode::ERROR, ctx @@ -1062,21 +1095,18 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T auto jt = Topics.find(intName); if (jt == Topics.end() || pipe != jt->second->PipeClient) { //this is message from old version of pipe - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " ignored ev lock for topic = " << converterIter->second->GetPrintableString() - << " path recognized, but topic is unknown, this is unexpected" - ); + YDB_LOG_DEBUG_CTX(ctx, "Ignored ev lock for topic path recognized, but topic is unknown, this is unexpected", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", converterIter->second->GetPrintableString()}); return; } auto partitionGraph = jt->second->GetPartitionGraph(); auto* partitionNode = partitionGraph->GetPartition(record.GetPartition()); if (!partitionNode) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " lock for unknown partition = " << record.GetPartition() - ); + YDB_LOG_DEBUG_CTX(ctx, "Lock for unknown partition", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"partition", record.GetPartition()}); Locks.push_back(ev->Release()); if (!AuthInflight) { SendAuthRequest(ctx); @@ -1124,7 +1154,9 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvLockPartition::TPtr& ev, const T it->second.PartitionsLocked.Inc(); it->second.PartitionsInfly.Inc(); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " lock: " << record); + YDB_LOG_INFO_CTX(ctx, "Lock partition", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"ev", record}); ctx.Send(actorId, new TEvPQProxy::TEvLockPartition(0, 0, false, !ClientsideLocksAllowed)); } @@ -1137,8 +1169,9 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionStatus::TPtr& ev, const T auto it = Partitions.find(std::make_pair(evTopic->GetClientsideName(), ev->Get()->Partition)); Y_ABORT_UNLESS(it != Partitions.end()); if (!it->second.LockGeneration) { - LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " the unlocked partition " << ev->Get()->Partition << " status has been requested"); + YDB_LOG_ALERT_CTX(ctx, "The unlocked partition status has been requested", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"partition", ev->Get()->Partition}); CloseSession( TStringBuilder() << "Internal server error, the unlocked partition " << ev->Get()->Partition << " status has been requested", NPersQueue::NErrorCode::ERROR, ctx @@ -1212,7 +1245,9 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons auto converterIter = FullPathToConverter.find(NPersQueue::NormalizeFullPath(topic)); if (converterIter.IsEnd()) { - LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Failed to parse balancer response: " << record.ShortDebugString()); + YDB_LOG_ALERT_CTX(ctx, "Failed to parse balancer", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"response", record.ShortDebugString()}); CloseSession( TStringBuilder() << "Internal server error, cannot parse release event: " << record.ShortDebugString() << ", path not recognized", NPersQueue::NErrorCode::ERROR, ctx @@ -1232,7 +1267,10 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons } auto onUnknownPartition = [&](auto& marker) { - LOG_ALERT_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " Releasing unknown partition: " << record.ShortDebugString() << " " << marker); + YDB_LOG_ALERT_CTX(ctx, "Releasing unknown", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"partition", record.ShortDebugString()}, + {"marker", marker}); CloseSession( TStringBuilder() << "Internal server error, releasing unknown partition: " << record.ShortDebugString() << " " << marker, NPersQueue::NErrorCode::ERROR, ctx @@ -1265,7 +1303,10 @@ void TReadSessionActor::Handle(TEvPersQueue::TEvReleasePartition::TPtr& ev, cons it->second.PartitionsToBeReleased.Inc(); } - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " releasing " << jt->first.first << ":" << jt->first.second); + YDB_LOG_INFO_CTX(ctx, "Releasing", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"key", jt->first.first}, + {"partition", jt->first.second}); jt->second.Releasing = true; ctx.Send(actorId, new TEvPQProxy::TEvReleasePartition()); @@ -1319,7 +1360,10 @@ void TReadSessionActor::InformBalancerAboutRelease(const THashMap<std::pair<TStr req.SetTopic(it->first.first); req.SetPartition(it->first.second); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " released: " << it->first.first << ":" << it->first.second); + YDB_LOG_INFO_CTX(ctx, "Released", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"key", it->first.first}, + {"partition", it->first.second}); NTabletPipe::SendData(ctx, jt->second->PipeClient, request.Release()); } @@ -1347,17 +1391,21 @@ void TReadSessionActor::CloseSession(const TString& errorReason, const NPersQueu error->SetDescription(errorReason); error->SetCode(errorCode); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " closed with error reason: " << errorReason); + YDB_LOG_INFO_CTX(ctx, "Closed with error", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"reason", errorReason}); if (!Handler->IsShuttingDown()) { ui64 diff = result.ByteSize(); BytesInflight_ += diff; if (BytesInflight) (*BytesInflight) += diff; Handler->Reply(std::move(result)); } else { - LOG_WARN_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " GRps is shutting dows, skip reply"); + YDB_LOG_WARN_CTX(ctx, "GRps is shutting down, skip reply", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); } } else { - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " closed"); + YDB_LOG_INFO_CTX(ctx, "Closed", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); } Die(ctx); @@ -1457,7 +1505,9 @@ bool TReadSessionActor::ProcessReleasePartition(const THashMap<std::pair<TString bool TReadSessionActor::ProcessBalancerDead(const ui64 tablet, const TActorContext& ctx) { for (auto& t : Topics) { if (t.second->TabletID == tablet) { - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " balancer for topic " << t.first << " is dead, restarting all from this topic"); + YDB_LOG_INFO_CTX(ctx, "Balancer for topic is dead, restarting all from this topic", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", t.first}); //Drop all partitions from this topic for (auto it = Partitions.begin(); it != Partitions.end();) { @@ -1520,7 +1570,10 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvRead::TPtr& ev, const TActorContex ProcessAuth(event->Request.GetCredentials()); event->Request.ClearCredentials(); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " got read request: " << event->Request.GetRead() << " with guid: " << event->Guid); + YDB_LOG_DEBUG_CTX(ctx, "Got read with", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"request", event->Request.GetRead()}, + {"guid", event->Guid}); Reads.emplace_back(event.Release()); @@ -1562,9 +1615,12 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvReadResponse::TPtr& ev, const TAct TFormedReadResponse::TPtr formedResponse = it->second; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " read done guid " << formedResponse->Guid - << " " << key.first << ":" << key.second - << " size " << event->Response.ByteSize()); + YDB_LOG_DEBUG_CTX(ctx, "Read done guid size", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"guid", formedResponse->Guid}, + {"keyFirst", key.first}, + {"keySecond", key.second}, + {"byteSize", event->Response.ByteSize()}); const i64 diff = formedResponse->ApplyResponse(std::move(event->Response)); if (event->FromDisk) { @@ -1609,7 +1665,10 @@ bool TReadSessionActor::ProcessAnswer(const TActorContext& ctx, TFormedReadRespo const bool hasMessages = HasMessages(formedResponse->Response.GetBatchedData()); if (hasMessages) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " assign read id " << ReadIdToResponse << " to read request " << formedResponse->Guid); + YDB_LOG_DEBUG_CTX(ctx, "Assign read id to read request", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"readIdToResponse", ReadIdToResponse}, + {"guid", formedResponse->Guid}); formedResponse->Response.MutableBatchedData()->SetCookie(ReadIdToResponse); // reply to client if (ProtocolVersion < NPersQueue::TReadRequest::Batching) { @@ -1618,7 +1677,9 @@ bool TReadSessionActor::ProcessAnswer(const TActorContext& ctx, TFormedReadRespo diff -= formedResponse->Response.ByteSize(); // Bytes will be tracked inside handler Handler->Reply(std::move(formedResponse->Response)); } else { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " empty read result " << formedResponse->Guid << ", start new reading"); + YDB_LOG_DEBUG_CTX(ctx, "Empty read result start new reading", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"guid", formedResponse->Guid}); } BytesInflight_ -= diff; @@ -1764,11 +1825,16 @@ bool TReadSessionActor::ProcessReads(const TActorContext& ctx) { readR->SetMaxTimeLagMs(MaxTimeLagMs); readR->SetReadTimestampMs(ReadTimestampMs); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX - << " performing read request: " << (*readR) << " with guid " << read->Guid - << " from " << part.Topic->GetPrintableString() << ", partition:" << part.Partition - << " count " << ccount << " size " << csize - << " partitionsAsked " << partitionsAsked << " maxTimeLag " << MaxTimeLagMs << "ms"); + YDB_LOG_DEBUG_CTX(ctx, "Performing read with guid from count size partitionsAsked maxTimeLag ms", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"request", (*readR)}, + {"guid", read->Guid}, + {"topic", part.Topic->GetPrintableString()}, + {"partition", part.Partition}, + {"ccount", ccount}, + {"csize", csize}, + {"partitionsAsked", partitionsAsked}, + {"maxTimeLagMs", MaxTimeLagMs}); Y_ABORT_UNLESS(!it->second.Reading); @@ -1807,10 +1873,14 @@ void TReadSessionActor::Handle(TEvPQProxy::TEvPartitionReady::TPtr& ev, const TA if (!ActualPartitionActor(ev->Sender)) return; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << ev->Get()->Topic->GetPrintableString() - << " partition:" << ev->Get()->Partition << " ready for read with readOffset " - << ev->Get()->ReadOffset << " endOffset " << ev->Get()->EndOffset << " WTime " - << ev->Get()->WTime << " sizeLag " << ev->Get()->SizeLag); + YDB_LOG_DEBUG_CTX(ctx, "Ready for read with readOffset endOffset WTime sizeLag", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", ev->Get()->Topic->GetPrintableString()}, + {"partition", ev->Get()->Partition}, + {"readOffset", ev->Get()->ReadOffset}, + {"endOffset", ev->Get()->EndOffset}, + {"WTime", ev->Get()->WTime}, + {"sizeLag", ev->Get()->SizeLag}); const auto it = PartitionToReadResponse.find(ev->Sender); // check whether this partition is taking part in read response auto& container = it != PartitionToReadResponse.end() ? it->second->PartitionsBecameAvailable : AvailablePartitions; @@ -1834,7 +1904,8 @@ void TReadSessionActor::HandleWakeup(const TActorContext& ctx) { ForceACLCheck = false; RequestNotChecked = false; Y_ABORT_UNLESS(!AuthInitActor); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " checking auth because of timeout"); + YDB_LOG_DEBUG_CTX(ctx, "Checking auth because of timeout", + {"PQLOGPREFIX", PQ_LOG_PREFIX}); SendAuthRequest(ctx); } @@ -1935,17 +2006,29 @@ void TPartitionActor::Bootstrap(const TActorContext&) { void TPartitionActor::CheckRelease(const TActorContext& ctx) { const bool hasUncommittedData = ReadOffset > ClientCommitOffset && ReadOffset > ClientReadOffset; if (NeedRelease) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " checking release readOffset " << ReadOffset << " committedOffset " << CommittedOffset << " ReadGuid " << ReadGuid - << " CommitsInfly.size " << CommitsInfly.size() << " Released " << Released); + YDB_LOG_DEBUG_CTX(ctx, "Checking release readOffset committedOffset ReadGuid CommitsInfly.size Released", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}, + {"readGuid", ReadGuid}, + {"commitsInfly", CommitsInfly.size()}, + {"released", Released}); } if (NeedRelease && ReadGuid.empty() && CommitsInfly.empty() && !hasUncommittedData && !Released) { Released = true; ctx.Send(ParentId, new TEvPQProxy::TEvPartitionReleased(Topic, Partition)); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " check release done - releasing; readOffset " << ReadOffset << " committedOffset " << CommittedOffset << " ReadGuid " << ReadGuid - << " CommitsInfly.size " << CommitsInfly.size() << " Released " << Released); + YDB_LOG_INFO_CTX(ctx, "Check release done - releasing; readOffset committedOffset ReadGuid CommitsInfly.size Released", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}, + {"readGuid", ReadGuid}, + {"commitsInfly", CommitsInfly.size()}, + {"released", Released}); } } @@ -1991,9 +2074,14 @@ void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TAc Y_ABORT_UNLESS(!Session.empty()); commit->SetSessionId(Session); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" - << Partition << " committing to position " << offset << " prev " << CommittedOffset - << " end " << EndOffset << " by cookie " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Committing to position prev end by cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"offset", offset}, + {"committedOffset", CommittedOffset}, + {"endOffset", EndOffset}, + {"readId", readId}); TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); req->Record.Swap(&request); @@ -2004,17 +2092,22 @@ void TPartitionActor::SendCommit(const ui64 readId, const ui64 offset, const TAc void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { if (CommitsInfly.empty()) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " unwaited commit-response with cookie " << cookie << "; waiting for nothing"); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited commit-response with cookie waiting for nothing", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", cookie}); return; } ui64 readId = CommitsInfly.front().first; if (cookie != readId) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " unwaited commit-response with cookie " << cookie << "; waiting for " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited commit-response with cookie waiting", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", cookie}, + {"readId", readId}); return; } @@ -2025,17 +2118,23 @@ void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { CommitsInfly.pop_front(); if (readId != Max<ui64>()) //this readId is reserved for upcommits on client skipping with ClientCommitOffset ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " commit done to position " << CommittedOffset << " endOffset " << EndOffset << " with cookie " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Commit done to position endOffset with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"committedOffset", CommittedOffset}, + {"endOffset", EndOffset}, + {"readId", readId}); while (!CommitsInfly.empty() && CommitsInfly.front().second == Max<ui64>()) { //this is cookies that have no effect on this partition readId = CommitsInfly.front().first; CommitsInfly.pop_front(); ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << "partition :" << Partition - << " commit done with no effect with cookie " << readId); + YDB_LOG_DEBUG_CTX(ctx, "Partition commit done with no effect with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readId", readId}); } CheckRelease(ctx); @@ -2043,8 +2142,12 @@ void TPartitionActor::CommitDone(ui64 cookie, const TActorContext& ctx) { } void TPartitionActor::SendPartitionReady(const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << Topic->GetPrintableString() << " partition:" << Partition - << " ready for read with readOffset " << ReadOffset << " endOffset " << EndOffset); + YDB_LOG_DEBUG_CTX(ctx, "Ready for read with readOffset endOffset", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}); if (FirstRead) { ctx.Send(ParentId, new TEvPQProxy::TEvReadingStarted(Topic->GetInternalName(), Partition)); FirstRead = false; @@ -2072,8 +2175,12 @@ void TPartitionActor::RestartPipe(const TActorContext& ctx, const TString& reaso ctx.Schedule(TDuration::MilliSeconds(RESTART_PIPE_DELAY_MS), new TEvPQProxy::TEvRestartPipe()); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " schedule pipe restart attempt " << PipeGeneration << " reason: " << reason); + YDB_LOG_INFO_CTX(ctx, "Schedule pipe restart attempt", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"pipeGeneration", PipeGeneration}, + {"reason", reason}); } @@ -2092,13 +2199,23 @@ void TPartitionActor::Handle(const TEvPQProxy::TEvRestartPipe::TPtr&, const TAct PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, TabletID, clientConfig)); Y_ABORT_UNLESS(TabletID); - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " pipe restart attempt " << PipeGeneration << " RequestInfly " << RequestInfly << " ReadOffset " << ReadOffset << " EndOffset " << EndOffset - << " InitDone " << InitDone << " WaitForData " << WaitForData); + YDB_LOG_INFO_CTX(ctx, "Pipe restart attempt RequestInfly ReadOffset EndOffset InitDone WaitForData", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"pipeGeneration", PipeGeneration}, + {"requestInfly", RequestInfly}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}, + {"initDone", InitDone}, + {"waitForData", WaitForData}); if (RequestInfly) { //got read infly - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " resend " << CurrentRequest); + YDB_LOG_INFO_CTX(ctx, "Resend", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"currentRequest", CurrentRequest}); TAutoPtr<TEvPersQueue::TEvRequest> event(new TEvPersQueue::TEvRequest); event->Record = CurrentRequest; @@ -2152,16 +2269,21 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo return; } - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " initDone " << InitDone << " event " << PartitionResponseToLog(result)); + YDB_LOG_DEBUG_CTX(ctx, "InitDone event", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"initDone", InitDone}, + {"partitionResponseToLog", PartitionResponseToLog(result)}); if (!InitDone) { if (result.GetCookie() != INIT_COOKIE) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << " partition:" << Partition - << " unwaited response in init with cookie " << result.GetCookie()); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited response in init with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", result.GetCookie()}); return; } Y_ABORT_UNLESS(RequestInfly); @@ -2183,9 +2305,13 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo ClientHasAnyCommits = resp.GetClientHasAnyCommits(); InitDone = true; PipeGeneration = 0; //reset tries counter - all ok - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " INIT DONE " << Topic->GetPrintableString() - << " partition:" << Partition - << " EndOffset " << EndOffset << " readOffset " << ReadOffset << " committedOffset " << CommittedOffset); + YDB_LOG_INFO_CTX(ctx, "INIT DONE EndOffset readOffset committedOffset", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"endOffset", EndOffset}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}); if (!StartReading) { @@ -2207,9 +2333,13 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo const auto& res = result.GetCmdReadResult(); if (result.GetCookie() != (ui64)ReadOffset) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_PROXY, PQ_LOG_PREFIX << " " << Topic->GetPrintableString() - << "partition :" << Partition - << " unwaited read-response with cookie " << result.GetCookie() << "; waiting for " << ReadOffset << "; current read guid is " << ReadGuid); + YDB_LOG_DEBUG_CTX(ctx, "Partition unwaited read-response with cookie waiting for current read guid is", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"cookie", result.GetCookie()}, + {"readOffset", ReadOffset}, + {"readGuid", ReadGuid}); return; } @@ -2250,11 +2380,14 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo TString sourceId = ""; if (!r.GetSourceId().empty()) { if (!NPQ::NSourceIdEncoding::IsValidEncoded(r.GetSourceId())) { - LOG_ERROR_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << "read bad sourceId from topic " << Topic->GetPrintableString() - << " partition:" << Partition - << " offset " << r.GetOffset() << " seqNo " << r.GetSeqNo() << " sourceId '" << r.GetSourceId() << "' ReadGuid " << ReadGuid); + YDB_LOG_ERROR_CTX(ctx, "Read bad sourceId from topic offset seqNo sourceId ReadGuid", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"offset", r.GetOffset()}, + {"seqNo", r.GetSeqNo()}, + {"sourceId", r.GetSourceId()}, + {"readGuid", ReadGuid}); } sourceId = NPQ::NSourceIdEncoding::Decode(r.GetSourceId()); } @@ -2311,13 +2444,14 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo if (proto.HasCodec()) { const auto codec = proto.GetCodec(); if (codec < Min<int>() || codec > Max<int>() || !NPersQueueCommon::ECodec_IsValid(codec)) { - LOG_ERROR_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << "data chunk (topic " << Topic->GetInternalName() << ", partition " << Partition - << ", offset " << r.GetOffset() << ", seqNo " << r.GetSeqNo() << ", sourceId " - << r.GetSourceId() << ") codec (id " << codec - << ") is not valid NPersQueueCommon::ECodec, loss of data compression codec information" - ); + YDB_LOG_ERROR_CTX(ctx, "Data chunk codec is not a valid NPersQueueCommon::ECodec; compression codec info may be lost", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetInternalName()}, + {"partition", Partition}, + {"offset", r.GetOffset()}, + {"seqNo", r.GetSeqNo()}, + {"sourceId", r.GetSourceId()}, + {"codec", codec}); } message->SetCodec((NPersQueueCommon::ECodec)proto.GetCodec()); } @@ -2341,11 +2475,13 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo WaitDataInPartition(ctx); } - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " after read state " << Topic->GetPrintableString() - << " partition:" << Partition - << " EndOffset " << EndOffset << " ReadOffset " << ReadOffset << " ReadGuid " << ReadGuid); + YDB_LOG_DEBUG_CTX(ctx, "After read state EndOffset ReadOffset ReadGuid", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"endOffset", EndOffset}, + {"readOffset", ReadOffset}, + {"readGuid", ReadGuid}); ReadGuid = TString(); @@ -2364,10 +2500,12 @@ void TPartitionActor::Handle(TEvPersQueue::TEvResponse::TPtr& ev, const TActorCo void TPartitionActor::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { TEvTabletPipe::TEvClientConnected *msg = ev->Get(); - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " pipe restart attempt " << PipeGeneration << " pipe creation result: " << msg->Status); + YDB_LOG_INFO_CTX(ctx, "Pipe restart attempt pipe creation", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"pipeGeneration", PipeGeneration}, + {"result", msg->Status}); if (msg->Status != NKikimrProto::OK) { RestartPipe(ctx, TStringBuilder() << "pipe to tablet is dead " << msg->TabletId, NPersQueue::NErrorCode::ERROR); @@ -2381,12 +2519,13 @@ void TPartitionActor::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const void TPartitionActor::Handle(TEvPQProxy::TEvReleasePartition::TPtr&, const TActorContext& ctx) { - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " (partition)releasing " << Topic->GetPrintableString() << " partition:" << Partition - << " ReadOffset " << ReadOffset << " ClientCommitOffset " << ClientCommitOffset - << " CommittedOffst " << CommittedOffset - ); + YDB_LOG_INFO_CTX(ctx, "(partition)releasing", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"committedOffset", CommittedOffset}); NeedRelease = true; CheckRelease(ctx); } @@ -2456,13 +2595,15 @@ void TPartitionActor::InitStartReading(const TActorContext& ctx) { Y_ABORT_UNLESS(AllPrepareInited); Y_ABORT_UNLESS(!WaitForData); - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " Start reading " << Topic->GetPrintableString() << " partition:" << Partition - << " EndOffset " << EndOffset << " readOffset " << ReadOffset << " committedOffset " - << CommittedOffset << " clientCommittedOffset " << ClientCommitOffset - << " clientReadOffset " << ClientReadOffset - ); + YDB_LOG_INFO_CTX(ctx, "Start reading EndOffset readOffset committedOffset clientCommittedOffset clientReadOffset", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"endOffset", EndOffset}, + {"readOffset", ReadOffset}, + {"committedOffset", CommittedOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"clientReadOffset", ClientReadOffset}); Counters.PartitionsToBeLocked.Dec(); LockCounted = false; @@ -2547,9 +2688,10 @@ void TPartitionActor::InitLockPartition(const TActorContext& ctx) { cmd->SetGeneration(Generation); cmd->SetStep(Step); - LOG_INFO_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " INITING " << Topic->GetPrintableString() << " partition:" << Partition); + YDB_LOG_INFO_CTX(ctx, "INITING", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}); TAutoPtr<TEvPersQueue::TEvRequest> req(new TEvPersQueue::TEvRequest); Y_ABORT_UNLESS(!RequestInfly); @@ -2588,11 +2730,11 @@ void TPartitionActor::WaitDataInPartition(const TActorContext& ctx) { event->Record.SetDeadline(deadline); event->Record.SetClientId(InternalClientId); - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " wait data in partition inited, cookie " << WaitDataCookie - ); + YDB_LOG_DEBUG_CTX(ctx, "Wait data in partition inited", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"waitDataCookie", WaitDataCookie}); NTabletPipe::SendData(ctx, PipeClient, event.Release()); @@ -2609,10 +2751,11 @@ void TPartitionActor::Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, con auto it = WaitDataInfly.find(ev->Get()->Record.GetCookie()); if (it == WaitDataInfly.end()) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " unwaited response for WaitData " << ev->Get()->Record); + YDB_LOG_DEBUG_CTX(ctx, "Unwaited response for WaitData", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"record", ev->Get()->Record}); return; } WaitDataInfly.erase(it); @@ -2626,13 +2769,16 @@ void TPartitionActor::Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, con Y_ABORT_UNLESS(EndOffset <= record.GetEndOffset()); //end offset could not be changed if no data arrived, but signal will be sended anyway after timeout Y_ABORT_UNLESS(ReadOffset >= EndOffset); //otherwise no WaitData were needed - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " wait for data done: " << " readOffset " << ReadOffset << " EndOffset " << EndOffset - << " newEndOffset " << record.GetEndOffset() << " commitOffset " << CommittedOffset - << " clientCommitOffset " << ClientCommitOffset << " cookie " << ev->Get()->Record.GetCookie() - ); + YDB_LOG_DEBUG_CTX(ctx, "Wait for data done: readOffset EndOffset newEndOffset commitOffset clientCommitOffset cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}, + {"recordEndOffset", record.GetEndOffset()}, + {"committedOffset", CommittedOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"cookie", ev->Get()->Record.GetCookie()}); EndOffset = record.GetEndOffset(); SizeLag = record.GetSizeLag(); @@ -2664,13 +2810,16 @@ void TPartitionActor::Handle(TEvPersQueue::TEvHasDataInfoResponse::TPtr& ev, con void TPartitionActor::Handle(TEvPQProxy::TEvRead::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " READ FROM " << Topic->GetPrintableString() << " partition:" << Partition - << " event " << ev->Get()->Request << " readOffset " << ReadOffset - << " EndOffset " << EndOffset << " ClientCommitOffset " << ClientCommitOffset - << " committedOffset " << CommittedOffset << " Guid " << ev->Get()->Guid - ); + YDB_LOG_DEBUG_CTX(ctx, "READ FROM event readOffset EndOffset ClientCommitOffset committedOffset Guid", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"request", ev->Get()->Request}, + {"readOffset", ReadOffset}, + {"endOffset", EndOffset}, + {"clientCommitOffset", ClientCommitOffset}, + {"committedOffset", CommittedOffset}, + {"guid", ev->Get()->Guid}); Y_ABORT_UNLESS(!NeedRelease); Y_ABORT_UNLESS(!Released); @@ -2727,12 +2876,13 @@ void TPartitionActor::Handle(TEvPQProxy::TEvCommit::TPtr& ev, const TActorContex const ui64 offset = ev->Get()->Offset; Y_ABORT_UNLESS(offset != Max<ui64>()); // has concreete offset if (offset < ClientCommitOffset) { - LOG_ERROR_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " commit done to too small position " << offset - << " committedOffset " << ClientCommitOffset << " cookie " << readId - ); + YDB_LOG_ERROR_CTX(ctx, "Commit done to too small position committedOffset cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"offset", offset}, + {"clientCommitOffset", ClientCommitOffset}, + {"readId", readId}); } Y_ABORT_UNLESS(offset >= ClientCommitOffset); @@ -2740,11 +2890,11 @@ void TPartitionActor::Handle(TEvPQProxy::TEvCommit::TPtr& ev, const TActorContex if (!hasProgress) {//nothing to commit for this partition if (CommitsInfly.empty()) { - LOG_DEBUG_S( - ctx, NKikimrServices::PQ_READ_PROXY, - PQ_LOG_PREFIX << " " << Topic->GetPrintableString() << " partition:" << Partition - << " commit done with no effect with cookie " << readId - ); + YDB_LOG_DEBUG_CTX(ctx, "Commit done with no effect with cookie", + {"PQLOGPREFIX", PQ_LOG_PREFIX}, + {"topic", Topic->GetPrintableString()}, + {"partition", Partition}, + {"readId", readId}); ctx.Send(ParentId, new TEvPQProxy::TEvCommitDone(readId, Topic, Partition)); CheckRelease(ctx); } else { diff --git a/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp b/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp index d9a9abc1dcb..3a409920805 100644 --- a/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp +++ b/ydb/services/deprecated/persqueue_v0/grpc_pq_write_actor.cpp @@ -17,6 +17,8 @@ #include <util/string/vector.h> #include <util/string/escape.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::PQ_WRITE_PROXY + using namespace NActors; using namespace NKikimrClient; @@ -133,7 +135,9 @@ void TWriteSessionActor::Die(const TActorContext& ctx) { BytesInflightTotal.Dec(BytesInflightTotal_); } - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " is DEAD"); + YDB_LOG_INFO_CTX(ctx, "Session is DEAD", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); if (PartitionChooser) { ctx.Send(PartitionChooser, new TEvents::TEvPoison()); @@ -161,7 +165,9 @@ void TWriteSessionActor::CheckFinish(const TActorContext& ctx) { } void TWriteSessionActor::Handle(TEvPQProxy::TEvDone::TPtr&, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " got TEvDone"); + YDB_LOG_INFO_CTX(ctx, "Session got TEvDone", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); WritesDone = true; CheckFinish(ctx); } @@ -231,7 +237,11 @@ void TWriteSessionActor::Handle(TEvPQProxy::TEvWriteInit::TPtr& ev, const TActor event->Request.ClearCredentials(); Y_PROTOBUF_SUPPRESS_NODISCARD Auth.SerializeToString(&AuthStr); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session request cookie: " << Cookie << " " << init << ", Database: '" << Database << "' from " << PeerName); + YDB_LOG_INFO_CTX(ctx, "Session request Database", + {"cookie", Cookie}, + {"init", init}, + {"database", Database}, + {"peerName", PeerName}); UserAgent = init.GetVersion(); LogSession(ctx); @@ -402,8 +412,10 @@ void TWriteSessionActor::Handle(TEvDescribeTopicsResponse::TPtr& ev, const TActo if (Auth.GetCredentialsCase() == NPersQueueCommon::TCredentials::CREDENTIALS_NOT_SET) { //ACLCheckInProgress is still true - no recheck will be done - LOG_WARN_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session without AuthInfo : " << DiscoveryConverter->GetPrintableString() - << " sourceId " << SourceId << " from " << PeerName); + YDB_LOG_WARN_CTX(ctx, "Session without AuthInfo sourceId", + {"discoveryConverter", DiscoveryConverter->GetPrintableString()}, + {"sourceId", SourceId}, + {"peerName", PeerName}); SessionsWithoutAuth.Inc(); if (FirstACLCheck) { FirstACLCheck = false; @@ -445,9 +457,10 @@ void TWriteSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& Y_ABORT_UNLESS(ACLCheckInProgress); TString ticket = ev->Get()->Ticket; TString maskedTicket = ticket.size() > 5 ? (ticket.substr(0, 5) + "***" + ticket.substr(ticket.size() - 5)) : "***"; - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "CheckACL ticket " << maskedTicket << " got result from TICKET_PARSER response: error: " - << ev->Get()->Error << " user: " - << (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")); + YDB_LOG_INFO_CTX(ctx, "CheckACL ticket got result from TICKET_PARSER response", + {"maskedTicket", maskedTicket}, + {"error", ev->Get()->Error}, + {"user", (!ev->Get()->HasError() ? ev->Get()->Token->GetUserSID() : "")}); if (ev->Get()->HasError()) { if (AppData()->EnforceUserTokenRequirement || AppData()->EnforceUserTokenCheckRequirement) { @@ -459,7 +472,9 @@ void TWriteSessionActor::Handle(TEvTicketParser::TEvAuthorizeTicketResult::TPtr& Y_ABORT_UNLESS(ACLCheckInProgress); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session cookie: " << Cookie << " sessionId: " << OwnerCookie << " describe result for acl check"); + YDB_LOG_INFO_CTX(ctx, "Session describe result for acl check", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); CheckACL(ctx); } @@ -548,13 +563,17 @@ void TWriteSessionActor::CloseSession(const TString& errorReason, const NPersQue error->SetDescription(errorReason); error->SetCode(errorCode); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, - "session error cookie: " << Cookie << " reason: \"" << errorReason << "\" code: " - << EErrorCode_Name(errorCode) << " sessionId: " << OwnerCookie); + YDB_LOG_INFO_CTX(ctx, "Session error reason", + {"cookie", Cookie}, + {"errorReason", errorReason}, + {"code", EErrorCode_Name(errorCode)}, + {"sessionId", OwnerCookie}); Handler->Reply(std::move(result)); } else { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session closed cookie: " << Cookie << " sessionId: " << OwnerCookie); + YDB_LOG_INFO_CTX(ctx, "Session closed", + {"cookie", Cookie}, + {"sessionId", OwnerCookie}); } Die(ctx); @@ -586,8 +605,11 @@ void TWriteSessionActor::Handle(NPQ::TEvPartitionWriter::TEvInitResult::TPtr& ev Y_ABORT_UNLESS(FullConverter); init->SetTopic(FullConverter->GetClientsideName()); - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "session inited cookie: " << Cookie << " partition: " << Partition - << " MaxSeqNo: " << maxSeqNo << " sessionId: " << OwnerCookie); + YDB_LOG_INFO_CTX(ctx, "Session inited", + {"cookie", Cookie}, + {"partition", Partition}, + {"maxSeqNo", maxSeqNo}, + {"sessionId", OwnerCookie}); Handler->Reply(std::move(response)); @@ -886,9 +908,14 @@ void TWriteSessionActor::HandlePoison(TEvPQProxy::TEvDieCommand::TPtr& ev, const void TWriteSessionActor::LogSession(const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::PQ_WRITE_PROXY, "write session: cookie=" << Cookie << " sessionId=" << OwnerCookie - << " userAgent=\"" << UserAgent << "\" ip=" << PeerName << " proto=v0 " - << " topic=" << DiscoveryConverter->GetPrintableString() << " durationSec=" << (ctx.Now() - StartTime).Seconds()); + YDB_LOG_INFO_CTX(ctx, "Write session", + {"cookie", Cookie}, + {"proto", "v0"}, + {"sessionId", OwnerCookie}, + {"userAgent", UserAgent}, + {"ip", PeerName}, + {"topic", DiscoveryConverter->GetPrintableString()}, + {"durationSec", (ctx.Now() - StartTime).Seconds()}); LogSessionDeadline = ctx.Now() + TDuration::Hours(1) + TDuration::Seconds(rand() % 60); } diff --git a/ydb/services/persqueue_v1/ut/topic_deferred_publish_ut.cpp b/ydb/services/persqueue_v1/ut/topic_deferred_publish_ut.cpp index cfbe9e4c964..686ffd7eb56 100644 --- a/ydb/services/persqueue_v1/ut/topic_deferred_publish_ut.cpp +++ b/ydb/services/persqueue_v1/ut/topic_deferred_publish_ut.cpp @@ -431,17 +431,6 @@ void GrantPublicationTableWrite(NPersQueue::TTestServer& server, const TString& } } -void GrantPublicationRegistryDelete(NPersQueue::TTestServer& server, const TString& subject) { - GrantPublicationTableWrite(server, subject); - if (SchemePathExists(server, "/Root/.metadata/topic_deferred_publications")) { - server.AnnoyingClient->TestGrant( - "/Root/.metadata", - "topic_deferred_publications", - subject, - NACLib::EAccessRights::GenericWrite); - } -} - void InsertDestinationRow( NPersQueue::TTestServer& server, const TString& authTicket, @@ -2035,7 +2024,6 @@ Y_UNIT_TEST_SUITE(TopicDeferredPublishFinalize) { Y_UNIT_TEST(PublishAfterStreamWriteClearsRegistryAndMakesDataVisible) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-publish-topic", "ext-publish"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); constexpr TStringBuf payload = "deferred-payload-visible"; { @@ -2059,7 +2047,6 @@ Y_UNIT_TEST(PublishAfterStreamWriteClearsRegistryAndMakesDataVisible) { Y_UNIT_TEST(PublishAfterStreamWriteToTwoPartitionsMakesDataVisible) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-two-partitions-topic", "ext-two-partitions"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); constexpr TStringBuf payload0 = "deferred-payload-part-0"; constexpr TStringBuf payload1 = "deferred-payload-part-1"; @@ -2094,7 +2081,6 @@ Y_UNIT_TEST(PublishAfterStreamWriteToTwoPartitionsMakesDataVisible) { Y_UNIT_TEST(CancelAfterStreamWriteToTwoPartitionsClearsRegistryWithoutData) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-cancel-two-partitions-topic", "ext-cancel-two-partitions"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); { auto session = fixture.OpenWriteStream("producer-cancel-0", 0); @@ -2123,7 +2109,6 @@ Y_UNIT_TEST(CancelAfterStreamWriteToTwoPartitionsClearsRegistryWithoutData) { Y_UNIT_TEST(CancelAfterStreamWriteClearsRegistryWithoutData) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-cancel-topic", "ext-cancel"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); { auto session = fixture.OpenWriteStream("producer-cancel"); @@ -2144,7 +2129,6 @@ Y_UNIT_TEST(CancelAfterStreamWriteClearsRegistryWithoutData) { Y_UNIT_TEST(RepeatFinalizeReturnsNotFound) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-repeat-topic", "ext-repeat"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); { auto session = fixture.OpenWriteStream("producer-repeat"); @@ -2209,7 +2193,6 @@ Y_UNIT_TEST(PublishMultipleDestinations) { auto topicStub = MakeTopicServiceStub(server); const ui64 intPublicationId = BeginPublicationIntId( CallBeginPublication(*deferredStub, "/Root", "ext-multi")); - GrantPublicationRegistryDelete(server, "root@builtin"); { auto session = TStreamWriteSession::Open(*topicStub, "finalize-multi-topic-a", "producer-a", 0); @@ -2233,7 +2216,6 @@ Y_UNIT_TEST(PublishMultipleDestinations) { Y_UNIT_TEST(PublishBeforeWriteAckKeepsRegistry) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-before-ack-topic", "ext-before-ack"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); TFinalizePublicationOutcome publishOutcome; std::thread publishThread([&]() { @@ -2253,7 +2235,6 @@ Y_UNIT_TEST(PublishBeforeWriteAckKeepsRegistry) { Y_UNIT_TEST(PublishFailureOnInvalidDestinationKeepsRegistry) { auto fixture = TDeferredStreamWriteFixture::Enabled("finalize-bad-dest-topic", "ext-bad-dest"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); { auto session = fixture.OpenWriteStream("producer-bad-dest"); @@ -2303,7 +2284,6 @@ Y_UNIT_TEST_SUITE(TopicDeferredPublishLifecycle) { Y_UNIT_TEST(PublishMakesDataVisible) { auto fixture = TDeferredStreamWriteFixture::Enabled("lifecycle-publish-topic", "ext-lifecycle-publish"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); constexpr TStringBuf payload = "lifecycle-publish-payload"; { @@ -2327,7 +2307,6 @@ Y_UNIT_TEST(PublishMakesDataVisible) { Y_UNIT_TEST(CancelDiscardsData) { auto fixture = TDeferredStreamWriteFixture::Enabled("lifecycle-cancel-topic", "ext-lifecycle-cancel"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); { auto session = fixture.OpenWriteStream("producer-lifecycle-cancel"); @@ -2346,7 +2325,6 @@ Y_UNIT_TEST(CancelDiscardsData) { Y_UNIT_TEST(StagingNotVisibleBeforePublish) { auto fixture = TDeferredStreamWriteFixture::Enabled("lifecycle-staging-topic", "ext-lifecycle-staging"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); constexpr TStringBuf payload = "lifecycle-staging-payload"; { @@ -2380,7 +2358,6 @@ Y_UNIT_TEST(MultiDestinationSinglePublication) { auto topicStub = MakeTopicServiceStub(server); const ui64 intPublicationId = BeginPublicationIntId( CallBeginPublication(*deferredStub, "/Root", "ext-lifecycle-multi-dest")); - GrantPublicationRegistryDelete(server, "root@builtin"); constexpr TStringBuf payloadA = "lifecycle-topic-a"; constexpr TStringBuf payloadB = "lifecycle-topic-b"; @@ -2409,7 +2386,6 @@ Y_UNIT_TEST(MultiDestinationSinglePublication) { Y_UNIT_TEST(RepeatFinalizeNotFound) { auto fixture = TDeferredStreamWriteFixture::Enabled("lifecycle-repeat-topic", "ext-lifecycle-repeat"); - GrantPublicationRegistryDelete(fixture.Server, "root@builtin"); constexpr TStringBuf payload = "lifecycle-repeat-payload"; { @@ -2449,7 +2425,6 @@ Y_UNIT_TEST(BeginOnlyPublishAborts) { const ui64 intPublicationId = BeginPublicationIntId( CallBeginPublication(*deferredStub, "/Root", "ext-begin-only-publish")); - GrantPublicationRegistryDelete(server, "root@builtin"); const auto publishOutcome = CallPublish(*deferredStub, "/Root", intPublicationId); UNIT_ASSERT_VALUES_EQUAL(publishOutcome.Operation.status(), Ydb::StatusIds::ABORTED); @@ -2467,7 +2442,6 @@ Y_UNIT_TEST(BeginOnlyCancelDeletes) { const ui64 intPublicationId = BeginPublicationIntId( CallBeginPublication(*deferredStub, "/Root", "ext-begin-only-cancel")); - GrantPublicationRegistryDelete(server, "root@builtin"); const auto cancelOutcome = CallCancelPublication(*deferredStub, "/Root", intPublicationId); UNIT_ASSERT_VALUES_EQUAL(cancelOutcome.Operation.status(), Ydb::StatusIds::SUCCESS); diff --git a/ydb/tests/compatibility/indexes/test_bloom_filter_index.py b/ydb/tests/compatibility/indexes/test_bloom_filter_index.py index b1bd6fee268..d4c9ec25b38 100644 --- a/ydb/tests/compatibility/indexes/test_bloom_filter_index.py +++ b/ydb/tests/compatibility/indexes/test_bloom_filter_index.py @@ -32,7 +32,7 @@ class TestBloomFilterIndex(RestartToAnotherVersionFixture): # Verify bloom filters exist as scheme objects via describe_table def callee(session): - return session.describe_table(f"/{self.database_root}/{self.table_name}") + return session.describe_table(f"/{self.database_path}/{self.table_name}") with ydb.SessionPool(self.driver, size=1) as pool: desc = pool.retry_operation_sync(callee) @@ -66,7 +66,7 @@ class TestBloomFilterIndex(RestartToAnotherVersionFixture): # Verify bloom filters still exist as scheme objects via describe_table after version change def callee_after(session): - return session.describe_table(f"/{self.database_root}/{self.table_name}") + return session.describe_table(f"/{self.database_path}/{self.table_name}") with ydb.SessionPool(self.driver, size=1) as pool: desc_after = pool.retry_operation_sync(callee_after) diff --git a/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp b/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp index b77e4c69504..4d634c17b60 100644 --- a/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp +++ b/ydb/tests/fq/control_plane_storage/in_memory_control_plane_storage_ut.cpp @@ -13,7 +13,6 @@ #include <ydb/core/fq/libs/control_plane_storage/events/events.h> #include <ydb/core/fq/libs/ydb/util.h> #include <ydb/core/fq/libs/ydb/ydb.h> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/tests/tools/fqrun/src/fq_setup.h> diff --git a/ydb/tests/fq/control_plane_storage/ya.make b/ydb/tests/fq/control_plane_storage/ya.make index 0d66ed059fc..9287a2e53f8 100644 --- a/ydb/tests/fq/control_plane_storage/ya.make +++ b/ydb/tests/fq/control_plane_storage/ya.make @@ -16,7 +16,6 @@ PEERDIR( library/cpp/testing/unittest ydb/core/base ydb/core/external_sources - ydb/core/fq/libs/actors/logging ydb/core/fq/libs/init ydb/core/fq/libs/quota_manager/events ydb/core/fq/libs/rate_limiter/control_plane_service diff --git a/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h b/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h index eab8e590c1a..d88b47553d8 100644 --- a/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h +++ b/ydb/tests/fq/control_plane_storage/ydb_test_bootstrap.h @@ -6,7 +6,6 @@ #include <ydb/core/fq/libs/control_plane_storage/schema.h> #include <yql/essentials/providers/common/proto/gateways_config.pb.h> -#include <ydb/core/fq/libs/actors/logging/log.h> #include <ydb/core/fq/libs/ydb/util.h> #include <ydb/core/fq/libs/ydb/ydb.h> #include <ydb/core/testlib/tablet_helpers.h> diff --git a/ydb/tests/fq/streaming/test_streaming.py b/ydb/tests/fq/streaming/test_streaming.py index 5e2f3f73765..0a9b50b0eba 100644 --- a/ydb/tests/fq/streaming/test_streaming.py +++ b/ydb/tests/fq/streaming/test_streaming.py @@ -9,6 +9,7 @@ from typing import Callable import ydb from ydb.tests.fq.streaming_common.common import Kikimr, StreamingTestBase +from ydb.tests.library.test_meta import link_test_case from ydb.tests.tools.datastreams_helpers.control_plane import create_read_rule logger = logging.getLogger(__name__) @@ -1543,3 +1544,248 @@ FROM `{table_name}`""" result_sets = kikimr.ydb_client.query(sql) assert result_sets[0].rows[0]['Data'] == message + + @link_test_case("#46136") + @link_test_case("#46137") + @pytest.mark.parametrize("local_topics", [True, False]) + @pytest.mark.parametrize("additional_operator", ["hop", "mr", "join"]) + def test_precompute_and_other_ops(self: StreamingTestBase, kikimr: Kikimr, entity_name: Callable[[str], str], local_topics: bool, additional_operator: str) -> None: + inp, out, endpoint = self.get_io_names(kikimr, f"test_precompute_and_other_ops_{local_topics!s:.1}_{additional_operator}", local_topics, entity_name) + + precompute_row_table = f"test_precompute_and_other_ops_precompute_row_table_{local_topics!s:.1}_{additional_operator}" + precompute_column_table = f"test_precompute_and_other_ops_precompute_column_table_{local_topics!s:.1}_{additional_operator}" + output_table = f"test_precompute_and_other_ops_output_table_{local_topics!s:.1}_{additional_operator}" + kikimr.ydb_client.query(f""" + CREATE TABLE `{precompute_row_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ); + CREATE TABLE `{precompute_column_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ) WITH ( + STORE = COLUMN + ); + CREATE TABLE `{output_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key, Value) + ); + """) + kikimr.ydb_client.query(f""" + UPSERT INTO `{precompute_row_table}` + (Key, Value) + VALUES + (1, "value-p-row"); + UPSERT INTO `{precompute_column_table}` + (Key, Value) + VALUES + (1, "value-p-column"); + """) + + if additional_operator == "hop": + process = """ + SELECT + Key, + SOME(Value) AS Value + FROM $in + GROUP BY + Key, + HOP(Ts, "PT10S", "PT10S", "PT10S") + """ + + expected_data1 = [] + expected_data2 = ["in1-value-p-row-value-p-column:1", "in1-value-p-row-value-p-column:2"] + elif additional_operator == "mr": + process = """ + SELECT * FROM $in MATCH_RECOGNIZE( + MEASURES + LAST(A.Key) as MatchKey + ALL ROWS PER MATCH + AFTER MATCH SKIP TO NEXT ROW + PATTERN ( A B ) + DEFINE + A as A.Key = 1, + B as B.Key = 2 + ); + """ + + expected_data1 = [] + expected_data2 = ["in1-value-p-row-value-p-column:1", "in1-value-p-row-value-p-column:2", "in2-value-p-row-value-p-column:1", "in2-value-p-row-value-p-column:2"] + elif additional_operator == "join": + join_table = f"test_precompute_and_other_ops_join_table_{local_topics!s:.1}" + kikimr.ydb_client.query(f""" + CREATE TABLE `{join_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ); + """) + kikimr.ydb_client.query(f""" + UPSERT INTO `{join_table}` + (Key, Value) + VALUES + (1, "value-j1"), + (2, "value-j2"); + """) + + process = f""" + SELECT + i.Key AS Key, + i.Value || "-" || j.Value AS Value + FROM $in AS i + LEFT JOIN `{join_table}` AS j ON i.Key = j.Key + """ + + expected_data1 = ["in1-value-p-row-value-j1-value-p-column:1", "in1-value-p-row-value-j2-value-p-column:2"] + expected_data2 = ["in2-value-p-row-value-j1-value-p-column:1", "in2-value-p-row-value-j2-value-p-column:2"] + + query_name = f"test_precompute_and_other_ops_query_{local_topics!s:.1}_{additional_operator}" + kikimr.ydb_client.query(f""" + CREATE STREAMING QUERY `{query_name}` AS + DO BEGIN + PRAGMA FeatureR010 = "prototype"; + + $p_key = SELECT Key FROM `{precompute_row_table}`; + $p_row = SELECT Value FROM `{precompute_row_table}`; + $p_column = SELECT Value FROM `{precompute_column_table}`; + + $in = SELECT + Key, + Unwrap(Value || "-" || $p_row) AS Value, + Unwrap(CAST(Ts AS Timestamp)) AS Ts + FROM {inp} WITH ( + FORMAT = json_each_row, + SCHEMA = ( + Key Int32 NOT NULL, + Value String NOT NULL, + Ts String NOT NULL + ) + ) + WHERE Key % $p_key == 0; + + $processed = {process}; + + INSERT INTO {out} + SELECT + Unwrap(Value || "-" || $p_column || ":" || CAST(Key AS String)) AS Value + FROM $processed; + + UPSERT INTO `{output_table}` SELECT + Unwrap(Key) AS Key, + Unwrap(Value) AS Value + FROM $processed; + END DO; + """) + + path = f"/Root/{query_name}" + self.wait_completed_checkpoints(kikimr, path) + + def validate_table(expected): + result_sets = kikimr.ydb_client.query(f""" + SELECT * FROM `{output_table}` + ORDER BY Value || ":" || CAST(Key AS String); + """) + assert len(result_sets[0].rows) == len(expected) + + for row, expected_value in zip(result_sets[0].rows, sorted(expected)): + value, key = expected_value.split(":") + assert value.startswith(row["Value"].decode("utf-8")), row["Value"].decode("utf-8") + " vs " + value + assert row["Key"] == int(key) + + result_sets = kikimr.ydb_client.query(f""" + DELETE FROM `{output_table}`; + """) + + self.write_stream(['{"Key": 1, "Value": "in1", "Ts": "2026-07-17T07:20:53.428176Z"}'], endpoint=endpoint) + self.write_stream(['{"Key": 2, "Value": "in1", "Ts": "2026-07-17T08:20:53.428176Z"}'], endpoint=endpoint) + if expected_data1: + assert sorted(self.read_stream(len(expected_data1), topic_path=self.output_topic, endpoint=endpoint)) == sorted(expected_data1) + self.wait_completed_checkpoints(kikimr, path) + validate_table(expected_data1) + + self.wait_completed_checkpoints(kikimr, path) + kikimr.ydb_client.query(f""" + ALTER STREAMING QUERY `{query_name}` SET (RUN = FALSE); + """) + + self.write_stream(['{"Key": 1, "Value": "in2", "Ts": "2026-07-17T09:20:53.428176Z"}'], endpoint=endpoint) + self.write_stream(['{"Key": 2, "Value": "in2", "Ts": "2026-07-17T10:20:53.428176Z"}'], endpoint=endpoint) + + kikimr.ydb_client.query(f""" + ALTER STREAMING QUERY `{query_name}` SET (RUN = TRUE); + """) + + assert sorted(self.read_stream(len(expected_data2), topic_path=self.output_topic, endpoint=endpoint)) == sorted(expected_data2) + self.wait_completed_checkpoints(kikimr, path) + validate_table(expected_data2) + + @link_test_case("#46139") + @pytest.mark.parametrize("local_topics", [True, False]) + def test_alter_query_with_precompute(self: StreamingTestBase, kikimr: Kikimr, entity_name: Callable[[str], str], local_topics: bool) -> None: + inp, out, endpoint = self.get_io_names(kikimr, f"test_alter_query_with_precompute_{local_topics!s:.1}", local_topics, entity_name) + + precompute_table = f"test_alter_query_with_precompute_table_{local_topics!s:.1}" + kikimr.ydb_client.query(f""" + CREATE TABLE `{precompute_table}` ( + Key Int32 NOT NULL, + Value String NOT NULL, + PRIMARY KEY (Key) + ); + """) + kikimr.ydb_client.query(f""" + UPSERT INTO `{precompute_table}` + (Key, Value) + VALUES + (1, "value-p-row"); + """) + + query_name = f"test_alter_query_with_precompute_query_{local_topics!s:.1}" + kikimr.ydb_client.query(f""" + CREATE STREAMING QUERY `{query_name}` AS + DO BEGIN + INSERT INTO {out} + SELECT * FROM {inp} + END DO; + """) + + path = f"/Root/{query_name}" + self.wait_completed_checkpoints(kikimr, path) + + expected_data = ["test_data1"] + self.write_stream(expected_data, endpoint=endpoint) + assert self.read_stream(len(expected_data), topic_path=self.output_topic, endpoint=endpoint) == expected_data + + self.wait_completed_checkpoints(kikimr, path) + kikimr.ydb_client.query(f""" + ALTER STREAMING QUERY `{query_name}` SET (FORCE = TRUE) AS + DO BEGIN + $p_value = SELECT Value FROM `{precompute_table}`; + INSERT INTO {out} + SELECT Unwrap(Data || $p_value) FROM {inp} + END DO; + """) + + self.wait_completed_checkpoints(kikimr, path) + self.write_stream(["test_data2"], endpoint=endpoint) + assert self.read_stream(1, topic_path=self.output_topic, endpoint=endpoint) == ["test_data2value-p-row"] + + self.wait_completed_checkpoints(kikimr, path) + kikimr.ydb_client.query(f""" + ALTER STREAMING QUERY `{query_name}` SET (FORCE = TRUE) AS + DO BEGIN + INSERT INTO {out} + SELECT * FROM {inp} + END DO; + """) + + self.wait_completed_checkpoints(kikimr, path) + expected_data = ["test_data3"] + self.write_stream(expected_data, endpoint=endpoint) + assert self.read_stream(len(expected_data), topic_path=self.output_topic, endpoint=endpoint) == expected_data + + kikimr.ydb_client.query(f""" + DROP STREAMING QUERY `{query_name}`; + """) diff --git a/ydb/tests/fq/streaming/ya.make b/ydb/tests/fq/streaming/ya.make index e1b14d8871e..97c37186f39 100644 --- a/ydb/tests/fq/streaming/ya.make +++ b/ydb/tests/fq/streaming/ya.make @@ -29,6 +29,7 @@ IF (SANITIZER_TYPE) ELSE() SIZE(MEDIUM) FORK_SUBTESTS() + SPLIT_FACTOR(20) ENDIF() PEERDIR( diff --git a/ydb/tests/fq/streaming_common/common.py b/ydb/tests/fq/streaming_common/common.py index f24a1aa1b46..b02339eb7e7 100644 --- a/ydb/tests/fq/streaming_common/common.py +++ b/ydb/tests/fq/streaming_common/common.py @@ -71,6 +71,7 @@ def get_ydb_config(request): "enable_watermarks_advanced": enable_watermarks_advanced, "enable_streaming_partition_balancing": enable_streaming_partition_balancing, "enable_compile_cache_warmup": False, + "enable_channel_memory_tracking": False, # Remove after fix https://github.com/ydb-platform/ydb/issues/46891 }, replication_config={ "iam_service_control": { diff --git a/ydb/tests/functional/config/test_distconf_self_heal.py b/ydb/tests/functional/config/test_distconf_self_heal.py index 91c9846795e..bd14fa0c77e 100644 --- a/ydb/tests/functional/config/test_distconf_self_heal.py +++ b/ydb/tests/functional/config/test_distconf_self_heal.py @@ -26,6 +26,28 @@ def get_ring_group(request_config, config_name): return config["Ring"] +def extract_node_ids(ring): + """Recursively collect all node ids referenced by a (possibly nested) TRing structure.""" + node_ids = set() + if "Node" in ring: + node_ids.update(ring["Node"]) + for nested in ring.get("Ring", []): + node_ids.update(extract_node_ids(nested)) + return node_ids + + +def extract_all_node_ids(config, config_name): + """Collect all node ids present in either the RingGroups or single Ring representation.""" + config = config[f"{config_name}Config"] + node_ids = set() + if "RingGroups" in config: + for rg in config["RingGroups"]: + node_ids.update(extract_node_ids(rg)) + elif "Ring" in config: + node_ids.update(extract_node_ids(config["Ring"])) + return node_ids + + class KiKiMRDistConfSelfHealTest(object): nodes_count = 9 erasure = Erasure.MIRROR_3_DC @@ -33,6 +55,11 @@ class KiKiMRDistConfSelfHealTest(object): separate_node_configs = True state_storage_rings = None n_to_select = None + # extra fields merged into the static self_management_config section of the initial + # (bootstrap) YAML config; NB: Cfg->SelfManagementConfig used by distconf self-heal is read + # once at node startup and is NOT affected by dynamic ReplaceConfig calls, so options like + # automatic_*_management / *_self_heal_allowed_nodes must be set here, before cluster start. + self_management_extra_options = {} metadata_section = { "kind": "MainConfig", "version": 0, @@ -59,6 +86,9 @@ class KiKiMRDistConfSelfHealTest(object): n_to_select=cls.n_to_select, state_storage_rings=cls.state_storage_rings) + if cls.self_management_extra_options: + cls.configurator.yaml_config["self_management_config"].update(cls.self_management_extra_options) + cls.cluster = KiKiMR(configurator=cls.configurator) cls.cluster.start() @@ -234,3 +264,73 @@ class TestKiKiMRDistConfSelfHealParallelCall2(KiKiMRDistConfSelfHealTest): rg = self.do_request_config()[f"{configName}Config"]["Ring"] assert_eq(rg["NToSelect"], 9) assert_eq(len(rg["Ring"]), 9) + + +class TestKiKiMRDistConfSelfHealAllowedNodes(KiKiMRDistConfSelfHealTest): + # 3 extra spare nodes (10, 11, 12) relative to the base 9-node MIRROR_3_DC topology so that + # forbidding one node still leaves enough nodes for a valid, fully-healthy configuration. + erasure = Erasure.MIRROR_3_DC + nodes_count = 12 + rgOffset = 1 + + forbidden_node_id = nodes_count + allowed_node_ids = sorted(set(range(1, nodes_count + 1)) - {forbidden_node_id}) + + self_management_extra_options = { + "automatic_state_storage_management": True, + "state_storage_self_heal_allowed_nodes": allowed_node_ids, + "automatic_state_storage_board_management": True, + "state_storage_board_self_heal_allowed_nodes": allowed_node_ids, + "automatic_scheme_board_management": True, + "scheme_board_self_heal_allowed_nodes": allowed_node_ids, + } + + def do_test(self, configName): + self.do_bad_config(configName) + + logger.info("Start SelfHeal with allowed nodes restriction (forbidding node %s) for %s", + self.forbidden_node_id, configName) + logger.info(self.do_request({"SelfHealStateStorage": {"WaitForConfigStep": 1, "ForceHeal": True}})) + time.sleep(10) + + healed_config = self.do_request_config() + node_ids = extract_all_node_ids(healed_config, configName) + + assert_that(len(node_ids) > 0, f"Healed {configName} config has no nodes: {healed_config}") + assert_that(self.forbidden_node_id not in node_ids, + f"Forbidden node {self.forbidden_node_id} present in healed {configName} config: {node_ids}") + assert_that(node_ids <= set(self.allowed_node_ids), + f"Healed {configName} config uses nodes outside the allowed list: {node_ids - set(self.allowed_node_ids)}") + + +class TestKiKiMRDistConfSelfHealAutomaticManagementDisabled(KiKiMRDistConfSelfHealTest): + erasure = Erasure.MIRROR_3_DC + nodes_count = 9 + rgOffset = 1 + + self_management_extra_options = { + "automatic_state_storage_management": False, + "automatic_state_storage_board_management": False, + "automatic_scheme_board_management": False, + } + + def check_failed(self, req, message): + resp = self.do_request(req) + assert_that(resp.get("ErrorReason", "").startswith(message), {"Response": resp, "Expected": message}) + + def do_test(self, configName): + # Manually push the config into a "bad"/suboptimal state (ReconfigStateStorage is a + # direct manual API and is unaffected by AutomaticManagement flags). + self.do_bad_config(configName) + bad_config = get_ring_group(self.do_request_config(), configName) + + logger.info("Attempting SelfHeal with automatic management disabled for %s", configName) + self.check_failed( + {"SelfHealStateStorage": {"WaitForConfigStep": 1, "ForceHeal": True}}, + "Current configuration is recommended. Nothing to self-heal.") + time.sleep(5) + + # The config must remain completely untouched: self-heal must not have modified it since + # automatic management for this subsystem is disabled. + unchanged_config = get_ring_group(self.do_request_config(), configName) + assert_eq(unchanged_config, bad_config) diff --git a/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json b/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json index 4743e515a2b..8873583c0d7 100644 --- a/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json +++ b/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_domain_sysviews_registry/root_sysviews.json @@ -1027,6 +1027,10 @@ { "name": "HasPath", "type": "Utf8?" + }, + { + "name": "HasStream", + "type": "Bool?" } ], "primary_key": [ diff --git a/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_tenant_sysviews_registry/tenant_sysviews.json b/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_tenant_sysviews_registry/tenant_sysviews.json index 32221b4c12c..9d8b2beb174 100644 --- a/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_tenant_sysviews_registry/tenant_sysviews.json +++ b/ydb/tests/functional/tenants/canondata/test_system_views.TestSysViewsRegistry.test_tenant_sysviews_registry/tenant_sysviews.json @@ -603,6 +603,10 @@ { "name": "HasPath", "type": "Utf8?" + }, + { + "name": "HasStream", + "type": "Bool?" } ], "primary_key": [ diff --git a/ydb/tests/functional/tenants/test_remove_storage_groups.py b/ydb/tests/functional/tenants/test_remove_storage_groups.py new file mode 100644 index 00000000000..f5a996ec2c5 --- /dev/null +++ b/ydb/tests/functional/tenants/test_remove_storage_groups.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +import copy +import logging +import os + +import pytest +from hamcrest import assert_that, equal_to + +import ydb.public.api.protos.ydb_cms_pb2 as cms_pb +from ydb.tests.library.common.wait_for import wait_for_and_assert +from ydb.tests.library.harness.util import LogLevels +from ydb.tests.oss.ydb_sdk_import import ydb + +logger = logging.getLogger(__name__) + +# local configuration for the ydb cluster (fetched by ydb_cluster_configuration fixture) +CLUSTER_CONFIG = dict( + additional_log_configs={ + # more logs + 'CMS_TENANTS': LogLevels.TRACE, + 'HIVE': LogLevels.TRACE, + # less logs + 'KQP_PROXY': LogLevels.CRIT, + 'KQP_WORKER': LogLevels.CRIT, + 'KQP_GATEWAY': LogLevels.CRIT, + 'GRPC_PROXY': LogLevels.ERROR, + 'TX_DATASHARD': LogLevels.ERROR, + 'TX_PROXY_SCHEME_CACHE': LogLevels.ERROR, + 'KQP_YQL': LogLevels.ERROR, + 'KQP_SESSION': LogLevels.CRIT, + 'KQP_COMPILE_ACTOR': LogLevels.CRIT, + 'PERSQUEUE_CLUSTER_TRACKER': LogLevels.CRIT, + 'SCHEME_BOARD_SUBSCRIBER': LogLevels.CRIT, + 'SCHEME_BOARD_POPULATOR': LogLevels.CRIT, + }, + extra_feature_flags=[ + 'enable_cut_history' + ], + # do not clutter logs with resource pools auto creation + enable_resource_pools=False, + hive_config={ + 'cut_history_allow_list': '' # allow all + }, +) + + +# fixtures.ydb_cluster_configuration local override [email protected](scope='module') +def ydb_cluster_configuration(): + conf = copy.deepcopy(CLUSTER_CONFIG) + return conf + + +def test_remove_storage_group(ydb_cluster, ydb_root, ydb_safe_test_name, ydb_client): + database = os.path.join(ydb_root, ydb_safe_test_name) + + ydb_cluster.create_database( + database, + storage_pool_units_count={'hdd': 1}, + ) + database_nodes = ydb_cluster.register_and_start_slots(database, count=1) + ydb_cluster.wait_tenant_up(database) + + ydb_cluster.alter_database( + database, + storage_units_to_add={'hdd': 2}, + ) + + driver = ydb_client(database) + with ydb.QuerySessionPool(driver, size=1) as pool: + table_name = f"`{database}/table`" + pool.execute_with_retries("CREATE TABLE " + table_name + " (key Int32, value String, primary key(key));") + + pool.execute_with_retries("""UPSERT INTO """ + table_name + """ (key, value) VALUES (1, "value1");""") + + ydb_cluster.alter_database( + database, + storage_units_to_remove={'hdd': 2}, + ) + + ydb_cluster.restart_slots() + + def get_storage_units(): + status = ydb_cluster.get_database_status(database) + units = sum([unit.count for unit in status.allocated_resources.storage_units]) + return units + + wait_for_and_assert( + get_storage_units, + equal_to(1), + timeout_seconds=120, + step_seconds=10, + ) + + status = ydb_cluster.get_database_status(database) + assert_that(status.state, equal_to(cms_pb.GetDatabaseStatusResult.RUNNING)) + + ydb_cluster.unregister_and_stop_slots(database_nodes) + ydb_cluster.remove_database(database) diff --git a/ydb/tests/functional/tenants/ya.make b/ydb/tests/functional/tenants/ya.make index 724adb69b22..03963bb090a 100644 --- a/ydb/tests/functional/tenants/ya.make +++ b/ydb/tests/functional/tenants/ya.make @@ -13,6 +13,7 @@ TEST_SRCS( test_system_views.py test_auth_system_views.py test_publish_into_schemeboard_with_common_ssring.py + test_remove_storage_groups.py test_user_administration.py test_users_groups_with_acl.py ) diff --git a/ydb/tests/library/common/protobuf_console.py b/ydb/tests/library/common/protobuf_console.py index 925fe5dc0a0..6a6bb1da857 100644 --- a/ydb/tests/library/common/protobuf_console.py +++ b/ydb/tests/library/common/protobuf_console.py @@ -108,6 +108,18 @@ class AlterTenantRequest(AbstractProtobufBuilder): if soft is not None: quotas.data_size_soft_quota = soft + def add_storage_groups_to_add(self, pool_type, pool_size): + pool = self.protobuf.AlterTenantRequest.Request.storage_units_to_add.add() + pool.unit_kind = pool_type + pool.count = pool_size + return self + + def add_storage_groups_to_remove(self, pool_type, pool_size): + pool = self.protobuf.AlterTenantRequest.Request.storage_units_to_remove.add() + pool.unit_kind = pool_type + pool.count = pool_size + return self + class GetTenantStatusRequest(AbstractProtobufBuilder): """ diff --git a/ydb/tests/library/harness/kikimr_cluster_interface.py b/ydb/tests/library/harness/kikimr_cluster_interface.py index a8cb0106891..52b6015045e 100644 --- a/ydb/tests/library/harness/kikimr_cluster_interface.py +++ b/ydb/tests/library/harness/kikimr_cluster_interface.py @@ -400,6 +400,40 @@ class KiKiMRClusterInterface(object): ) return database_name + def alter_database( + self, + database_name, + storage_units_to_add=None, + storage_units_to_remove=None, + timeout_seconds=120, + token=None, + ): + req = AlterTenantRequest(database_name) + + if token is not None: + req.set_user_token(token) + + assert storage_units_to_add or storage_units_to_remove + if storage_units_to_add: + for pool_type, count in storage_units_to_add.items(): + req.add_storage_groups_to_add(pool_type, count) + + if storage_units_to_remove: + for pool_type, count in storage_units_to_remove.items(): + req.add_storage_groups_to_remove(pool_type, count) + + response = self.client.send_request(req.protobuf, method='ConsoleRequest') + operation = response.AlterTenantResponse.Response.operation + if not operation.ready and response.Status.Code != StatusIds.STATUS_CODE_UNSPECIFIED: + raise RuntimeError( + 'alter_database_storage_units failed: %s: %s' % (response.Status.Code, response.Status.Reason) + ) + if not operation.ready: + operation = self.__wait_console_op(operation.id, timeout_seconds=timeout_seconds, token=token) + if operation.status != StatusIds.SUCCESS: + raise RuntimeError('alter_database_storage_units failed: %s' % (operation.status,)) + return database_name + def remove_database( self, database_name, diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py index 4829367047d..b0daf035c9e 100644 --- a/ydb/tests/library/harness/kikimr_config.py +++ b/ydb/tests/library/harness/kikimr_config.py @@ -174,6 +174,7 @@ class KikimrConfigGenerator(object): pg_compatible_expirement=False, generic_connector_config=None, # typing.Optional[TGenericConnectorConfig] kafka_api_port=None, + kafka_listen_address=None, metadata_section=None, column_shard_config=None, use_config_store=False, @@ -597,6 +598,8 @@ class KikimrConfigGenerator(object): kafka_proxy_config = dict() kafka_proxy_config['enable_kafka_proxy'] = True kafka_proxy_config["listening_port"] = self.get_kafka_api_port(node_id) + if kafka_listen_address is not None: + kafka_proxy_config["listening_address"] = kafka_listen_address self.yaml_config["kafka_proxy_config"] = kafka_proxy_config diff --git a/ydb/tests/library/harness/kikimr_runner.py b/ydb/tests/library/harness/kikimr_runner.py index 79d269c8959..784b0edccc0 100644 --- a/ydb/tests/library/harness/kikimr_runner.py +++ b/ydb/tests/library/harness/kikimr_runner.py @@ -805,6 +805,11 @@ class KiKiMR(kikimr_cluster_interface.KiKiMRClusterInterface): node.stop() node.start() + def restart_slots(self): + for slot in list(self.slots.values()): + slot.stop() + slot.start() + def prepare_node(self, configurator=None, seed_nodes_file=None): try: new_node_object = self.__register_node(configurator, seed_nodes_file) diff --git a/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py b/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py index 66817523528..95985bb74e8 100644 --- a/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py +++ b/ydb/tests/olap/scenario/helpers/scenario_tests_helper.py @@ -773,7 +773,7 @@ class ScenarioTestHelper: def get_volumes_columns(self, table_name: str, name_column: str) -> tuple[int, int]: path = table_name if table_name.startswith('/') else self.get_full_path(table_name) - query = f'''SELECT * FROM `{path}/.sys/primary_index_stats` WHERE Activity == 1''' + query = f'''SELECT SUM(RawBytes) AS RawBytes, SUM(BlobRangeSize) AS BlobRangeSize FROM `{path}/.sys/primary_index_stats` WHERE Activity == 1''' if (len(name_column)): query += f' AND EntityName = \"{name_column}\"' result_sets = self.execute_query(query) diff --git a/ydb/tools/stress_tool/README.md b/ydb/tools/stress_tool/README.md index e0608e7afab..d636240ee2b 100644 --- a/ydb/tools/stress_tool/README.md +++ b/ydb/tools/stress_tool/README.md @@ -84,10 +84,42 @@ Each `DDiskTestList` entry contains one or more `DDiskLoad` sources. These describe network load generated through the actors library interconnect subsystem (`ydb/library/actors/interconnect/load.h`), reusing the same `TInterconnectLoad` load actor used by the `InterconnectLoad` load type of the -regular YDB load actor service. Since the stress tool runs a single node, the -load actor and its counterpart (the load responder) both live in the same -process; traffic is routed through loopback using `NodeHops`, so no real -network connection is required to generate and measure interconnect traffic. +regular YDB load actor service. + +By default (no `--server`/`--client` options), the stress tool runs a single +node; the load actor and its counterpart (the load responder) both live in the +same process, and traffic is routed through loopback using `NodeHops`, so no +real network connection is required to generate and measure interconnect +traffic. + +To measure interconnect performance over a **real network** between two +hosts, run two copies of `ydb_stress_tool`: + +- On the **second** (remote) host, start a responder-only instance. Its + `--server NODE_ID` must match the NodeId that the client will assign this + server via `--endpoint` (see below) -- for a single server this is `1`: + ``` + ydb_stress_tool --cfg cfg.txt --server 1 --client 2 --ic-port 19001 + ``` + This registers the interconnect load responder for node 1 and listens on + port 19001 for an incoming connection from the client (node 2). No device + `--path` is required. + +- On the **first** host, run the actual `InterconnectTestList` from the config + as a client that connects out to the responder: + ``` + ydb_stress_tool --cfg cfg.txt --client 2 --endpoint remote-host:19001 + ``` + The client's own NodeId is `2` (from `--client`, must differ from all + server NodeIds); the server passed via `--endpoint` is assigned NodeId `1` + (the i-th `--endpoint` gets NodeId `i + 1`, matching the `--server NODE_ID` + used on the remote host). The config's `InterconnectLoad.NodeHops` must + reference this NodeId (e.g. `NodeHops: [1]`) so that load messages are + routed to the remote responder over the network instead of looping back + locally. + + Optional: `--use-uring` selects the io_uring transport instead of epoll for + the interconnect connection itself (Linux 5.19+). Each `InterconnectTestList` entry contains one or more `InterconnectLoad` sources, run sequentially. Detailed periodic throughput/RTT statistics are diff --git a/ydb/tools/stress_tool/device_test_tool.cpp b/ydb/tools/stress_tool/device_test_tool.cpp index 6362368ef29..53311967ed1 100644 --- a/ydb/tools/stress_tool/device_test_tool.cpp +++ b/ydb/tools/stress_tool/device_test_tool.cpp @@ -88,6 +88,15 @@ static void InstallServerSignalHandler() { signal(SIGTERM, ServerSignalHandler); } +static void InterconnectServerSignalHandler(int) { + NKikimr::InterconnectServerStopEvent.Signal(); +} + +static void InstallInterconnectServerSignalHandler() { + signal(SIGINT, InterconnectServerSignalHandler); + signal(SIGTERM, InterconnectServerSignalHandler); +} + static const char* ydb_logo = R"__( @@ -158,7 +167,9 @@ int main(int argc, char **argv) { // DDisk client/server options are hidden from the auto-generated --help list and // re-rendered below as a dedicated "DDisk client/server options" section. Keep the // section text in sync with the option definitions here. - opts.AddLongOption("server", "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect)") + opts.AddLongOption("server", "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect); " + "if the config has no DDiskTestList but has InterconnectTestList, runs as an interconnect load responder instead " + "(for testing interconnect over a real network between two ydb_stress_tool instances)") .RequiredArgument("NODE_ID").StoreResult(&serverNodeId).Hidden(); opts.AddLongOption("client", "client node ID; in server mode this is the expected client's ID (used in the nameserver), in client mode this is the client's own ID") .RequiredArgument("NODE_ID").StoreResult(&clientNodeId).Hidden(); @@ -166,7 +177,7 @@ int main(int argc, char **argv) { .RequiredArgument("HOST:PORT").AppendTo(&clientEndpoints).Hidden(); opts.AddLongOption("ic-port", "interconnect port for server to listen on (server only)") .RequiredArgument("PORT").StoreResult(&icPort).Hidden(); - opts.AddLongOption("num-server-devices", "number of devices per server (default 1)") + opts.AddLongOption("num-server-devices", "number of devices per server (default 1); ignored in interconnect server/client mode") .RequiredArgument("N").DefaultValue("1").Hidden(); bool useUring = false; opts.AddLongOption("use-uring", "use io_uring transport for interconnect instead of epoll (Linux 5.19+)") @@ -185,7 +196,9 @@ int main(int argc, char **argv) { TStringBuilder ddiskHelp; ddiskHelp << row("server NODE_ID", - "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect)") + "run as DDisk server with the given node ID (sets up PDisks/DDisks, listens via interconnect); " + "if the config has InterconnectTestList instead of DDiskTestList, runs as an interconnect load " + "responder for the network interconnect test (see InterconnectTestList in README)") << row("client NODE_ID", "client node ID; in server mode this is the expected client's ID (used in the nameserver), " "in client mode this is the client's own ID") @@ -194,11 +207,11 @@ int main(int argc, char **argv) { << row("ic-port PORT", "interconnect port for server to listen on (server only)") << row("num-server-devices N", - "number of devices per server (default 1)") + "number of devices per server (default 1); ignored for InterconnectTestList") << row("use-uring", "use io_uring transport for interconnect instead of epoll (Linux 5.19+)"); - opts.AddSection("DDisk client/server options", ddiskHelp); + opts.AddSection("DDisk / Interconnect client/server options", ddiskHelp); } TOptsParseResult res(&opts, argc, argv); @@ -206,6 +219,9 @@ int main(int argc, char **argv) { // Server mode is selected by --server. Client mode by --client without --server. // In server mode, --client is also required and provides the expected client's NodeID // so the interconnect handshake can resolve the peer in the static nameserver table. + // Both modes are shared between DDisk (real device paths, PDisk/DDisk actors) and + // Interconnect (no device paths, just a load responder / load actor) tests; which + // one is used is decided later based on the config file contents. const bool serverMode = res.Has("server"); const bool clientMode = !serverMode && res.Has("client"); if (serverMode) { @@ -221,10 +237,6 @@ int main(int argc, char **argv) { Cerr << "Error: --server and --client NODE_IDs must differ" << Endl; return 1; } - if (paths.empty()) { - Cerr << "Error: --server requires at least one --path" << Endl; - return 1; - } if (!icPort) { Cerr << "Error: --server requires --ic-port" << Endl; return 1; @@ -241,7 +253,7 @@ int main(int argc, char **argv) { } } - if (!clientMode && paths.empty()) { + if (!clientMode && !serverMode && paths.empty()) { Cerr << "Error: at least one --path must be specified" << Endl; return 1; } @@ -250,8 +262,11 @@ int main(int argc, char **argv) { Cout << ydb_logo << Flush; } - // For client mode, paths can be empty; use a dummy for TPerfTestConfig - if (clientMode && paths.empty()) { + const bool pathsProvided = !paths.empty(); + + // For client mode, and for server mode when running as an interconnect + // responder (no real devices), paths can be empty; use a dummy for TPerfTestConfig. + if ((clientMode || serverMode) && paths.empty()) { paths.push_back("unused"); } @@ -288,10 +303,77 @@ int main(int argc, char **argv) { } } - // Client-server mode dispatch + // Client-server mode dispatch. Two independent flavors share --server/--client/ + // --endpoint/--ic-port: DDisk (real device paths, PDisk/DDisk actors) is used when + // the config has DDiskTestList; Interconnect (no device paths, just a load + // responder / load actor talking over a real network connection) is used when the + // config has InterconnectTestList instead. This lets a second ydb_stress_tool + // instance on another host act as the interconnect load responder. if (serverMode || clientMode) { - if (protoTests.DDiskTestListSize() == 0) { - Cerr << "Error: --server/--client mode requires DDiskTestList in config" << Endl; + const bool hasDDiskTests = protoTests.DDiskTestListSize() > 0; + const bool hasInterconnectTests = protoTests.InterconnectTestListSize() > 0; + + if (!hasDDiskTests && !hasInterconnectTests) { + Cerr << "Error: --server/--client mode requires DDiskTestList or InterconnectTestList in config" << Endl; + return 1; + } + if (hasDDiskTests && hasInterconnectTests) { + Cerr << "Error: --server/--client mode requires exactly one of DDiskTestList or InterconnectTestList in config" << Endl; + return 1; + } + + if (hasInterconnectTests) { + // Interconnect network test: no real device paths are used. + NDevicePerfTest::TInterconnectTest testProto = protoTests.GetInterconnectTestList(0); + + if (serverMode) { + InstallInterconnectServerSignalHandler(); + auto printer = MakeIntrusive<NKikimr::TResultPrinter>(config.OutputFormat, config.RunCount); + THolder<NKikimr::TPerfTest> test(new NKikimr::TInterconnectServer(config, serverNodeId, clientNodeId, icPort, useUring)); + test->SetPrinter(printer); + test->RunTest(); + return 0; + } + + if (clientMode) { + // Build server peer list. Server with index i (0-based) gets NodeId = i + 1. + // Config's InterconnectLoad.NodeHops entries should reference these NodeIds + // to route traffic to the corresponding remote server. + TVector<NKikimr::TInterconnectPeer> serverPeers; + serverPeers.reserve(clientEndpoints.size()); + for (size_t i = 0; i < clientEndpoints.size(); ++i) { + try { + auto [host, port] = ParseHostPort(clientEndpoints[i]); + serverPeers.push_back({static_cast<ui32>(i + 1), host, port}); + } catch (const yexception& ex) { + Cerr << "Error: --endpoint #" << (i + 1) << ": " << ex.what() << Endl; + return 1; + } + } + for (const auto& peer : serverPeers) { + if (peer.NodeId == clientNodeId) { + Cerr << "Error: --client NODE_ID (" << clientNodeId + << ") collides with server NodeId " << peer.NodeId + << "; client NodeId must differ from all server NodeIds" << Endl; + return 1; + } + } + + auto printer = MakeIntrusive<NKikimr::TResultPrinter>(config.OutputFormat, config.RunCount); + for (ui32 run = 0; run < config.RunCount; ++run) { + THolder<NKikimr::TPerfTest> test( + new NKikimr::TInterconnectClient(config, testProto, clientNodeId, serverPeers, useUring)); + test->SetPrinter(printer); + test->RunTest(); + } + printer->EndTest(); + return 0; + } + } + + // DDisk test: real device paths (--path) are required. + if (serverMode && !pathsProvided) { + Cerr << "Error: --server requires at least one --path (DDisk mode)" << Endl; return 1; } NDevicePerfTest::TDDiskTest testProto = protoTests.GetDDiskTestList(0); @@ -320,6 +402,14 @@ int main(int argc, char **argv) { return 1; } } + for (const auto& peer : serverPeers) { + if (peer.NodeId == clientNodeId) { + Cerr << "Error: --client NODE_ID (" << clientNodeId + << ") collides with server NodeId " << peer.NodeId + << "; client NodeId must differ from all server NodeIds" << Endl; + return 1; + } + } auto overrideDDiskInFlight = [](NDevicePerfTest::TDDiskTest& tp, ui32 inFlight) { for (size_t j = 0; j < tp.DDiskTestListSize(); ++j) { diff --git a/ydb/tools/stress_tool/device_test_tool_interconnect_test.h b/ydb/tools/stress_tool/device_test_tool_interconnect_test.h index cdf4356b59e..57920621f43 100644 --- a/ydb/tools/stress_tool/device_test_tool_interconnect_test.h +++ b/ydb/tools/stress_tool/device_test_tool_interconnect_test.h @@ -30,11 +30,13 @@ #include <util/system/event.h> #include "device_test_tool.h" +#include "device_test_tool_ddisk_client_server.h" namespace NKikimr { static TSystemEvent InterconnectDoneEvent(TSystemEvent::rAuto); static TSystemEvent InterconnectResultsPrintedEvent(TSystemEvent::rAuto); +static TSystemEvent InterconnectServerStopEvent(TSystemEvent::rAuto); // Runs a sequence of interconnect load tests (NKikimr::TEvLoadTestRequest::TInterconnectLoad) // inside a single actor system, one after another, printing a short summary for each. @@ -258,7 +260,7 @@ struct TInterconnectTest : public TPerfTest { Printer->PrintResults(); InterconnectResultsPrintedEvent.Signal(); } - } catch (yexception ex) { + } catch (yexception& ex) { LastException = ex; IsLastExceptionSet = true; VERBOSE_COUT(ex.what()); @@ -282,4 +284,265 @@ struct TInterconnectTest : public TPerfTest { } }; +//////////////////////////////////////////////////////////////////////////////// +// Network (multi-host) interconnect test: a real TCP interconnect connection is +// established between two separate ydb_stress_tool processes running on +// different hosts. One side runs as a responder-only "server" (registers just +// the load responder actor and listens for incoming connections); the other +// side runs as a "client" that drives the InterconnectTestList load and +// connects out to the server via NodeHops pointing at the server's NodeId. +//////////////////////////////////////////////////////////////////////////////// + +// Server: registers the interconnect load responder actor for this node and +// listens for an incoming connection from the client. Does not run any load +// itself; it only replies to messages forwarded through TLoadResponderActor. +struct TInterconnectServer : public TPerfTest { + THolder<TActorSystemSetup> Setup; + TIntrusivePtr<NActors::NLog::TSettings> LogSettings; + THolder<TActorSystem> ActorSystem; + TAppData AppData; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + volatile bool IsLastExceptionSet = false; + ui32 ServerNodeId; + ui32 ClientNodeId; + ui16 Port; + bool UseUring; + + TInterconnectServer(const TPerfTestConfig& cfg, ui32 serverNodeId, ui32 clientNodeId, ui16 port, bool useUring = false) + : TPerfTest(cfg) + , Setup(new TActorSystemSetup()) + , LogSettings(new NActors::NLog::TSettings(NActors::TActorId(serverNodeId, "logger"), + NActorsServices::LOGGER, + NActors::NLog::PRI_ERROR, + NActors::NLog::PRI_ERROR, + 0)) + , AppData(0, 1, 3, 2, TMap<TString, ui32>(), nullptr, nullptr, nullptr, nullptr) + , ServerNodeId(serverNodeId) + , ClientNodeId(clientNodeId) + , Port(port) + , UseUring(useUring) + { + } + + void Init() override { + try { + Counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + + Setup->NodeId = ServerNodeId; + Setup->ExecutorsCount = 4; + Setup->Executors.Reset(new TAutoPtr<IExecutorPool>[4]); + Setup->Executors[0].Reset(new TBasicExecutorPool(0, 2, 20, "interconnect")); + Setup->Executors[1].Reset(new TBasicExecutorPool(1, 2, 20, "load_responder")); + Setup->Executors[2].Reset(new TBasicExecutorPool(2, 2, 20, "perf_actors")); + Setup->Executors[3].Reset(new TIOExecutorPool(3, 1, "IO")); + Setup->Scheduler.Reset(new TBasicSchedulerThread(TSchedulerConfig(64, 20))); + + // Register the interconnect load responder actor for this node; it acts as + // the remote endpoint that the client's load actor routes messages to via + // NodeHops. + Setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( + NInterconnect::MakeLoadResponderActorId(ServerNodeId), + TActorSetupCmd(NInterconnect::CreateLoadResponderActor(), TMailboxType::ReadAsFilled, 1))); + + // Server only accepts incoming connections, but the interconnect handshake + // still resolves the peer NodeId via the local nameserver, so register the + // client with a placeholder address (port 0; server never initiates a + // connection to the client). + auto common = MakeInterconnectCommon(Counters, ServerNodeId, UseUring); + TVector<TInterconnectPeer> peers = {{ClientNodeId, "::", 0}}; + SetupInterconnectServices(Setup.Get(), common, ServerNodeId, "::", Port, peers, /*listen=*/true); + + // Logger + LogSettings->Append( + NActorsServices::EServiceCommon_MIN, + NActorsServices::EServiceCommon_MAX, + NActorsServices::EServiceCommon_Name + ); + LogSettings->Append( + NKikimrServices::EServiceKikimr_MIN, + NKikimrServices::EServiceKikimr_MAX, + NKikimrServices::EServiceKikimr_Name + ); + + TString explanation; + const NLog::EPriority icLevel = std::min<NLog::EPriority>(Cfg.LogLevel, NLog::PRI_INFO); + LogSettings->SetLevel(Cfg.LogLevel, NKikimrServices::BS_LOAD_TEST, explanation); + LogSettings->SetLevel(icLevel, NActorsServices::INTERCONNECT, explanation); + + NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor(LogSettings, + NActors::CreateStderrBackend(), + GetServiceCounters(Counters, "utils")); + NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::Simple, 3); + Setup->LocalServices.emplace_back( + NActors::TActorId(ServerNodeId, "logger"), std::move(loggerActorCmd)); + + ActorSystem.Reset(new TActorSystem(Setup, &AppData, LogSettings)); + ActorSystem->Start(); + } catch (yexception& ex) { + IsLastExceptionSet = true; + VERBOSE_COUT("Error on interconnect server init, what# " << ex.what()); + } + } + + void Run() override { + if (IsLastExceptionSet) { + return; + } + + Cerr << "Interconnect load responder (node " << ServerNodeId << ", expecting client node " << ClientNodeId << ")" + << " is listening on port " << Port << ". Press Ctrl+C to stop." << Endl; + + InterconnectServerStopEvent.Wait(); + + if (ActorSystem.Get()) { + ActorSystem->Stop(); + ActorSystem.Destroy(); + } + } + + void Finish() override { + } + + ~TInterconnectServer() override { + } +}; + +// Client: connects out to a set of remote interconnect servers and drives the +// InterconnectTestList load. The load actor's NodeHops must reference the +// server's NodeId (see ServerPeers) so that traffic is routed over the real +// network connection rather than looped back locally. +struct TInterconnectClient : public TPerfTest { + THolder<TActorSystemSetup> Setup; + TIntrusivePtr<NActors::NLog::TSettings> LogSettings; + THolder<TActorSystem> ActorSystem; + TAppData AppData; + TIntrusivePtr<NMonitoring::TDynamicCounters> Counters; + yexception LastException; + volatile bool IsLastExceptionSet = false; + TActorId TestId; + NDevicePerfTest::TInterconnectTest TestProto; + ui32 ClientNodeId; + TVector<TInterconnectPeer> ServerPeers; + bool UseUring; + + TInterconnectClient(const TPerfTestConfig& cfg, const NDevicePerfTest::TInterconnectTest& testProto, + ui32 clientNodeId, const TVector<TInterconnectPeer>& serverPeers, bool useUring = false) + : TPerfTest(cfg) + , Setup(new TActorSystemSetup()) + , LogSettings(new NActors::NLog::TSettings(NActors::TActorId(clientNodeId, "logger"), + NActorsServices::LOGGER, + NActors::NLog::PRI_ERROR, + NActors::NLog::PRI_ERROR, + 0)) + , AppData(0, 1, 3, 2, TMap<TString, ui32>(), nullptr, nullptr, nullptr, nullptr) + , TestProto(testProto) + , ClientNodeId(clientNodeId) + , ServerPeers(serverPeers) + , UseUring(useUring) + { + } + + void Init() override { + try { + Counters = MakeIntrusive<NMonitoring::TDynamicCounters>(); + + Setup->NodeId = ClientNodeId; + Setup->ExecutorsCount = 4; + Setup->Executors.Reset(new TAutoPtr<IExecutorPool>[4]); + Setup->Executors[0].Reset(new TBasicExecutorPool(0, 4, 20, "interconnect")); + Setup->Executors[1].Reset(new TBasicExecutorPool(1, 2, 20, "load_responder")); + Setup->Executors[2].Reset(new TBasicExecutorPool(2, 4, 20, "perf_actors")); + Setup->Executors[3].Reset(new TIOExecutorPool(3, 1, "IO")); + Setup->Scheduler.Reset(new TBasicSchedulerThread(TSchedulerConfig(64, 20))); + + // Set up interconnect with all server peers + auto common = MakeInterconnectCommon(Counters, ClientNodeId, UseUring); + SetupInterconnectServices(Setup.Get(), common, ClientNodeId, "::", 0, ServerPeers, /*listen=*/false); + + // The load actor queries a local load responder for a shared traffic + // counter on Bootstrap (see NInterconnect::TLoadActor::Bootstrap), so a + // load responder must be registered on this node too, even though the + // actual load messages are routed to the remote server via NodeHops. + Setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( + NInterconnect::MakeLoadResponderActorId(ClientNodeId), + TActorSetupCmd(NInterconnect::CreateLoadResponderActor(), TMailboxType::ReadAsFilled, 1))); + + // Logger + LogSettings->Append( + NActorsServices::EServiceCommon_MIN, + NActorsServices::EServiceCommon_MAX, + NActorsServices::EServiceCommon_Name + ); + LogSettings->Append( + NKikimrServices::EServiceKikimr_MIN, + NKikimrServices::EServiceKikimr_MAX, + NKikimrServices::EServiceKikimr_Name + ); + + TString explanation; + // INTERCONNECT is too chatty at DEBUG/TRACE; floor it at INFO regardless of --log-level. + const NLog::EPriority icLevel = std::min<NLog::EPriority>(Cfg.LogLevel, NLog::PRI_INFO); + LogSettings->SetLevel(Cfg.LogLevel, NKikimrServices::BS_LOAD_TEST, explanation); + LogSettings->SetLevel(icLevel, NActorsServices::INTERCONNECT, explanation); + + NActors::TLoggerActor *loggerActor = new NActors::TLoggerActor(LogSettings, + NActors::CreateStderrBackend(), + GetServiceCounters(Counters, "utils")); + NActors::TActorSetupCmd loggerActorCmd(loggerActor, NActors::TMailboxType::Simple, 3); + Setup->LocalServices.emplace_back( + NActors::TActorId(ClientNodeId, "logger"), std::move(loggerActorCmd)); + + TestId = MakeBlobStorageProxyID(1); + TActorSetupCmd testSetup(new TInterconnectPerfTestActor(Cfg, TestProto, Printer, Counters), + TMailboxType::ReadAsFilled, 2); + Setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(TestId, std::move(testSetup))); + + ActorSystem.Reset(new TActorSystem(Setup, &AppData, LogSettings)); + ActorSystem->Start(); + + // Give interconnect some time to establish connections to the server(s) + Sleep(TDuration::Seconds(3)); + } catch (yexception& ex) { + IsLastExceptionSet = true; + VERBOSE_COUT("Error on interconnect client init, what# " << ex.what()); + } + } + + void Run() override { + if (IsLastExceptionSet) { + return; + } + + try { + ActorSystem->Send(TestId, new TEvTablet::TEvBoot(MakeTabletID(0, 0, 1), 0, nullptr, TActorId(), nullptr)); + + for (ui32 i = 0; i < TestProto.InterconnectTestListSize(); ++i) { + InterconnectDoneEvent.Wait(); + Printer->PrintResults(); + InterconnectResultsPrintedEvent.Signal(); + } + } catch (yexception& ex) { + LastException = ex; + IsLastExceptionSet = true; + VERBOSE_COUT(ex.what()); + } + + if (ActorSystem.Get()) { + ActorSystem->Stop(); + ActorSystem.Destroy(); + } + InterconnectDoneEvent.Reset(); + if (IsLastExceptionSet) { + IsLastExceptionSet = false; + ythrow LastException; + } + } + + void Finish() override { + } + + ~TInterconnectClient() override { + } +}; + } // namespace NKikimr diff --git a/yql/essentials/tools/sql2yql/sql2yql.cpp b/yql/essentials/tools/sql2yql/sql2yql.cpp index 85ff6111c49..948cf86ef63 100644 --- a/yql/essentials/tools/sql2yql/sql2yql.cpp +++ b/yql/essentials/tools/sql2yql/sql2yql.cpp @@ -275,7 +275,12 @@ int BuildAST(int argc, char** argv) { opts.AddLongOption("mem-limit", "Set memory limit in megabytes").Handler1T<ui32>(0, NYql::SetAddressSpaceLimit); opts.AddLongOption("gateways-cfg", "Gateways configuration file").Optional().RequiredArgument("FILE").Handler1T<TString>([&gatewaysConfig, &clusterMapping](const TString& file) { gatewaysConfig = ParseProtoConfig<NYql::TGatewaysConfig>(file); - GetClusterMappingFromGateways(*gatewaysConfig, clusterMapping); + + THashMap<TString, TString> local; + GetClusterMappingFromGateways(*gatewaysConfig, local); + for (const auto& [cluster, service] : local) { + clusterMapping.emplace(cluster, service); // priority to argv + } }); opts.AddLongOption("pg-ext", "Pg extensions config file").Optional().RequiredArgument("FILE").Handler1T<TString>([](const TString& file) { auto pgExtConfig = ParseProtoConfig<NYql::NProto::TPgExtensions>(file); diff --git a/yt/yql/providers/yt/provider/yql_yt_io_discovery_walk_folders.cpp b/yt/yql/providers/yt/provider/yql_yt_io_discovery_walk_folders.cpp index 1e798159ed3..9b3845fdede 100644 --- a/yt/yql/providers/yt/provider/yql_yt_io_discovery_walk_folders.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_io_discovery_walk_folders.cpp @@ -211,6 +211,7 @@ IGraphTransformer::TStatus TWalkFoldersImpl::AfterListFolderOp(TExprContext& ctx ProcessingState_ = PreHandling; } else { folderListVal.ReportIssues(ctx.IssueManager); + return IGraphTransformer::TStatus::Error; } BatchFolderListFuture_ = Nothing(); diff --git a/yt/yt/client/formats/parser.h b/yt/yt/client/formats/parser.h index 6993e7e0b9c..61acf3aa517 100644 --- a/yt/yt/client/formats/parser.h +++ b/yt/yt/client/formats/parser.h @@ -13,8 +13,7 @@ namespace NYT::NFormats { struct IParser : public TNonCopyable { - virtual ~IParser() - { } + virtual ~IParser() = default; virtual void Read(TStringBuf data) = 0; virtual void Finish() = 0; diff --git a/yt/yt/core/misc/persistent_queue-inl.h b/yt/yt/core/misc/persistent_queue-inl.h index 829d13d1728..ee9e6ecbb16 100644 --- a/yt/yt/core/misc/persistent_queue-inl.h +++ b/yt/yt/core/misc/persistent_queue-inl.h @@ -118,6 +118,7 @@ void TPersistentQueue<T, ChunkSize>::Enqueue(T value) auto& size = this->Size_; if (!head.CurrentChunk_) { + ++this->ChunkCount_; auto chunk = New<TChunk>(); head.CurrentChunk_ = tail.CurrentChunk_ = chunk; head.CurrentIndex_ = tail.CurrentIndex_ = 0; @@ -127,6 +128,7 @@ void TPersistentQueue<T, ChunkSize>::Enqueue(T value) ++size; if (head.CurrentIndex_ == ChunkSize) { + ++this->ChunkCount_; auto chunk = New<TChunk>(); head.CurrentChunk_->Next = chunk; head.CurrentChunk_ = chunk; @@ -148,6 +150,9 @@ T TPersistentQueue<T, ChunkSize>::Dequeue() if (tail.CurrentIndex_ == ChunkSize) { tail.CurrentChunk_ = tail.CurrentChunk_->Next; tail.CurrentIndex_ = 0; + + YT_VERIFY(this->ChunkCount_ > 0); + --this->ChunkCount_; } return result; @@ -159,6 +164,13 @@ void TPersistentQueue<T, ChunkSize>::Clear() this->Head_ = TPersistentQueueIterator<T, ChunkSize>(); this->Tail_ = TPersistentQueueIterator<T, ChunkSize>(); this->Size_ = 0; + this->ChunkCount_ = 0; +} + +template <class T, size_t ChunkSize> +i64 TPersistentQueue<T, ChunkSize>::GetByteSize() const +{ + return this->ChunkCount_ * sizeof(TChunk); } template <class T, size_t ChunkSize> @@ -211,6 +223,12 @@ void TIndexedPersistentQueue<T, ChunkSize>::Clear() } template <class T, size_t ChunkSize> +i64 TIndexedPersistentQueue<T, ChunkSize>::GetByteSize() const +{ + return TBase::GetByteSize() + Chunks_.size() * sizeof(TChunk*); +} + +template <class T, size_t ChunkSize> void TIndexedPersistentQueue<T, ChunkSize>::Freeze() { auto& tail = TBase::Tail_; diff --git a/yt/yt/core/misc/persistent_queue.h b/yt/yt/core/misc/persistent_queue.h index d5d4458ae4c..f3925461bf8 100644 --- a/yt/yt/core/misc/persistent_queue.h +++ b/yt/yt/core/misc/persistent_queue.h @@ -116,6 +116,9 @@ public: virtual T Dequeue(); virtual void Clear(); + // NB: Chunks kept only by snapshots of the persistent queue do not get accounted here. + virtual i64 GetByteSize() const; + using TSnapshot = TPersistentQueueSnapshot<T, ChunkSize>; TSnapshot MakeSnapshot() const; @@ -125,6 +128,8 @@ public: private: using TChunk = TPersistentQueueChunk<T, ChunkSize>; using TChunkPtr = TIntrusivePtr<TChunk>; + + int ChunkCount_ = 0; }; // Almost zero-cost until used extension over TPersistentQueue. @@ -144,6 +149,8 @@ public: T Dequeue() override; void Clear() override; + i64 GetByteSize() const override; + void Freeze(); const T& operator[](int index) const; |