diff options
389 files changed, 23754 insertions, 4982 deletions
diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 54cd989bd70..e82af2ae762 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -35,12 +35,17 @@ ydb/core/mind/ut TDynamicNameserverTest.OnlyAliveDynamicNodesStaticNodeChangeSub ydb/core/tx/schemeshard/ut_background_cleaning TSchemeshardBackgroundCleaningTest.SchemeshardBackgroundCleaningTestCreateCleanManyTables ydb/core/tx/schemeshard/ut_compaction_reboots SchemeshardForcedCompactionTestReboots.ForceCompactWithIndexCreation[TabletRebootsBucket0] ydb/core/tx/schemeshard/ut_compaction_reboots unittest.[*/*] chunk +ydb/core/tx/schemeshard/ut_export_reboots_s3 TExportToS3WithRebootsTests.ShouldSucceedOnManyTablesWithParquet[TabletRebootsBucket0] ydb/library/actors/core/ut TestThreadContextQueueTimestamps.CurrentQueueTimestamps ydb/library/actors/http/ut Http2Integration.Http2UpgradeFrom101 ydb/library/actors/interconnect/ut InterconnectDirectSession.UnregisterRestoresActorSystemDelivery +ydb/library/actors/interconnect/ut InterconnectSessionV2.ActorSystemRoundTrip ydb/library/actors/interconnect/ut InterconnectSessionV2.ActorSystemTeardownIdleSessions +ydb/library/actors/interconnect/ut InterconnectSessionV2.ClosePeerSocketAndReconnect +ydb/library/actors/interconnect/ut InterconnectSessionV2.DirectSessionRoundTrip ydb/library/actors/interconnect/ut InterconnectSessionV2.FallbackWithoutBufRing ydb/library/actors/interconnect/ut InterconnectSessionV2.LargePayloadRoundTrip +ydb/library/actors/interconnect/ut InterconnectSessionV2.PeerStopTriggersDisconnect ydb/public/sdk/cpp/tests/integration/topic gtest.[*/*] chunk ydb/public/sdk/cpp/tests/integration/topic/with_direct_read gtest.[*/*] chunk ydb/services/persqueue_v1/ut TPersQueueTest.TopicServiceCommitOffset diff --git a/.github/config/muted_ya_asan.txt b/.github/config/muted_ya_asan.txt index f394756449e..b23bb0ac81b 100644 --- a/.github/config/muted_ya_asan.txt +++ b/.github/config/muted_ya_asan.txt @@ -93,6 +93,7 @@ ydb/core/http_proxy/ut/inside_ydb_ut TestKinesisHttpProxy.ListShardsToken ydb/core/http_proxy/ut/inside_ydb_ut TestKinesisHttpProxy.MissingAction ydb/core/http_proxy/ut/inside_ydb_ut TestKinesisHttpProxy.PartitionSplit_AutosplitByLoad_KllSketchBasedSplit_ExplicitHashKey ydb/core/http_proxy/ut/inside_ydb_ut TestKinesisHttpProxy.PutRecordsWithIncorrectHashKey +ydb/core/http_proxy/ut/inside_ydb_ut TestKinesisHttpProxy.PutRecordsWithLongExplicitHashKey ydb/core/http_proxy/ut/inside_ydb_ut TestKinesisHttpProxy.TestConsumersEmptyNames ydb/core/http_proxy/ut/inside_ydb_ut TestKinesisHttpProxy.TestCounters ydb/core/http_proxy/ut/inside_ydb_ut TestKinesisHttpProxy.TestEmptyHttpBody @@ -455,10 +456,12 @@ ydb/core/kqp/executer_actor/ut unittest.sole chunk ydb/core/kqp/proxy_service/ut ScriptExecutionsTest.GetOperationDuringAtomicUploadFinalization ydb/core/kqp/ut/federated_query/datastreams KqpFederatedQueryDatastreams.ReadFromLocalTopicsWithAuth ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.CreateAndAlterStreamingQuery +ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.OffsetsAndStateRecoveryOnInternalRetry ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.OffsetsRecoveryAfterManualAndInternalRetry ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.StreamingQueryWithLocalYdbJoin ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.StreamingQueryWithMultipleWrites ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.StreamingQueryWithPrecompute +ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.StreamingQueryWithTwoGroupByHops ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.StreamingQueryWithTwoGroupByHopsOnSameKey ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.WritingInLocalYdbTablesWithCheckpoints ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesSysView.ReadSysViewWithRowCountBackPressure @@ -545,6 +548,7 @@ ydb/core/load_test/ut_ycsb ReadLoad.ShouldReadIterate ydb/core/mind/hive/ut THiveTestWithTenants.TestDrain ydb/core/mind/hive/ut unittest.[*/*] chunk ydb/core/mind/ut TDynamicNameserverTest.OnlyAliveDynamicNodesStaticNodeChangeSubscribers +ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_ut TPartitionDirectTest.ShouldSuicideOnPoisonByBlockedGeneration ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut TVChunkTest.ShouldSwitchHostToOfflineAndBack ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut TWriteRequestWithPBufferReplicationTest.ShouldWorkWithMultipleResponsesAndHedge ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut unittest.sole chunk @@ -556,6 +560,7 @@ ydb/core/persqueue/public/describer/ut TDescriberTests.TopicNotCanonizedPath ydb/core/persqueue/public/describer/ut TDescriberTests.TopicNotTopic ydb/core/persqueue/public/describer/ut TDescriberTests.TopicWithoutDatabase ydb/core/persqueue/public/describer/ut unittest.sole chunk +ydb/core/persqueue/public/mlp/ut TMLPReaderTests.ReceiveAttemptIdPartitionRouting ydb/core/persqueue/public/mlp/ut TMLPWriterTests.Deduplicated_Reboot ydb/core/persqueue/public/mlp/ut TMLPWriterTests.WriteToAutopartitioningTopic ydb/core/persqueue/public/mlp/ut TMLPWriterTests.WriteTwoMessage_Deduplicated @@ -569,9 +574,12 @@ ydb/core/statistics/service/ut unittest.[*/*] chunk ydb/core/transfer/ut/column_table Transfer_ColumnTable.BigBatchSize_Local ydb/core/transfer/ut/column_table Transfer_ColumnTable.BigBatchSize_Remote ydb/core/transfer/ut/column_table unittest.sole chunk +ydb/core/transfer/ut/functional Transfer.CheckCommittedOffsetEmptyBatch_Local ydb/core/transfer/ut/functional Transfer.CheckCommittedOffsetEmptyBatch_Remote ydb/core/transfer/ut/functional Transfer.LocalTopic_BigMessage +ydb/core/transfer/ut/large TransferLarge.Transfer100KM_10P_ColumnTable ydb/core/transfer/ut/large TransferLarge.Transfer100KM_10P_LocalRead_TopicAutoPartitioning +ydb/core/transfer/ut/large unittest.sole chunk ydb/core/tx/columnshard/backup/async_jobs/ut unittest.sole chunk ydb/core/tx/columnshard/backup/iscan/ut IScan.ShouldRejectParquetExportWithEncryption ydb/core/tx/columnshard/ut_rw TColumnShardTestReadWrite.CompactionSplitGranuleStrKey_PKUtf8 @@ -588,14 +596,19 @@ ydb/core/tx/datashard/build_index/ut TTxDataShardPrefixKMeansScan.BuildToPosting ydb/core/tx/datashard/build_index/ut TTxDataShardPrefixKMeansScan.DimensionMismatchError ydb/core/tx/datashard/build_index/ut TTxDataShardPrefixKMeansScan.EmptyEmbeddingSkipped ydb/core/tx/datashard/build_index/ut TTxDataShardPrefixKMeansScan.NullEmbedding +ydb/core/tx/datashard/build_index/ut TTxDataShardRecomputeKMeansScan.BadRequest ydb/core/tx/datashard/build_index/ut TTxDataShardRecomputeKMeansScan.BuildTable+WithForeign ydb/core/tx/datashard/build_index/ut TTxDataShardRecomputeKMeansScan.DimensionMismatchError ydb/core/tx/datashard/build_index/ut TTxDataShardRecomputeKMeansScan.NullEmbedding +ydb/core/tx/datashard/build_index/ut TTxDataShardReshuffleKMeansScan.MainToPostingResumeFromLastKeyAck ydb/core/tx/datashard/build_index/ut TTxDataShardSampleKScan.DimensionMismatchError ydb/core/tx/datashard/build_index/ut TTxDataShardSampleKScan.NullEmbedding +ydb/core/tx/datashard/build_index/ut TTxDataShardSampleKScan.RunScan +ydb/core/tx/datashard/build_index/ut TTxDataShardSampleKScan.SkipForeign ydb/core/tx/datashard/build_index/ut TTxDataShardValidateUniqueIndexScan.BadRequest ydb/core/tx/datashard/build_index/ut TTxDataShardValidateUniqueIndexScan.RunScan ydb/core/tx/datashard/build_index/ut unittest.sole chunk +ydb/core/tx/datashard/ut_export ExportS3BufferTest.MinBufferSizeWithCompressionAndEncryption-parquet ydb/core/tx/datashard/ut_export unittest.sole chunk ydb/core/tx/datashard/ut_lock_rows DataShardLockRows.RandomizedDeadlockDetection ydb/core/tx/datashard/ut_lock_rows unittest.[*/*] chunk @@ -638,8 +651,11 @@ ydb/library/actors/interconnect/ut DynamicProxy.RaceCheck10 ydb/library/actors/interconnect/ut InterconnectDirectSession.DoesNotReceiveUnexpectedReplies ydb/library/actors/interconnect/ut InterconnectDirectSession.ReceivesExpectedReplies ydb/library/actors/interconnect/ut InterconnectDirectSession.UnregisterRestoresActorSystemDelivery +ydb/library/actors/interconnect/ut InterconnectSessionV2.ActorSystemRoundTrip +ydb/library/actors/interconnect/ut InterconnectSessionV2.ClosePeerSocketAndReconnect ydb/library/actors/interconnect/ut InterconnectSessionV2.DirectSessionRoundTrip ydb/library/actors/interconnect/ut InterconnectSessionV2.FallbackWithoutBufRing +ydb/library/actors/interconnect/ut InterconnectSessionV2.LargePayloadRoundTrip ydb/library/actors/interconnect/ut InterconnectSessionV2.PeerStopTriggersDisconnect ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll ydb/library/actors/interconnect/ut_huge_cluster unittest.sole chunk @@ -663,6 +679,7 @@ ydb/services/persqueue_v1/ut TPersQueueTest.ConsumerAvailabilityPeriodCleanup ydb/services/persqueue_v1/ut/new_schemecache_ut TPersQueueNewSchemeCacheTest.TestWriteStat1stClassTopicAPI ydb/services/scheme_secret/ut/fast DescribeSchemaSecretsService.GetUpdatedValue ydb/services/scheme_secret/ut/fast DescribeSchemaSecretsService.GroupGrants +ydb/services/test_shard/ut unittest.sole chunk ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_NoConnectionStringRelativeTableTopicTransferPath ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_NoConnectionStringRelativeTopicAbsoluteTransferPath ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_NoTokenNoUserPassword @@ -676,6 +693,7 @@ ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_UseUserPasswordWi ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_WithConnectionStringFakeTopicPath ydb/services/ydb/backup_ut BackupRestore.ReplicasAreNotBackedUp+UseSchemeSecret ydb/services/ydb/backup_ut BackupRestore.ReplicasAreNotBackedUp-UseSchemeSecret +ydb/services/ydb/backup_ut BackupRestore.RestoreReplicationThatDoesNotUseSecret ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_stable-26-3-1_to_stable-26-3-1-row] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_prestable-26-4-COLUMN] ydb/tests/compatibility/common test_data_type.py.TestDataType.test_parametrized_data_type[restart_prestable-26-4_to_stable-26-3-1-COLUMN] @@ -957,6 +975,7 @@ ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_doc ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_docs_search_list_with_path_filter ydb/tests/functional/ydb_cli test_ydb_interactive_ai.py.TestToolsOpenAI.test_docs_search_tool_schema_in_request ydb/tests/olap data_read_correctness.py.TestDataReadPerformanceNoIntersections.test +ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_from_compression[zstd_11_compression-algorithm=zstd,level=11] ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_from_compression[zstd_20_compression-algorithm=zstd,level=20] ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_from_compression[zstd_8_compression-algorithm=zstd,level=8] ydb/tests/olap/column_compression/alter alter_compression.py.TestAlterColumnCompression.test_alter_to_compression[zstd_20_compression-algorithm=zstd,level=20] diff --git a/build/conf/ts/node_modules.conf b/build/conf/ts/node_modules.conf index cbfc402c7f2..a70fc2cd9df 100644 --- a/build/conf/ts/node_modules.conf +++ b/build/conf/ts/node_modules.conf @@ -10,6 +10,10 @@ _NODE_MODULES_BUNDLE_ARG= _YATOOL_PREBUILDER_ARG= _USE_LEGACY_PNPM_VIRTUAL_STORE_ARG= _INJECT_PEERS_ARG= +_INJECT_PEERS= +_HERMETIC_NODE_MODULES_ELIGIBLE= +_HERMETIC_NODE_MODULES_DISABLED= +_HERMETIC_NODE_MODULES_ARG= macro _TS_ADD_NODE_MODULES_FOR_BUILDER() { # Calculate inputs and outputs of node_modules, fill `_NODE_MODULES_INOUTS` variable @@ -66,7 +70,7 @@ module _PREPARE_DEPS_BASE: _BARE_UNIT { SET(MODULE_TAG TS_PREPARE_DEPS) SET(MODULE_LANG TS) # what modules it can PEERDIR to - SET(PEERDIR_TAGS TS_PREPARE_DEPS RESOURCE_LIB) + SET(PEERDIR_TAGS TS TS_PROTO TS_PROTO_FROM_SCHEMA RESOURCE_LIB) # do not include it into "results" of graph DISABLE(START_TARGET) @@ -82,5 +86,13 @@ macro USE_LEGACY_PNPM_VIRTUAL_STORE() { } macro INJECT_PEERS() { + SET(_INJECT_PEERS yes) SET(_INJECT_PEERS_ARG --inject-peers yes) } + +### @usage: DISABLE_HERMETIC_NODE_MODULES() +### +### Restores the legacy node_modules installation flow for this target. +macro DISABLE_HERMETIC_NODE_MODULES() { + SET(_HERMETIC_NODE_MODULES_DISABLED yes) +} diff --git a/build/conf/ts/ts.conf b/build/conf/ts/ts.conf index d04aeeccbe8..a9224d054a9 100644 --- a/build/conf/ts/ts.conf +++ b/build/conf/ts/ts.conf @@ -1,4 +1,5 @@ TS_USE_PREBUILT_NOTS_TOOL=yes +TS_HERMETIC_NODE_MODULES=yes TS_FAKEID=${FAKEID}.2025-06-05 NOTS_TOOL=${tool:"devtools/frontend_build_platform/nots/builder"} ${hide:TS_FAKEID} @@ -53,6 +54,7 @@ NOTS_TOOL_BASE_ARGS=\ --pm-type $PM_TYPE \ $_LD_LIBRARY_PATH_ARGS \ $_INJECT_PEERS_ARG \ + $_HERMETIC_NODE_MODULES_ARG \ $_YATOOL_PREBUILDER_ARG $_NODE_MODULES_BUNDLE_ARG $_USE_LEGACY_PNPM_VIRTUAL_STORE_ARG NOTS_TOOL_BUILD_ENV= @@ -196,6 +198,7 @@ TS_GLOB_EXCLUDE=$TS_CONFIG_PATH \ a.yaml \ output.tar \ output.tar.uuid \ + node_modules.layer \ workspace_node_modules.tar \ $TS_EXCLUDE_DIR_GLOB \ $TS_COMMON_OUTDIR_GLOB \ diff --git a/build/conf/ts/ts_check.conf b/build/conf/ts/ts_check.conf index 430c9b57572..2b42377fd43 100644 --- a/build/conf/ts/ts_check.conf +++ b/build/conf/ts/ts_check.conf @@ -29,6 +29,7 @@ multimodule TS_TEST_FOR { SET(MODULE_TAG TS_TEST_FOR) SET(MODULE_LANG TS) + ENABLE(_HERMETIC_NODE_MODULES_ELIGIBLE) # for multimodule peers we should choose TS SET(PEERDIR_TAGS TS TS_PROTO TS_PROTO_FROM_SCHEMA) @@ -61,6 +62,7 @@ multimodule TS_TEST_FOR { SET(MODULE_SUFFIX .test_for.prepare_deps.fake) # lang makrup for statistics SET(MODULE_LANG TS) + ENABLE(_HERMETIC_NODE_MODULES_ELIGIBLE) # what modules it can PEERDIR to SET(PEERDIR_TAGS TS_PREPARE_DEPS) # do not include it into "results" of graph diff --git a/build/conf/ts/ts_library.conf b/build/conf/ts/ts_library.conf index 08d72714de2..4ccaaf0f4ef 100644 --- a/build/conf/ts/ts_library.conf +++ b/build/conf/ts/ts_library.conf @@ -47,6 +47,7 @@ multimodule TS_LIBRARY { # by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case) # but we have to set it to TS for include processor to work SET(MODULE_TAG TS) + ENABLE(_HERMETIC_NODE_MODULES_ELIGIBLE) SET(_TS_MOD_NAME TS_LIBRARY) @@ -58,6 +59,7 @@ multimodule TS_LIBRARY { } module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE { + ENABLE(_HERMETIC_NODE_MODULES_ELIGIBLE) } } @@ -86,6 +88,7 @@ macro _TS_LIBRARY_EPILOGUE() { _SET_TS_INPUTS_BASE_EXCLUDES=a.yaml .npmrc package.json pnpm-lock.yaml \ output.tar \ output.tar.uuid \ + node_modules.layer \ workspace_node_modules.tar \ ${TS_EXCLUDE_DIR_GLOB} \ ${TS_GLOB_EXCLUDE_ADDITIONAL} diff --git a/build/conf/ts/ts_package.conf b/build/conf/ts/ts_package.conf index cb242bc9dc5..dbd90dfdf21 100644 --- a/build/conf/ts/ts_package.conf +++ b/build/conf/ts/ts_package.conf @@ -21,6 +21,7 @@ multimodule TS_PACKAGE { # by default multimodule overrides inherited MODULE_TAG to submodule name (BUILD in this case) # but we have to set it to TS for include processor to work SET(MODULE_TAG TS) + ENABLE(_HERMETIC_NODE_MODULES_ELIGIBLE) # flag TS_PACKAGE for on_ts_library_configure ENABLE(_TS_PACKAGE) @@ -36,6 +37,7 @@ multimodule TS_PACKAGE { } module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE { + ENABLE(_HERMETIC_NODE_MODULES_ELIGIBLE) } } @@ -56,4 +58,4 @@ macro _FORBID_PKG_TS_FILES(_p...) { macro _FORBID_PKG_TS_FILES_GLOB(_p...) { MESSAGE(TS_FILES_GLOB does nothing in TS_PACKAGE. You can remove TS_FILES_GLOB macro from ya.make) -}
\ No newline at end of file +} diff --git a/build/conf/ts/ts_test.conf b/build/conf/ts/ts_test.conf index dffc513d5f1..bed30acc6ed 100644 --- a/build/conf/ts/ts_test.conf +++ b/build/conf/ts/ts_test.conf @@ -213,6 +213,11 @@ macro _SETUP_EXTRACT_NODE_MODULES_RECIPE(FOR_PATH) { DEPENDS(devtools/frontend_build_platform/nots/recipes/extract_node_modules) } +macro _SETUP_EXTRACT_NODE_MODULES_LAYER_RECIPE(FOR_PATH) { + USE_RECIPE(devtools/frontend_build_platform/nots/recipes/extract_node_modules/recipe $FOR_PATH node_modules.layer) + DEPENDS(devtools/frontend_build_platform/nots/recipes/extract_node_modules) +} + macro _SETUP_EXTRACT_OUTPUT_TARS_RECIPE(FOR_PATH) { USE_RECIPE(devtools/frontend_build_platform/nots/recipes/extract_output_tars/extract_output_tars $FOR_PATH) DEPENDS(devtools/frontend_build_platform/nots/recipes/extract_output_tars) diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 685cb5b6154..063dd4202c9 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -2119,6 +2119,7 @@ "12811590910": "{registry_endpoint}/12811590910", "12826608531": "{registry_endpoint}/12826608531", "12831547000": "{registry_endpoint}/12831547000", + "12863699273": "{registry_endpoint}/12863699273", "6662972834": "{registry_endpoint}/6662972834", "11823170312": "{registry_endpoint}/11823170312", "11879814525": "{registry_endpoint}/11879814525", @@ -2142,6 +2143,7 @@ "12811589861": "{registry_endpoint}/12811589861", "12826606922": "{registry_endpoint}/12826606922", "12831544610": "{registry_endpoint}/12831544610", + "12863697692": "{registry_endpoint}/12863697692", "6662965433": "{registry_endpoint}/6662965433", "11823173798": "{registry_endpoint}/11823173798", "11879819941": "{registry_endpoint}/11879819941", @@ -2165,6 +2167,7 @@ "12811592057": "{registry_endpoint}/12811592057", "12826610495": "{registry_endpoint}/12826610495", "12831549374": "{registry_endpoint}/12831549374", + "12863701354": "{registry_endpoint}/12863701354", "6662980150": "{registry_endpoint}/6662980150", "5476908047": "{registry_endpoint}/5476908047", "5509380757": "{registry_endpoint}/5509380757", @@ -4553,6 +4556,7 @@ "12811590910": "yfm-docs for darwin", "12826608531": "yfm-docs for darwin", "12831547000": "yfm-docs for darwin", + "12863699273": "yfm-docs for darwin", "6662972834": "yfm-docs for darwin", "11823170312": "yfm-docs for linux", "11879814525": "yfm-docs for linux", @@ -4576,6 +4580,7 @@ "12811589861": "yfm-docs for linux", "12826606922": "yfm-docs for linux", "12831544610": "yfm-docs for linux", + "12863697692": "yfm-docs for linux", "6662965433": "yfm-docs for linux", "11823173798": "yfm-docs for win32", "11879819941": "yfm-docs for win32", @@ -4599,6 +4604,7 @@ "12811592057": "yfm-docs for win32", "12826610495": "yfm-docs for win32", "12831549374": "yfm-docs for win32", + "12863701354": "yfm-docs for win32", "6662980150": "yfm-docs for win32", "5476908047": "ymake for darwin", "5509380757": "ymake for darwin", diff --git a/build/platform/yfm/resources.json b/build/platform/yfm/resources.json index 3e3d2a2cd23..7d709040ef8 100644 --- a/build/platform/yfm/resources.json +++ b/build/platform/yfm/resources.json @@ -1,16 +1,16 @@ { "by_platform": { "win32-x86_64": { - "uri": "sbr:12831549374" + "uri": "sbr:12863701354" }, "darwin-x86_64": { - "uri": "sbr:12831547000" + "uri": "sbr:12863699273" }, "linux-x86_64": { - "uri": "sbr:12831544610" + "uri": "sbr:12863697692" }, "darwin-arm64": { - "uri": "sbr:12831547000" + "uri": "sbr:12863699273" } } } diff --git a/build/plugins/lib/nots/package_manager/constants.py b/build/plugins/lib/nots/package_manager/constants.py index af031face1f..4138db2a53d 100644 --- a/build/plugins/lib/nots/package_manager/constants.py +++ b/build/plugins/lib/nots/package_manager/constants.py @@ -3,6 +3,7 @@ BUILD_DIRNAME = "build" BUNDLE_DIRNAME = "bundle" NODE_MODULES_DIRNAME = "node_modules" NODE_MODULES_WORKSPACE_BUNDLE_FILENAME = "workspace_node_modules.tar" +NODE_MODULES_LAYER_FILENAME = "node_modules.layer" NPM_REGISTRY_URL = "http://npm.yandex-team.ru" OUTPUT_TAR_FILENAME = "output.tar" OUTPUT_TAR_UUID_FILENAME = "output.tar.uuid" diff --git a/build/plugins/lib/nots/package_manager/package_manager.py b/build/plugins/lib/nots/package_manager/package_manager.py index 36ea6c099ce..3297ab922f0 100644 --- a/build/plugins/lib/nots/package_manager/package_manager.py +++ b/build/plugins/lib/nots/package_manager/package_manager.py @@ -8,7 +8,6 @@ from .constants import ( LOCAL_PNPM_INSTALL_MUTEX_FILENAME, NODE_MODULES_DIRNAME, VIRTUAL_STORE_DIRNAME, - NODE_MODULES_WORKSPACE_BUNDLE_FILENAME, NPM_REGISTRY_URL, ) from .lockfile import Lockfile @@ -17,14 +16,12 @@ from .utils import ( build_pre_lockfile_path, build_ws_config_path, b_rooted, - build_nm_bundle_path, build_nm_path, build_pj_path, build_pnpm_store_path, s_rooted, ) from .pnpm_workspace import PnpmWorkspace -from .node_modules_bundler import bundle_node_modules from .timeit import timeit from .package_json import PackageJson @@ -319,7 +316,6 @@ class PackageManager(object): yatool_prebuilder_path=None, use_legacy_pnpm_virtual_store=False, local_cli=False, - nm_bundle=False, original_lf_path=None, ): """ @@ -342,15 +338,7 @@ class PackageManager(object): self._run_apply_addons_if_need(yatool_prebuilder_path, virtual_store_dir or global_virtual_store_dir) - if nm_bundle: - # TODO: how to bundle node_modules with GVS? - bundle_node_modules( - build_root=self.build_root, - node_modules_path=self._nm_path(), - peers=ws.get_paths(base_path=self.module_path, ignore_self=True), - bundle_path=os.path.join(self.build_path, NODE_MODULES_WORKSPACE_BUNDLE_FILENAME), - inject_peers=self.inject_peers, - ) + return ws """ Runs pnpm install command with specified parameters in an exclusive and hashed manner. @@ -428,7 +416,7 @@ class PackageManager(object): @timeit def calc_prepare_deps_inouts_and_resources( - self, store_path: str, has_deps: bool, local_cli: bool + self, store_path: str, has_deps: bool, local_cli: bool, include_peer_outputs: bool = False ) -> tuple[list[str], list[str], list[str]]: ins = [ s_rooted(build_pj_path(self.module_path)), @@ -439,6 +427,15 @@ class PackageManager(object): ] resources = [] + # Hermetic node_modules preparation merges generated workspace metadata + # from peers. Keep these inputs out of the legacy graph. + # TS_PROTO_AUTO has no source package.json to inspect here. + if include_peer_outputs and os.path.exists(build_pj_path(self.sources_path)): + for dep_path in self.get_local_peers_from_package_json(): + ins.append(b_rooted(build_ws_config_path(dep_path))) + if not self.inject_peers: + ins.append(b_rooted(build_pre_lockfile_path(dep_path))) + if has_deps and not local_cli: for pkg in self.extract_packages_meta_from_lockfiles([build_lockfile_path(self.sources_path)]): resources.append(pkg.to_uri()) @@ -460,6 +457,8 @@ class PackageManager(object): outs = [] if nm_bundle: + from .utils import build_nm_bundle_path + outs.append(b_rooted(build_nm_bundle_path(self.module_path))) return ins, outs diff --git a/build/plugins/nots.py b/build/plugins/nots.py index f9085969aba..b433b8d3f8c 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -179,6 +179,11 @@ class NotsUnitType(UnitType): Setup test recipe to extract workspace-node_modules.tar before running tests """ + def on_setup_extract_node_modules_layer_recipe(self, args: UnitType.PluginArgs) -> None: + """ + Setup test recipe to extract the internal node_modules layer before running tests + """ + def on_setup_extract_output_tars_recipe(self, args: UnitType.PluginArgs) -> None: """ Setup test recipe to extract peer's output before running tests @@ -450,6 +455,30 @@ def _create_pm(unit: NotsUnitType) -> 'PackageManager': ) +def _use_hermetic_node_modules(unit: NotsUnitType) -> bool: + return ( + unit.get("TS_HERMETIC_NODE_MODULES") != "no" + and unit.get("_HERMETIC_NODE_MODULES_ELIGIBLE") == "yes" + and unit.get("_INJECT_PEERS") == "yes" + and unit.get("_HERMETIC_NODE_MODULES_DISABLED") != "yes" + ) + + +def _enable_hermetic_node_modules(unit: NotsUnitType) -> None: + if _use_hermetic_node_modules(unit): + unit.set(["_HERMETIC_NODE_MODULES_ARG", "--hermetic-node-modules yes"]) + + +def _setup_prebuilder_resource(unit: NotsUnitType) -> None: + unit.on_peerdir_ts_resource("@yatool/prebuilder") + unit.set( + [ + "_YATOOL_PREBUILDER_ARG", + "--yatool-prebuilder-path $YATOOL_PREBUILDER_ROOT/node_modules/@yatool/prebuilder", + ] + ) + + @ymake.macro @_with_report_configure_error def _SET_APPEND_WITH_DIRECTIVE(unit: NotsUnitType, var_name: str, directive: str, *values: tuple[str, ...]) -> None: @@ -1009,18 +1038,43 @@ def _PREPARE_DEPS_CONFIGURE(unit: NotsUnitType) -> None: pj = pm.load_package_json_from_dir(pm.sources_path) has_deps = pj.has_dependencies() local_cli = unit.get("TS_LOCAL_CLI") == "yes" - ins, outs, resources = pm.calc_prepare_deps_inouts_and_resources(unit.get("_TARBALLS_STORE"), has_deps, local_cli) + use_hermetic_node_modules = _use_hermetic_node_modules(unit) + ins, outs, resources = pm.calc_prepare_deps_inouts_and_resources( + unit.get("_TARBALLS_STORE"), has_deps, local_cli, include_peer_outputs=use_hermetic_node_modules + ) + if use_hermetic_node_modules: + from lib.nots.package_manager import constants + from lib.nots.package_manager.utils import b_rooted, s_rooted + + _enable_hermetic_node_modules(unit) + if pj.get_use_prebuilder(): + _setup_prebuilder_resource(unit) + __set_append(unit, "_PREPARE_DEPS_INOUTS", "${hide:PEERS}") + ins.extend( + s_rooted(os.path.normpath(os.path.join(pm.module_path, patch_path))) + for patch_path in pj.get_pnpm_patched_dependencies().values() + ) + outs.append(b_rooted(os.path.join(pm.module_path, constants.NODE_MODULES_LAYER_FILENAME))) if has_deps: - unit.onpeerdir(pm.get_local_peers_from_package_json()) + local_peers = pm.get_local_peers_from_package_json() + unit.onpeerdir(local_peers) + if use_hermetic_node_modules: + # The cached injected snapshot must contain built workspace peers, + # not their source/pre-build state. + unit.ondepends(local_peers) __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives(["hide", "input"], sorted(ins))) __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives(["hide", "output"], sorted(outs))) unit.set(["_PREPARE_DEPS_RESOURCES", " ".join([f'${{resource:"{uri}"}}' for uri in sorted(resources)])]) unit.set(["_PREPARE_DEPS_USE_RESOURCES_FLAG", "--resource-root $(RESOURCE_ROOT)"]) else: - __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives(["output"], sorted(outs))) - unit.set(["_PREPARE_DEPS_CMD", "$_PREPARE_NO_DEPS_CMD"]) + if use_hermetic_node_modules: + __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives(["hide", "input"], sorted(ins))) + __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives(["hide", "output"], sorted(outs))) + else: + __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives(["output"], sorted(outs))) + unit.set(["_PREPARE_DEPS_CMD", "$_PREPARE_NO_DEPS_CMD"]) @ymake.macro @@ -1139,7 +1193,10 @@ def _TS_CHECK_CONFIGURE(unit: NotsUnitType, validation_mode: str) -> None: return pm = _create_pm(unit) - unit.on_setup_install_node_modules_recipe(pm.module_path) + if _use_hermetic_node_modules(unit): + unit.on_setup_extract_node_modules_layer_recipe(pm.module_path) + else: + unit.on_setup_install_node_modules_recipe(pm.module_path) unit.on_setup_extract_output_tars_recipe(pm.module_path) peers = pm.get_local_peers_from_package_json() @@ -1187,6 +1244,8 @@ def _NODE_MODULES_CONFIGURE(unit: NotsUnitType) -> None: pm = _create_pm(unit) pj = pm.load_package_json_from_dir(pm.sources_path) has_deps = pj.has_dependencies() + if _use_hermetic_node_modules(unit): + _enable_hermetic_node_modules(unit) if has_deps: unit.onpeerdir(pm.get_local_peers_from_package_json()) @@ -1196,6 +1255,17 @@ def _NODE_MODULES_CONFIGURE(unit: NotsUnitType) -> None: ins, outs = pm.calc_node_modules_inouts(nm_bundle_needed) + if not _use_hermetic_node_modules(unit): + from lib.nots.package_manager.utils import s_rooted + + # Legacy builders materialize node_modules in the build action and + # copy pnpm patches from the source tree there. Declare those files + # explicitly so they are available in a distbuild sandbox. + ins.extend( + s_rooted(os.path.normpath(os.path.join(pm.module_path, patch_path))) + for patch_path in pj.get_pnpm_patched_dependencies().values() + ) + __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives(["hide", "input"], sorted(ins))) if not unit.get("TS_TEST_FOR"): __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives(["hide", "output"], sorted(outs))) @@ -1206,13 +1276,7 @@ def _NODE_MODULES_CONFIGURE(unit: NotsUnitType) -> None: lf.validate_importers() if pj.get_use_prebuilder(): - unit.on_peerdir_ts_resource("@yatool/prebuilder") - unit.set( - [ - "_YATOOL_PREBUILDER_ARG", - "--yatool-prebuilder-path $YATOOL_PREBUILDER_ROOT/node_modules/@yatool/prebuilder", - ] - ) + _setup_prebuilder_resource(unit) # YATOOL_PREBUILDER_0_7_0_RESOURCE_GLOBAL prebuilder_major = unit.get("YATOOL_PREBUILDER-ROOT-VAR-NAME").split("_")[2] diff --git a/contrib/python/clickhouse-connect/.dist-info/METADATA b/contrib/python/clickhouse-connect/.dist-info/METADATA index 656e95416df..73c6ef16c80 100644 --- a/contrib/python/clickhouse-connect/.dist-info/METADATA +++ b/contrib/python/clickhouse-connect/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: clickhouse-connect -Version: 1.4.1 +Version: 1.4.2 Summary: ClickHouse Database Core Driver for Python, Pandas, and Superset Home-page: https://github.com/ClickHouse/clickhouse-connect Author: ClickHouse Inc. diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/_version.py b/contrib/python/clickhouse-connect/clickhouse_connect/_version.py index 6abaa204304..841aad2c609 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/_version.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/_version.py @@ -1 +1 @@ -version = "1.4.1" +version = "1.4.2" diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py index d7bc03884d4..8c7caeb3539 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py @@ -2044,8 +2044,10 @@ class AsyncClient(Client): else: form.add_field(field_name, field_value, content_type="text/plain") request_kwargs["data"] = form - elif isinstance(data, dict): - request_kwargs["data"] = data + elif isinstance(data, (bytes, bytearray, memoryview)): + request_kwargs["data"] = io.BytesIO(data) + elif isinstance(data, str): + request_kwargs["data"] = io.BytesIO(data.encode()) else: request_kwargs["data"] = data diff --git a/contrib/python/clickhouse-connect/ya.make b/contrib/python/clickhouse-connect/ya.make index 800f19f6818..cdbe686ae0b 100644 --- a/contrib/python/clickhouse-connect/ya.make +++ b/contrib/python/clickhouse-connect/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.4.1) +VERSION(1.4.2) LICENSE(Apache-2.0) diff --git a/contrib/python/typer/.dist-info/METADATA b/contrib/python/typer/.dist-info/METADATA index 0340df72d94..c041852efde 100644 --- a/contrib/python/typer/.dist-info/METADATA +++ b/contrib/python/typer/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: typer -Version: 0.25.1 +Version: 0.26.8 Summary: Typer, build great CLIs. Easy to code. Based on Python type hints. Author-Email: =?utf-8?q?Sebasti=C3=A1n_Ram=C3=ADrez?= <[email protected]> License-Expression: MIT @@ -29,10 +29,10 @@ Project-URL: Repository, https://github.com/fastapi/typer Project-URL: Issues, https://github.com/fastapi/typer/issues Project-URL: Changelog, https://typer.tiangolo.com/release-notes/ Requires-Python: >=3.10 -Requires-Dist: click>=8.2.1 Requires-Dist: shellingham>=1.3.0 Requires-Dist: rich>=13.8.0 Requires-Dist: annotated-doc>=0.0.2 +Requires-Dist: colorama; platform_system == "Windows" Description-Content-Type: text/markdown <p align="center"> @@ -43,24 +43,21 @@ Description-Content-Type: text/markdown <em>Typer, build great CLIs. Easy to code. Based on Python type hints.</em> </p> <p align="center"> -<a href="https://github.com/fastapi/typer/actions?query=workflow%3ATest+event%3Apush+branch%3Amaster" target="_blank"> +<a href="https://github.com/fastapi/typer/actions?query=workflow%3ATest+event%3Apush+branch%3Amaster"> <img src="https://github.com/fastapi/typer/actions/workflows/test.yml/badge.svg?event=push&branch=master" alt="Test"> </a> -<a href="https://github.com/fastapi/typer/actions?query=workflow%3APublish" target="_blank"> - <img src="https://github.com/fastapi/typer/workflows/Publish/badge.svg" alt="Publish"> -</a> -<a href="https://coverage-badge.samuelcolvin.workers.dev/redirect/fastapi/typer" target="_blank"> +<a href="https://coverage-badge.samuelcolvin.workers.dev/redirect/fastapi/typer"> <img src="https://coverage-badge.samuelcolvin.workers.dev/fastapi/typer.svg" alt="Coverage"> -<a href="https://pypi.org/project/typer" target="_blank"> +<a href="https://pypi.org/project/typer"> <img src="https://img.shields.io/pypi/v/typer?color=%2334D058&label=pypi%20package" alt="Package version"> </a> </p> --- -**Documentation**: <a href="https://typer.tiangolo.com" target="_blank">https://typer.tiangolo.com</a> +**Documentation**: [https://typer.tiangolo.com](https://typer.tiangolo.com) -**Source Code**: <a href="https://github.com/fastapi/typer" target="_blank">https://github.com/fastapi/typer</a> +**Source Code**: [https://github.com/fastapi/typer](https://github.com/fastapi/typer) --- @@ -79,11 +76,11 @@ The key features are: ## FastAPI of CLIs -**Typer** is <a href="https://fastapi.tiangolo.com" class="external-link" target="_blank">FastAPI</a>'s little sibling, it's the FastAPI of CLIs. +**Typer** is [FastAPI](https://fastapi.tiangolo.com)'s little sibling, it's the FastAPI of CLIs. ## Installation -Create and activate a <a href="https://typer.tiangolo.com/virtual-environments/" class="external-link" target="_blank">virtual environment</a> and then install **Typer**: +Create and activate a [virtual environment](https://typer.tiangolo.com/virtual-environments/) and then install **Typer**: <div class="termy"> @@ -389,11 +386,20 @@ For a more complete example including more features, see the <a href="https://ty ## Dependencies -**Typer** stands on the shoulders of giants. It has three required dependencies: +**Typer** requires only a few dependencies (most are tiny): + +* [`rich`](https://rich.readthedocs.io/en/stable/index.html): to show nicely formatted errors automatically. +* [`shellingham`](https://github.com/sarugaku/shellingham): to automatically detect the current shell when installing completion. +* [`annotated-doc`](https://github.com/fastapi/annotated-doc): to generate documentation from Python type annotations. +* [`colorama`](https://github.com/tartley/colorama) (only on Windows): for producing colored terminal text on Windows. + +### Click code + +Typer used to depend on [Click](https://click.palletsprojects.com/) as well, a popular tool for building CLIs in Python. + +Since version 0.26.0, Typer has vendored Click (included Click's source code internally, instead of installing it as a third party package) and has unified the code interactions between Typer and the embedded Click source code for easier maintainability in the future. -* <a href="https://click.palletsprojects.com/" class="external-link" target="_blank">Click</a>: a popular tool for building CLIs in Python. Typer is based on it. -* <a href="https://rich.readthedocs.io/en/stable/index.html" class="external-link" target="_blank"><code>rich</code></a>: to show nicely formatted errors automatically. -* <a href="https://github.com/sarugaku/shellingham" class="external-link" target="_blank"><code>shellingham</code></a>: to automatically detect the current shell when installing completion. +Note that some Click functionality will not be available anymore in the future, as we continue to improve and extend Typer's codebase. ### `typer-slim` diff --git a/contrib/python/typer/README.md b/contrib/python/typer/README.md index 87b5f031188..8f7c49f7ebb 100644 --- a/contrib/python/typer/README.md +++ b/contrib/python/typer/README.md @@ -6,24 +6,21 @@ <em>Typer, build great CLIs. Easy to code. Based on Python type hints.</em> </p> <p align="center"> -<a href="https://github.com/fastapi/typer/actions?query=workflow%3ATest+event%3Apush+branch%3Amaster" target="_blank"> +<a href="https://github.com/fastapi/typer/actions?query=workflow%3ATest+event%3Apush+branch%3Amaster"> <img src="https://github.com/fastapi/typer/actions/workflows/test.yml/badge.svg?event=push&branch=master" alt="Test"> </a> -<a href="https://github.com/fastapi/typer/actions?query=workflow%3APublish" target="_blank"> - <img src="https://github.com/fastapi/typer/workflows/Publish/badge.svg" alt="Publish"> -</a> -<a href="https://coverage-badge.samuelcolvin.workers.dev/redirect/fastapi/typer" target="_blank"> +<a href="https://coverage-badge.samuelcolvin.workers.dev/redirect/fastapi/typer"> <img src="https://coverage-badge.samuelcolvin.workers.dev/fastapi/typer.svg" alt="Coverage"> -<a href="https://pypi.org/project/typer" target="_blank"> +<a href="https://pypi.org/project/typer"> <img src="https://img.shields.io/pypi/v/typer?color=%2334D058&label=pypi%20package" alt="Package version"> </a> </p> --- -**Documentation**: <a href="https://typer.tiangolo.com" target="_blank">https://typer.tiangolo.com</a> +**Documentation**: [https://typer.tiangolo.com](https://typer.tiangolo.com) -**Source Code**: <a href="https://github.com/fastapi/typer" target="_blank">https://github.com/fastapi/typer</a> +**Source Code**: [https://github.com/fastapi/typer](https://github.com/fastapi/typer) --- @@ -42,11 +39,11 @@ The key features are: ## FastAPI of CLIs -**Typer** is <a href="https://fastapi.tiangolo.com" class="external-link" target="_blank">FastAPI</a>'s little sibling, it's the FastAPI of CLIs. +**Typer** is [FastAPI](https://fastapi.tiangolo.com)'s little sibling, it's the FastAPI of CLIs. ## Installation -Create and activate a <a href="https://typer.tiangolo.com/virtual-environments/" class="external-link" target="_blank">virtual environment</a> and then install **Typer**: +Create and activate a [virtual environment](https://typer.tiangolo.com/virtual-environments/) and then install **Typer**: <div class="termy"> @@ -352,11 +349,20 @@ For a more complete example including more features, see the <a href="https://ty ## Dependencies -**Typer** stands on the shoulders of giants. It has three required dependencies: +**Typer** requires only a few dependencies (most are tiny): + +* [`rich`](https://rich.readthedocs.io/en/stable/index.html): to show nicely formatted errors automatically. +* [`shellingham`](https://github.com/sarugaku/shellingham): to automatically detect the current shell when installing completion. +* [`annotated-doc`](https://github.com/fastapi/annotated-doc): to generate documentation from Python type annotations. +* [`colorama`](https://github.com/tartley/colorama) (only on Windows): for producing colored terminal text on Windows. + +### Click code + +Typer used to depend on [Click](https://click.palletsprojects.com/) as well, a popular tool for building CLIs in Python. + +Since version 0.26.0, Typer has vendored Click (included Click's source code internally, instead of installing it as a third party package) and has unified the code interactions between Typer and the embedded Click source code for easier maintainability in the future. -* <a href="https://click.palletsprojects.com/" class="external-link" target="_blank">Click</a>: a popular tool for building CLIs in Python. Typer is based on it. -* <a href="https://rich.readthedocs.io/en/stable/index.html" class="external-link" target="_blank"><code>rich</code></a>: to show nicely formatted errors automatically. -* <a href="https://github.com/sarugaku/shellingham" class="external-link" target="_blank"><code>shellingham</code></a>: to automatically detect the current shell when installing completion. +Note that some Click functionality will not be available anymore in the future, as we continue to improve and extend Typer's codebase. ### `typer-slim` diff --git a/contrib/python/typer/typer/.agents/skills/typer/SKILL.md b/contrib/python/typer/typer/.agents/skills/typer/SKILL.md index 19b63c491f8..60cf47a3919 100644 --- a/contrib/python/typer/typer/.agents/skills/typer/SKILL.md +++ b/contrib/python/typer/typer/.agents/skills/typer/SKILL.md @@ -112,7 +112,7 @@ def main(name: str = typer.Argument(default="World")): print(f"Hello {name}") ``` -Similarly, the old style could use ellipsis (...) to explicitely mark an argument as required. +Similarly, the old style could use ellipsis (...) to explicitly mark an argument as required. ```python # DO NOT DO THIS: old style. Use Annotated without a default value instead. @@ -125,7 +125,29 @@ def main(name: str = typer.Argument(default=...)): ## CLI Options -CLI options are declared in a similar fashion as arguments, but will be called on the CLI with a single dash (single letter) or 2 dashes (full name): +CLI options are declared in a similar fashion as arguments, but will be called on the CLI with a single dash (single letter) or 2 dashes (full name). + +The CLI option name is automatically generated from the variable name, so `user_name` becomes `--user-name` automatically: + +```python +from typing import Annotated + +import typer + +app = typer.Typer() + + +def main(user_name: Annotated[str, typer.Option()]): + # On the CLI, the required user name can be specified with --user-name + print(f"Hello {user_name}") + + +if __name__ == "__main__": + app() +``` + +If you want to specify a different name, or want to add a short version, declare them in the `typer.Option`: ```python from typing import Annotated @@ -259,7 +281,7 @@ if __name__ == "__main__": ## Click -Originally, Typer was built on Click. However, going forward Typer will vendor Click. As such, Click extensions should not be used anymore. +Originally, Typer was built on Click. However, since version 0.26.0, Typer has vendored Click. As such, Click extensions should not be used anymore. Other settings of `Option` and `Argument` that came from Click but shouldn't be used in Typer anymore, include: `expose_value`, `shell_complete`, `show_choices`, `errors`, `prompt_required`, `is_flag`, `flag_value` and `allow_from_autoenv`. diff --git a/contrib/python/typer/typer/__init__.py b/contrib/python/typer/typer/__init__.py index 548408fe98d..cbc8f36a1d9 100644 --- a/contrib/python/typer/typer/__init__.py +++ b/contrib/python/typer/typer/__init__.py @@ -1,31 +1,24 @@ """Typer, build great CLIs. Easy to code. Based on Python type hints.""" -__version__ = "0.25.1" +__version__ = "0.26.8" from shutil import get_terminal_size as get_terminal_size -from click.exceptions import Abort as Abort -from click.exceptions import BadParameter as BadParameter -from click.exceptions import Exit as Exit -from click.termui import clear as clear -from click.termui import confirm as confirm -from click.termui import echo_via_pager as echo_via_pager -from click.termui import edit as edit -from click.termui import getchar as getchar -from click.termui import pause as pause -from click.termui import progressbar as progressbar -from click.termui import prompt as prompt -from click.termui import secho as secho -from click.termui import style as style -from click.termui import unstyle as unstyle -from click.utils import echo as echo -from click.utils import format_filename as format_filename -from click.utils import get_app_dir as get_app_dir -from click.utils import get_binary_stream as get_binary_stream -from click.utils import get_text_stream as get_text_stream -from click.utils import open_file as open_file - from . import colors as colors +from ._click.exceptions import Abort as Abort +from ._click.exceptions import BadParameter as BadParameter +from ._click.exceptions import Exit as Exit +from ._click.termui import confirm as confirm +from ._click.termui import getchar as getchar +from ._click.termui import progressbar as progressbar +from ._click.termui import prompt as prompt +from ._click.termui import secho as secho +from ._click.termui import style as style +from ._click.utils import echo as echo +from ._click.utils import format_filename as format_filename +from ._click.utils import get_app_dir as get_app_dir +from ._click.utils import get_binary_stream as get_binary_stream +from ._click.utils import get_text_stream as get_text_stream from .main import Typer as Typer from .main import launch as launch from .main import run as run diff --git a/contrib/python/typer/typer/_click/LICENSE.txt b/contrib/python/typer/typer/_click/LICENSE.txt new file mode 100644 index 00000000000..d12a8491869 --- /dev/null +++ b/contrib/python/typer/typer/_click/LICENSE.txt @@ -0,0 +1,28 @@ +Copyright 2014 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/python/typer/typer/_click/__init__.py b/contrib/python/typer/typer/_click/__init__.py new file mode 100644 index 00000000000..48c9edcddd7 --- /dev/null +++ b/contrib/python/typer/typer/_click/__init__.py @@ -0,0 +1,11 @@ +""" +Code taken and adapted from Click: https://github.com/pallets/click/releases/tag/8.3.1 +""" + +from .core import Command as Command +from .core import Context as Context +from .core import Parameter as Parameter +from .exceptions import ClickException as ClickException +from .formatting import HelpFormatter as HelpFormatter +from .termui import launch as launch +from .utils import echo as echo diff --git a/contrib/python/typer/typer/_click/_compat.py b/contrib/python/typer/typer/_click/_compat.py new file mode 100644 index 00000000000..6ed0ceb8ac6 --- /dev/null +++ b/contrib/python/typer/typer/_click/_compat.py @@ -0,0 +1,569 @@ +import codecs +import io +import os +import re +import sys +from collections.abc import Callable, Mapping, MutableMapping +from types import TracebackType +from typing import ( + IO, + Any, + BinaryIO, + TextIO, + cast, +) +from weakref import WeakKeyDictionary + +CYGWIN = sys.platform.startswith("cygwin") +WIN = sys.platform.startswith("win") +auto_wrap_for_ansi: Callable[[TextIO], TextIO] | None = None +_ansi_re = re.compile(r"\033\[[;?0-9]*[a-zA-Z]") + + +def _make_text_stream( + stream: BinaryIO, + encoding: str | None, + errors: str, +) -> TextIO: + if encoding is None: + encoding = get_best_encoding(stream) + return _NonClosingTextIOWrapper( + stream, + encoding, + errors, + line_buffering=True, + ) + + +def is_ascii_encoding(encoding: str) -> bool: + """Checks if a given encoding is ascii.""" + try: + return codecs.lookup(encoding).name == "ascii" + except LookupError: + return False + + +def get_best_encoding(stream: IO[Any]) -> str: + """Returns the default stream encoding if not found.""" + rv = getattr(stream, "encoding", None) or sys.getdefaultencoding() + if is_ascii_encoding(rv): + return "utf-8" + return rv + + +class _NonClosingTextIOWrapper(io.TextIOWrapper): + def __init__( + self, + stream: BinaryIO, + encoding: str | None, + errors: str | None, + **extra: Any, + ) -> None: + self._stream = stream = cast(BinaryIO, _FixupStream(stream)) + super().__init__(stream, encoding, errors, **extra) + + def __del__(self) -> None: + try: + self.detach() + except Exception: # pragma: no cover + pass + + def isatty(self) -> bool: + # https://bitbucket.org/pypy/pypy/issue/1803 + return self._stream.isatty() + + +class _FixupStream: + """The new io interface needs more from streams than streams + traditionally implement. As such, this fix-up code is necessary in + some circumstances. + """ + + def __init__( + self, + stream: BinaryIO, + ): + self._stream = stream + + def __getattr__(self, name: str) -> Any: + return getattr(self._stream, name) + + def read1(self, size: int) -> bytes: + f = getattr(self._stream, "read1", None) + + if f is not None: + return cast(bytes, f(size)) + + return self._stream.read(size) + + def readable(self) -> bool: + return True + + def writable(self) -> bool: + return True + + def seekable(self) -> bool: + x = getattr(self._stream, "seekable", None) + if x is not None: + return cast(bool, x()) + return False + + +def _is_binary_reader(stream: IO[Any], default: bool = False) -> bool: + try: + return isinstance(stream.read(0), bytes) + except Exception: # pragma: no cover + return default + # This happens in some cases where the stream was already + # closed. In this case, we assume the default. + + +def _is_binary_writer(stream: IO[Any], default: bool = False) -> bool: + try: + stream.write(b"") + except Exception: # pragma: no cover + try: + stream.write("") + return False + except Exception: + pass + return default + return True + + +def _find_binary_reader(stream: IO[Any]) -> BinaryIO | None: + # We need to figure out if the given stream is already binary. + # This can happen because the official docs recommend detaching + # the streams to get binary streams. Some code might do this, so + # we need to deal with this case explicitly. + if _is_binary_reader(stream, False): + return cast(BinaryIO, stream) + + buf = getattr(stream, "buffer", None) + + # Same situation here; this time we assume that the buffer is + # actually binary in case it's closed. + if buf is not None and _is_binary_reader(buf, True): + return cast(BinaryIO, buf) + + return None + + +def _find_binary_writer(stream: IO[Any]) -> BinaryIO | None: + # We need to figure out if the given stream is already binary. + # This can happen because the official docs recommend detaching + # the streams to get binary streams. Some code might do this, so + # we need to deal with this case explicitly. + if _is_binary_writer(stream, False): + return cast(BinaryIO, stream) + + buf = getattr(stream, "buffer", None) + + # Same situation here; this time we assume that the buffer is + # actually binary in case it's closed. + if buf is not None and _is_binary_writer(buf, True): + return cast(BinaryIO, buf) + + return None + + +def _stream_is_misconfigured(stream: TextIO) -> bool: + """A stream is misconfigured if its encoding is ASCII.""" + # If the stream does not have an encoding set, we assume it's set + # to ASCII. This appears to happen in certain unittest + # environments. It's not quite clear what the correct behavior is + # but this at least will force Click to recover somehow. + return is_ascii_encoding(getattr(stream, "encoding", None) or "ascii") + + +def _is_compat_stream_attr(stream: TextIO, attr: str, value: str | None) -> bool: + """A stream attribute is compatible if it is equal to the + desired value or the desired value is unset and the attribute + has a value. + """ + stream_value = getattr(stream, attr, None) + return stream_value == value or (value is None and stream_value is not None) + + +def _is_compatible_text_stream( + stream: TextIO, encoding: str | None, errors: str | None +) -> bool: + """Check if a stream's encoding and errors attributes are + compatible with the desired values. + """ + return _is_compat_stream_attr( + stream, "encoding", encoding + ) and _is_compat_stream_attr(stream, "errors", errors) + + +def _force_correct_text_stream( + text_stream: IO[Any], + encoding: str | None, + errors: str | None, + is_binary: Callable[[IO[Any], bool], bool], + find_binary: Callable[[IO[Any]], BinaryIO | None], +) -> TextIO: + if is_binary(text_stream, False): + binary_reader = cast(BinaryIO, text_stream) + else: + text_stream = cast(TextIO, text_stream) + # If the stream looks compatible, and won't default to a + # misconfigured ascii encoding, return it as-is. + if _is_compatible_text_stream(text_stream, encoding, errors) and not ( + encoding is None and _stream_is_misconfigured(text_stream) + ): + return text_stream + + # Otherwise, get the underlying binary reader. + possible_binary_reader = find_binary(text_stream) + + # If that's not possible, silently use the original reader + # and get mojibake instead of exceptions. + if possible_binary_reader is None: + return text_stream + + binary_reader = possible_binary_reader + + # Default errors to replace instead of strict in order to get + # something that works. + if errors is None: + errors = "replace" + + # Wrap the binary stream in a text stream with the correct + # encoding parameters. + return _make_text_stream( + binary_reader, + encoding, + errors, + ) + + +def _force_correct_text_reader( + text_reader: IO[Any], + encoding: str | None, + errors: str | None, +) -> TextIO: + return _force_correct_text_stream( + text_reader, + encoding, + errors, + _is_binary_reader, + _find_binary_reader, + ) + + +def _force_correct_text_writer( + text_writer: IO[Any], + encoding: str | None, + errors: str | None, +) -> TextIO: + return _force_correct_text_stream( + text_writer, + encoding, + errors, + _is_binary_writer, + _find_binary_writer, + ) + + +def get_binary_stdin() -> BinaryIO: + reader = _find_binary_reader(sys.stdin) + if reader is None: # pragma: no cover + raise RuntimeError("Was not able to determine binary stream for sys.stdin.") + return reader + + +def get_binary_stdout() -> BinaryIO: + writer = _find_binary_writer(sys.stdout) + if writer is None: # pragma: no cover + raise RuntimeError("Was not able to determine binary stream for sys.stdout.") + return writer + + +def get_binary_stderr() -> BinaryIO: + writer = _find_binary_writer(sys.stderr) + if writer is None: # pragma: no cover + raise RuntimeError("Was not able to determine binary stream for sys.stderr.") + return writer + + +def get_text_stdin(encoding: str | None = None, errors: str | None = None) -> TextIO: + rv = _get_windows_console_stream(sys.stdin, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_reader(sys.stdin, encoding, errors) + + +def get_text_stdout(encoding: str | None = None, errors: str | None = None) -> TextIO: + rv = _get_windows_console_stream(sys.stdout, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_writer(sys.stdout, encoding, errors) + + +def get_text_stderr(encoding: str | None = None, errors: str | None = None) -> TextIO: + rv = _get_windows_console_stream(sys.stderr, encoding, errors) + if rv is not None: + return rv + return _force_correct_text_writer(sys.stderr, encoding, errors) + + +def _wrap_io_open( + file: str | os.PathLike[str] | int, + mode: str, + encoding: str | None, + errors: str | None, +) -> IO[Any]: + """Handles not passing ``encoding`` and ``errors`` in binary mode.""" + if "b" in mode: + return open(file, mode) + + return open(file, mode, encoding=encoding, errors=errors) + + +def open_stream( + filename: str | os.PathLike[str], + mode: str = "r", + encoding: str | None = None, + errors: str | None = "strict", + atomic: bool = False, +) -> tuple[IO[Any], bool]: + binary = "b" in mode + filename = os.fspath(filename) + + # Standard streams first, ignoring the atomic flag. + if os.fsdecode(filename) == "-": + if any(m in mode for m in ["w", "a", "x"]): + if binary: + return get_binary_stdout(), False + return get_text_stdout(encoding=encoding, errors=errors), False + if binary: + return get_binary_stdin(), False + return get_text_stdin(encoding=encoding, errors=errors), False + + # Non-atomic writes directly go out through the regular open functions. + if not atomic: + return _wrap_io_open(filename, mode, encoding, errors), True + + # Some usability stuff for atomic writes + if "a" in mode: + raise ValueError( + "Appending to an existing file is not supported, because that" + " would involve an expensive `copy`-operation to a temporary" + " file. Open the file in normal `w`-mode and copy explicitly" + " if that's what you're after." + ) + if "x" in mode: + raise ValueError("Use the `overwrite`-parameter instead.") + if "w" not in mode: + raise ValueError("Atomic writes only make sense with `w`-mode.") + + # Atomic writes are more complicated. They work by opening a file + # as a proxy in the same folder and then using the fdopen + # functionality to wrap it in a Python file. Then we wrap it in an + # atomic file that moves the file over on close. + import errno + import random + + try: + perm: int | None = os.stat(filename).st_mode + except OSError: # pragma: no cover + perm = None + + flags = os.O_RDWR | os.O_CREAT | os.O_EXCL + + if binary: + flags |= getattr(os, "O_BINARY", 0) + + while True: + tmp_filename = os.path.join( + os.path.dirname(filename), + f".__atomic-write{random.randrange(1 << 32):08x}", + ) + try: + fd = os.open(tmp_filename, flags, 0o666 if perm is None else perm) + break + except OSError as e: # pragma: no cover + if e.errno == errno.EEXIST or ( + os.name == "nt" + and e.errno == errno.EACCES + and os.path.isdir(e.filename) + and os.access(e.filename, os.W_OK) + ): + continue + raise + + if perm is not None: + os.chmod(tmp_filename, perm) # in case perm includes bits in umask + + f = _wrap_io_open(fd, mode, encoding, errors) + af = _AtomicFile(f, tmp_filename, os.path.realpath(filename)) + return cast(IO[Any], af), True + + +class _AtomicFile: + def __init__(self, f: IO[Any], tmp_filename: str, real_filename: str) -> None: + self._f = f + self._tmp_filename = tmp_filename + self._real_filename = real_filename + self.closed = False + + @property + def name(self) -> str: + return self._real_filename + + def close(self, delete: bool = False) -> None: + if self.closed: + return # pragma: no cover + self._f.close() + os.replace(self._tmp_filename, self._real_filename) + self.closed = True + + def __getattr__(self, name: str) -> Any: + return getattr(self._f, name) + + def __enter__(self) -> "_AtomicFile": + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + tb: TracebackType | None, + ) -> None: + self.close(delete=exc_type is not None) + + def __repr__(self) -> str: + return repr(self._f) + + +def strip_ansi(value: str) -> str: + return _ansi_re.sub("", value) + + +def _is_jupyter_kernel_output(stream: IO[Any]) -> bool: + while isinstance(stream, (_FixupStream, _NonClosingTextIOWrapper)): + stream = stream._stream + + return stream.__class__.__module__.startswith("ipykernel.") + + +def should_strip_ansi(stream: IO[Any] | None = None, color: bool | None = None) -> bool: + if color is None: + if stream is None: + stream = sys.stdin + return not isatty(stream) and not _is_jupyter_kernel_output(stream) + return not color + + +# On Windows, wrap the output streams with colorama to support ANSI +# color codes. +# NOTE: double check is needed so mypy does not analyze this on Linux +if sys.platform.startswith("win") and WIN: + from ._winconsole import _get_windows_console_stream + + def _get_argv_encoding() -> str: + import locale + + return locale.getpreferredencoding() + + _ansi_stream_wrappers: MutableMapping[TextIO, TextIO] = WeakKeyDictionary() + + def auto_wrap_for_ansi(stream: TextIO, color: bool | None = None) -> TextIO: + """Support ANSI color and style codes on Windows by wrapping a + stream with colorama. + """ + try: + cached = _ansi_stream_wrappers.get(stream) + except Exception: # pragma: no cover + cached = None + + if cached is not None: + return cached + + import colorama + + strip = should_strip_ansi(stream, color) + ansi_wrapper = colorama.AnsiToWin32(stream, strip=strip) + rv = cast(TextIO, ansi_wrapper.stream) + _write = rv.write + + def _safe_write(s: str) -> int: + try: + return _write(s) + except BaseException: # pragma: no cover + ansi_wrapper.reset_all() + raise + + rv.write = _safe_write # type: ignore[method-assign] # ty: ignore[invalid-assignment] + + try: + _ansi_stream_wrappers[stream] = rv + except Exception: # pragma: no cover + pass + + return rv + +else: + + def _get_argv_encoding() -> str: + return getattr(sys.stdin, "encoding", None) or sys.getfilesystemencoding() + + def _get_windows_console_stream( + f: TextIO, encoding: str | None, errors: str | None + ) -> TextIO | None: + return None + + +def term_len(x: str) -> int: + return len(strip_ansi(x)) + + +def isatty(stream: IO[Any]) -> bool: + try: + return stream.isatty() + except Exception: # pragma: no cover + return False + + +def _make_cached_stream_func( + src_func: Callable[[], TextIO], + wrapper_func: Callable[[], TextIO], +) -> Callable[[], TextIO]: + cache: MutableMapping[TextIO, TextIO] = WeakKeyDictionary() + + def func() -> TextIO: + stream = src_func() + + try: + rv = cache.get(stream) + except Exception: # pragma: no cover + rv = None + if rv is not None: + return rv + rv = wrapper_func() + try: + cache[stream] = rv + except Exception: # pragma: no cover + pass + return rv + + return func + + +_default_text_stdin = _make_cached_stream_func(lambda: sys.stdin, get_text_stdin) +_default_text_stdout = _make_cached_stream_func(lambda: sys.stdout, get_text_stdout) +_default_text_stderr = _make_cached_stream_func(lambda: sys.stderr, get_text_stderr) + + +binary_streams: Mapping[str, Callable[[], BinaryIO]] = { + "stdin": get_binary_stdin, + "stdout": get_binary_stdout, + "stderr": get_binary_stderr, +} + +text_streams: Mapping[str, Callable[[str | None, str | None], TextIO]] = { + "stdin": get_text_stdin, + "stdout": get_text_stdout, + "stderr": get_text_stderr, +} diff --git a/contrib/python/typer/typer/_click/_termui_impl.py b/contrib/python/typer/typer/_click/_termui_impl.py new file mode 100644 index 00000000000..c621810cbf8 --- /dev/null +++ b/contrib/python/typer/typer/_click/_termui_impl.py @@ -0,0 +1,522 @@ +""" +To keep the import times down, some infrequently used termui functionality +is placed here and only imported as needed. +""" + +import contextlib +import math +import os +import sys +import time +from collections.abc import Callable, Iterable, Iterator +from io import StringIO +from types import TracebackType +from typing import Generic, TextIO, TypeVar, cast + +from ._compat import ( + CYGWIN, + WIN, + _default_text_stdout, + get_best_encoding, + isatty, + term_len, +) +from .utils import echo + +V = TypeVar("V") + +if os.name == "nt": + BEFORE_BAR = "\r" + AFTER_BAR = "\n" +else: + BEFORE_BAR = "\r\033[?25l" + AFTER_BAR = "\033[?25h\n" + + +class ProgressBar(Generic[V]): + def __init__( + self, + iterable: Iterable[V] | None, + length: int | None = None, + fill_char: str = "#", + empty_char: str = " ", + bar_template: str = "%(bar)s", + info_sep: str = " ", + hidden: bool = False, + show_eta: bool = True, + show_percent: bool | None = None, + show_pos: bool = False, + item_show_func: Callable[[V | None], str | None] | None = None, + label: str | None = None, + file: TextIO | None = None, + color: bool | None = None, + update_min_steps: int = 1, + width: int = 30, + ) -> None: + self.fill_char = fill_char + self.empty_char = empty_char + self.bar_template = bar_template + self.info_sep = info_sep + self.hidden = hidden + self.show_eta = show_eta + self.show_percent = show_percent + self.show_pos = show_pos + self.item_show_func = item_show_func + self.label: str = label or "" + + if file is None: + file = _default_text_stdout() + + # There are no standard streams attached to write to. For example, + # pythonw on Windows. + if file is None: # pragma: no cover + file = StringIO() + + self.file = file + self.color = color + self.update_min_steps = update_min_steps + self._completed_intervals = 0 + self.width: int = width + self.autowidth: bool = width == 0 + + if length is None: + from operator import length_hint + + length = length_hint(iterable, -1) + + if length == -1: # pragma: no cover + length = None + if iterable is None: + if length is None: # pragma: no cover + raise TypeError("iterable or length is required") + iterable = cast("Iterable[V]", range(length)) + self.iter: Iterable[V] = iter(iterable) + self.length = length + self.pos: int = 0 + self.avg: list[float] = [] + self.last_eta: float + self.start: float + self.start = self.last_eta = time.time() + self.eta_known: bool = False + self.finished: bool = False + self.max_width: int | None = None + self.entered: bool = False + self.current_item: V | None = None + self._is_atty = isatty(self.file) + self._last_line: str | None = None + + def __enter__(self) -> "ProgressBar[V]": + self.entered = True + self.render_progress() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + tb: TracebackType | None, + ) -> None: + self.render_finish() + + def __iter__(self) -> Iterator[V]: + if not self.entered: + raise RuntimeError("You need to use progress bars in a with block.") + self.render_progress() + return self.generator() + + def __next__(self) -> V: + # Iteration is defined in terms of a generator function, + # returned by iter(self); use that to define next(). This works + # because `self.iter` is an iterable consumed by that generator, + # so it is re-entry safe. Calling `next(self.generator())` + # twice works and does "what you want". + return next(iter(self)) + + def render_finish(self) -> None: + if self.hidden or not self._is_atty: + return + self.file.write(AFTER_BAR) + self.file.flush() + + @property + def pct(self) -> float: + if self.finished: + return 1.0 + return min(self.pos / (float(self.length or 1) or 1), 1.0) + + @property + def time_per_iteration(self) -> float: + if not self.avg: + return 0.0 + return sum(self.avg) / float(len(self.avg)) + + @property + def eta(self) -> float: + if self.length is not None and not self.finished: + return self.time_per_iteration * (self.length - self.pos) + return 0.0 + + def format_eta(self) -> str: + if self.eta_known: + t = int(self.eta) + seconds = t % 60 + t //= 60 + minutes = t % 60 + t //= 60 + hours = t % 24 + t //= 24 + if t > 0: + return f"{t}d {hours:02}:{minutes:02}:{seconds:02}" + else: + return f"{hours:02}:{minutes:02}:{seconds:02}" + return "" + + def format_pos(self) -> str: + pos = str(self.pos) + if self.length is not None: + pos += f"/{self.length}" + return pos + + def format_pct(self) -> str: + return f"{int(self.pct * 100): 4}%"[1:] + + def format_bar(self) -> str: + if self.length is not None: + bar_length = int(self.pct * self.width) + bar = self.fill_char * bar_length + bar += self.empty_char * (self.width - bar_length) + elif self.finished: + bar = self.fill_char * self.width + else: + chars = list(self.empty_char * (self.width or 1)) + if self.time_per_iteration != 0: + chars[ + int( + (math.cos(self.pos * self.time_per_iteration) / 2.0 + 0.5) + * self.width + ) + ] = self.fill_char + bar = "".join(chars) + return bar + + def format_progress_line(self) -> str: + show_percent = self.show_percent + + info_bits = [] + if self.length is not None and show_percent is None: + show_percent = not self.show_pos + + if self.show_pos: + info_bits.append(self.format_pos()) + if show_percent: + info_bits.append(self.format_pct()) + if self.show_eta and self.eta_known and not self.finished: + info_bits.append(self.format_eta()) + if self.item_show_func is not None: + item_info = self.item_show_func(self.current_item) + if item_info is not None: + info_bits.append(item_info) + + return ( + self.bar_template + % { + "label": self.label, + "bar": self.format_bar(), + "info": self.info_sep.join(info_bits), + } + ).rstrip() + + def render_progress(self) -> None: + if self.hidden: + return + + if not self._is_atty: + # Only output the label once if the output is not a TTY. + if self._last_line != self.label: + self._last_line = self.label + echo(self.label, file=self.file, color=self.color) + return + + buf = [] + # Update width in case the terminal has been resized + if self.autowidth: + import shutil + + old_width = self.width + self.width = 0 + clutter_length = term_len(self.format_progress_line()) + new_width = max(0, shutil.get_terminal_size().columns - clutter_length) + if new_width < old_width and self.max_width is not None: + buf.append(BEFORE_BAR) + buf.append(" " * self.max_width) + self.max_width = new_width + self.width = new_width + + clear_width = self.width + if self.max_width is not None: + clear_width = self.max_width + + buf.append(BEFORE_BAR) + line = self.format_progress_line() + line_len = term_len(line) + if self.max_width is None or self.max_width < line_len: + self.max_width = line_len + + buf.append(line) + buf.append(" " * (clear_width - line_len)) + line = "".join(buf) + # Render the line only if it changed. + + if line != self._last_line: + self._last_line = line + echo(line, file=self.file, color=self.color, nl=False) + self.file.flush() + + def make_step(self, n_steps: int) -> None: + self.pos += n_steps + if self.length is not None and self.pos >= self.length: + self.finished = True + + if (time.time() - self.last_eta) < 1.0: + return + + self.last_eta = time.time() + + # self.avg is a rolling list of length <= 7 of steps where steps are + # defined as time elapsed divided by the total progress through + # self.length. + if self.pos: + step = (time.time() - self.start) / self.pos + else: + step = time.time() - self.start + + self.avg = self.avg[-6:] + [step] + + self.eta_known = self.length is not None + + def update(self, n_steps: int) -> None: + """Update the progress bar by advancing a specified number of steps.""" + self._completed_intervals += n_steps + + if self._completed_intervals >= self.update_min_steps: + self.make_step(self._completed_intervals) + self.render_progress() + self._completed_intervals = 0 + + def finish(self) -> None: + self.eta_known = False + self.current_item = None + self.finished = True + + def generator(self) -> Iterator[V]: + """Return a generator which yields the items added to the bar + during construction, and updates the progress bar *after* the + yielded block returns. + """ + # WARNING: the iterator interface for `ProgressBar` relies on + # this and only works because this is a simple generator which + # doesn't create or manage additional state. If this function + # changes, the impact should be evaluated both against + # `iter(bar)` and `next(bar)`. `next()` in particular may call + # `self.generator()` repeatedly, and this must remain safe in + # order for that interface to work. + if not self.entered: # pragma: no cover + raise RuntimeError("You need to use progress bars in a with block.") + + if not self._is_atty: + yield from self.iter + else: + for rv in self.iter: + self.current_item = rv + + # This allows show_item_func to be updated before the + # item is processed. Only trigger at the beginning of + # the update interval. + if self._completed_intervals == 0: + self.render_progress() + + yield rv + self.update(1) + + self.finish() + self.render_progress() + + +def open_url(url: str, wait: bool = False, locate: bool = False) -> int: + import subprocess + + def _unquote_file(url: str) -> str: + from urllib.parse import unquote + + if url.startswith("file://"): + url = unquote(url[7:]) + + return url + + if sys.platform == "darwin": + args = ["open"] + if wait: + args.append("-W") + if locate: + args.append("-R") + args.append(_unquote_file(url)) + null = open("/dev/null", "w") + try: + return subprocess.Popen(args, stderr=null).wait() + finally: + null.close() + elif WIN: + if locate: + url = _unquote_file(url) + args = ["explorer", f"/select,{url}"] + else: + args = ["start"] + if wait: + args.append("/WAIT") + args.append("") + args.append(url) + try: + return subprocess.call(args) + except OSError: + # Command not found + return 127 + elif CYGWIN: # pragma: no cover + if locate: + url = _unquote_file(url) + args = ["cygstart", os.path.dirname(url)] + else: + args = ["cygstart"] + if wait: + args.append("-w") + args.append(url) + try: + return subprocess.call(args) + except OSError: + # Command not found + return 127 + + try: + if locate: + url = os.path.dirname(_unquote_file(url)) or "." + else: + url = _unquote_file(url) + c = subprocess.Popen(["xdg-open", url]) + if wait: + return c.wait() + return 0 + except OSError: # pragma: no cover + # TODO: remove this part, doesn't get hit by Typer code paths? + if url.startswith(("http://", "https://")) and not locate and not wait: + import webbrowser + + webbrowser.open(url) + return 0 + return 1 + + +def _translate_ch_to_exc(ch: str) -> None: + if ch == "\x03": + raise KeyboardInterrupt() + + if ch == "\x04" and not WIN: # Unix-like, Ctrl+D + raise EOFError() + + if ch == "\x1a" and WIN: # Windows, Ctrl+Z + raise EOFError() + + return None + + +if sys.platform == "win32": + import msvcrt + + @contextlib.contextmanager + def raw_terminal() -> Iterator[int]: + yield -1 + + def getchar(echo: bool) -> str: + # The function `getch` will return a bytes object corresponding to + # the pressed character. Since Windows 10 build 1803, it will also + # return \x00 when called a second time after pressing a regular key. + # + # `getwch` does not share this probably-bugged behavior. Moreover, it + # returns a Unicode object by default, which is what we want. + # + # Either of these functions will return \x00 or \xe0 to indicate + # a special key, and you need to call the same function again to get + # the "rest" of the code. The fun part is that \u00e0 is + # "latin small letter a with grave", so if you type that on a French + # keyboard, you _also_ get a \xe0. + # E.g., consider the Up arrow. This returns \xe0 and then \x48. The + # resulting Unicode string reads as "a with grave" + "capital H". + # This is indistinguishable from when the user actually types + # "a with grave" and then "capital H". + # + # When \xe0 is returned, we assume it's part of a special-key sequence + # and call `getwch` again, but that means that when the user types + # the \u00e0 character, `getchar` doesn't return until a second + # character is typed. + # The alternative is returning immediately, but that would mess up + # cross-platform handling of arrow keys and others that start with + # \xe0. Another option is using `getch`, but then we can't reliably + # read non-ASCII characters, because return values of `getch` are + # limited to the current 8-bit codepage. + # + # Anyway, Click doesn't claim to do this Right(tm), and using `getwch` + # is doing the right thing in more situations than with `getch`. + + if echo: + func = cast(Callable[[], str], msvcrt.getwche) + else: + func = cast(Callable[[], str], msvcrt.getwch) + + rv = func() + + if rv in ("\x00", "\xe0"): + # \x00 and \xe0 are control characters that indicate special key, + # see above. + rv += func() + + _translate_ch_to_exc(rv) + return rv + +else: + import termios + import tty + + @contextlib.contextmanager + def raw_terminal() -> Iterator[int]: + f: TextIO | None + fd: int + + if not isatty(sys.stdin): + f = open("/dev/tty") + fd = f.fileno() + else: + fd = sys.stdin.fileno() + f = None + + try: + old_settings = termios.tcgetattr(fd) + + try: + tty.setraw(fd) + yield fd + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + sys.stdout.flush() + + if f is not None: + f.close() + except termios.error: # pragma: no cover + pass + + def getchar(echo: bool) -> str: + with raw_terminal() as fd: + ch = os.read(fd, 32).decode(get_best_encoding(sys.stdin), "replace") + + if echo and isatty(sys.stdout): # pragma: no cover + sys.stdout.write(ch) + + _translate_ch_to_exc(ch) + return ch diff --git a/contrib/python/typer/typer/_click/_textwrap.py b/contrib/python/typer/typer/_click/_textwrap.py new file mode 100644 index 00000000000..9f9636b31f4 --- /dev/null +++ b/contrib/python/typer/typer/_click/_textwrap.py @@ -0,0 +1,46 @@ +import textwrap +from collections.abc import Iterator +from contextlib import contextmanager + + +class TextWrapper(textwrap.TextWrapper): + def _handle_long_word( + self, + reversed_chunks: list[str], + cur_line: list[str], + cur_len: int, + width: int, + ) -> None: + space_left = max(width - cur_len, 1) + + last = reversed_chunks[-1] + cut = last[:space_left] + res = last[space_left:] + cur_line.append(cut) + reversed_chunks[-1] = res + + @contextmanager + def extra_indent(self, indent: str) -> Iterator[None]: + old_initial_indent = self.initial_indent + old_subsequent_indent = self.subsequent_indent + self.initial_indent += indent + self.subsequent_indent += indent + + try: + yield + finally: + self.initial_indent = old_initial_indent + self.subsequent_indent = old_subsequent_indent + + def indent_only(self, text: str) -> str: + rv = [] + + for idx, line in enumerate(text.splitlines()): + indent = self.initial_indent + + if idx > 0: + indent = self.subsequent_indent + + rv.append(f"{indent}{line}") + + return "\n".join(rv) diff --git a/contrib/python/typer/typer/_click/_winconsole.py b/contrib/python/typer/typer/_click/_winconsole.py new file mode 100644 index 00000000000..f6dedbfd6ef --- /dev/null +++ b/contrib/python/typer/typer/_click/_winconsole.py @@ -0,0 +1,300 @@ +# This module is based on the excellent work by Adam Bartoš who +# provided a lot of what went into the implementation here in +# the discussion to issue1602 in the Python bug tracker. +# +# There are some general differences in regards to how this works +# compared to the original patches as we do not need to patch +# the entire interpreter but just work in our little world of +# echo and prompt. +import io +import sys +import time +from collections.abc import Callable, Iterable, Mapping +from ctypes import ( + POINTER, + Array, + Structure, + byref, + c_char, + c_char_p, + c_int, + c_ssize_t, + c_ulong, + c_void_p, + py_object, +) +from ctypes.wintypes import DWORD, HANDLE, LPCWSTR, LPWSTR +from typing import ( + TYPE_CHECKING, + Any, + AnyStr, + BinaryIO, + Literal, + TextIO, + cast, +) + +from ._compat import _NonClosingTextIOWrapper + +assert sys.platform == "win32" +import msvcrt # noqa: E402 +from ctypes import WINFUNCTYPE, windll # noqa: E402 + +c_ssize_p = POINTER(c_ssize_t) + +kernel32 = windll.kernel32 +GetStdHandle = kernel32.GetStdHandle +ReadConsoleW = kernel32.ReadConsoleW +WriteConsoleW = kernel32.WriteConsoleW +GetConsoleMode = kernel32.GetConsoleMode +GetLastError = kernel32.GetLastError +GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) +CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( + ("CommandLineToArgvW", windll.shell32) +) +LocalFree = WINFUNCTYPE(c_void_p, c_void_p)(("LocalFree", windll.kernel32)) + +STDIN_HANDLE = GetStdHandle(-10) +STDOUT_HANDLE = GetStdHandle(-11) +STDERR_HANDLE = GetStdHandle(-12) + +PyBUF_SIMPLE = 0 +PyBUF_WRITABLE = 1 + +ERROR_SUCCESS = 0 +ERROR_NOT_ENOUGH_MEMORY = 8 +ERROR_OPERATION_ABORTED = 995 + +STDIN_FILENO = 0 +STDOUT_FILENO = 1 +STDERR_FILENO = 2 + +EOF = b"\x1a" +MAX_BYTES_WRITTEN = 32767 + +if TYPE_CHECKING: + try: + # Using `typing_extensions.Buffer` instead of `collections.abc` + # on Windows for some reason does not have `Sized` implemented. + from collections.abc import Buffer # type: ignore + except ImportError: + from typing_extensions import Buffer + +try: + from ctypes import pythonapi +except ImportError: # pragma: no cover + # On PyPy we cannot get buffers so our ability to operate here is + # severely limited. + get_buffer = None +else: + + class Py_buffer(Structure): + _fields_ = [ # noqa: RUF012 + ("buf", c_void_p), + ("obj", py_object), + ("len", c_ssize_t), + ("itemsize", c_ssize_t), + ("readonly", c_int), + ("ndim", c_int), + ("format", c_char_p), + ("shape", c_ssize_p), + ("strides", c_ssize_p), + ("suboffsets", c_ssize_p), + ("internal", c_void_p), + ] + + PyObject_GetBuffer = pythonapi.PyObject_GetBuffer + PyBuffer_Release = pythonapi.PyBuffer_Release + + def get_buffer(obj: "Buffer", writable: bool = False) -> Array[c_char]: + buf = Py_buffer() + flags: int = PyBUF_WRITABLE if writable else PyBUF_SIMPLE + PyObject_GetBuffer(py_object(obj), byref(buf), flags) + + try: + buffer_type = c_char * buf.len + out: Array[c_char] = buffer_type.from_address(buf.buf) + return out + finally: + PyBuffer_Release(byref(buf)) + + +class _WindowsConsoleRawIOBase(io.RawIOBase): + def __init__(self, handle: int | None) -> None: + self.handle = handle + + def isatty(self) -> Literal[True]: + super().isatty() + return True + + +class _WindowsConsoleReader(_WindowsConsoleRawIOBase): + def readable(self) -> Literal[True]: + return True + + def readinto(self, b: "Buffer") -> int: + bytes_to_be_read = len(b) + if not bytes_to_be_read: + return 0 + elif bytes_to_be_read % 2: + raise ValueError( + "cannot read odd number of bytes from UTF-16-LE encoded console" + ) + + buffer = get_buffer(b, writable=True) + code_units_to_be_read = bytes_to_be_read // 2 + code_units_read = c_ulong() + + rv = ReadConsoleW( + HANDLE(self.handle), + buffer, + code_units_to_be_read, + byref(code_units_read), + None, + ) + if GetLastError() == ERROR_OPERATION_ABORTED: + # wait for KeyboardInterrupt + time.sleep(0.1) + if not rv: + raise OSError(f"Windows error: {GetLastError()}") + + if buffer[0] == EOF: + return 0 + return 2 * code_units_read.value + + +class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): + def writable(self) -> Literal[True]: + return True + + @staticmethod + def _get_error_message(errno: int) -> str: + if errno == ERROR_SUCCESS: + return "ERROR_SUCCESS" + elif errno == ERROR_NOT_ENOUGH_MEMORY: + return "ERROR_NOT_ENOUGH_MEMORY" + return f"Windows error {errno}" + + def write(self, b: "Buffer") -> int: + bytes_to_be_written = len(b) + buf = get_buffer(b) + code_units_to_be_written = min(bytes_to_be_written, MAX_BYTES_WRITTEN) // 2 + code_units_written = c_ulong() + + WriteConsoleW( + HANDLE(self.handle), + buf, + code_units_to_be_written, + byref(code_units_written), + None, + ) + bytes_written = 2 * code_units_written.value + + if bytes_written == 0 and bytes_to_be_written > 0: + raise OSError(self._get_error_message(GetLastError())) # pragma: no cover + return bytes_written + + +class ConsoleStream: + def __init__(self, text_stream: TextIO, byte_stream: BinaryIO) -> None: + self._text_stream = text_stream + self.buffer = byte_stream + + @property + def name(self) -> str: + return self.buffer.name + + def write(self, x: AnyStr) -> int: + if isinstance(x, str): + return self._text_stream.write(x) + try: + self.flush() + except Exception: # pragma: no cover + pass + return self.buffer.write(x) + + def writelines(self, lines: Iterable[AnyStr]) -> None: + for line in lines: + self.write(line) + + def __getattr__(self, name: str) -> Any: + return getattr(self._text_stream, name) + + def isatty(self) -> bool: + return self.buffer.isatty() + + def __repr__(self) -> str: + return f"<ConsoleStream name={self.name!r} encoding={self.encoding!r}>" + + +def _get_text_stdin(buffer_stream: BinaryIO) -> TextIO: + text_stream = _NonClosingTextIOWrapper( + io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return cast(TextIO, ConsoleStream(text_stream, buffer_stream)) + + +def _get_text_stdout(buffer_stream: BinaryIO) -> TextIO: + text_stream = _NonClosingTextIOWrapper( + io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return cast(TextIO, ConsoleStream(text_stream, buffer_stream)) + + +def _get_text_stderr(buffer_stream: BinaryIO) -> TextIO: + text_stream = _NonClosingTextIOWrapper( + io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return cast(TextIO, ConsoleStream(text_stream, buffer_stream)) + + +_stream_factories: Mapping[int, Callable[[BinaryIO], TextIO]] = { + 0: _get_text_stdin, + 1: _get_text_stdout, + 2: _get_text_stderr, +} + + +def _is_console(f: TextIO) -> bool: + if not hasattr(f, "fileno"): + return False + + try: + fileno = f.fileno() + except (OSError, io.UnsupportedOperation): + return False + + handle = msvcrt.get_osfhandle(fileno) + return bool(GetConsoleMode(handle, byref(DWORD()))) + + +def _get_windows_console_stream( + f: TextIO, encoding: str | None, errors: str | None +) -> TextIO | None: + if ( + get_buffer is None + or encoding not in {"utf-16-le", None} + or errors not in {"strict", None} + or not _is_console(f) + ): + return None + + func = _stream_factories.get(f.fileno()) + if func is None: + return None + + b = getattr(f, "buffer", None) + + if b is None: + return None + + return func(b) diff --git a/contrib/python/typer/typer/_click/core.py b/contrib/python/typer/typer/_click/core.py new file mode 100644 index 00000000000..580b558b9fc --- /dev/null +++ b/contrib/python/typer/typer/_click/core.py @@ -0,0 +1,1111 @@ +import enum +import inspect +import os +from abc import ABC, abstractmethod +from collections.abc import Callable, Iterator, Mapping, MutableMapping, Sequence +from contextlib import AbstractContextManager, ExitStack, contextmanager +from types import TracebackType +from typing import ( + TYPE_CHECKING, + Any, + Literal, + NoReturn, + TypeVar, + Union, + cast, + overload, +) + +from . import types +from .exceptions import ( + Abort, + BadParameter, + Exit, + MissingParameter, + NoArgsIsHelpError, + UsageError, +) +from .formatting import HelpFormatter +from .globals import pop_context, push_context +from .parser import _OptionParser +from .termui import style +from .utils import echo, make_default_short_help + +if TYPE_CHECKING: + from ..core import TyperOption + from .shell_completion import CompletionItem + +F = TypeVar("F", bound="Callable[..., Any]") +V = TypeVar("V") + + +def _complete_visible_commands( + ctx: "Context", incomplete: str +) -> Iterator[tuple[str, "Command"]]: + """List all the subcommands of a group that start with the + incomplete value and aren't hidden. + """ + # avoid circular imports + from ..core import TyperGroup + + multi = cast(TyperGroup, ctx.command) + + for name in multi.list_commands(ctx): + if name.startswith(incomplete): + command = multi.get_command(ctx, name) + + if command is not None and not command.hidden: + yield name, command + + +@contextmanager +def augment_usage_errors( + ctx: "Context", param: Union["Parameter", None] = None +) -> Iterator[None]: + """Context manager that attaches extra information to exceptions.""" + try: + yield + except BadParameter as e: + if e.ctx is None: + e.ctx = ctx + if param is not None and e.param is None: + e.param = param + raise + except UsageError as e: # pragma: no cover + if e.ctx is None: + e.ctx = ctx + raise + + +def iter_params_for_processing( + invocation_order: Sequence["Parameter"], + declaration_order: Sequence["Parameter"], +) -> list["Parameter"]: + """Returns all declared parameters in the order they should be processed. + + The declared parameters are re-shuffled depending on the order in which + they were invoked, as well as the eagerness of each parameters. + + The invocation order takes precedence over the declaration order. I.e. the + order in which the user provided them to the CLI is respected. + + This behavior and its effect on callback evaluation is detailed at: + https://click.palletsprojects.com/en/stable/advanced/#callback-evaluation-order + """ + + def sort_key(item: Parameter) -> tuple[bool, float]: + try: + idx: float = invocation_order.index(item) + except ValueError: + idx = float("inf") + + return not item.is_eager, idx + + return sorted(declaration_order, key=sort_key) + + +class ParameterSource(enum.Enum): + """This is an `Enum` that indicates the source of a + parameter's value. + """ + + COMMANDLINE = enum.auto() + """The value was provided by the command line args.""" + ENVIRONMENT = enum.auto() + """The value was provided with an environment variable.""" + DEFAULT = enum.auto() + """Used the default specified by the parameter.""" + DEFAULT_MAP = enum.auto() + """Used a default provided by `Context.default_map`.""" + PROMPT = enum.auto() + """Used a prompt to confirm a default or provide a value.""" + + +class Context: + """The context is a special internal object that holds state relevant + for the script execution at every single level. It's normally invisible + to commands unless they opt-in to getting access to it. + + The context is useful as it can pass internal objects around and can + control special execution features such as reading data from + environment variables. + + A context can be used as context manager in which case it will call + `close` on teardown. + """ + + formatter_class: type[HelpFormatter] = HelpFormatter + + def __init__( + self, + command: "Command", + parent: Union["Context", None] = None, + info_name: str | None = None, + obj: Any | None = None, + auto_envvar_prefix: str | None = None, + default_map: MutableMapping[str, Any] | None = None, + terminal_width: int | None = None, + max_content_width: int | None = None, + resilient_parsing: bool = False, + allow_extra_args: bool | None = None, + allow_interspersed_args: bool | None = None, + ignore_unknown_options: bool | None = None, + help_option_names: list[str] | None = None, + token_normalize_func: Callable[[str], str] | None = None, + color: bool | None = None, + show_default: bool | None = None, + ) -> None: + self.parent = parent + self.command = command + self.info_name = info_name + # Map of parameter names to their parsed values. + self.params: dict[str, Any] = {} + # the leftover arguments. + self.args: list[str] = [] + # protected arguments. used to implement nested parsing. + self._protected_args: list[str] = [] + # the collected prefixes of the command's options. + self._opt_prefixes: set[str] = set(parent._opt_prefixes) if parent else set() + + if obj is None and parent is not None: + obj = parent.obj + + self.obj: Any = obj + self._meta: dict[str, Any] = getattr(parent, "meta", {}) + + # A dictionary (-like object) with defaults for parameters. + if ( + default_map is None + and info_name is not None + and parent is not None + and parent.default_map is not None + ): + default_map = parent.default_map.get(info_name) + + self.default_map: MutableMapping[str, Any] | None = default_map + + # This flag indicates if a subcommand is going to be executed. + self.invoked_subcommand: str | None = None + + if terminal_width is None and parent is not None: + terminal_width = parent.terminal_width + + # The width of the terminal (None is autodetection). + self.terminal_width: int | None = terminal_width + + if max_content_width is None and parent is not None: + max_content_width = parent.max_content_width + + self.max_content_width: int | None = max_content_width + + if allow_extra_args is None: + allow_extra_args = command.allow_extra_args + + self.allow_extra_args = allow_extra_args + + if allow_interspersed_args is None: + allow_interspersed_args = command.allow_interspersed_args + + self.allow_interspersed_args: bool = allow_interspersed_args + + if ignore_unknown_options is None: + ignore_unknown_options = command.ignore_unknown_options + + self.ignore_unknown_options: bool = ignore_unknown_options + + if help_option_names is None: + if parent is not None: + help_option_names = parent.help_option_names + else: + help_option_names = ["--help"] + + self.help_option_names: list[str] = help_option_names + + if token_normalize_func is None and parent is not None: + token_normalize_func = parent.token_normalize_func + + # An optional normalization function for tokens. (options, choices, commands etc.) + self.token_normalize_func: Callable[[str], str] | None = token_normalize_func + + # Indicates if resilient parsing is enabled. + self.resilient_parsing: bool = resilient_parsing + + # If there is no envvar prefix yet, but the parent has one and + # the command on this level has a name, we can expand the envvar + # prefix automatically. + if auto_envvar_prefix is None: + if ( + parent is not None + and parent.auto_envvar_prefix is not None + and self.info_name is not None + ): + auto_envvar_prefix = ( + f"{parent.auto_envvar_prefix}_{self.info_name.upper()}" + ) + else: + auto_envvar_prefix = auto_envvar_prefix.upper() + + if auto_envvar_prefix is not None: + auto_envvar_prefix = auto_envvar_prefix.replace("-", "_") + + self.auto_envvar_prefix: str | None = auto_envvar_prefix + + if color is None and parent is not None: + color = parent.color + + # Controls if styling output is wanted or not. + self.color: bool | None = color + + if show_default is None and parent is not None: + show_default = parent.show_default + + # Show option default values when formatting help text. + self.show_default: bool | None = show_default + + self._close_callbacks: list[Callable[[], Any]] = [] + self._depth = 0 + self._parameter_source: dict[str, ParameterSource] = {} + self._exit_stack = ExitStack() + + def __enter__(self) -> "Context": + self._depth += 1 + push_context(self) + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + tb: TracebackType | None, + ) -> bool | None: + self._depth -= 1 + exit_result: bool | None = None + if self._depth == 0: + exit_result = self._close_with_exception_info(exc_type, exc_value, tb) + pop_context() + + return exit_result + + @contextmanager + def scope(self, cleanup: bool = True) -> Iterator["Context"]: + """This helper method can be used with the context object to promote + it to the current thread local (see `get_current_context`). + The default behavior of this is to invoke the cleanup functions which + can be disabled by setting `cleanup` to `False`. The cleanup + functions are typically used for things such as closing file handles. + + If the cleanup is intended the context object can also be directly + used as a context manager. + """ + if not cleanup: + self._depth += 1 + try: + with self as rv: + yield rv + finally: + if not cleanup: + self._depth -= 1 + + @property + def meta(self) -> dict[str, Any]: + """This is a dictionary which is shared with all the contexts + that are nested. It exists so that click utilities can store some + state here if they need to. It is however the responsibility of + that code to manage this dictionary well. + + The keys are supposed to be unique dotted strings. For instance + module paths are a good choice for it. What is stored in there is + irrelevant for the operation of click. However what is important is + that code that places data here adheres to the general semantics of + the system. + """ + return self._meta + + def make_formatter(self) -> HelpFormatter: + """Creates the HelpFormatter for the help and + usage output. + """ + return self.formatter_class( + width=self.terminal_width, max_width=self.max_content_width + ) + + def with_resource(self, context_manager: AbstractContextManager[V]) -> V: + """Register a resource as if it were used in a ``with`` + statement. The resource will be cleaned up when the context is + popped. + + Uses `contextlib.ExitStack.enter_context`. It calls the + resource's ``__enter__()`` method and returns the result. When + the context is popped, it closes the stack, which calls the + resource's ``__exit__()`` method. + + To register a cleanup function for something that isn't a + context manager, use `call_on_close`. Or use something + from `contextlib` to turn it into a context manager first. + """ + return self._exit_stack.enter_context(context_manager) + + def call_on_close(self, f: Callable[..., Any]) -> Callable[..., Any]: + """Register a function to be called when the context tears down. + + This can be used to close resources opened during the script + execution. Resources that support Python's context manager + protocol which would be used in a ``with`` statement should be + registered with `with_resource` instead. + """ + return self._exit_stack.callback(f) + + def close(self) -> None: + """Invoke all close callbacks registered with `call_on_close`, + and exit all context managers entered with `with_resource`. + """ + self._close_with_exception_info(None, None, None) + + def _close_with_exception_info( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + tb: TracebackType | None, + ) -> bool | None: + """Unwind the exit stack by calling its `__exit__` providing the exception + information to allow for exception handling by the various resources registered + using `with_resource` + """ + exit_result = self._exit_stack.__exit__(exc_type, exc_value, tb) + # In case the context is reused, create a new exit stack. + self._exit_stack = ExitStack() + + return exit_result + + @property + def command_path(self) -> str: + """The computed command path. This is used for the ``usage`` + information on the help page. It's automatically created by + combining the info names of the chain of contexts to the root. + """ + rv = "" + if self.info_name is not None: + rv = self.info_name + if self.parent is not None: + parent_command_path = [self.parent.command_path] + + if isinstance(self.parent.command, Command): + for param in self.parent.command.get_params(self): + parent_command_path.extend(param.get_usage_pieces(self)) + + rv = f"{' '.join(parent_command_path)} {rv}" + return rv.lstrip() + + def find_root(self) -> "Context": + """Finds the outermost context.""" + node = self + while node.parent is not None: + node = node.parent + return node + + def find_object(self, object_type: type[V]) -> V | None: + """Finds the closest object of a given type.""" + node: Context | None = self + + while node is not None: + if isinstance(node.obj, object_type): + return node.obj + + node = node.parent + + return None + + def ensure_object(self, object_type: type[V]) -> V: + """Like `find_object` but sets the innermost object to a + new instance of `object_type` if it does not exist. + """ + rv = self.find_object(object_type) + if rv is None: + self.obj = rv = object_type() + return rv + + @overload + def lookup_default(self, name: str, call: Literal[True] = True) -> Any | None: ... + + @overload + def lookup_default( + self, name: str, call: Literal[False] = ... + ) -> Any | Callable[[], Any] | None: ... + + def lookup_default(self, name: str, call: bool = True) -> Any | None: + """Get the default for a parameter from `default_map`.""" + if self.default_map is not None: + value = self.default_map.get(name) + + if call and callable(value): + return value() + + return value + + return None + + def fail(self, message: str) -> NoReturn: + """Aborts the execution of the program with a specific error + message. + """ + raise UsageError(message, self) + + def abort(self) -> NoReturn: + """Aborts the script.""" + raise Abort() + + def exit(self, code: int = 0) -> NoReturn: + """Exits the application with a given exit code.""" + self.close() + raise Exit(code) + + def get_usage(self) -> str: + """Helper method to get formatted usage string for the current + context and command. + """ + return self.command.get_usage(self) + + def get_help(self) -> str: + """Helper method to get formatted help page for the current + context and command. + """ + return self.command.get_help(self) + + def invoke(self, callback: Callable[..., V], /, *args: Any, **kwargs: Any) -> V: + """Invokes a command callback in exactly the way it expects. There + are two ways to invoke this method: + + 1. the first argument can be a callback and all other arguments and + keyword arguments are forwarded directly to the function. + 2. the first argument is a click command object. In that case all + arguments are forwarded as well but proper click parameters + (options and click arguments) must be keyword arguments and Click + will fill in defaults. + """ + ctx = self + + with augment_usage_errors(self): + with ctx: + return callback(*args, **kwargs) + + def set_parameter_source(self, name: str, source: ParameterSource) -> None: + """Set the source of a parameter. This indicates the location + from which the value of the parameter was obtained. + """ + self._parameter_source[name] = source + + def get_parameter_source(self, name: str) -> ParameterSource | None: + """Get the source of a parameter. This indicates the location + from which the value of the parameter was obtained. + + This can be useful for determining when a user specified a value + on the command line that is the same as the default value. It + will be `ParameterSource.DEFAULT` only if the + value was actually taken from the default. + """ + return self._parameter_source.get(name) + + +class Command(ABC): + """Commands are the basic building block of command line interfaces in + Click. A basic command handles command line parsing and might dispatch + more parsing to commands nested below it. + """ + + context_class: type[Context] = Context + allow_extra_args = False + allow_interspersed_args = True + ignore_unknown_options = False + + def __init__( + self, + name: str | None, + context_settings: MutableMapping[str, Any] | None = None, + callback: Callable[..., Any] | None = None, + params: list["Parameter"] | None = None, + help: str | None = None, + epilog: str | None = None, + short_help: str | None = None, + options_metavar: str | None = "[OPTIONS]", + add_help_option: bool = True, + no_args_is_help: bool = False, + hidden: bool = False, + deprecated: bool | str = False, + ) -> None: + self.name = name + + if context_settings is None: + context_settings = {} + + self.context_settings: MutableMapping[str, Any] = context_settings + + self.callback = callback + self.params: list[Parameter] = params or [] + self.help = help + self.epilog = epilog + self.options_metavar = options_metavar + self.short_help = short_help + self.add_help_option = add_help_option + self._help_option: TyperOption | None = None + self.no_args_is_help = no_args_is_help + self.hidden = hidden + self.deprecated = deprecated + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self.name}>" + + def get_usage(self, ctx: Context) -> str: + """Formats the usage line into a string and returns it.""" + formatter = ctx.make_formatter() + self.format_usage(ctx, formatter) + return formatter.getvalue().rstrip("\n") + + def get_params(self, ctx: Context) -> list["Parameter"]: + params = self.params + help_option = self.get_help_option(ctx) + + if help_option is not None: + params = [*params, help_option] + + return params + + def format_usage(self, ctx: Context, formatter: HelpFormatter) -> None: + """Writes the usage line into the formatter.""" + pieces = self.collect_usage_pieces(ctx) + formatter.write_usage(ctx.command_path, " ".join(pieces)) + + def collect_usage_pieces(self, ctx: Context) -> list[str]: + """Returns all the pieces that go into the usage line and returns + it as a list of strings. + """ + rv = [self.options_metavar] if self.options_metavar else [] + + for param in self.get_params(ctx): + rv.extend(param.get_usage_pieces(ctx)) + + return rv + + def get_help_option_names(self, ctx: Context) -> list[str]: + """Returns the names for the help option.""" + all_names = set(ctx.help_option_names) + for param in self.params: + all_names.difference_update(param.opts) + all_names.difference_update(param.secondary_opts) + return list(all_names) + + def get_help_option(self, ctx: Context) -> Union["TyperOption", None]: + """Returns the help option object.""" + help_option_names = self.get_help_option_names(ctx) + + if not help_option_names or not self.add_help_option: + return None + + # Cache the help option object in private _help_option attribute to + # avoid creating it multiple times. Not doing this will break the + # callback ordering by iter_params_for_processing(), which relies on + # object comparison. + if self._help_option is None: + # Avoid circular import. + from .decorators import help_option + + # Apply help_option decorator and pop resulting option + help_option(help_option_names)(self) + self._help_option = cast("TyperOption", self.params.pop()) + + return self._help_option + + def make_parser(self, ctx: Context) -> _OptionParser: + """Creates the underlying option parser for this command.""" + parser = _OptionParser(ctx) + for param in self.get_params(ctx): + param.add_to_parser(parser, ctx) + return parser + + def get_help(self, ctx: Context) -> str: + """Formats the help into a string and returns it.""" + formatter = ctx.make_formatter() + self.format_help(ctx, formatter) + return formatter.getvalue().rstrip("\n") + + def get_short_help_str(self, limit: int = 45) -> str: + """Gets short help for the command or makes it by shortening the + long help string. + """ + if self.short_help: + text = inspect.cleandoc(self.short_help) + elif self.help: + text = make_default_short_help(self.help, limit) + else: + text = "" + + if self.deprecated: + deprecated_message = ( + f"(DEPRECATED: {self.deprecated})" + if isinstance(self.deprecated, str) + else "(DEPRECATED)" + ) + text = f"{text} {deprecated_message}" + + return text.strip() + + def format_help(self, ctx: Context, formatter: HelpFormatter) -> None: + """Writes the help into the formatter if it exists.""" + self.format_usage(ctx, formatter) + self.format_help_text(ctx, formatter) + self.format_options(ctx, formatter) + self.format_epilog(ctx, formatter) + + def format_help_text(self, ctx: Context, formatter: HelpFormatter) -> None: + """Writes the help text to the formatter if it exists.""" + if self.help is not None: + # truncate the help text to the first form feed + text = inspect.cleandoc(self.help).partition("\f")[0] + else: + text = "" + + if self.deprecated: + deprecated_message = ( + f"(DEPRECATED: {self.deprecated})" + if isinstance(self.deprecated, str) + else "(DEPRECATED)" + ) + text = f"{text} {deprecated_message}" + + if text: + formatter.write_paragraph() + + with formatter.indentation(): + formatter.write_text(text) + + @abstractmethod + def format_options(self, ctx: Context, formatter: HelpFormatter) -> None: + pass # pragma: no cover + + def format_epilog(self, ctx: Context, formatter: HelpFormatter) -> None: + """Writes the epilog into the formatter if it exists.""" + if self.epilog: + epilog = inspect.cleandoc(self.epilog) + formatter.write_paragraph() + + with formatter.indentation(): + formatter.write_text(epilog) + + def make_context( + self, + info_name: str | None, + args: list[str], + parent: Context | None = None, + **extra: Any, + ) -> Context: + """This function when given an info name and arguments will kick + off the parsing and create a new `Context`. It does not + invoke the actual command callback though. + + To quickly customize the context class used without overriding + this method, set the `context_class` attribute. + """ + for key, value in self.context_settings.items(): + if key not in extra: + extra[key] = value + + ctx = self.context_class(self, info_name=info_name, parent=parent, **extra) + + with ctx.scope(cleanup=False): + self.parse_args(ctx, args) + return ctx + + def parse_args(self, ctx: Context, args: list[str]) -> list[str]: + if not args and self.no_args_is_help and not ctx.resilient_parsing: + raise NoArgsIsHelpError(ctx) # pragma: no cover + + parser = self.make_parser(ctx) + opts, args, param_order = parser.parse_args(args=args) + + for param in iter_params_for_processing(param_order, self.get_params(ctx)): + _, args = param.handle_parse_result(ctx, opts, args) + + if args and not ctx.allow_extra_args and not ctx.resilient_parsing: + ctx.fail(f"Got unexpected extra argument(s) ({' '.join(map(str, args))})") + + ctx.args = args + ctx._opt_prefixes.update(parser._opt_prefixes) + return args + + def invoke(self, ctx: Context) -> Any: + """Given a context, this invokes the attached callback (if it exists) + in the right way. + """ + if self.deprecated: + extra_message = ( + f" {self.deprecated}" if isinstance(self.deprecated, str) else "" + ) + message = f"DeprecationWarning: The command {self.name!r} is deprecated.{extra_message}" + echo(style(message, fg="red"), err=True) + + if self.callback is not None: + return ctx.invoke(self.callback, **ctx.params) + + def shell_complete(self, ctx: Context, incomplete: str) -> list["CompletionItem"]: + """Return a list of completions for the incomplete value. Looks + at the names of options and chained multi-commands. + + Any command could be part of a chained multi-command, so sibling + commands are valid at any point during command completion. + """ + # avoid circular imports + from .shell_completion import CompletionItem + + results: list[CompletionItem] = [] + + if incomplete and not incomplete[0].isalnum(): + # avoid circular imports + from ..core import TyperOption + + for param in self.get_params(ctx): + if ( + not isinstance(param, TyperOption) + or param.hidden + or ( + not param.multiple + and ctx.get_parameter_source(param.name) # type: ignore + is ParameterSource.COMMANDLINE + ) + ): + continue + + results.extend( + CompletionItem(name, help=param.help) + for name in [*param.opts, *param.secondary_opts] + if name.startswith(incomplete) + ) + + return results + + @abstractmethod + def main( + self, + args: Sequence[str] | None = None, + prog_name: str | None = None, + complete_var: str | None = None, + standalone_mode: bool = True, + windows_expand_args: bool = True, + **extra: Any, + ) -> Any: + pass # pragma: no cover + + @abstractmethod + def _main_shell_completion( + self, + ctx_args: MutableMapping[str, Any], + prog_name: str, + complete_var: str | None = None, + ) -> None: + pass # pragma: no cover + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + """Alias for self.main""" + return self.main(*args, **kwargs) + + +class Parameter(ABC): + r"""A parameter to a command comes in two versions: they are either + `Option`\s or `Argument`\s. + + Some settings are supported by both options and arguments. + """ + + param_type_name = "parameter" + + def __init__( + self, + param_decls: Sequence[str] | None = None, + type: types.ParamType | Any | None = None, + required: bool = False, + default: Any | Callable[[], Any] | None = None, + callback: Callable[[Context, "Parameter", Any], Any] | None = None, + nargs: int | None = None, + multiple: bool = False, + metavar: str | None = None, + expose_value: bool = True, + is_eager: bool = False, + envvar: str | Sequence[str] | None = None, + shell_complete: Callable[ + [Context, "Parameter", str], list["CompletionItem"] | list[str] + ] + | None = None, + ) -> None: + self.name: str | None + self.opts: list[str] + self.secondary_opts: list[str] + self.name, self.opts, self.secondary_opts = self._parse_decls( + param_decls or (), expose_value + ) + self.type: types.ParamType = types.convert_type(type, default) + + # Default nargs to what the type tells us if we have that + # information available. + if nargs is None: + if self.type.is_composite: + nargs = self.type.arity + else: + nargs = 1 + + self.required = required + self.callback = callback + self.nargs = nargs + self.multiple = multiple + self.expose_value = expose_value + self.default: Any | Callable[[], Any] | None = default + self.is_eager = is_eager + self.metavar = metavar + self.envvar = envvar + self._custom_shell_complete = shell_complete + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self.name}>" + + @abstractmethod + def _parse_decls( + self, decls: Sequence[str], expose_value: bool + ) -> tuple[str | None, list[str], list[str]]: + pass # pragma: no cover + + @property + def human_readable_name(self) -> str: + """Returns the human readable name of this parameter. This is the + same as the name for options, but the metavar for arguments. + """ + assert self.name is not None, "self.name should be set" + return self.name + + def make_metavar(self, ctx: Context) -> str: + if self.metavar is not None: + return self.metavar + + metavar = self.type.get_metavar(param=self, ctx=ctx) + + if metavar is None: + metavar = self.type.name.upper() + + if self.nargs != 1: + metavar += "..." + + return metavar + + @overload + def get_default(self, ctx: Context, call: Literal[True] = True) -> Any | None: ... + + @overload + def get_default( + self, ctx: Context, call: bool = ... + ) -> Any | Callable[[], Any] | None: ... + + def get_default( + self, ctx: Context, call: bool = True + ) -> Any | Callable[[], Any] | None: + """Get the default for the parameter""" + value = ctx.lookup_default(self.name, call=False) # type: ignore + + if value is None: + value = self.default + + if call and callable(value): + value = value() + + return value + + @abstractmethod + def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None: + pass # pragma: no cover + + def consume_value( + self, ctx: Context, opts: Mapping[str, Any] + ) -> tuple[Any, ParameterSource]: + value = opts.get(self.name) # type: ignore + source = ParameterSource.COMMANDLINE + + if value is None: + value = self.value_from_envvar(ctx) + source = ParameterSource.ENVIRONMENT + + if value is None: + value = ctx.lookup_default(self.name) # type: ignore + source = ParameterSource.DEFAULT_MAP + + if value is None: + value = self.get_default(ctx) + source = ParameterSource.DEFAULT + + return value, source + + def type_cast_value(self, ctx: Context, value: Any) -> Any: + """Convert and validate a value against the parameter's + `type`, `multiple`, and `nargs`. + """ + if value is None: + return () if self.multiple or self.nargs == -1 else None + + def check_iter(value: Any) -> Iterator[Any]: + if isinstance(value, str): + raise BadParameter("Value must be an iterable.", ctx=ctx, param=self) + else: + return iter(value) + + # Define the conversion function based on nargs and type. + if self.nargs == 1 or self.type.is_composite: + + def convert(value: Any) -> Any: + return self.type(value, param=self, ctx=ctx) + + elif self.nargs == -1: + + def convert(value: Any) -> Any: # tuple[t.Any, ...] + return tuple(self.type(x, self, ctx) for x in check_iter(value)) + + # TODO: evaluate whether we need to keep this in Typer + else: # nargs > 1 + + def convert(value: Any) -> Any: # tuple[t.Any, ...] + value = tuple(check_iter(value)) + + if len(value) != self.nargs: + raise BadParameter( + f"Takes {self.nargs} values but {len(value)} given.", + ctx=ctx, + param=self, + ) + + return tuple(self.type(x, self, ctx) for x in value) + + if self.multiple: + return tuple(convert(x) for x in check_iter(value)) + + return convert(value) + + @abstractmethod + def value_is_missing(self, value: Any) -> bool: + pass # pragma: no cover + + def process_value(self, ctx: Context, value: Any) -> Any: + """Process the value of this parameter""" + value = self.type_cast_value(ctx, value) + + if self.required and self.value_is_missing(value): + raise MissingParameter(ctx=ctx, param=self) + + if self.callback is not None: + value = self.callback(ctx, self, value) + + return value + + def resolve_envvar_value(self, ctx: Context) -> str | None: + """Returns the value found in the environment variable(s) attached to this + parameter. + + Environment variables values are `always returned as strings + <https://docs.python.org/3/library/os.html#os.environ>`_. + + This method returns ``None`` if: + + - the `envvar` property is not set on `Parameter`, + - the environment variable is not found in the environment, + - the variable is found in the environment but its value is empty (i.e. the + environment variable is present but has an empty string). + + If `envvar` is setup with multiple environment variables, + then only the first non-empty value is returned. + """ + if self.envvar is None: + return None + + if isinstance(self.envvar, str): + rv = os.environ.get(self.envvar) + + if rv: + return rv + else: + for envvar in self.envvar: + rv = os.environ.get(envvar) + + # Return the first non-empty value of the list of environment variables. + if rv: + return rv + # Else, absence of value is interpreted as an environment variable that + # is not set, so proceed to the next one. + + return None + + def value_from_envvar(self, ctx: Context) -> str | Sequence[str] | None: + """Process the raw environment variable string for this parameter. + + Returns the string as-is or splits it into a sequence of strings if the + parameter is expecting multiple values (i.e. its `nargs` property is set + to a value other than ``1``). + """ + rv: Any | None = self.resolve_envvar_value(ctx) + + if rv is not None and self.nargs != 1: + rv = self.type.split_envvar_value(rv) + + return rv + + def handle_parse_result( + self, ctx: Context, opts: Mapping[str, Any], args: list[str] + ) -> tuple[Any, list[str]]: + """Process the value produced by the parser from user input. + + Always process the value through the Parameter's `type`, wherever it + comes from. + + If the parameter is deprecated, this method warn the user about it. But only if + the value has been explicitly set by the user (and as such, is not coming from + a default). + """ + with augment_usage_errors(ctx, param=self): + value, source = self.consume_value(ctx, opts) + + ctx.set_parameter_source(self.name, source) # type: ignore + + # Process the value through the parameter's type. + try: + value = self.process_value(ctx, value) + except Exception: + if not ctx.resilient_parsing: + raise + value = None + + if self.expose_value: + ctx.params[self.name] = value # type: ignore + + return value, args + + @abstractmethod + def get_help_record(self, ctx: Context) -> tuple[str, str] | None: + pass # pragma: no cover + + def get_usage_pieces(self, ctx: Context) -> list[str]: + return [] + + def get_error_hint(self, ctx: Context) -> str: + """Get a stringified version of the param for use in error messages to + indicate which param caused the error. + """ + hint_list = self.opts or [self.human_readable_name] + return " / ".join(f"'{x}'" for x in hint_list) + + def shell_complete(self, ctx: Context, incomplete: str) -> list["CompletionItem"]: + """Return a list of completions for the incomplete value. If a + ``shell_complete`` function was given during init, it is used. + Otherwise, the `type` `ParamType.shell_complete` function is used. + """ + if self._custom_shell_complete is not None: + results = self._custom_shell_complete(ctx, self, incomplete) + + if results and isinstance(results[0], str): + from .shell_completion import CompletionItem + + results = [CompletionItem(c) for c in results] + + return cast("list[CompletionItem]", results) + + return self.type.shell_complete(ctx, self, incomplete) diff --git a/contrib/python/typer/typer/_click/decorators.py b/contrib/python/typer/typer/_click/decorators.py new file mode 100644 index 00000000000..28ad656a8c7 --- /dev/null +++ b/contrib/python/typer/typer/_click/decorators.py @@ -0,0 +1,60 @@ +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar + +from .core import Command, Context, Parameter +from .utils import echo + +if TYPE_CHECKING: + from ..core import TyperGroup, TyperOption + + GrpType = TypeVar("GrpType", bound=TyperGroup) + + +P = ParamSpec("P") + +R = TypeVar("R") +T = TypeVar("T") +_AnyCallable = Callable[..., Any] + + +CmdType = TypeVar("CmdType", bound=Command) + + +def option( + param_decls: list[str], cls: type["TyperOption"] | None = None, **attrs: Any +) -> Callable[[Command], Command]: + """Attaches an option to the command.""" + if cls is None: + # avoid circular imports + from ..core import TyperOption + + cls = TyperOption + + def decorator(f: Command) -> Command: + param = cls(param_decls=param_decls, **attrs) + f.params.append(param) + return f + + return decorator + + +def help_option(param_decls: list[str]) -> Callable[[Command], Command]: + """Help option which prints the help page and exits the program.""" + + def show_help(ctx: Context, param: Parameter, value: bool) -> None: + """Callback that print the help page on ``<stdout>`` and exits.""" + if value and not ctx.resilient_parsing: + echo(ctx.get_help(), color=ctx.color) + ctx.exit() + + assert len(param_decls) > 0, "At least one help option should be provided" + + return option( + param_decls, + is_flag=True, + expose_value=False, + is_eager=True, + help="Show this message and exit.", + callback=show_help, + required=False, + ) diff --git a/contrib/python/typer/typer/_click/exceptions.py b/contrib/python/typer/typer/_click/exceptions.py new file mode 100644 index 00000000000..75ba2296bbb --- /dev/null +++ b/contrib/python/typer/typer/_click/exceptions.py @@ -0,0 +1,260 @@ +from collections.abc import Sequence +from typing import IO, TYPE_CHECKING, Any, Union + +from ._compat import get_text_stderr +from .globals import resolve_color_default +from .utils import echo, format_filename + +if TYPE_CHECKING: + from .core import Command, Context, Parameter + + +def _join_param_hints(param_hint: Sequence[str] | str | None) -> str | None: + if param_hint is not None and not isinstance(param_hint, str): + return " / ".join(repr(x) for x in param_hint) + + return param_hint + + +class ClickException(Exception): + """An exception that Click can handle and show to the user.""" + + exit_code = 1 + + def __init__(self, message: str) -> None: + super().__init__(message) + # The context will be removed by the time we print the message, so cache + # the color settings here to be used later on (in `show`) + self.show_color: bool | None = resolve_color_default() + self.message = message + + def format_message(self) -> str: + return self.message + + def __str__(self) -> str: + return self.message + + def show(self, file: IO[Any] | None = None) -> None: + if file is None: + file = get_text_stderr() + + echo( + f"Error: {self.format_message()}", + file=file, + color=self.show_color, + ) + + +class UsageError(ClickException): + """An internal exception that signals a usage error. This typically + aborts any further handling. + """ + + exit_code = 2 + + def __init__(self, message: str, ctx: Union["Context", None] = None) -> None: + super().__init__(message) + self.ctx = ctx + self.cmd: Command | None = self.ctx.command if self.ctx else None + + def show(self, file: IO[Any] | None = None) -> None: + if file is None: + file = get_text_stderr() + color = None + hint = "" + if ( + self.ctx is not None + and self.ctx.command.get_help_option(self.ctx) is not None + ): + command = self.ctx.command_path + option = self.ctx.help_option_names[0] + hint = f"Try '{command} {option}' for help.\n" + if self.ctx is not None: + color = self.ctx.color + echo(f"{self.ctx.get_usage()}\n{hint}", file=file, color=color) + echo( + f"Error: {self.format_message()}", + file=file, + color=color, + ) + + +class BadParameter(UsageError): + """An exception that formats out a standardized error message for a + bad parameter. This is useful when thrown from a callback or type as + Click will attach contextual information to it (for instance, which + parameter it is). + """ + + def __init__( + self, + message: str, + ctx: Union["Context", None] = None, + param: Union["Parameter", None] = None, + param_hint: Sequence[str] | str | None = None, + ) -> None: + super().__init__(message, ctx) + self.param = param + self.param_hint = param_hint + + def format_message(self) -> str: + if self.param_hint is not None: + param_hint = self.param_hint + elif self.param is not None: + param_hint = self.param.get_error_hint(self.ctx) # type: ignore + else: + return f"Invalid value: {self.message}" + + hint = _join_param_hints(param_hint) + return f"Invalid value for {hint}: {self.message}" + + +class MissingParameter(BadParameter): + """Raised if click required an option or argument but it was not + provided when invoking the script. + """ + + def __init__( + self, + message: str | None = None, + ctx: Union["Context", None] = None, + param: Union["Parameter", None] = None, + param_hint: Sequence[str] | str | None = None, + param_type: str | None = None, + ) -> None: + super().__init__(message or "", ctx, param, param_hint) + self.param_type = param_type + + def format_message(self) -> str: + if self.param_hint is not None: + param_hint: Sequence[str] | str | None = self.param_hint + elif self.param is not None: + param_hint = self.param.get_error_hint(self.ctx) # type: ignore + else: + param_hint = None + + param_hint = _join_param_hints(param_hint) + param_hint = f" {param_hint}" if param_hint else "" + + param_type = self.param_type + if param_type is None and self.param is not None: + param_type = self.param.param_type_name + + msg = self.message + if self.param is not None: + msg_extra = self.param.type.get_missing_message( + param=self.param, ctx=self.ctx + ) + if msg_extra: + if msg: + msg += f". {msg_extra}" + else: + msg = msg_extra + + msg = f" {msg}" if msg else "" + + # Translate param_type for known types. + if param_type == "argument": + missing = "Missing argument" + elif param_type == "option": + missing = "Missing option" + elif param_type == "parameter": + missing = "Missing parameter" + else: + missing = f"Missing {param_type}" + + return f"{missing}{param_hint}.{msg}" + + def __str__(self) -> str: + if not self.message: + param_name = self.param.name if self.param else None + return f"Missing parameter: {param_name}" + else: + return self.message + + +class NoSuchOption(UsageError): + """Raised if click attempted to handle an option that does not + exist. + """ + + def __init__( + self, + option_name: str, + message: str | None = None, + possibilities: Sequence[str] | None = None, + ctx: Union["Context", None] = None, + ) -> None: + if message is None: + message = f"No such option: {option_name}" + + super().__init__(message, ctx) + self.option_name = option_name + self.possibilities = possibilities + + def format_message(self) -> str: + if not self.possibilities: + return self.message + + possibility_str = ", ".join(sorted(self.possibilities)) + suggest = f"(Possible options: {possibility_str})" + return f"{self.message} {suggest}" + + +class BadOptionUsage(UsageError): + """Raised if an option is generally supplied but the use of the option + was incorrect. This is for instance raised if the number of arguments + for an option is not correct. + """ + + def __init__( + self, option_name: str, message: str, ctx: Union["Context", None] = None + ) -> None: + super().__init__(message, ctx) + self.option_name = option_name + + +class BadArgumentUsage(UsageError): + """Raised if an argument is generally supplied but the use of the argument + was incorrect. This is for instance raised if the number of values + for an argument is not correct. + """ + + +class NoArgsIsHelpError(UsageError): + def __init__(self, ctx: "Context") -> None: + self.ctx: Context + super().__init__(ctx.get_help(), ctx=ctx) + + def show(self, file: IO[Any] | None = None) -> None: + echo(self.format_message(), file=file, err=True, color=self.ctx.color) + + +class FileError(ClickException): + """Raised if a file cannot be opened.""" + + def __init__(self, filename: str, hint: str | None = None) -> None: + if hint is None: + hint = "unknown error" + + super().__init__(hint) + self.ui_filename: str = format_filename(filename) + self.filename = filename + + def format_message(self) -> str: + return f"Could not open file {self.ui_filename!r}: {self.message}" + + +class Abort(RuntimeError): + """An internal signalling exception that signals Click to abort.""" + + +class Exit(RuntimeError): + """An exception that indicates that the application should exit with some + status code. + """ + + __slots__ = ("exit_code",) + + def __init__(self, code: int = 0) -> None: + self.exit_code: int = code diff --git a/contrib/python/typer/typer/_click/formatting.py b/contrib/python/typer/typer/_click/formatting.py new file mode 100644 index 00000000000..b5eaab3bd0b --- /dev/null +++ b/contrib/python/typer/typer/_click/formatting.py @@ -0,0 +1,272 @@ +from collections.abc import Iterable, Iterator, Sequence +from contextlib import contextmanager + +from ._compat import term_len +from .parser import _split_opt + +# Can force a width. This is used by the test system +FORCED_WIDTH: int | None = None + + +def measure_table(rows: Iterable[tuple[str, str]]) -> tuple[int, ...]: + widths: dict[int, int] = {} + + for row in rows: + for idx, col in enumerate(row): + widths[idx] = max(widths.get(idx, 0), term_len(col)) + + return tuple(y for x, y in sorted(widths.items())) + + +def iter_rows( + rows: Iterable[tuple[str, str]], col_count: int +) -> Iterator[tuple[str, ...]]: + for row in rows: + yield row + ("",) * (col_count - len(row)) + + +def wrap_text( + text: str, + width: int = 78, + initial_indent: str = "", + subsequent_indent: str = "", + preserve_paragraphs: bool = False, +) -> str: + """A helper function that intelligently wraps text. By default, it + assumes that it operates on a single paragraph of text but if the + `preserve_paragraphs` parameter is provided it will intelligently + handle paragraphs (defined by two empty lines). + + If paragraphs are handled, a paragraph can be prefixed with an empty + line containing the ``\\b`` character (``\\x08``) to indicate that + no rewrapping should happen in that block. + """ + from ._textwrap import TextWrapper + + text = text.expandtabs() + wrapper = TextWrapper( + width, + initial_indent=initial_indent, + subsequent_indent=subsequent_indent, + replace_whitespace=False, + ) + if not preserve_paragraphs: + return wrapper.fill(text) + + p: list[tuple[int, bool, str]] = [] + buf: list[str] = [] + indent = None + + def _flush_par() -> None: + if not buf: + return + if buf[0].strip() == "\b": + p.append((indent or 0, True, "\n".join(buf[1:]))) + else: + p.append((indent or 0, False, " ".join(buf))) + del buf[:] + + for line in text.splitlines(): + if not line: + _flush_par() + indent = None + else: + if indent is None: + orig_len = term_len(line) + line = line.lstrip() + indent = orig_len - term_len(line) + buf.append(line) + _flush_par() + + rv = [] + for indent, raw, text in p: + with wrapper.extra_indent(" " * indent): + if raw: + rv.append(wrapper.indent_only(text)) + else: + rv.append(wrapper.fill(text)) + + return "\n\n".join(rv) + + +class HelpFormatter: + """This class helps with formatting text-based help pages. It's + usually just needed for very special internal cases, but it's also + exposed so that developers can write their own fancy outputs. + + At present, it always writes into memory. + """ + + def __init__( + self, + indent_increment: int = 2, + width: int | None = None, + max_width: int | None = None, + ) -> None: + self.indent_increment = indent_increment + if max_width is None: + max_width = 80 + if width is None: + import shutil + + width = FORCED_WIDTH + if width is None: + width = max(min(shutil.get_terminal_size().columns, max_width) - 2, 50) + self.width = width + self.current_indent: int = 0 + self.buffer: list[str] = [] + + def write(self, string: str) -> None: + """Writes a unicode string into the internal buffer.""" + self.buffer.append(string) + + def indent(self) -> None: + """Increases the indentation.""" + self.current_indent += self.indent_increment + + def dedent(self) -> None: + """Decreases the indentation.""" + self.current_indent -= self.indent_increment + + def write_usage(self, prog: str, args: str = "", prefix: str | None = None) -> None: + """Writes a usage line into the buffer.""" + if prefix is None: + prefix = "Usage: " + + usage_prefix = f"{prefix:>{self.current_indent}}{prog} " + text_width = self.width - self.current_indent + + if text_width >= (term_len(usage_prefix) + 20): + # The arguments will fit to the right of the prefix. + indent = " " * term_len(usage_prefix) + self.write( + wrap_text( + args, + text_width, + initial_indent=usage_prefix, + subsequent_indent=indent, + ) + ) + else: + # The prefix is too long, put the arguments on the next line. + self.write(usage_prefix) + self.write("\n") + indent = " " * (max(self.current_indent, term_len(prefix)) + 4) + self.write( + wrap_text( + args, text_width, initial_indent=indent, subsequent_indent=indent + ) + ) + + self.write("\n") + + def write_heading(self, heading: str) -> None: + """Writes a heading into the buffer.""" + self.write(f"{'':>{self.current_indent}}{heading}:\n") + + def write_paragraph(self) -> None: + """Writes a paragraph into the buffer.""" + if self.buffer: + self.write("\n") + + def write_text(self, text: str) -> None: + """Writes re-indented text into the buffer. This rewraps and + preserves paragraphs. + """ + indent = " " * self.current_indent + self.write( + wrap_text( + text, + self.width, + initial_indent=indent, + subsequent_indent=indent, + preserve_paragraphs=True, + ) + ) + self.write("\n") + + def write_dl( + self, + rows: Sequence[tuple[str, str]], + col_max: int = 30, + col_spacing: int = 2, + ) -> None: + """Writes a definition list into the buffer. This is how options + and commands are usually formatted. + """ + rows = list(rows) + widths = measure_table(rows) + if len(widths) != 2: # pragma: no cover + raise TypeError("Expected two columns for definition list") + + first_col = min(widths[0], col_max) + col_spacing + + for first, second in iter_rows(rows, len(widths)): + self.write(f"{'':>{self.current_indent}}{first}") + if not second: + self.write("\n") + continue + if term_len(first) <= first_col - col_spacing: + self.write(" " * (first_col - term_len(first))) + else: + self.write("\n") + self.write(" " * (first_col + self.current_indent)) + + text_width = max(self.width - first_col - 2, 10) + wrapped_text = wrap_text(second, text_width, preserve_paragraphs=True) + lines = wrapped_text.splitlines() + + if lines: + self.write(f"{lines[0]}\n") + + for line in lines[1:]: + self.write(f"{'':>{first_col + self.current_indent}}{line}\n") + else: # pragma: no cover + self.write("\n") + + @contextmanager + def section(self, name: str) -> Iterator[None]: + """Helpful context manager that writes a paragraph, a heading, + and the indents. + """ + self.write_paragraph() + self.write_heading(name) + self.indent() + try: + yield + finally: + self.dedent() + + @contextmanager + def indentation(self) -> Iterator[None]: + """A context manager that increases the indentation.""" + self.indent() + try: + yield + finally: + self.dedent() + + def getvalue(self) -> str: + """Returns the buffer contents.""" + return "".join(self.buffer) + + +def join_options(options: Sequence[str]) -> tuple[str, bool]: + """Given a list of option strings this joins them in the most appropriate + way and returns them in the form ``(formatted_string, + any_prefix_is_slash)`` where the second item in the tuple is a flag that + indicates if any of the option prefixes was a slash. + """ + rv = [] + any_prefix_is_slash = False + + for opt in options: + prefix = _split_opt(opt)[0] + + if prefix == "/": + any_prefix_is_slash = True + + rv.append((len(prefix), opt)) + + rv.sort(key=lambda x: x[0]) + return ", ".join(x[1] for x in rv), any_prefix_is_slash diff --git a/contrib/python/typer/typer/_click/globals.py b/contrib/python/typer/typer/_click/globals.py new file mode 100644 index 00000000000..372dc40749b --- /dev/null +++ b/contrib/python/typer/typer/_click/globals.py @@ -0,0 +1,61 @@ +from threading import local +from typing import TYPE_CHECKING, Literal, Union, cast, overload + +if TYPE_CHECKING: + from .core import Context + +_local = local() + + +@overload +def get_current_context(silent: Literal[False] = False) -> "Context": ... + + +@overload +def get_current_context(silent: bool = ...) -> Union["Context", None]: ... + + +def get_current_context(silent: bool = False) -> Union["Context", None]: + """Returns the current click context. This can be used as a way to + access the current context object from anywhere. This is a more implicit + alternative to the `pass_context` decorator. This function is + primarily useful for helpers such as `echo` which might be + interested in changing its behavior based on the current context. + + To push the current context, `Context.scope` can be used. + """ + try: + return cast("Context", _local.stack[-1]) + except (AttributeError, IndexError) as e: + if not silent: + raise RuntimeError( + "There is no active click context." + ) from e # pragma: no cover + + return None + + +def push_context(ctx: "Context") -> None: + """Pushes a new context to the current stack.""" + _local.__dict__.setdefault("stack", []).append(ctx) + + +def pop_context() -> None: + """Removes the top level from the stack.""" + _local.stack.pop() + + +def resolve_color_default(color: bool | None = None) -> bool | None: + """Internal helper to get the default value of the color flag. If a + value is passed it's returned unchanged, otherwise it's looked up from + the current context. + """ + if color is not None: + return color + + ctx = get_current_context(silent=True) + + if ctx is not None: + return ctx.color + + return None diff --git a/contrib/python/typer/typer/_click/parser.py b/contrib/python/typer/typer/_click/parser.py new file mode 100644 index 00000000000..3eb5acd632a --- /dev/null +++ b/contrib/python/typer/typer/_click/parser.py @@ -0,0 +1,459 @@ +""" +This module started out as largely a copy paste from the stdlib's +optparse module with the features removed that we do not need from +optparse because we implement them in Click on a higher level (for +instance type handling, help formatting and a lot more). + +The plan is to remove more and more from here over time. + +The reason this is a different module and not optparse from the stdlib +is that there are differences in 2.x and 3.x about the error messages +generated and optparse in the stdlib uses gettext for no good reason +and might cause us issues. + +Click uses parts of optparse written by Gregory P. Ward and maintained +by the Python Software Foundation. This is limited to code in parser.py. + +Copyright 2001-2006 Gregory P. Ward. All rights reserved. +Copyright 2002-2006 Python Software Foundation. All rights reserved. +""" + +# This code uses parts of optparse written by Gregory P. Ward and +# maintained by the Python Software Foundation. +# Copyright 2001-2006 Gregory P. Ward +# Copyright 2002-2006 Python Software Foundation +from collections import deque +from collections.abc import Sequence +from typing import ( + TYPE_CHECKING, + Any, + TypeVar, + Union, +) + +from .exceptions import BadArgumentUsage, BadOptionUsage, NoSuchOption, UsageError + +if TYPE_CHECKING: + from typer.core import TyperArgument as CoreArgument + from typer.core import TyperOption as CoreOption + + from .core import Context + from .core import Parameter as CoreParameter + +V = TypeVar("V") + + +def _unpack_args( + args: Sequence[str], nargs_spec: Sequence[int] +) -> tuple[Sequence[str | Sequence[str | None] | None], list[str]]: + """Given an iterable of arguments and an iterable of nargs specifications, + it returns a tuple with all the unpacked arguments at the first index + and all remaining arguments as the second. + + The nargs specification is the number of arguments that should be consumed + or `-1` to indicate that this position should eat up all the remainders. + """ + args = deque(args) + nargs_spec = deque(nargs_spec) + rv: list[str | tuple[str | None, ...] | None] = [] + spos: int | None = None + + def _fetch(c: deque[V]) -> V | None: + try: + if spos is None: + return c.popleft() + else: + return c.pop() + except IndexError: + return None + + while nargs_spec: + nargs = _fetch(nargs_spec) + assert nargs is not None + + if nargs == 1: + rv.append(_fetch(args)) + elif nargs > 1: + x = [_fetch(args) for _ in range(nargs)] + + # If we're reversed, we're pulling in the arguments in reverse, + # so we need to turn them around. + if spos is not None: + x.reverse() + + rv.append(tuple(x)) + elif nargs < 0: + if spos is not None: # pragma: no cover + raise TypeError("Cannot have two nargs < 0") + + spos = len(rv) + rv.append(None) + + # spos is the position of the wildcard (star). If it's not `None`, + # we fill it with the remainder. + if spos is not None: + rv[spos] = tuple(args) + args = [] + rv[spos + 1 :] = reversed(rv[spos + 1 :]) + + return tuple(rv), list(args) + + +def _split_opt(opt: str) -> tuple[str, str]: + first = opt[:1] + if first.isalnum(): + return "", opt + if opt[1:2] == first: + return opt[:2], opt[2:] + return first, opt[1:] + + +def _normalize_opt(opt: str, ctx: Union["Context", None]) -> str: + if ctx is None or ctx.token_normalize_func is None: + return opt + prefix, opt = _split_opt(opt) + return f"{prefix}{ctx.token_normalize_func(opt)}" + + +class _Option: + def __init__( + self, + obj: "CoreOption", + opts: Sequence[str], + dest: str | None, + action: str = "store", + nargs: int = 1, + const: Any | None = None, + ): + self._short_opts = [] + self._long_opts = [] + self.prefixes: set[str] = set() + + for opt in opts: + prefix, value = _split_opt(opt) + if not prefix: # pragma: no cover + raise ValueError(f"Invalid start character for option ({opt})") + self.prefixes.add(prefix[0]) + if len(prefix) == 1 and len(value) == 1: + self._short_opts.append(opt) + else: + self._long_opts.append(opt) + self.prefixes.add(prefix) + + self.dest = dest + self.action = action + self.nargs = nargs + self.const = const + self.obj = obj + + @property + def takes_value(self) -> bool: + return self.action in ("store", "append") + + def process(self, value: Any, state: "_ParsingState") -> None: + if self.action == "store": + state.opts[self.dest] = value # type: ignore + elif self.action == "store_const": + state.opts[self.dest] = self.const # type: ignore + elif self.action == "append": + state.opts.setdefault(self.dest, []).append(value) # type: ignore + elif self.action == "append_const": + state.opts.setdefault(self.dest, []).append(self.const) # type: ignore + elif self.action == "count": + state.opts[self.dest] = state.opts.get(self.dest, 0) + 1 # type: ignore + else: # pragma: no cover + raise ValueError(f"unknown action '{self.action}'") + state.order.append(self.obj) + + +class _Argument: + def __init__(self, obj: "CoreArgument", dest: str | None, nargs: int = 1): + self.dest = dest + self.nargs = nargs + self.obj = obj + + def process( + self, + value: str | Sequence[str | None] | None, + state: "_ParsingState", + ) -> None: + if self.nargs > 1: + assert value is not None + holes = sum(1 for x in value if x is None) + if holes == len(value): + value = None + elif holes != 0: + raise BadArgumentUsage( + f"Argument {self.dest!r} takes {self.nargs} values." + ) + + if self.nargs == -1 and value == (): + # Replace empty tuple with None so regular default resolution + # (env var, default map, and parameter default) can be tried. + value = None + + state.opts[self.dest] = value # type: ignore + state.order.append(self.obj) + + +class _ParsingState: + def __init__(self, rargs: list[str]) -> None: + self.opts: dict[str, Any] = {} + self.largs: list[str] = [] + self.rargs = rargs + self.order: list[CoreParameter] = [] + + +class _OptionParser: + """The option parser is an internal class that is ultimately used to + parse options and arguments. It's modelled after optparse and brings + a similar but vastly simplified API. It should generally not be used + directly as the high level Click classes wrap it for you. + + It's not nearly as extensible as optparse or argparse as it does not + implement features that are implemented on a higher level (such as + types or defaults). + """ + + def __init__(self, ctx: Union["Context", None] = None) -> None: + self.ctx = ctx + # This controls how the parser deals with interspersed arguments. + # If this is set to `False`, the parser will stop on the first + # non-option. Click uses this to implement nested subcommands + # safely. + self.allow_interspersed_args: bool = True + # This tells the parser how to deal with unknown options. By + # default it will error out (which is sensible), but there is a + # second mode where it will ignore it and continue processing + # after shifting all the unknown options into the resulting args. + self.ignore_unknown_options: bool = False + + if ctx is not None: + self.allow_interspersed_args = ctx.allow_interspersed_args + self.ignore_unknown_options = ctx.ignore_unknown_options + + self._short_opt: dict[str, _Option] = {} + self._long_opt: dict[str, _Option] = {} + self._opt_prefixes = {"-", "--"} + self._args: list[_Argument] = [] + + def add_option( + self, + obj: "CoreOption", + opts: Sequence[str], + dest: str | None, + action: str = "store", + nargs: int = 1, + const: Any | None = None, + ) -> None: + """Adds a new option named `dest` to the parser. The destination + is not inferred (unlike with optparse) and needs to be explicitly + provided. Action can be any of ``store``, ``store_const``, + ``append``, ``append_const`` or ``count``. + + The `obj` can be used to identify the option in the order list + that is returned from the parser. + """ + opts = [_normalize_opt(opt, self.ctx) for opt in opts] + option = _Option(obj, opts, dest, action=action, nargs=nargs, const=const) + self._opt_prefixes.update(option.prefixes) + for opt in option._short_opts: + self._short_opt[opt] = option + for opt in option._long_opts: + self._long_opt[opt] = option + + def add_argument( + self, obj: "CoreArgument", dest: str | None, nargs: int = 1 + ) -> None: + """Adds a positional argument named `dest` to the parser. + + The `obj` can be used to identify the option in the order list + that is returned from the parser. + """ + self._args.append(_Argument(obj, dest=dest, nargs=nargs)) + + def parse_args( + self, args: list[str] + ) -> tuple[dict[str, Any], list[str], list["CoreParameter"]]: + """Parses positional arguments and returns ``(values, args, order)`` + for the parsed options and arguments as well as the leftover + arguments if there are any. The order is a list of objects as they + appear on the command line. If arguments appear multiple times they + will be memorized multiple times as well. + """ + state = _ParsingState(args) + try: + self._process_args_for_options(state) + self._process_args_for_args(state) + except UsageError: + if self.ctx is None or not self.ctx.resilient_parsing: + raise + return state.opts, state.largs, state.order + + def _process_args_for_args(self, state: _ParsingState) -> None: + pargs, args = _unpack_args( + state.largs + state.rargs, [x.nargs for x in self._args] + ) + + for idx, arg in enumerate(self._args): + arg.process(pargs[idx], state) + + state.largs = args + state.rargs = [] + + def _process_args_for_options(self, state: _ParsingState) -> None: + while state.rargs: + arg = state.rargs.pop(0) + arglen = len(arg) + # Double dashes always handled explicitly regardless of what + # prefixes are valid. + if arg == "--": + return + elif arg[:1] in self._opt_prefixes and arglen > 1: + self._process_opts(arg, state) + elif self.allow_interspersed_args: + state.largs.append(arg) + else: + state.rargs.insert(0, arg) + return + + # Say this is the original argument list: + # [arg0, arg1, ..., arg(i-1), arg(i), arg(i+1), ..., arg(N-1)] + # ^ + # (we are about to process arg(i)). + # + # Then rargs is [arg(i), ..., arg(N-1)] and largs is a *subset* of + # [arg0, ..., arg(i-1)] (any options and their arguments will have + # been removed from largs). + # + # The while loop will usually consume 1 or more arguments per pass. + # If it consumes 1 (eg. arg is an option that takes no arguments), + # then after _process_arg() is done the situation is: + # + # largs = subset of [arg0, ..., arg(i)] + # rargs = [arg(i+1), ..., arg(N-1)] + # + # If allow_interspersed_args is false, largs will always be + # *empty* -- still a subset of [arg0, ..., arg(i-1)], but + # not a very interesting subset! + + def _match_long_opt( + self, opt: str, explicit_value: str | None, state: _ParsingState + ) -> None: + if opt not in self._long_opt: + from difflib import get_close_matches + + possibilities = get_close_matches(opt, self._long_opt) + raise NoSuchOption(opt, possibilities=possibilities, ctx=self.ctx) + + option = self._long_opt[opt] + if option.takes_value: + # At this point it's safe to modify rargs by injecting the + # explicit value, because no exception is raised in this + # branch. This means that the inserted value will be fully + # consumed. + if explicit_value is not None: + state.rargs.insert(0, explicit_value) + + value = self._get_value_from_state(opt, option, state) + + elif explicit_value is not None: # pragma: no cover + raise BadOptionUsage(opt, f"Option {opt!r} does not take a value.") + + else: + value = None + + option.process(value, state) + + def _match_short_opt(self, arg: str, state: _ParsingState) -> None: + stop = False + i = 1 + prefix = arg[0] + unknown_options = [] + + for ch in arg[1:]: + opt = _normalize_opt(f"{prefix}{ch}", self.ctx) + option = self._short_opt.get(opt) + i += 1 + + if not option: + if self.ignore_unknown_options: + unknown_options.append(ch) + continue + raise NoSuchOption(opt, ctx=self.ctx) + if option.takes_value: + # Any characters left in arg? Pretend they're the + # next arg, and stop consuming characters of arg. + if i < len(arg): + state.rargs.insert(0, arg[i:]) + stop = True + + value = self._get_value_from_state(opt, option, state) + + else: + value = None + + option.process(value, state) + + if stop: + break + + # If we got any unknown options we recombine the string of the + # remaining options and re-attach the prefix, then report that + # to the state as new 'largs'. This way there is basic combinatorics + # that can be achieved while still ignoring unknown arguments. + if self.ignore_unknown_options and unknown_options: + state.largs.append(f"{prefix}{''.join(unknown_options)}") + + def _get_value_from_state( + self, option_name: str, option: _Option, state: _ParsingState + ) -> str | Sequence[str]: + nargs = option.nargs + + value: str | Sequence[str] + + if len(state.rargs) < nargs: + msg = "an argument." if nargs == 1 else f"{nargs} arguments." + raise BadOptionUsage( + option_name, + f"Option {option_name!r} requires {msg}", + ) + elif nargs == 1: + value = state.rargs.pop(0) + else: + value = tuple(state.rargs[:nargs]) + del state.rargs[:nargs] + + return value + + def _process_opts(self, arg: str, state: _ParsingState) -> None: + explicit_value = None + # Long option handling happens in two parts. The first part is + # supporting explicitly attached values. In any case, we will try + # to long match the option first. + if "=" in arg: + long_opt, explicit_value = arg.split("=", 1) + else: + long_opt = arg + norm_long_opt = _normalize_opt(long_opt, self.ctx) + + # At this point we will match the (assumed) long option through + # the long option matching code. Note that this allows options + # like "-foo" to be matched as long options. + try: + self._match_long_opt(norm_long_opt, explicit_value, state) + except NoSuchOption: + # At this point the long option matching failed, and we need + # to try with short options. However there is a special rule + # which says, that if we have a two character options prefix + # (applies to "--foo" for instance), we do not dispatch to the + # short option code and will instead raise the no option + # error. + if arg[:2] not in self._opt_prefixes: + self._match_short_opt(arg, state) + return + + if not self.ignore_unknown_options: + raise + + state.largs.append(arg) diff --git a/contrib/python/typer/typer/_click/py.typed b/contrib/python/typer/typer/_click/py.typed new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/contrib/python/typer/typer/_click/py.typed diff --git a/contrib/python/typer/typer/_click/shell_completion.py b/contrib/python/typer/typer/_click/shell_completion.py new file mode 100644 index 00000000000..4490f188109 --- /dev/null +++ b/contrib/python/typer/typer/_click/shell_completion.py @@ -0,0 +1,306 @@ +import re +from abc import ABC, abstractmethod +from collections.abc import MutableMapping +from typing import Any, ClassVar, TypeVar + +from .core import Command, Context, Parameter, ParameterSource + + +class CompletionItem: + """Represents a completion value and metadata about the value. The + default metadata is ``type`` to indicate special shell handling, + and ``help`` if a shell supports showing a help string next to the + value. + + Arbitrary parameters can be passed when creating the object, and + accessed using ``item.attr``. If an attribute wasn't passed, + accessing it returns ``None``. + """ + + __slots__ = ("value", "type", "help", "_info") + + def __init__( + self, + value: Any, + type: str = "plain", + help: str | None = None, + **kwargs: Any, + ) -> None: + self.value: Any = value + self.type: str = type + self.help: str | None = help + self._info = kwargs + + def __getattr__(self, name: str) -> Any: + return self._info.get(name) + + +class ShellComplete(ABC): + """Base class for providing shell completion support. A subclass for + a given shell will override attributes and methods to implement the + completion instructions (``source`` and ``complete``). + """ + + name: ClassVar[str] + """Name to register the shell as with `add_completion_class`. + This is used in completion instructions (``{name}_source`` and + ``{name}_complete``). + """ + + source_template: ClassVar[str] + """Completion script template formatted by `source`. This must + be provided by subclasses. + """ + + def __init__( + self, + cli: Command, + ctx_args: MutableMapping[str, Any], + prog_name: str, + complete_var: str, + ) -> None: + self.cli = cli + self.ctx_args = ctx_args + self.prog_name = prog_name + self.complete_var = complete_var + + @property + def func_name(self) -> str: + """The name of the shell function defined by the completion + script. + """ + safe_name = re.sub(r"\W*", "", self.prog_name.replace("-", "_"), flags=re.ASCII) + return f"_{safe_name}_completion" + + @abstractmethod + def source_vars(self) -> dict[str, Any]: + """Vars for formatting `source_template`.""" + pass # pragma: no cover + + def source(self) -> str: + """Produce the shell script that defines the completion + function. By default this ``%``-style formats + `source_template` with the dict returned by `source_vars`. + """ + return self.source_template % self.source_vars() + + @abstractmethod + def get_completion_args(self) -> tuple[list[str], str]: + """Use the env vars defined by the shell script to return a + tuple of ``args, incomplete``. This must be implemented by + subclasses. + """ + pass # pragma: no cover + + def get_completions(self, args: list[str], incomplete: str) -> list[CompletionItem]: + """Determine the context and last complete command or parameter + from the complete args. Call that object's ``shell_complete`` + method to get the completions for the incomplete value. + """ + ctx = _resolve_context(self.cli, self.ctx_args, self.prog_name, args) + obj, incomplete = _resolve_incomplete(ctx, args, incomplete) + return obj.shell_complete(ctx, incomplete) + + @abstractmethod + def format_completion(self, item: CompletionItem) -> str: + """Format a completion item into the form recognized by the + shell script. This must be implemented by subclasses. + """ + pass # pragma: no cover + + def complete(self) -> str: + """Produce the completion data to send back to the shell. + + By default this calls `get_completion_args`, gets the + completions, then calls `format_completion` for each + completion. + """ + args, incomplete = self.get_completion_args() + completions = self.get_completions(args, incomplete) + out = [self.format_completion(item) for item in completions] + return "\n".join(out) + + +ShellCompleteType = TypeVar("ShellCompleteType", bound="type[ShellComplete]") + + +_available_shells: dict[str, type[ShellComplete]] = {} + + +def add_completion_class(cls: ShellCompleteType, name: str) -> ShellCompleteType: + """Register a `ShellComplete` subclass under the given name. + The name will be provided by the completion instruction environment + variable during completion. + """ + _available_shells[name] = cls + + return cls + + +def get_completion_class(shell: str) -> type[ShellComplete] | None: + """Look up a registered `ShellComplete` subclass by the name + provided by the completion instruction environment variable. If the + name isn't registered, returns ``None``. + """ + return _available_shells.get(shell) + + +def split_arg_string(string: str) -> list[str]: + """Split an argument string as with `shlex.split`, but don't + fail if the string is incomplete. Ignores a missing closing quote or + incomplete escape sequence and uses the partial token as-is. + """ + import shlex + + lex = shlex.shlex(string, posix=True) + lex.whitespace_split = True + lex.commenters = "" + out = [] + + try: + for token in lex: + out.append(token) + except ValueError: # pragma: no cover + # Raised when end-of-string is reached in an invalid state. Use + # the partial token as-is. The quote or escape character is in + # lex.state, not lex.token. + out.append(lex.token) + + return out + + +def _is_incomplete_argument(ctx: Context, param: Parameter) -> bool: + """Determine if the given parameter is an argument that can still + accept values. + """ + # avoid circular imports + from ..core import TyperArgument + + if not isinstance(param, TyperArgument): + return False + + assert param.name is not None + # Will be None if expose_value is False. + value = ctx.params.get(param.name) + return ( + param.nargs == -1 + or ctx.get_parameter_source(param.name) is not ParameterSource.COMMANDLINE + or ( + param.nargs > 1 + and isinstance(value, (tuple, list)) + and len(value) < param.nargs + ) + ) + + +def _start_of_option(ctx: Context, value: str) -> bool: + """Check if the value looks like the start of an option.""" + if not value: + return False + + c = value[0] + return c in ctx._opt_prefixes + + +def _is_incomplete_option(ctx: Context, args: list[str], param: Parameter) -> bool: + """Determine if the given parameter is an option that needs a value.""" + # avoid circular imports + from ..core import TyperOption + + if not isinstance(param, TyperOption): + return False + + if param.is_flag or param.count: + return False + + last_option = None + + for index, arg in enumerate(reversed(args)): + if index + 1 > param.nargs: + break + + if _start_of_option(ctx, arg): + last_option = arg + break + + return last_option is not None and last_option in param.opts + + +def _resolve_context( + cli: Command, + ctx_args: MutableMapping[str, Any], + prog_name: str, + args: list[str], +) -> Context: + """Produce the context hierarchy starting with the command and + traversing the complete arguments. This only follows the commands, + it doesn't trigger input prompts or callbacks. + """ + # avoid circular imports + from ..core import TyperGroup + + ctx_args["resilient_parsing"] = True + with cli.make_context(prog_name, args.copy(), **ctx_args) as ctx: + args = ctx._protected_args + ctx.args + + while args: + command = ctx.command + + if isinstance(command, TyperGroup): + # if not command.chain: + name, cmd, args = command.resolve_command(ctx, args) + + if cmd is None: + return ctx + + with cmd.make_context( + name, args, parent=ctx, resilient_parsing=True + ) as sub_ctx: + ctx = sub_ctx + args = ctx._protected_args + ctx.args + else: # pragma: no cover + break + + return ctx + + +def _resolve_incomplete( + ctx: Context, args: list[str], incomplete: str +) -> tuple[Command | Parameter, str]: + """Find the Click object that will handle the completion of the + incomplete value. Return the object and the incomplete value. + """ + # Different shells treat an "=" between a long option name and + # value differently. Might keep the value joined, return the "=" + # as a separate item, or return the split name and value. Always + # split and discard the "=" to make completion easier. + if incomplete == "=": + incomplete = "" + elif "=" in incomplete and _start_of_option(ctx, incomplete): + name, _, incomplete = incomplete.partition("=") + args.append(name) + + # The "--" marker tells Click to stop treating values as options + # even if they start with the option character. If it hasn't been + # given and the incomplete arg looks like an option, the current + # command will provide option name completions. + if "--" not in args and _start_of_option(ctx, incomplete): + return ctx.command, incomplete + + params = ctx.command.get_params(ctx) + + # If the last complete arg is an option name with an incomplete + # value, the option will provide value completions. + for param in params: + if _is_incomplete_option(ctx, args, param): + return param, incomplete + + # It's not an option name or value. The first argument without a + # parsed value will provide value completions. + for param in params: + if _is_incomplete_argument(ctx, param): + return param, incomplete + + # There were no unparsed arguments, the command may be a group that + # will provide command name completions. + return ctx.command, incomplete diff --git a/contrib/python/typer/typer/_click/termui.py b/contrib/python/typer/typer/_click/termui.py new file mode 100644 index 00000000000..0a8c82574d2 --- /dev/null +++ b/contrib/python/typer/typer/_click/termui.py @@ -0,0 +1,430 @@ +import io +from collections.abc import Callable, Iterable +from contextlib import AbstractContextManager +from typing import IO, TYPE_CHECKING, Any, AnyStr, TextIO, TypeVar, overload + +from .exceptions import Abort, UsageError +from .globals import resolve_color_default +from .types import ParamType, convert_type +from .utils import LazyFile, echo + +if TYPE_CHECKING: + from ._termui_impl import ProgressBar + +V = TypeVar("V") + +# The prompt functions to use. The doc tools currently override these +# functions to customize how they work. +visible_prompt_func: Callable[[str], str] = input + +_ansi_colors = { + "black": 30, + "red": 31, + "green": 32, + "yellow": 33, + "blue": 34, + "magenta": 35, + "cyan": 36, + "white": 37, + "reset": 39, + "bright_black": 90, + "bright_red": 91, + "bright_green": 92, + "bright_yellow": 93, + "bright_blue": 94, + "bright_magenta": 95, + "bright_cyan": 96, + "bright_white": 97, +} +_ansi_reset_all = "\033[0m" + + +def hidden_prompt_func(prompt: str) -> str: + import getpass + + return getpass.getpass(prompt) + + +def _build_prompt( + text: str, + suffix: str, + show_default: bool = False, + default: Any | None = None, + show_choices: bool = True, + type: ParamType | None = None, +) -> str: + # prevent circular imports + from .._types import TyperChoice + + prompt = text + if type is not None and show_choices and isinstance(type, TyperChoice): + prompt += f" ({', '.join(map(str, type.choices))})" + if default is not None and show_default: + prompt = f"{prompt} [{_format_default(default)}]" + return f"{prompt}{suffix}" + + +def _format_default(default: Any) -> Any: + if isinstance(default, (io.IOBase, LazyFile)) and hasattr(default, "name"): + return default.name + + return default + + +def prompt( + text: str, + default: Any | None = None, + hide_input: bool = False, + confirmation_prompt: bool | str = False, + type: ParamType | Any | None = None, + value_proc: Callable[[str], Any] | None = None, + prompt_suffix: str = ": ", + show_default: bool = True, + err: bool = False, + show_choices: bool = True, +) -> Any: + """Prompts a user for input. This is a convenience function that can + be used to prompt a user for input later. + + If the user aborts the input by sending an interrupt signal, this + function will catch it and raise an `Abort` exception. + """ + + def prompt_func(text: str) -> str: + f = hidden_prompt_func if hide_input else visible_prompt_func + try: + # Write the prompt separately so that we get nice + # coloring through colorama on Windows + echo(text[:-1], nl=False, err=err) + # Echo the last character to stdout to work around an issue where + # readline causes backspace to clear the whole line. + return f(text[-1:]) + except (KeyboardInterrupt, EOFError): # pragma: no cover + # getpass doesn't print a newline if the user aborts input with ^C. + # Allegedly this behavior is inherited from getpass(3). + # A doc bug has been filed at https://bugs.python.org/issue24711 + if hide_input: + echo(None, err=err) + raise Abort() from None + + if value_proc is None: + value_proc = convert_type(type, default) + + prompt = _build_prompt( + text, prompt_suffix, show_default, default, show_choices, type + ) + + if confirmation_prompt: + if confirmation_prompt is True: + confirmation_prompt = "Repeat for confirmation" + + confirmation_prompt = _build_prompt(confirmation_prompt, prompt_suffix) + + while True: + while True: + value = prompt_func(prompt) + if value: + break + elif default is not None: + value = default + break + try: + result = value_proc(value) + except UsageError as e: # pragma: no cover + if hide_input: + echo("Error: The value you entered was invalid.", err=err) + else: + echo(f"Error: {e.message}", err=err) + continue + if not confirmation_prompt: + return result + while True: + value2 = prompt_func(confirmation_prompt) + is_empty = not value and not value2 + if value2 or is_empty: + break + if value == value2: + return result + echo("Error: The two entered values do not match.", err=err) + + +def confirm( + text: str, + default: bool | None = False, + abort: bool = False, + prompt_suffix: str = ": ", + show_default: bool = True, + err: bool = False, +) -> bool: + """Prompts for confirmation (yes/no question). + + If the user aborts the input by sending a interrupt signal this + function will catch it and raise an `Abort` exception. + """ + prompt = _build_prompt( + text, + prompt_suffix, + show_default, + "y/n" if default is None else ("Y/n" if default else "y/N"), + ) + + while True: + try: + # Write the prompt separately so that we get nice + # coloring through colorama on Windows + echo(prompt[:-1], nl=False, err=err) + # Echo the last character to stdout to work around an issue where + # readline causes backspace to clear the whole line. + value = visible_prompt_func(prompt[-1:]).lower().strip() + except (KeyboardInterrupt, EOFError): # pragma: no cover + raise Abort() from None + if value in ("y", "yes"): + rv = True + elif value in ("n", "no"): + rv = False + elif default is not None and value == "": + rv = default + else: # pragma: no cover + echo("Error: invalid input", err=err) + continue + break + if abort and not rv: + raise Abort() + return rv + + +@overload +def progressbar( + *, + length: int, + label: str | None = None, + hidden: bool = False, + show_eta: bool = True, + show_percent: bool | None = None, + show_pos: bool = False, + fill_char: str = "#", + empty_char: str = "-", + bar_template: str = "%(label)s [%(bar)s] %(info)s", + info_sep: str = " ", + width: int = 36, + file: TextIO | None = None, + color: bool | None = None, + update_min_steps: int = 1, +) -> "ProgressBar[int]": ... + + +@overload +def progressbar( + iterable: Iterable[V] | None = None, + length: int | None = None, + label: str | None = None, + hidden: bool = False, + show_eta: bool = True, + show_percent: bool | None = None, + show_pos: bool = False, + item_show_func: Callable[[V | None], str | None] | None = None, + fill_char: str = "#", + empty_char: str = "-", + bar_template: str = "%(label)s [%(bar)s] %(info)s", + info_sep: str = " ", + width: int = 36, + file: TextIO | None = None, + color: bool | None = None, + update_min_steps: int = 1, +) -> "ProgressBar[V]": ... + + +def progressbar( + iterable: Iterable[V] | None = None, + length: int | None = None, + label: str | None = None, + hidden: bool = False, + show_eta: bool = True, + show_percent: bool | None = None, + show_pos: bool = False, + item_show_func: Callable[[V | None], str | None] | None = None, + fill_char: str = "#", + empty_char: str = "-", + bar_template: str = "%(label)s [%(bar)s] %(info)s", + info_sep: str = " ", + width: int = 36, + file: TextIO | None = None, + color: bool | None = None, + update_min_steps: int = 1, +) -> "ProgressBar[V]": + """This function creates an iterable context manager that can be used + to iterate over something while showing a progress bar. It will + either iterate over the `iterable` or `length` items (that are counted + up). While iteration happens, this function will print a rendered + progress bar to the given `file` (defaults to stdout) and will attempt + to calculate remaining time and more. By default, this progress bar + will not be rendered if the file is not a terminal. + + The context manager creates the progress bar. When the context + manager is entered the progress bar is already created. With every + iteration over the progress bar, the iterable passed to the bar is + advanced and the bar is updated. When the context manager exits, + a newline is printed and the progress bar is finalized on screen. + + Note: The progress bar is currently designed for use cases where the + total progress can be expected to take at least several seconds. + Because of this, the ProgressBar class object won't display + progress that is considered too fast, and progress where the time + between steps is less than a second. + + No printing must happen or the progress bar will be unintentionally + destroyed. + """ + from ._termui_impl import ProgressBar + + color = resolve_color_default(color) + return ProgressBar( + iterable=iterable, + length=length, + hidden=hidden, + show_eta=show_eta, + show_percent=show_percent, + show_pos=show_pos, + item_show_func=item_show_func, + fill_char=fill_char, + empty_char=empty_char, + bar_template=bar_template, + info_sep=info_sep, + file=file, + label=label, + width=width, + color=color, + update_min_steps=update_min_steps, + ) + + +def _interpret_color(color: int | tuple[int, int, int] | str, offset: int = 0) -> str: + if isinstance(color, int): + return f"{38 + offset};5;{color:d}" + + if isinstance(color, (tuple, list)): + r, g, b = color + return f"{38 + offset};2;{r:d};{g:d};{b:d}" + + return str(_ansi_colors[color] + offset) + + +def style( + text: Any, + fg: int | tuple[int, int, int] | str | None = None, + bg: int | tuple[int, int, int] | str | None = None, + bold: bool | None = None, + dim: bool | None = None, + underline: bool | None = None, + overline: bool | None = None, + italic: bool | None = None, + blink: bool | None = None, + reverse: bool | None = None, + strikethrough: bool | None = None, + reset: bool = True, +) -> str: + """Styles a text with ANSI styles and returns the new string. By + default the styling is self contained which means that at the end + of the string a reset code is issued. This can be prevented by + passing ``reset=False``. + """ + if not isinstance(text, str): + text = str(text) + + bits = [] + + if fg: + try: + bits.append(f"\033[{_interpret_color(fg)}m") + except KeyError: + raise TypeError(f"Unknown color {fg!r}") from None + + if bg: + try: + bits.append(f"\033[{_interpret_color(bg, 10)}m") + except KeyError: + raise TypeError(f"Unknown color {bg!r}") from None + + if bold is not None: + bits.append(f"\033[{1 if bold else 22}m") + if dim is not None: + bits.append(f"\033[{2 if dim else 22}m") + if underline is not None: + bits.append(f"\033[{4 if underline else 24}m") + if overline is not None: + bits.append(f"\033[{53 if overline else 55}m") + if italic is not None: + bits.append(f"\033[{3 if italic else 23}m") + if blink is not None: + bits.append(f"\033[{5 if blink else 25}m") + if reverse is not None: + bits.append(f"\033[{7 if reverse else 27}m") + if strikethrough is not None: + bits.append(f"\033[{9 if strikethrough else 29}m") + bits.append(text) + if reset: + bits.append(_ansi_reset_all) + return "".join(bits) + + +def secho( + message: Any | None = None, + file: IO[AnyStr] | None = None, + nl: bool = True, + err: bool = False, + color: bool | None = None, + **styles: Any, +) -> None: + """This function combines `echo` and `style` into one call.""" + if message is not None and not isinstance(message, (bytes, bytearray)): + message = style(message, **styles) + + return echo(message, file=file, nl=nl, err=err, color=color) + + +def launch(url: str, wait: bool = False, locate: bool = False) -> int: + """This function launches the given URL (or filename) in the default + viewer application for this file type. If this is an executable, it + might launch the executable in a new session. The return value is + the exit code of the launched application. Usually, ``0`` indicates + success. + """ + from ._termui_impl import open_url + + return open_url(url, wait=wait, locate=locate) + + +# If this is provided, getchar() calls into this instead. This is used +# for unittesting purposes. +_getchar: Callable[[bool], str] | None = None + + +def getchar(echo: bool = False) -> str: + """Fetches a single character from the terminal and returns it. This + will always return a unicode character and under certain rare + circumstances this might return more than one character. The + situations which more than one character is returned is when for + whatever reason multiple characters end up in the terminal buffer or + standard input was not actually a terminal. + + Note that this will always read from the terminal, even if something + is piped into the standard input. + + Note for Windows: in rare cases when typing non-ASCII characters, this + function might wait for a second character and then return both at once. + This is because certain Unicode characters look like special-key markers. + """ + global _getchar + + if _getchar is None: + from ._termui_impl import getchar as f + + _getchar = f + + return _getchar(echo) + + +def raw_terminal() -> AbstractContextManager[int]: + from ._termui_impl import raw_terminal as f + + return f() diff --git a/contrib/python/typer/typer/_click/types.py b/contrib/python/typer/typer/_click/types.py new file mode 100644 index 00000000000..5ccf15fe1bd --- /dev/null +++ b/contrib/python/typer/typer/_click/types.py @@ -0,0 +1,695 @@ +import os +import sys +from collections.abc import Callable, Sequence +from datetime import datetime +from typing import ( + IO, + TYPE_CHECKING, + Any, + ClassVar, + Literal, + NoReturn, + TypedDict, + TypeGuard, + TypeVar, + Union, + cast, +) + +from ._compat import _get_argv_encoding, open_stream +from .exceptions import BadParameter +from .utils import LazyFile, format_filename, safecall + +if TYPE_CHECKING: + from .core import Context, Parameter + from .shell_completion import CompletionItem + +ParamTypeValue = TypeVar("ParamTypeValue") + + +class ParamType: + """Represents the type of a parameter. Validates and converts values + from the command line or Python into the correct type. + + To implement a custom type, subclass and implement at least the + following: + + - The `name` class attribute must be set. + - Calling an instance of the type with ``None`` must return + ``None``. This is already implemented by default. + - `convert` must convert string values to the correct type. + - `convert` must accept values that are already the correct + type. + - It must be able to convert a value if the ``ctx`` and ``param`` + arguments are ``None``. This can occur when converting prompt + input. + """ + + is_composite: ClassVar[bool] = False + arity: ClassVar[int] = 1 + name: str + + # if a list of this type is expected and the value is pulled from a + # string environment variable, this is what splits it up. `None` + # means any whitespace. For all parameters the general rule is that + # whitespace splits them up. The exception are paths and files which + # are split by ``os.path.pathsep`` by default (":" on Unix and ";" on + # Windows). + envvar_list_splitter: ClassVar[str | None] = None + + def __call__( + self, + value: Any, + param: Union["Parameter", None] = None, + ctx: Union["Context", None] = None, + ) -> Any: + if value is not None: + return self.convert(value, param, ctx) + + def get_metavar(self, param: "Parameter", ctx: "Context") -> str | None: + """Returns the metavar default for this param if it provides one.""" + pass # pragma: no cover + + def get_missing_message( + self, param: "Parameter", ctx: Union["Context", None] + ) -> str | None: + """Optionally might return extra information about a missing + parameter. + """ + pass # pragma: no cover + + def convert( + self, value: Any, param: Union["Parameter", None], ctx: Union["Context", None] + ) -> Any: + pass # pragma: no cover + + def split_envvar_value(self, rv: str) -> Sequence[str]: + """Given a value from an environment variable this splits it up + into small chunks depending on the defined envvar list splitter. + + If the splitter is set to `None`, which means that whitespace splits, + then leading and trailing whitespace is ignored. Otherwise, leading + and trailing splitters usually lead to empty items being included. + """ + return (rv or "").split(self.envvar_list_splitter) + + def fail( + self, + message: str, + param: Union["Parameter", None] = None, + ctx: Union["Context", None] = None, + ) -> NoReturn: + """Helper method to fail with an invalid value message.""" + raise BadParameter(message, ctx=ctx, param=param) + + def shell_complete( + self, ctx: "Context", param: "Parameter", incomplete: str + ) -> list["CompletionItem"]: + """Return a list of `CompletionItem` objects for the + incomplete value. Most types do not provide completions, but + some do, and this allows custom types to provide custom + completions as well. + """ + return [] + + +class CompositeParamType(ParamType): + is_composite = True + + @property + def arity(self) -> int: # type: ignore + raise NotImplementedError() # pragma: no cover + + +class FuncParamType(ParamType): + def __init__(self, func: Callable[[Any], Any]) -> None: + self.name: str = getattr(func, "__name__", "function") + self.func = func + + def convert( + self, value: Any, param: Union["Parameter", None], ctx: Union["Context", None] + ) -> Any: + try: + return self.func(value) + except ValueError: + try: + value = str(value) + except UnicodeError: # pragma: no cover + assert isinstance(value, bytes) + value = value.decode("utf-8", "replace") + + self.fail(value, param, ctx) + + +class StringParamType(ParamType): + name = "text" + + def convert( + self, value: Any, param: Union["Parameter", None], ctx: Union["Context", None] + ) -> Any: + if isinstance(value, bytes): + enc = _get_argv_encoding() + try: + value = value.decode(enc) + except UnicodeError: + fs_enc = sys.getfilesystemencoding() + if fs_enc != enc: + try: + value = value.decode(fs_enc) + except UnicodeError: + value = value.decode("utf-8", "replace") + else: + value = value.decode("utf-8", "replace") + return value + return str(value) + + def __repr__(self) -> str: + return "STRING" + + +class DateTime(ParamType): + """The DateTime type converts date strings into `datetime` objects. + + The format strings which are checked are configurable, but default to some + common (non-timezone aware) ISO 8601 formats. + + When specifying *DateTime* formats, you should only pass a list or a tuple. + Other iterables, like generators, may lead to surprising results. + + The format strings are processed using ``datetime.strptime``, and this + consequently defines the format strings which are allowed. + + Parsing is tried using each format, in order, and the first format which + parses successfully is used. + """ + + name = "datetime" + + def __init__(self, formats: Sequence[str] | None = None): + self.formats: Sequence[str] = formats or [ + "%Y-%m-%d", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S", + ] + + def get_metavar(self, param: "Parameter", ctx: "Context") -> str | None: + return f"[{'|'.join(self.formats)}]" + + def _try_to_convert_date(self, value: Any, format: str) -> datetime | None: + try: + return datetime.strptime(value, format) + except ValueError: + return None + + def convert( + self, value: Any, param: Union["Parameter", None], ctx: Union["Context", None] + ) -> Any: + if isinstance(value, datetime): + return value + + for format in self.formats: + converted = self._try_to_convert_date(value, format) + + if converted is not None: + return converted + + formats_str = ", ".join(map(repr, self.formats)) + self.fail( + f"{value!r} does not match the formats {formats_str}.", + param, + ctx, + ) + + def __repr__(self) -> str: + return "DateTime" + + +class _NumberParamTypeBase(ParamType): + _number_class: ClassVar[type[Any]] + + def convert( + self, value: Any, param: Union["Parameter", None], ctx: Union["Context", None] + ) -> Any: + try: + return self._number_class(value) + except ValueError: + self.fail( + f"{value!r} is not a valid {self.name}.", + param, + ctx, + ) + + +class _NumberRangeBase(_NumberParamTypeBase): + def __init__( + self, + min: float | None = None, + max: float | None = None, + min_open: bool = False, + max_open: bool = False, + clamp: bool = False, + ) -> None: + self.min = min + self.max = max + self.min_open = min_open + self.max_open = max_open + self.clamp = clamp + + def convert( + self, value: Any, param: Union["Parameter", None], ctx: Union["Context", None] + ) -> Any: + import operator + + rv = super().convert(value, param, ctx) + lt_min: bool = self.min is not None and ( + operator.le if self.min_open else operator.lt + )(rv, self.min) + gt_max: bool = self.max is not None and ( + operator.ge if self.max_open else operator.gt + )(rv, self.max) + + if self.clamp: + if lt_min: + return self._clamp(self.min, 1, self.min_open) # type: ignore[arg-type] + + if gt_max: + return self._clamp(self.max, -1, self.max_open) # type: ignore[arg-type] + + if lt_min or gt_max: + self.fail( + f"{rv} is not in the range {self._describe_range()}.", + param, + ctx, + ) + + return rv + + def _clamp(self, bound: float, dir: Literal[1, -1], open: bool) -> float: + """Find the valid value to clamp to bound in the given + direction. + """ + raise NotImplementedError # pragma: no cover + + def _describe_range(self) -> str: + """Describe the range for use in help text.""" + if self.min is None: + op = "<" if self.max_open else "<=" + return f"x{op}{self.max}" + + if self.max is None: + op = ">" if self.min_open else ">=" + return f"x{op}{self.min}" + + lop = "<" if self.min_open else "<=" + rop = "<" if self.max_open else "<=" + return f"{self.min}{lop}x{rop}{self.max}" + + def __repr__(self) -> str: + clamp = " clamped" if self.clamp else "" + return f"<{type(self).__name__} {self._describe_range()}{clamp}>" + + +class IntParamType(_NumberParamTypeBase): + name = "integer" + _number_class = int + + def __repr__(self) -> str: + return "INT" + + +class IntRange(_NumberRangeBase, IntParamType): + """Restrict an `INT` value to a range of accepted values. See + + If ``min`` or ``max`` are not passed, any value is accepted in that + direction. If ``min_open`` or ``max_open`` are enabled, the + corresponding boundary is not included in the range. + + If ``clamp`` is enabled, a value outside the range is clamped to the + boundary instead of failing. + """ + + name = "integer range" + + def _clamp( # type: ignore + self, bound: int, dir: Literal[1, -1], open: bool + ) -> int: + if not open: + return bound + + return bound + dir + + +class FloatParamType(_NumberParamTypeBase): + name = "float" + _number_class = float + + def __repr__(self) -> str: + return "FLOAT" + + +class FloatRange(_NumberRangeBase, FloatParamType): + """Restrict a `FLOAT` value to a range of accepted + values. See `ranges`. + + If ``min`` or ``max`` are not passed, any value is accepted in that + direction. If ``min_open`` or ``max_open`` are enabled, the + corresponding boundary is not included in the range. + + If ``clamp`` is enabled, a value outside the range is clamped to the + boundary instead of failing. This is not supported if either + boundary is marked ``open``. + """ + + name = "float range" + + def __init__( + self, + min: float | None = None, + max: float | None = None, + min_open: bool = False, + max_open: bool = False, + clamp: bool = False, + ) -> None: + super().__init__( + min=min, max=max, min_open=min_open, max_open=max_open, clamp=clamp + ) + + if (min_open or max_open) and clamp: + raise TypeError("Clamping is not supported for open bounds.") + + def _clamp(self, bound: float, dir: Literal[1, -1], open: bool) -> float: + if not open: + return bound + + # Could use math.nextafter here, but clamping an + # open float range doesn't seem to be particularly useful. It's + # left up to the user to write a callback to do it if needed. + raise RuntimeError( + "Clamping is not supported for open bounds." + ) # pragma: no cover + + +class BoolParamType(ParamType): + name = "boolean" + + bool_states: dict[str, bool] = { + "1": True, + "0": False, + "yes": True, + "no": False, + "true": True, + "false": False, + "on": True, + "off": False, + "t": True, + "f": False, + "y": True, + "n": False, + # Absence of value is considered False. + "": False, + } + """A mapping of string values to boolean states. + + Mapping is inspired by `configparser.ConfigParser.BOOLEAN_STATES` + and extends it. + """ + + @staticmethod + def str_to_bool(value: str | bool) -> bool | None: + """Convert a string to a boolean value. + + If the value is already a boolean, it is returned as-is. If the value is a + string, it is stripped of whitespaces and lower-cased, then checked against + the known boolean states pre-defined in the `BoolParamType.bool_states` mapping + above. + + Returns `None` if the value does not match any known boolean state. + """ + if isinstance(value, bool): + return value + return BoolParamType.bool_states.get(value.strip().lower()) + + def convert( + self, value: Any, param: Union["Parameter", None], ctx: Union["Context", None] + ) -> bool: + normalized = self.str_to_bool(value) + if normalized is None: + states = ", ".join(sorted(self.bool_states)) + self.fail( + f"{value!r} is not a valid boolean. Recognized values: {states}", + param, + ctx, + ) + return normalized + + def __repr__(self) -> str: + return "BOOL" + + +class UUIDParameterType(ParamType): + name = "uuid" + + def convert( + self, value: Any, param: Union["Parameter", None], ctx: Union["Context", None] + ) -> Any: + import uuid + + if isinstance(value, uuid.UUID): + return value + + value = value.strip() + + try: + return uuid.UUID(value) + except ValueError: + self.fail(f"{value!r} is not a valid UUID.", param, ctx) + + def __repr__(self) -> str: + return "UUID" + + +class File(ParamType): + """Declares a parameter to be a file for reading or writing. The file + is automatically closed once the context tears down (after the command + finished working). + + Files can be opened for reading or writing. The special value ``-`` + indicates stdin or stdout depending on the mode. + + By default, the file is opened for reading text data, but it can also be + opened in binary mode or for writing. The encoding parameter can be used + to force a specific encoding. + + The `lazy` flag controls if the file should be opened immediately or upon + first IO. The default is to be non-lazy for standard input and output + streams as well as files opened for reading, `lazy` otherwise. When opening a + file lazily for reading, it is still opened temporarily for validation, but + will not be held open until first IO. lazy is mainly useful when opening + for writing to avoid creating the file until it is needed. + + Files can also be opened atomically in which case all writes go into a + separate file in the same folder and upon completion the file will + be moved over to the original location. This is useful if a file + regularly read by other users is modified. + """ + + name = "filename" + envvar_list_splitter: ClassVar[str] = os.path.pathsep + + def __init__( + self, + mode: str = "r", + encoding: str | None = None, + errors: str | None = "strict", + lazy: bool | None = None, + atomic: bool = False, + ) -> None: + self.mode = mode + self.encoding = encoding + self.errors = errors + self.lazy = lazy + self.atomic = atomic + + def resolve_lazy_flag(self, value: str | os.PathLike[str]) -> bool: + if self.lazy is not None: + return self.lazy + if os.fspath(value) == "-": + return False + elif "w" in self.mode: + return True + return False + + def convert( + self, + value: str | os.PathLike[str] | IO[Any], + param: Union["Parameter", None], + ctx: Union["Context", None], + ) -> IO[Any]: + if _is_file_like(value): + return value + + value = cast("str | os.PathLike[str]", value) + + try: + lazy = self.resolve_lazy_flag(value) + + if lazy: + lf = LazyFile( + value, self.mode, self.encoding, self.errors, atomic=self.atomic + ) + + if ctx is not None: + ctx.call_on_close(lf.close_intelligently) + + return cast("IO[Any]", lf) + + f, should_close = open_stream( + value, self.mode, self.encoding, self.errors, atomic=self.atomic + ) + + # If a context is provided, we automatically close the file + # at the end of the context execution (or flush out). If a + # context does not exist, it's the caller's responsibility to + # properly close the file. This for instance happens when the + # type is used with prompts. + if ctx is not None: + if should_close: + ctx.call_on_close(safecall(f.close)) + else: + ctx.call_on_close(safecall(f.flush)) + + return f + except OSError as e: # pragma: no cover + self.fail(f"'{format_filename(value)}': {e.strerror}", param, ctx) + + def shell_complete( + self, ctx: "Context", param: "Parameter", incomplete: str + ) -> list["CompletionItem"]: + """Return a special completion marker that tells the completion + system to use the shell to provide file path completions. + """ + from .shell_completion import CompletionItem + + return [CompletionItem(incomplete, type="file")] + + +def _is_file_like(value: Any) -> TypeGuard[IO[Any]]: + return hasattr(value, "read") or hasattr(value, "write") + + +class Tuple(CompositeParamType): + """The default behavior of Click is to apply a type on a value directly. + This works well in most cases, except for when `nargs` is set to a fixed + count and different types should be used for different items. In this + case the `Tuple` type can be used. This type can only be used + if `nargs` is set to a fixed number. + + For more information see `tuple-type`. + + This can be selected by using a Python tuple literal as a type. + """ + + def __init__(self, types: Sequence[type[Any] | ParamType]) -> None: + self.types: Sequence[ParamType] = [convert_type(ty) for ty in types] + + @property + def name(self) -> str: # type: ignore[override] + return f"<{' '.join(ty.name for ty in self.types)}>" + + @property + def arity(self) -> int: # type: ignore + return len(self.types) + + def convert( + self, value: Any, param: Union["Parameter", None], ctx: Union["Context", None] + ) -> Any: + len_type = len(self.types) + len_value = len(value) + + if len_value != len_type: + self.fail( + f"{len_type} values are required, but {len_value} given.", + param=param, + ctx=ctx, + ) + + return tuple( + ty(x, param, ctx) for ty, x in zip(self.types, value, strict=False) + ) + + +def convert_type(ty: Any | None, default: Any | None = None) -> ParamType: + """Find the most appropriate `ParamType` for the given Python + type. If the type isn't provided, it can be inferred from a default + value. + """ + guessed_type = False + + if ty is None and default is not None: + if isinstance(default, (tuple, list)): + # If the default is empty, ty will remain None and will + # return STRING. + if default: + item = default[0] + + # A tuple of tuples needs to detect the inner types. + # Can't call convert recursively because that would + # incorrectly unwind the tuple to a single type. + if isinstance(item, (tuple, list)): + ty = tuple(map(type, item)) + else: + ty = type(item) + else: + ty = type(default) + + guessed_type = True + + if isinstance(ty, tuple): + return Tuple(ty) + + if isinstance(ty, ParamType): + return ty + + if ty is str or ty is None: + return STRING + + if ty is int: + return INT + + if ty is float: + return FLOAT + + if ty is bool: + return BOOL + + if guessed_type: + return STRING + + return FuncParamType(ty) + + +# A unicode string parameter type which is the implicit default. This +# can also be selected by using ``str`` as type. +STRING = StringParamType() + +# An integer parameter. This can also be selected by using ``int`` as +# type. +INT = IntParamType() + +# A floating point value parameter. This can also be selected by using +# ``float`` as type. +FLOAT = FloatParamType() + +# A boolean parameter. This is the default for boolean flags. This can +# also be selected by using ``bool`` as a type. +BOOL = BoolParamType() + +# A UUID parameter. +UUID = UUIDParameterType() + + +class OptionHelpExtra(TypedDict, total=False): + envvars: tuple[str, ...] + default: str + range: str + required: str diff --git a/contrib/python/typer/typer/_click/utils.py b/contrib/python/typer/typer/_click/utils.py new file mode 100644 index 00000000000..ac8e5ba3f25 --- /dev/null +++ b/contrib/python/typer/typer/_click/utils.py @@ -0,0 +1,470 @@ +import os +import re +import sys +from collections.abc import Callable, Iterable, Iterator +from functools import update_wrapper +from types import ModuleType, TracebackType +from typing import ( + IO, + Any, + AnyStr, + BinaryIO, + Literal, + ParamSpec, + TextIO, + TypeVar, + cast, +) + +from ._compat import ( + WIN, + _default_text_stderr, + _default_text_stdout, + _find_binary_writer, + auto_wrap_for_ansi, + binary_streams, + open_stream, + should_strip_ansi, + strip_ansi, + text_streams, +) +from .globals import resolve_color_default + +P = ParamSpec("P") +R = TypeVar("R") + + +def _posixify(name: str) -> str: + return "-".join(name.split()).lower() + + +def safecall(func: Callable[P, R]) -> Callable[P, R | None]: + """Wraps a function so that it swallows exceptions.""" + + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R | None: + try: + return func(*args, **kwargs) + except Exception: # pragma: no cover + pass + return None # pragma: no cover + + return update_wrapper(wrapper, func) + + +def make_default_short_help(help: str, max_length: int = 45) -> str: + """Returns a condensed version of help string.""" + # Consider only the first paragraph. + paragraph_end = help.find("\n\n") + + if paragraph_end != -1: + help = help[:paragraph_end] + + # Collapse newlines, tabs, and spaces. + words = help.split() + + if not words: + return "" + + # The first paragraph started with a "no rewrap" marker, ignore it. + if words[0] == "\b": + words = words[1:] + + total_length = 0 + last_index = len(words) - 1 + + for i, word in enumerate(words): + total_length += len(word) + (i > 0) + + if total_length > max_length: # too long, truncate + break + + if word[-1] == ".": # sentence end, truncate without "..." + return " ".join(words[: i + 1]) + + if total_length == max_length and i != last_index: + break # not at sentence end, truncate with "..." + else: + return " ".join(words) # no truncation needed + + # Account for the length of the suffix. + total_length += len("...") + + # remove words until the length is short enough + while i > 0: + total_length -= len(words[i]) + (i > 0) + + if total_length <= max_length: + break + + i -= 1 + + return " ".join(words[:i]) + "..." + + +class LazyFile: + """A lazy file works like a regular file but it does not fully open + the file but it does perform some basic checks early to see if the + filename parameter does make sense. This is useful for safely opening + files for writing. + """ + + def __init__( + self, + filename: str | os.PathLike[str], + mode: str = "r", + encoding: str | None = None, + errors: str | None = "strict", + atomic: bool = False, + ): + self.name: str = os.fspath(filename) + self.mode = mode + self.encoding = encoding + self.errors = errors + self.atomic = atomic + self._f: IO[Any] | None + self.should_close: bool + + if self.name == "-": + self._f, self.should_close = open_stream(filename, mode, encoding, errors) + else: + if "r" in mode: + # Open and close the file in case we're opening it for + # reading so that we can catch at least some errors in + # some cases early. + open(filename, mode).close() + self._f = None + self.should_close = True + + def __getattr__(self, name: str) -> Any: + return getattr(self.open(), name) + + def __repr__(self) -> str: + if self._f is not None: + return repr(self._f) + return f"<unopened file '{format_filename(self.name)}' {self.mode}>" + + def open(self) -> IO[Any]: + """Opens the file if it's not yet open. This call might fail with + a `FileError`. Not handling this error will produce an error + that Click shows. + """ + if self._f is not None: + return self._f + try: + rv, self.should_close = open_stream( + self.name, self.mode, self.encoding, self.errors, atomic=self.atomic + ) + except OSError as e: + from .exceptions import FileError + + raise FileError(self.name, hint=e.strerror) from e + self._f = rv + return rv + + def close(self) -> None: + """Closes the underlying file, no matter what.""" + if self._f is not None: + self._f.close() + + def close_intelligently(self) -> None: + """This function only closes the file if it was opened by the lazy + file wrapper. For instance this will never close stdin. + """ + if self.should_close: + self.close() + + def __enter__(self) -> "LazyFile": + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + tb: TracebackType | None, + ) -> None: + self.close_intelligently() + + def __iter__(self) -> Iterator[AnyStr]: + self.open() + return iter(self._f) # type: ignore + + +def echo( + message: Any | None = None, + file: IO[Any] | None = None, + nl: bool = True, + err: bool = False, + color: bool | None = None, +) -> None: + """Print a message and newline to stdout or a file. This should be + used instead of `print` because it provides better support + for different data, files, and environments. + + Compared to `print`, this does the following: + + - Ensures that the output encoding is not misconfigured on Linux. + - Supports Unicode in the Windows console. + - Supports writing to binary outputs, and supports writing bytes + to text outputs. + - Supports colors and styles on Windows. + - Removes ANSI color and style codes if the output does not look + like an interactive terminal. + - Always flushes the output. + """ + if file is None: + if err: + file = _default_text_stderr() + else: + file = _default_text_stdout() + + # There are no standard streams attached to write to. For example, + # pythonw on Windows. + if file is None: + return + + # Convert non bytes/text into the native string type. + if message is not None and not isinstance(message, (str, bytes, bytearray)): + out: str | bytes | bytearray | None = str(message) + else: + out = message + + if nl: + out = out or "" + if isinstance(out, str): + out += "\n" + else: + out += b"\n" + + if not out: + file.flush() + return + + # If there is a message and the value looks like bytes, we manually + # need to find the binary stream and write the message in there. + # This is done separately so that most stream types will work as you + # would expect. Eg: you can write to StringIO for other cases. + if isinstance(out, (bytes, bytearray)): + binary_file = _find_binary_writer(file) + + if binary_file is not None: + file.flush() + binary_file.write(out) + binary_file.flush() + return + + # ANSI style code support. For no message or bytes, nothing happens. + # When outputting to a file instead of a terminal, strip codes. + else: + color = resolve_color_default(color) + + if should_strip_ansi(file, color): + out = strip_ansi(out) + elif WIN: + if auto_wrap_for_ansi is not None: + file = auto_wrap_for_ansi(file, color) # type: ignore[arg-type,call-arg] + elif not color: + out = strip_ansi(out) + + file.write(out) # type: ignore[arg-type] + file.flush() + + +def get_binary_stream(name: Literal["stdin", "stdout", "stderr"]) -> BinaryIO: + """Returns a system stream for byte processing.""" + opener = binary_streams.get(name) + if opener is None: + raise TypeError(f"Unknown standard stream '{name}'") + return opener() + + +def get_text_stream( + name: Literal["stdin", "stdout", "stderr"], + encoding: str | None = None, + errors: str | None = "strict", +) -> TextIO: + """Returns a system stream for text processing. This usually returns + a wrapped stream around a binary stream returned from + `get_binary_stream` but it also can take shortcuts for already + correctly configured streams. + """ + opener = text_streams.get(name) + if opener is None: + raise TypeError(f"Unknown standard stream '{name}'") + return opener(encoding, errors) + + +def format_filename( + filename: str | bytes | os.PathLike[str] | os.PathLike[bytes], + shorten: bool = False, +) -> str: + """Format a filename as a string for display. Ensures the filename can be + displayed by replacing any invalid bytes or surrogate escapes in the name + with the replacement character ``�``. + + Invalid bytes or surrogate escapes will raise an error when written to a + stream with ``errors="strict"``. This will typically happen with ``stdout`` + when the locale is something like ``en_GB.UTF-8``. + + Many scenarios *are* safe to write surrogates though, due to PEP 538 and + PEP 540, including: + + - Writing to ``stderr``, which uses ``errors="backslashreplace"``. + - The system has ``LANG=C.UTF-8``, ``C``, or ``POSIX``. Python opens + stdout and stderr with ``errors="surrogateescape"``. + - None of ``LANG/LC_*`` are set. Python assumes ``LANG=C.UTF-8``. + - Python is started in UTF-8 mode with ``PYTHONUTF8=1`` or ``-X utf8``. + Python opens stdout and stderr with ``errors="surrogateescape"``. + """ + if shorten: + filename = os.path.basename(filename) + else: + filename = os.fspath(filename) + + if isinstance(filename, bytes): + filename = filename.decode(sys.getfilesystemencoding(), "replace") + else: + filename = filename.encode("utf-8", "surrogateescape").decode( + "utf-8", "replace" + ) + + return filename + + +def get_app_dir(app_name: str, roaming: bool = True, force_posix: bool = False) -> str: + r"""Returns the config folder for the application. The default behavior + is to return whatever is most appropriate for the operating system. + + To give you an idea, for an app called ``"Foo Bar"``, something like + the following folders could be returned: + + Mac OS X: + ``~/Library/Application Support/Foo Bar`` + Mac OS X (POSIX): + ``~/.foo-bar`` + Unix: + ``~/.config/foo-bar`` + Unix (POSIX): + ``~/.foo-bar`` + Windows (roaming): + ``C:\Users\<user>\AppData\Roaming\Foo Bar`` + Windows (not roaming): + ``C:\Users\<user>\AppData\Local\Foo Bar`` + """ + if WIN: + key = "APPDATA" if roaming else "LOCALAPPDATA" + folder = os.environ.get(key) + if folder is None: + folder = os.path.expanduser("~") + return os.path.join(folder, app_name) + if force_posix: + return os.path.join(os.path.expanduser(f"~/.{_posixify(app_name)}")) + if sys.platform == "darwin": + return os.path.join( + os.path.expanduser("~/Library/Application Support"), app_name + ) + return os.path.join( + os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), + _posixify(app_name), + ) + + +class PacifyFlushWrapper: + """This wrapper is used to catch and suppress BrokenPipeErrors resulting + from ``.flush()`` being called on broken pipe during the shutdown/final-GC + of the Python interpreter. Notably ``.flush()`` is always called on + ``sys.stdout`` and ``sys.stderr``. So as to have minimal impact on any + other cleanup code, and the case where the underlying file is not a broken + pipe, all calls and attributes are proxied. + """ + + def __init__(self, wrapped: IO[Any]) -> None: + self.wrapped = wrapped + + def flush(self) -> None: + try: + self.wrapped.flush() + except OSError as e: # pragma: no cover + import errno + + if e.errno != errno.EPIPE: + raise + + def __getattr__(self, attr: str) -> Any: + return getattr(self.wrapped, attr) + + +def _detect_program_name( + path: str | None = None, _main: ModuleType | None = None +) -> str: + """Determine the command used to run the program, for use in help + text. If a file or entry point was executed, the file name is + returned. If ``python -m`` was used to execute a module or package, + ``python -m name`` is returned. + + This doesn't try to be too precise, the goal is to give a concise + name for help text. Files are only shown as their name without the + path. ``python`` is only shown for modules, and the full path to + ``sys.executable`` is not shown. + """ + if _main is None: + _main = sys.modules["__main__"] + + if not path: + path = sys.argv[0] + + # The value of __package__ indicates how Python was called. It may + # not exist if a setuptools script is installed as an egg. It may be + # set incorrectly for entry points created with pip on Windows. + # It is set to "" inside a Shiv or PEX zipapp. + if getattr(_main, "__package__", None) in {None, ""} or ( + os.name == "nt" + and _main.__package__ == "" + and not os.path.exists(path) + and os.path.exists(f"{path}.exe") + ): + # Executed a file, like "python app.py". + return os.path.basename(path) + + # Executed a module, like "python -m example". + # Rewritten by Python from "-m script" to "/path/to/script.py". + # Need to look at main module to determine how it was executed. + py_module = cast(str, _main.__package__) + name = os.path.splitext(os.path.basename(path))[0] + + # A submodule like "example.cli". + if name != "__main__": + py_module = f"{py_module}.{name}" + + return f"python -m {py_module.lstrip('.')}" + + +def _expand_args( + args: Iterable[str], + *, + user: bool = True, + env: bool = True, + glob_recursive: bool = True, +) -> list[str]: + """Simulate Unix shell expansion with Python functions.""" + from glob import glob + + out = [] + + for arg in args: + if user: + arg = os.path.expanduser(arg) + + if env: + arg = os.path.expandvars(arg) + + try: + matches = glob(arg, recursive=glob_recursive) + except re.error: # pragma: no cover + matches = [] + + if not matches: + out.append(arg) + else: + out.extend(matches) + + return out diff --git a/contrib/python/typer/typer/_completion_classes.py b/contrib/python/typer/typer/_completion_classes.py index 8548fb4d6a3..cfae02c9bf1 100644 --- a/contrib/python/typer/typer/_completion_classes.py +++ b/contrib/python/typer/typer/_completion_classes.py @@ -4,11 +4,9 @@ import re import sys from typing import Any -import click -import click.parser -import click.shell_completion -from click.shell_completion import split_arg_string as click_split_arg_string - +from . import _click +from ._click.shell_completion import CompletionItem, ShellComplete, add_completion_class +from ._click.shell_completion import split_arg_string as click_split_arg_string from ._completion_shared import ( COMPLETION_SCRIPT_BASH, COMPLETION_SCRIPT_FISH, @@ -27,7 +25,7 @@ def _sanitize_help_text(text: str) -> str: return rich_utils.rich_render_text(text) -class BashComplete(click.shell_completion.BashComplete): +class BashComplete(ShellComplete): name = Shells.bash.value source_template = COMPLETION_SCRIPT_BASH @@ -50,7 +48,7 @@ class BashComplete(click.shell_completion.BashComplete): return args, incomplete - def format_completion(self, item: click.shell_completion.CompletionItem) -> str: + def format_completion(self, item: CompletionItem) -> str: # TODO: Explore replicating the new behavior from Click, with item types and # triggering completion for files and directories # return f"{item.type},{item.value}" @@ -62,8 +60,42 @@ class BashComplete(click.shell_completion.BashComplete): out = [self.format_completion(item) for item in completions] return "\n".join(out) + @staticmethod + def _check_version() -> None: + import shutil + import subprocess + + bash_exe = shutil.which("bash") + + if bash_exe is None: + match = None # pragma: no cover + else: + output = subprocess.run( + [bash_exe, "--norc", "-c", 'echo "${BASH_VERSION}"'], + stdout=subprocess.PIPE, + ) + match = re.search(r"^(\d+)\.(\d+)\.\d+", output.stdout.decode()) + + if match is not None: + major, minor = match.groups() + + if major < "4" or major == "4" and minor < "4": + _click.utils.echo( + "Shell completion is not supported for Bash versions older than 4.4.", + err=True, + ) + else: + _click.utils.echo( + "Couldn't detect Bash version, shell completion is not supported.", + err=True, + ) # pragma: no cover + + def source(self) -> str: + self._check_version() + return super().source() + -class ZshComplete(click.shell_completion.ZshComplete): +class ZshComplete(ShellComplete): name = Shells.zsh.value source_template = COMPLETION_SCRIPT_ZSH @@ -85,7 +117,7 @@ class ZshComplete(click.shell_completion.ZshComplete): incomplete = "" return args, incomplete - def format_completion(self, item: click.shell_completion.CompletionItem) -> str: + def format_completion(self, item: CompletionItem) -> str: def escape(s: str) -> str: return ( s.replace('"', '""') @@ -114,7 +146,7 @@ class ZshComplete(click.shell_completion.ZshComplete): return "_files" -class FishComplete(click.shell_completion.FishComplete): +class FishComplete(ShellComplete): name = Shells.fish.value source_template = COMPLETION_SCRIPT_FISH @@ -136,7 +168,7 @@ class FishComplete(click.shell_completion.FishComplete): incomplete = "" return args, incomplete - def format_completion(self, item: click.shell_completion.CompletionItem) -> str: + def format_completion(self, item: CompletionItem) -> str: # TODO: Explore replicating the new behavior from Click, pay attention to # the difference with and without formatted help # if item.help: @@ -167,7 +199,7 @@ class FishComplete(click.shell_completion.FishComplete): return "" # pragma: no cover -class PowerShellComplete(click.shell_completion.ShellComplete): +class PowerShellComplete(ShellComplete): name = Shells.powershell.value source_template = COMPLETION_SCRIPT_POWER_SHELL @@ -185,15 +217,13 @@ class PowerShellComplete(click.shell_completion.ShellComplete): args = cwords[1:-1] if incomplete else cwords[1:] return args, incomplete - def format_completion(self, item: click.shell_completion.CompletionItem) -> str: + def format_completion(self, item: CompletionItem) -> str: return f"{item.value}:::{_sanitize_help_text(item.help) if item.help else ' '}" def completion_init() -> None: - click.shell_completion.add_completion_class(BashComplete, Shells.bash.value) - click.shell_completion.add_completion_class(ZshComplete, Shells.zsh.value) - click.shell_completion.add_completion_class(FishComplete, Shells.fish.value) - click.shell_completion.add_completion_class( - PowerShellComplete, Shells.powershell.value - ) - click.shell_completion.add_completion_class(PowerShellComplete, Shells.pwsh.value) + add_completion_class(BashComplete, Shells.bash.value) + add_completion_class(ZshComplete, Shells.zsh.value) + add_completion_class(FishComplete, Shells.fish.value) + add_completion_class(PowerShellComplete, Shells.powershell.value) + add_completion_class(PowerShellComplete, Shells.pwsh.value) diff --git a/contrib/python/typer/typer/_completion_shared.py b/contrib/python/typer/typer/_completion_shared.py index 5a81dcf68cd..8d2c19715c5 100644 --- a/contrib/python/typer/typer/_completion_shared.py +++ b/contrib/python/typer/typer/_completion_shared.py @@ -4,9 +4,11 @@ import subprocess from enum import Enum from pathlib import Path -import click import shellingham +from . import _click +from ._click.globals import get_current_context + class Shells(str, Enum): bash = "bash" @@ -78,8 +80,8 @@ def get_completion_script(*, prog_name: str, complete_var: str, shell: str) -> s cf_name = _invalid_ident_char_re.sub("", prog_name.replace("-", "_")) script = _completion_scripts.get(shell) if script is None: - click.echo(f"Shell {shell} not supported.", err=True) - raise click.exceptions.Exit(1) + _click.echo(f"Shell {shell} not supported.", err=True) + raise _click.exceptions.Exit(1) return ( script % { @@ -172,8 +174,8 @@ def install_powershell(*, prog_name: str, complete_var: str, shell: str) -> Path stdout=subprocess.PIPE, ) if result.returncode != 0: # pragma: no cover - click.echo("Couldn't get PowerShell user profile", err=True) - raise click.exceptions.Exit(result.returncode) + _click.echo("Couldn't get PowerShell user profile", err=True) + raise _click.exceptions.Exit(result.returncode) path_str = "" if isinstance(result.stdout, str): # pragma: no cover path_str = result.stdout @@ -185,8 +187,8 @@ def install_powershell(*, prog_name: str, complete_var: str, shell: str) -> Path except UnicodeDecodeError: # pragma: no cover pass if not path_str: # pragma: no cover - click.echo("Couldn't decode the path automatically", err=True) - raise click.exceptions.Exit(1) + _click.echo("Couldn't decode the path automatically", err=True) + raise _click.exceptions.Exit(1) path_obj = Path(path_str.strip()) parent_dir: Path = path_obj.parent parent_dir.mkdir(parents=True, exist_ok=True) @@ -203,7 +205,7 @@ def install( prog_name: str | None = None, complete_var: str | None = None, ) -> tuple[str, Path]: - prog_name = prog_name or click.get_current_context().find_root().info_name + prog_name = prog_name or get_current_context().find_root().info_name assert prog_name if complete_var is None: complete_var = "_{}_COMPLETE".format(prog_name.replace("-", "_").upper()) @@ -231,8 +233,8 @@ def install( ) return shell, installed_path else: - click.echo(f"Shell {shell} is not supported.") - raise click.exceptions.Exit(1) + _click.echo(f"Shell {shell} is not supported.") + raise _click.exceptions.Exit(1) def _get_shell_name() -> str | None: diff --git a/contrib/python/typer/typer/_types.py b/contrib/python/typer/typer/_types.py index dc9fc63220d..09b38afb3f0 100644 --- a/contrib/python/typer/typer/_types.py +++ b/contrib/python/typer/typer/_types.py @@ -1,21 +1,42 @@ +from collections.abc import Iterable, Mapping, Sequence from enum import Enum -from typing import TypeVar +from typing import Any, Generic, TypeVar -import click +from . import _click +from ._click import types +from ._click.shell_completion import CompletionItem ParamTypeValue = TypeVar("ParamTypeValue") -class TyperChoice(click.Choice[ParamTypeValue]): +class TyperChoice(types.ParamType, Generic[ParamTypeValue]): + # Code adapted from Click 8.3.1, with Typer using enum values in normalize_choice + name = "choice" + + def __init__( + self, choices: Iterable[ParamTypeValue], case_sensitive: bool = True + ) -> None: + self.choices: Sequence[ParamTypeValue] = tuple(choices) + self.case_sensitive = case_sensitive + + def _normalized_mapping( + self, ctx: _click.Context | None = None + ) -> Mapping[ParamTypeValue, str]: + """ + Returns mapping where keys are the original choices and the values are + the normalized values that are accepted via the command line. + """ + return { + choice: self.normalize_choice( + choice=choice, + ctx=ctx, + ) + for choice in self.choices + } + def normalize_choice( - self, choice: ParamTypeValue, ctx: click.Context | None + self, choice: ParamTypeValue, ctx: _click.Context | None ) -> str: - # Click 8.2.0 added a new method `normalize_choice` to the `Choice` class - # to support enums, but it uses the enum names, while Typer has always used the - # enum values. - # This class overrides that method to maintain the previous behavior. - # In Click: - # normed_value = choice.name if isinstance(choice, Enum) else str(choice) normed_value = str(choice.value) if isinstance(choice, Enum) else str(choice) if ctx is not None and ctx.token_normalize_func is not None: @@ -25,3 +46,75 @@ class TyperChoice(click.Choice[ParamTypeValue]): normed_value = normed_value.casefold() return normed_value + + def get_metavar(self, param: _click.Parameter, ctx: _click.Context) -> str | None: + if param.param_type_name == "option" and not param.show_choices: # type: ignore + choice_metavars = [ + types.convert_type(type(choice)).name.upper() for choice in self.choices + ] + choices_str = "|".join([*dict.fromkeys(choice_metavars)]) + else: + choices_str = "|".join( + [str(i) for i in self._normalized_mapping(ctx=ctx).values()] + ) + + # Use curly braces to indicate a required argument. + if param.required and param.param_type_name == "argument": + return f"{{{choices_str}}}" + + # Use square braces to indicate an option or optional argument. + return f"[{choices_str}]" + + def get_missing_message( + self, param: _click.Parameter, ctx: _click.Context | None + ) -> str: + """Message shown when no choice is passed.""" + choices = ",\n\t".join(self._normalized_mapping(ctx=ctx).values()) + return f"Choose from:\n\t{choices}" + + def convert( + self, value: Any, param: _click.Parameter | None, ctx: _click.Context | None + ) -> ParamTypeValue: + """ + For a given value from the parser, normalize it and find its + matching normalized value in the list of choices. Then return the + matched "original" choice. + """ + normed_value = self.normalize_choice(choice=value, ctx=ctx) + normalized_mapping = self._normalized_mapping(ctx=ctx) + + try: + return next( + original + for original, normalized in normalized_mapping.items() + if normalized == normed_value + ) + except StopIteration: + self.fail( + self.get_invalid_choice_message(value=value, ctx=ctx), + param=param, + ctx=ctx, + ) + + def get_invalid_choice_message(self, value: Any, ctx: _click.Context | None) -> str: + """Get the error message when the given choice is invalid.""" + choices_str = ", ".join(map(repr, self._normalized_mapping(ctx=ctx).values())) + return f"{value!r} is not one of {choices_str}." + + def __repr__(self) -> str: + return f"Choice({list(self.choices)})" + + def shell_complete( + self, ctx: _click.Context, param: _click.Parameter, incomplete: str + ) -> list[CompletionItem]: + """Complete choices that start with the incomplete value.""" + + str_choices = map(str, self.choices) + + if self.case_sensitive: + matched = (c for c in str_choices if c.startswith(incomplete)) + else: + incomplete = incomplete.lower() + matched = (c for c in str_choices if c.lower().startswith(incomplete)) + + return [CompletionItem(c) for c in matched] diff --git a/contrib/python/typer/typer/cli.py b/contrib/python/typer/typer/cli.py index 2a7d78c3a49..665bcf5a594 100644 --- a/contrib/python/typer/typer/cli.py +++ b/contrib/python/typer/typer/cli.py @@ -4,13 +4,12 @@ import sys from pathlib import Path from typing import Any -import click import typer import typer.core -from click import Command, Group, Option -from . import __version__ -from .core import HAS_RICH, MARKUP_MODE_KEY +from . import __version__, _click +from ._click import Command +from .core import HAS_RICH, MARKUP_MODE_KEY, TyperGroup, TyperOption default_app_names = ("app", "cli", "main") default_func_names = ("main", "cli", "app") @@ -31,7 +30,7 @@ class State: state = State() -def maybe_update_state(ctx: click.Context) -> None: +def maybe_update_state(ctx: _click.Context) -> None: path_or_module = ctx.params.get("path_or_module") if path_or_module: file_path = Path(path_or_module) @@ -53,19 +52,19 @@ def maybe_update_state(ctx: click.Context) -> None: class TyperCLIGroup(typer.core.TyperGroup): - def list_commands(self, ctx: click.Context) -> list[str]: + def list_commands(self, ctx: _click.Context) -> list[str]: self.maybe_add_run(ctx) return super().list_commands(ctx) - def get_command(self, ctx: click.Context, name: str) -> Command | None: # ty: ignore[invalid-method-override] + def get_command(self, ctx: _click.Context, name: str) -> Command | None: # ty: ignore[invalid-method-override] self.maybe_add_run(ctx) return super().get_command(ctx, name) - def invoke(self, ctx: click.Context) -> Any: + def invoke(self, ctx: _click.Context) -> Any: self.maybe_add_run(ctx) return super().invoke(ctx) - def maybe_add_run(self, ctx: click.Context) -> None: + def maybe_add_run(self, ctx: _click.Context) -> None: maybe_update_state(ctx) maybe_add_run_to_cli(self) @@ -138,7 +137,7 @@ def get_typer_from_state() -> typer.Typer | None: return obj -def maybe_add_run_to_cli(cli: click.Group) -> None: +def maybe_add_run_to_cli(cli: TyperGroup) -> None: if "run" not in cli.commands: if state.file or state.module: obj = get_typer_from_state() @@ -151,7 +150,7 @@ def maybe_add_run_to_cli(cli: click.Group) -> None: cli.add_command(click_obj) -def print_version(ctx: click.Context, param: Option, value: bool) -> None: +def print_version(ctx: _click.Context, param: TyperOption, value: bool) -> None: if not value or ctx.resilient_parsing: return typer.echo(f"Typer version: {__version__}") @@ -242,7 +241,7 @@ def get_docs_for_click( docs += "\n" if obj.epilog: docs += f"{obj.epilog}\n\n" - if isinstance(obj, Group): + if isinstance(obj, TyperGroup): group = obj commands = group.list_commands(ctx) if commands: diff --git a/contrib/python/typer/typer/completion.py b/contrib/python/typer/typer/completion.py index 0d621e411d7..f63692ddf3d 100644 --- a/contrib/python/typer/typer/completion.py +++ b/contrib/python/typer/typer/completion.py @@ -3,8 +3,8 @@ import sys from collections.abc import MutableMapping from typing import Any -import click - +from . import _click +from ._click import shell_completion from ._completion_classes import completion_init from ._completion_shared import Shells, _get_shell_name, get_completion_script, install from .models import ParamMeta @@ -27,19 +27,19 @@ def get_completion_inspect_parameters() -> tuple[ParamMeta, ParamMeta]: return install_param, show_param -def install_callback(ctx: click.Context, param: click.Parameter, value: Any) -> Any: +def install_callback(ctx: _click.Context, param: _click.Parameter, value: Any) -> Any: if not value or ctx.resilient_parsing: return value # pragma: no cover if isinstance(value, str): shell, path = install(shell=value) else: shell, path = install() - click.secho(f"{shell} completion installed in {path}", fg="green") - click.echo("Completion will take effect once you restart the terminal") + _click.termui.secho(f"{shell} completion installed in {path}", fg="green") + _click.echo("Completion will take effect once you restart the terminal") sys.exit(0) -def show_callback(ctx: click.Context, param: click.Parameter, value: Any) -> Any: +def show_callback(ctx: _click.Context, param: _click.Parameter, value: Any) -> Any: if not value or ctx.resilient_parsing: return value # pragma: no cover prog_name = ctx.find_root().info_name @@ -56,7 +56,7 @@ def show_callback(ctx: click.Context, param: click.Parameter, value: Any) -> Any script_content = get_completion_script( prog_name=prog_name, complete_var=complete_var, shell=shell ) - click.echo(script_content) + _click.echo(script_content) sys.exit(0) @@ -103,17 +103,16 @@ def _install_completion_no_auto_placeholder_function( # And to add extra error messages, for compatibility with Typer in previous versions # This is only called in new Command method, only used by Click 8.x+ def shell_complete( - cli: click.Command, + cli: _click.Command, ctx_args: MutableMapping[str, Any], prog_name: str, complete_var: str, instruction: str, ) -> int: - import click - import click.shell_completion + from . import _click if "_" not in instruction: - click.echo("Invalid completion instruction.", err=True) + _click.echo("Invalid completion instruction.", err=True) return 1 # Click 8 changed the order/style of shell instructions from e.g. @@ -124,23 +123,23 @@ def shell_complete( instruction, _, shell = instruction.partition("_") # Typer override end - comp_cls = click.shell_completion.get_completion_class(shell) + comp_cls = shell_completion.get_completion_class(shell) if comp_cls is None: - click.echo(f"Shell {shell} not supported.", err=True) + _click.echo(f"Shell {shell} not supported.", err=True) return 1 comp = comp_cls(cli, ctx_args, prog_name, complete_var) if instruction == "source": - click.echo(comp.source()) + _click.echo(comp.source()) return 0 # Typer override to print the completion help msg with Rich if instruction == "complete": - click.echo(comp.complete()) + _click.echo(comp.complete()) return 0 # Typer override end - click.echo(f'Completion instruction "{instruction}" not supported.', err=True) + _click.echo(f'Completion instruction "{instruction}" not supported.', err=True) return 1 diff --git a/contrib/python/typer/typer/core.py b/contrib/python/typer/typer/core.py index 48fee64e349..6868ab43559 100644 --- a/contrib/python/typer/typer/core.py +++ b/contrib/python/typer/typer/core.py @@ -2,7 +2,7 @@ import errno import inspect import os import sys -from collections.abc import Callable, MutableMapping, Sequence +from collections.abc import Callable, Mapping, MutableMapping, Sequence from difflib import get_close_matches from enum import Enum from gettext import gettext as _ @@ -13,13 +13,10 @@ from typing import ( cast, ) -import click -import click.core -import click.formatting -import click.shell_completion -import click.types -import click.utils - +from . import _click +from ._click import types +from ._click.parser import _OptionParser +from ._click.shell_completion import CompletionItem from ._typing import Literal from .utils import parse_boolean_env_var @@ -34,7 +31,7 @@ else: DEFAULT_MARKUP_MODE = None -# Copy from click.parser._split_opt +# Copy from _click.parser._split_opt def _split_opt(opt: str) -> tuple[str, str]: first = opt[:1] if first.isalnum(): @@ -45,10 +42,10 @@ def _split_opt(opt: str) -> tuple[str, str]: def _typer_param_setup_autocompletion_compat( - self: click.Parameter, + self: _click.Parameter, *, autocompletion: Callable[ - [click.Context, list[str], str], list[tuple[str, str] | str] + [_click.Context, list[str], str], list[tuple[str, str] | str] ] | None = None, ) -> None: @@ -65,10 +62,8 @@ def _typer_param_setup_autocompletion_compat( if autocompletion is not None: def compat_autocompletion( - ctx: click.Context, param: click.core.Parameter, incomplete: str - ) -> list["click.shell_completion.CompletionItem"]: - from click.shell_completion import CompletionItem - + ctx: _click.Context, param: _click.core.Parameter, incomplete: str + ) -> list[CompletionItem]: out = [] for c in autocompletion(ctx, [], incomplete): @@ -89,11 +84,11 @@ def _typer_param_setup_autocompletion_compat( def _get_default_string( obj: Union["TyperArgument", "TyperOption"], *, - ctx: click.Context, + ctx: _click.Context, show_default_is_str: bool, default_value: list[Any] | tuple[Any, ...] | str | Callable[..., Any] | Any, ) -> str: - # Extracted from click.core.Option.get_help_record() to be reused by + # Extracted from _click.core.Option.get_help_record() to be reused by # rich_utils avoiding RegEx hacks if show_default_is_str: default_string = f"({obj.show_default})" @@ -112,7 +107,7 @@ def _get_default_string( # For boolean flags that have distinct True/False opts, # use the opt without prefix instead of the value. # Typer override, original commented - # default_string = click.parser.split_opt( + # default_string = _click.parser.split_opt( # (self.opts if self.default else self.secondary_opts)[0] # )[1] if obj.default: @@ -136,9 +131,9 @@ def _get_default_string( def _extract_default_help_str( - obj: Union["TyperArgument", "TyperOption"], *, ctx: click.Context + obj: Union["TyperArgument", "TyperOption"], *, ctx: _click.Context ) -> Any | Callable[[], Any] | None: - # Extracted from click.core.Option.get_help_record() to be reused by + # Extracted from _click.core.Option.get_help_record() to be reused by # rich_utils avoiding RegEx hacks # Temporarily enable resilient parsing to avoid type casting # failing for the default. Might be possible to extend this to @@ -154,7 +149,7 @@ def _extract_default_help_str( def _main( - self: click.Command, + self: _click.Command, *, args: Sequence[str] | None = None, prog_name: str | None = None, @@ -164,7 +159,7 @@ def _main( rich_markup_mode: MarkupMode = DEFAULT_MARKUP_MODE, **extra: Any, ) -> Any: - # Typer override, duplicated from click.main() to handle custom rich exceptions + # Typer override, duplicated from _click.main() to handle custom rich exceptions # Verify that the environment is configured correctly, or reject # further execution to avoid a broken script. if args is None: @@ -172,12 +167,12 @@ def _main( # Covered in Click tests if os.name == "nt" and windows_expand_args: # pragma: no cover - args = click.utils._expand_args(args) + args = _click.utils._expand_args(args) else: args = list(args) if prog_name is None: - prog_name = click.utils._detect_program_name() + prog_name = _click.utils._detect_program_name() # Process shell completion requests and exit early. self._main_shell_completion(extra, prog_name, complete_var) @@ -197,11 +192,11 @@ def _main( # by its truthiness/falsiness ctx.exit() except EOFError as e: - click.echo(file=sys.stderr) - raise click.Abort() from e + _click.echo(file=sys.stderr) + raise _click.exceptions.Abort() from e except KeyboardInterrupt as e: - raise click.exceptions.Exit(130) from e - except click.ClickException as e: + raise _click.exceptions.Exit(130) from e + except _click.exceptions.ClickException as e: if not standalone_mode: raise # Typer override @@ -215,12 +210,12 @@ def _main( sys.exit(e.exit_code) except OSError as e: if e.errno == errno.EPIPE: - sys.stdout = cast(TextIO, click.utils.PacifyFlushWrapper(sys.stdout)) - sys.stderr = cast(TextIO, click.utils.PacifyFlushWrapper(sys.stderr)) + sys.stdout = cast(TextIO, _click.utils.PacifyFlushWrapper(sys.stdout)) + sys.stderr = cast(TextIO, _click.utils.PacifyFlushWrapper(sys.stderr)) sys.exit(1) else: raise - except click.exceptions.Exit as e: + except _click.exceptions.Exit as e: if standalone_mode: sys.exit(e.exit_code) else: @@ -233,7 +228,7 @@ def _main( # `ctx.exit(1)` and to `return 1`, the caller won't be able to # tell the difference between the two return e.exit_code - except click.Abort: + except _click.exceptions.Abort: if not standalone_mode: raise # Typer override @@ -242,19 +237,21 @@ def _main( rich_utils.rich_abort_error() else: - click.echo(_("Aborted!"), file=sys.stderr) + _click.echo(_("Aborted!"), file=sys.stderr) # Typer override end sys.exit(1) -class TyperArgument(click.core.Argument): +class TyperArgument(_click.core.Parameter): + param_type_name = "argument" + def __init__( self, *, # Parameter param_decls: list[str], type: Any | None = None, - required: bool | None = None, + required: bool = False, default: Any | None = None, callback: Callable[..., Any] | None = None, nargs: int | None = None, @@ -265,8 +262,8 @@ class TyperArgument(click.core.Argument): # Note that shell_complete is not fully supported and will be removed in future versions # TODO: Remove shell_complete in a future version (after 0.16.0) shell_complete: Callable[ - [click.Context, click.Parameter, str], - list["click.shell_completion.CompletionItem"] | list[str], + [_click.Context, _click.Parameter, str], + list[CompletionItem] | list[str], ] | None = None, autocompletion: Callable[..., Any] | None = None, @@ -301,10 +298,17 @@ class TyperArgument(click.core.Argument): ) _typer_param_setup_autocompletion_compat(self, autocompletion=autocompletion) + @property + def human_readable_name(self) -> str: + if self.metavar is not None: + return self.metavar + assert self.name is not None, "self.name or self.metavar should be set" + return self.name.upper() + def _get_default_string( self, *, - ctx: click.Context, + ctx: _click.Context, show_default_is_str: bool, default_value: list[Any] | tuple[Any, ...] | str | Callable[..., Any] | Any, ) -> str: @@ -316,12 +320,12 @@ class TyperArgument(click.core.Argument): ) def _extract_default_help_str( - self, *, ctx: click.Context + self, *, ctx: _click.Context ) -> Any | Callable[[], Any] | None: return _extract_default_help_str(self, ctx=ctx) - def get_help_record(self, ctx: click.Context) -> tuple[str, str] | None: - # Modified version of click.core.Option.get_help_record() + def get_help_record(self, ctx: _click.Context) -> tuple[str, str] | None: + # Modified version of _click.core.Option.get_help_record() # to support Arguments if self.hidden: return None @@ -376,8 +380,8 @@ class TyperArgument(click.core.Argument): help = f"{help} {extra_str}" if help else f"{extra_str}" return name, help - def make_metavar(self, ctx: click.Context) -> str: - # Modified version of click.core.Argument.make_metavar() + def make_metavar(self, ctx: _click.Context) -> str: + # Modified version of _click.core.Argument.make_metavar() # to include Argument name if self.metavar is not None: var = self.metavar @@ -397,15 +401,45 @@ class TyperArgument(click.core.Argument): def value_is_missing(self, value: Any) -> bool: return _value_is_missing(self, value) + def _parse_decls( + self, decls: Sequence[str], expose_value: bool + ) -> tuple[str | None, list[str], list[str]]: + if not decls: + if not expose_value: + return None, [], [] + raise TypeError("Argument is marked as exposed, but does not have a name.") + if len(decls) == 1: + name = arg = decls[0] + name = name.replace("-", "_").lower() + else: + raise TypeError( + "Arguments take exactly one parameter declaration, got" + f" {len(decls)}: {decls}." + ) + return name, [arg], [] + + def get_usage_pieces(self, ctx: _click.Context) -> list[str]: + return [self.make_metavar(ctx)] + + def get_error_hint(self, ctx: _click.Context) -> str: + return f"'{self.make_metavar(ctx)}'" + + def add_to_parser(self, parser: _OptionParser, ctx: _click.Context) -> None: + parser.add_argument(dest=self.name, nargs=self.nargs, obj=self) + + +class TyperOption(_click.Parameter): + param_type_name = "option" + + _depr_flag_value: bool | None -class TyperOption(click.core.Option): def __init__( self, *, # Parameter param_decls: list[str], - type: click.types.ParamType | Any | None = None, - required: bool | None = None, + type: types.ParamType | Any | None = None, + required: bool = False, default: Any | None = None, callback: Callable[..., Any] | None = None, nargs: int | None = None, @@ -416,8 +450,8 @@ class TyperOption(click.core.Option): # Note that shell_complete is not fully supported and will be removed in future versions # TODO: Remove shell_complete in a future version (after 0.16.0) shell_complete: Callable[ - [click.Context, click.Parameter, str], - list["click.shell_completion.CompletionItem"] | list[str], + [_click.Context, _click.Parameter, str], + list[CompletionItem] | list[str], ] | None = None, autocompletion: Callable[..., Any] | None = None, @@ -438,9 +472,13 @@ class TyperOption(click.core.Option): # Rich settings rich_help_panel: str | None = None, ): + if help: + help = inspect.cleandoc(help) + super().__init__( - param_decls=param_decls, + param_decls, type=type, + multiple=multiple, required=required, default=default, callback=callback, @@ -449,28 +487,246 @@ class TyperOption(click.core.Option): expose_value=expose_value, is_eager=is_eager, envvar=envvar, - show_default=show_default, - prompt=prompt, - confirmation_prompt=confirmation_prompt, - hide_input=hide_input, - is_flag=is_flag, - multiple=multiple, - count=count, - allow_from_autoenv=allow_from_autoenv, - help=help, - hidden=hidden, - show_choices=show_choices, - show_envvar=show_envvar, - prompt_required=prompt_required, shell_complete=shell_complete, ) + + if prompt is True: + if self.name is None: + raise TypeError("'name' is required with 'prompt=True'.") + + prompt_text: str | None = self.name.replace("_", " ").capitalize() + elif prompt is False: + prompt_text = None + else: + prompt_text = prompt + + self.prompt = prompt_text + self.confirmation_prompt = confirmation_prompt + self.prompt_required = prompt_required + self.hide_input = hide_input + self.hidden = hidden + + # TODO: revisit all of this flag stuff + if is_flag and type is None: + self.type: types.ParamType = types.BoolParamType() + + self.is_flag: bool = bool(is_flag) + self.is_bool_flag: bool = bool( + is_flag and isinstance(self.type, types.BoolParamType) + ) + + if self.is_flag: + self._depr_flag_value = True + else: + self._depr_flag_value = None + + # Counting. TODO: test or remove? Not currently in coverage. + self.count = count + if count and type is None: + self.type = types.IntRange(min=0) + + self.allow_from_autoenv = allow_from_autoenv + self.help = help + self.show_default = show_default + self.show_choices = show_choices + self.show_envvar = show_envvar + _typer_param_setup_autocompletion_compat(self, autocompletion=autocompletion) self.rich_help_panel = rich_help_panel + def get_error_hint(self, ctx: _click.Context) -> str: + result = super().get_error_hint(ctx) + if self.show_envvar and self.envvar is not None: + result += f" (env var: '{self.envvar}')" + return result + + def _parse_decls( + self, decls: Sequence[str], expose_value: bool + ) -> tuple[str | None, list[str], list[str]]: + opts = [] + secondary_opts = [] + name = None + possible_names = [] + + for decl in decls: + if decl.isidentifier(): + if name is not None: + raise TypeError(f"Name '{name}' defined twice") + name = decl + else: + split_char = ";" if decl[:1] == "/" else "/" + if split_char in decl: + first, second = decl.split(split_char, 1) + first = first.rstrip() + if first: + possible_names.append(_split_opt(first)) + opts.append(first) + second = second.lstrip() + if second: + secondary_opts.append(second.lstrip()) + if first == second: + raise ValueError( + f"Boolean option {decl!r} cannot use the" + " same flag for true/false." + ) + else: + possible_names.append(_split_opt(decl)) + opts.append(decl) + + if name is None and possible_names: + possible_names.sort(key=lambda x: -len(x[0])) # group long options first + name = possible_names[0][1].replace("-", "_").lower() + if not name.isidentifier(): + name = None + + return name, opts, secondary_opts + + def add_to_parser(self, parser: _OptionParser, ctx: _click.Context) -> None: + if self.multiple: + action = "append" + elif self.count: + action = "count" + else: + action = "store" + + if self.is_flag: + action = f"{action}_const" + + if self.is_bool_flag and self.secondary_opts: + parser.add_option( + obj=self, opts=self.opts, dest=self.name, action=action, const=True + ) + parser.add_option( + obj=self, + opts=self.secondary_opts, + dest=self.name, + action=action, + const=False, + ) + else: + parser.add_option( + obj=self, + opts=self.opts, + dest=self.name, + action=action, + const=self._depr_flag_value, + ) + else: + parser.add_option( + obj=self, + opts=self.opts, + dest=self.name, + action=action, + nargs=self.nargs, + ) + + def prompt_for_value(self, ctx: _click.Context) -> Any: + """This is an alternative flow that can be activated in the full + value processing if a value does not exist. It will prompt the + user until a valid value exists and then returns the processed + value as result. + """ + assert self.prompt is not None + + # Calculate the default before prompting anything to lock in the value before + # attempting any user interaction. + default = self.get_default(ctx) + + # A boolean flag can use a simplified [y/n] confirmation prompt. + if self.is_bool_flag: + # Nothing prevent you to declare an option that is simultaneously: + # 1) auto-detected as a boolean flag, + # 2) allowed to prompt, and + # 3) still declare a non-boolean default. + # This forced casting into a boolean is necessary to align any non-boolean + # default to the prompt, which is going to be a [y/n]-style confirmation + # because the option is still a boolean flag. That way, instead of [y/n], + # we get [Y/n] or [y/N] depending on the truthy value of the default. + # Refs: https://github.com/pallets/click/pull/3030#discussion_r2289180249 + if default is not None: + default = bool(default) + return _click.termui.confirm(self.prompt, default) + + # If show_default is set to True/False, provide this to `prompt` as well. For + # non-bool values of `show_default`, we use `prompt`'s default behavior + prompt_kwargs: Any = {} + if isinstance(self.show_default, bool): + prompt_kwargs["show_default"] = self.show_default + + return _click.termui.prompt( + self.prompt, + # Use ``None`` to inform the prompt() function to reiterate until a valid + # value is provided by the user if we have no default. + default=default, + type=self.type, + hide_input=self.hide_input, + show_choices=self.show_choices, + confirmation_prompt=self.confirmation_prompt, + value_proc=lambda x: self.process_value(ctx, x), + **prompt_kwargs, + ) + + def value_from_envvar(self, ctx: _click.Context) -> Any: + # TODO: clean up + rv = self.resolve_envvar_value(ctx) + + # Absent environment variable or an empty string is interpreted as unset. + if rv is None: + return None + + if self.nargs != 1 or self.multiple: + return self.type.split_envvar_value(rv) + + return rv + + def resolve_envvar_value(self, ctx: _click.Context) -> str | None: + rv = super().resolve_envvar_value(ctx) + + if rv is not None: + return rv + + if ( + self.allow_from_autoenv + and ctx.auto_envvar_prefix is not None + and self.name is not None + ): + envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}" + rv = os.environ.get(envvar) + + if rv: + return rv + + return None + + def consume_value( + self, ctx: _click.Context, opts: Mapping[str, _click.Parameter] + ) -> tuple[Any, _click.core.ParameterSource]: + """For `Option`, the value can be collected from an interactive prompt + if the option is a flag that needs a value (and the `prompt` property is + set). + + Additionally, this method handles flag option that are activated without a + value, in which case the `flag_value` is returned. + """ + value, source = super().consume_value(ctx, opts) + + # The value wasn't set, or used the param's default, prompt for one to the user + # if prompting is enabled. + if ( + source in {None, _click.core.ParameterSource.DEFAULT} + and self.prompt is not None + and (self.required or self.prompt_required) + and not ctx.resilient_parsing + ): + value = self.prompt_for_value(ctx) + source = _click.core.ParameterSource.PROMPT + + return value, source + def _get_default_string( self, *, - ctx: click.Context, + ctx: _click.Context, show_default_is_str: bool, default_value: list[Any] | tuple[Any, ...] | str | Callable[..., Any] | Any, ) -> str: @@ -482,14 +738,14 @@ class TyperOption(click.core.Option): ) def _extract_default_help_str( - self, *, ctx: click.Context + self, *, ctx: _click.Context ) -> Any | Callable[[], Any] | None: return _extract_default_help_str(self, ctx=ctx) - def make_metavar(self, ctx: click.Context) -> str: + def make_metavar(self, ctx: _click.Context) -> str: return super().make_metavar(ctx=ctx) - def get_help_record(self, ctx: click.Context) -> tuple[str, str] | None: + def get_help_record(self, ctx: _click.Context) -> tuple[str, str] | None: # Duplicate all of Click's logic only to modify a single line, to allow boolean # flags with only names for False values as it's currently supported by Typer # Ref: https://typer.tiangolo.com/tutorial/parameter-types/bool/#only-names-for-false @@ -501,7 +757,7 @@ class TyperOption(click.core.Option): def _write_opts(opts: Sequence[str]) -> str: nonlocal any_prefix_is_slash - rv, any_slashes = click.formatting.join_options(opts) + rv, any_slashes = _click.formatting.join_options(opts) if any_slashes: any_prefix_is_slash = True @@ -559,7 +815,7 @@ class TyperOption(click.core.Option): if default_string: extra.append(_("default: {default}").format(default=default_string)) - if isinstance(self.type, click.types._NumberRangeBase): + if isinstance(self.type, types._NumberRangeBase): range_str = self.type._describe_range() if range_str: @@ -588,14 +844,10 @@ class TyperOption(click.core.Option): return _value_is_missing(self, value) -def _value_is_missing(param: click.Parameter, value: Any) -> bool: +def _value_is_missing(param: _click.Parameter, value: Any) -> bool: if value is None: return True - # Click 8.3 and beyond - # if value is UNSET: - # return True - if (param.nargs != 1 or param.multiple) and value == (): return True # pragma: no cover @@ -603,7 +855,7 @@ def _value_is_missing(param: click.Parameter, value: Any) -> bool: def _typer_format_options( - self: click.core.Command, *, ctx: click.Context, formatter: click.HelpFormatter + self: _click.core.Command, *, ctx: _click.Context, formatter: _click.HelpFormatter ) -> None: args = [] opts = [] @@ -624,7 +876,7 @@ def _typer_format_options( def _typer_main_shell_completion( - self: click.core.Command, + self: _click.core.Command, *, ctx_args: MutableMapping[str, Any], prog_name: str, @@ -644,14 +896,14 @@ def _typer_main_shell_completion( sys.exit(rv) -class TyperCommand(click.core.Command): +class TyperCommand(_click.core.Command): def __init__( self, name: str | None, *, context_settings: dict[str, Any] | None = None, callback: Callable[..., Any] | None = None, - params: list[click.Parameter] | None = None, + params: list[_click.Parameter] | None = None, help: str | None = None, epilog: str | None = None, short_help: str | None = None, @@ -682,7 +934,7 @@ class TyperCommand(click.core.Command): self.rich_help_panel = rich_help_panel def format_options( - self, ctx: click.Context, formatter: click.HelpFormatter + self, ctx: _click.Context, formatter: _click.HelpFormatter ) -> None: _typer_format_options(self, ctx=ctx, formatter=formatter) @@ -716,7 +968,7 @@ class TyperCommand(click.core.Command): **extra, ) - def format_help(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: + def format_help(self, ctx: _click.Context, formatter: _click.HelpFormatter) -> None: if not HAS_RICH or self.rich_markup_mode is None: if not hasattr(ctx, "obj") or ctx.obj is None: ctx.ensure_object(dict) @@ -732,25 +984,153 @@ class TyperCommand(click.core.Command): ) -class TyperGroup(click.core.Group): +class TyperGroup(_click.Command): + allow_extra_args = True + allow_interspersed_args = False + command_class: type[_click.Command] | None = None + group_class: type["TyperGroup"] | type[type] | None = None + def __init__( self, *, name: str | None = None, - commands: dict[str, click.Command] | Sequence[click.Command] | None = None, + commands: dict[str, _click.Command] | Sequence[_click.Command] | None = None, # Rich settings rich_markup_mode: MarkupMode = DEFAULT_MARKUP_MODE, rich_help_panel: str | None = None, suggest_commands: bool = True, + # Click settings + invoke_without_command: bool = False, + no_args_is_help: bool = False, + subcommand_metavar: str | None = None, + result_callback: Callable[..., Any] | None = None, **attrs: Any, ) -> None: - super().__init__(name=name, commands=commands, **attrs) + super().__init__(name=name, **attrs) self.rich_markup_mode: MarkupMode = rich_markup_mode self.rich_help_panel = rich_help_panel self.suggest_commands = suggest_commands + # copied from Click's init + if commands is None: + commands = {} + elif isinstance(commands, Sequence): + commands = { + c.name: c + for c in commands + if isinstance(c, _click.Command) and c.name is not None + } + + self.commands = cast(MutableMapping[str, _click.Command], commands) + self.no_args_is_help = no_args_is_help + self.invoke_without_command = invoke_without_command + + if subcommand_metavar is None: + subcommand_metavar = "COMMAND [ARGS]..." + + self.subcommand_metavar = subcommand_metavar + self._result_callback = result_callback + + def add_command(self, cmd: _click.Command, name: str | None = None) -> None: + name = name or cmd.name + if name is None: + raise TypeError("Command has no name.") + self.commands[name] = cmd + + def get_command(self, ctx: _click.Context, cmd_name: str) -> _click.Command | None: + return self.commands.get(cmd_name) + + def collect_usage_pieces(self, ctx: _click.Context) -> list[str]: + rv = super().collect_usage_pieces(ctx) + rv.append(self.subcommand_metavar) + return rv + + def format_commands( + self, ctx: _click.Context, formatter: _click.HelpFormatter + ) -> None: + commands = [] + for subcommand in self.list_commands(ctx): + cmd = self.get_command(ctx, subcommand) + if cmd is None or cmd.hidden: + continue + + commands.append((subcommand, cmd)) + + # allow for 3 times the default spacing + if len(commands): + limit = formatter.width - 6 - max(len(cmd[0]) for cmd in commands) + + rows = [] + for subcommand, cmd in commands: + assert cmd is not None + help = cmd.get_short_help_str(limit) + rows.append((subcommand, help)) + + if rows: + with formatter.section(_("Commands")): + formatter.write_dl(rows) + + def parse_args(self, ctx: _click.Context, args: list[str]) -> list[str]: + if not args and self.no_args_is_help and not ctx.resilient_parsing: + raise _click.exceptions.NoArgsIsHelpError(ctx) + + rest = super().parse_args(ctx, args) + + if rest: + ctx._protected_args, ctx.args = rest[:1], rest[1:] + + return ctx.args + + def invoke(self, ctx: _click.Context) -> Any: + def _process_result(value: Any) -> Any: + if self._result_callback is not None: + value = ctx.invoke(self._result_callback, value, **ctx.params) + return value + + if not ctx._protected_args: + if self.invoke_without_command: + # No subcommand was invoked, so the result callback is + # invoked with the group return value for regular + # groups, or an empty list for chained groups. + with ctx: + rv = super().invoke(ctx) + # return _process_result([] if self.chain else rv) + return _process_result(rv) + ctx.fail(_("Missing command.")) + + # Fetch args back out + args = [*ctx._protected_args, *ctx.args] + ctx.args = [] + ctx._protected_args = [] + + # Make sure the context is entered so we do not clean up + # resources until the result processor has worked. + with ctx: + cmd_name, cmd, args = self.resolve_command(ctx, args) + assert cmd is not None + ctx.invoked_subcommand = cmd_name + super().invoke(ctx) + sub_ctx = cmd.make_context(cmd_name, args, parent=ctx) + with sub_ctx: + return _process_result(sub_ctx.command.invoke(sub_ctx)) + + def shell_complete( + self, ctx: _click.Context, incomplete: str + ) -> list[CompletionItem]: + """Return a list of completions for the incomplete value. Looks + at the names of options, subcommands, and chained + multi-commands. + """ + + results = [ + CompletionItem(name, help=command.get_short_help_str()) + for name, command in _click.core._complete_visible_commands(ctx, incomplete) + ] + results.extend(super().shell_complete(ctx, incomplete)) + return results + def format_options( - self, ctx: click.Context, formatter: click.HelpFormatter + self, ctx: _click.Context, formatter: _click.HelpFormatter ) -> None: _typer_format_options(self, ctx=ctx, formatter=formatter) self.format_commands(ctx, formatter) @@ -765,12 +1145,31 @@ class TyperGroup(click.core.Group): self, ctx_args=ctx_args, prog_name=prog_name, complete_var=complete_var ) + def _click_resolve_command( + self, ctx: _click.Context, args: list[str] + ) -> tuple[str | None, _click.Command | None, list[str]]: + cmd_name = args[0] + original_cmd_name = cmd_name + + # Get the command + cmd = self.get_command(ctx, cmd_name) + + if cmd is None and ctx.token_normalize_func is not None: + cmd_name = ctx.token_normalize_func(cmd_name) + cmd = self.get_command(ctx, cmd_name) + + if cmd is None and not ctx.resilient_parsing: + if _split_opt(cmd_name)[0]: + self.parse_args(ctx, args) + ctx.fail(_("No such command {name!r}.").format(name=original_cmd_name)) + return cmd_name if cmd else None, cmd, args[1:] + def resolve_command( - self, ctx: click.Context, args: list[str] - ) -> tuple[str | None, click.Command | None, list[str]]: + self, ctx: _click.Context, args: list[str] + ) -> tuple[str | None, _click.Command | None, list[str]]: try: - return super().resolve_command(ctx, args) - except click.UsageError as e: + return self._click_resolve_command(ctx, args) + except _click.exceptions.UsageError as e: if self.suggest_commands: available_commands = list(self.commands.keys()) if available_commands and args: @@ -802,7 +1201,7 @@ class TyperGroup(click.core.Group): **extra, ) - def format_help(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: + def format_help(self, ctx: _click.Context, formatter: _click.HelpFormatter) -> None: if not HAS_RICH or self.rich_markup_mode is None: return super().format_help(ctx, formatter) from . import rich_utils @@ -813,8 +1212,6 @@ class TyperGroup(click.core.Group): markup_mode=self.rich_markup_mode, ) - def list_commands(self, ctx: click.Context) -> list[str]: - """Returns a list of subcommand names. - Note that in Click's Group class, these are sorted. - In Typer, we wish to maintain the original order of creation (cf Issue #933)""" + def list_commands(self, ctx: _click.Context) -> list[str]: + """Returns a list of subcommand names, maintaining the original order of creation (cf Issue #933)""" return [n for n, c in self.commands.items()] diff --git a/contrib/python/typer/typer/main.py b/contrib/python/typer/typer/main.py index ebcf639a2c3..a825c1b14e9 100644 --- a/contrib/python/typer/typer/main.py +++ b/contrib/python/typer/typer/main.py @@ -15,10 +15,12 @@ from types import TracebackType from typing import Annotated, Any from uuid import UUID -import click from annotated_doc import Doc from typer._types import TyperChoice +from . import _click +from ._click import types +from ._click.globals import get_current_context from ._typing import get_args, get_origin, is_literal_type, is_union, literal_values from .completion import get_completion_inspect_parameters from .core import ( @@ -73,7 +75,7 @@ def except_hook( _original_except_hook(exc_type, exc_value, tb) return typer_path = os.path.dirname(__file__) - click_path = os.path.dirname(click.__file__) + click_path = os.path.dirname(_click.__file__) internal_dir_names = [typer_path, click_path] exc = exc_value if HAS_RICH: @@ -107,7 +109,7 @@ def except_hook( return -def get_install_completion_arguments() -> tuple[click.Parameter, click.Parameter]: +def get_install_completion_arguments() -> tuple[_click.Parameter, _click.Parameter]: install_param, show_param = get_completion_inspect_parameters() click_install_param, _ = get_click_param(install_param) click_show_param, _ = get_click_param(show_param) @@ -1168,7 +1170,7 @@ def get_group(typer_instance: Typer) -> TyperGroup: return group -def get_command(typer_instance: Typer) -> click.Command: +def get_command(typer_instance: Typer) -> _click.Command: if typer_instance._add_completion: click_install_param, click_show_param = get_install_completion_arguments() if ( @@ -1178,7 +1180,7 @@ def get_command(typer_instance: Typer) -> click.Command: or len(typer_instance.registered_commands) > 1 ): # Create a Group - click_command: click.Command = get_group(typer_instance) + click_command: _click.Command = get_group(typer_instance) if typer_instance._add_completion: click_command.params.append(click_install_param) click_command.params.append(click_show_param) @@ -1288,7 +1290,7 @@ def get_group_from_info( assert group_info.typer_instance, ( "A Typer instance is needed to generate a Click Group" ) - commands: dict[str, click.Command] = {} + commands: dict[str, _click.Command] = {} for command_info in group_info.typer_instance.registered_commands: command = get_command_from_info( command_info=command_info, @@ -1330,7 +1332,6 @@ def get_group_from_info( invoke_without_command=solved_info.invoke_without_command, no_args_is_help=solved_info.no_args_is_help, subcommand_metavar=solved_info.subcommand_metavar, - chain=solved_info.chain, result_callback=solved_info.result_callback, context_settings=solved_info.context_settings, callback=get_callback( @@ -1362,14 +1363,14 @@ def get_command_name(name: str) -> str: def get_params_convertors_ctx_param_name_from_function( callback: Callable[..., Any] | None, -) -> tuple[list[click.Argument | click.Option], dict[str, Any], str | None]: +) -> tuple[list[TyperArgument | TyperOption], dict[str, Any], str | None]: params = [] convertors = {} context_param_name = None if callback: parameters = get_params_from_function(callback) for param_name, param in parameters.items(): - if lenient_issubclass(param.annotation, click.Context): + if lenient_issubclass(param.annotation, _click.Context): context_param_name = param_name continue click_param, convertor = get_click_param(param) @@ -1384,7 +1385,7 @@ def get_command_from_info( *, pretty_exceptions_short: bool, rich_markup_mode: MarkupMode, -) -> click.Command: +) -> _click.Command: assert command_info.callback, "A command must have a callback function" name = command_info.name or get_command_name(command_info.callback.__name__) # ty: ignore use_help = command_info.help @@ -1486,7 +1487,7 @@ def generate_tuple_convertor( def get_callback( *, callback: Callable[..., Any] | None = None, - params: Sequence[click.Parameter] = [], + params: Sequence[_click.Parameter] = [], convertors: dict[str, Callable[[str], Any]] | None = None, context_param_name: str | None = None, pretty_exceptions_short: bool, @@ -1510,7 +1511,7 @@ def get_callback( else: use_params[k] = v if context_param_name: - use_params[context_param_name] = click.get_current_context() + use_params[context_param_name] = get_current_context() return callback(**use_params) update_wrapper(wrapper, callback) @@ -1519,15 +1520,15 @@ def get_callback( def get_click_type( *, annotation: Any, parameter_info: ParameterInfo -) -> click.ParamType: +) -> types.ParamType: if parameter_info.click_type is not None: return parameter_info.click_type elif parameter_info.parser is not None: - return click.types.FuncParamType(parameter_info.parser) + return types.FuncParamType(parameter_info.parser) elif annotation is str: - return click.STRING + return types.STRING elif annotation is int: if parameter_info.min is not None or parameter_info.max is not None: min_ = None @@ -1536,24 +1537,24 @@ def get_click_type( min_ = int(parameter_info.min) if parameter_info.max is not None: max_ = int(parameter_info.max) - return click.IntRange(min=min_, max=max_, clamp=parameter_info.clamp) + return types.IntRange(min=min_, max=max_, clamp=parameter_info.clamp) else: - return click.INT + return types.INT elif annotation is float: if parameter_info.min is not None or parameter_info.max is not None: - return click.FloatRange( + return types.FloatRange( min=parameter_info.min, max=parameter_info.max, clamp=parameter_info.clamp, ) else: - return click.FLOAT + return types.FLOAT elif annotation is bool: - return click.BOOL + return types.BOOL elif annotation == UUID: - return click.UUID + return types.UUID elif annotation == datetime: - return click.DateTime(formats=parameter_info.formats) + return types.DateTime(formats=parameter_info.formats) elif ( annotation == Path or parameter_info.allow_dash @@ -1571,7 +1572,7 @@ def get_click_type( path_type=parameter_info.path_type, ) elif lenient_issubclass(annotation, FileTextWrite): - return click.File( + return types.File( mode=parameter_info.mode or "w", encoding=parameter_info.encoding, errors=parameter_info.errors, @@ -1579,7 +1580,7 @@ def get_click_type( atomic=parameter_info.atomic, ) elif lenient_issubclass(annotation, FileText): - return click.File( + return types.File( mode=parameter_info.mode or "r", encoding=parameter_info.encoding, errors=parameter_info.errors, @@ -1587,7 +1588,7 @@ def get_click_type( atomic=parameter_info.atomic, ) elif lenient_issubclass(annotation, FileBinaryRead): - return click.File( + return types.File( mode=parameter_info.mode or "rb", encoding=parameter_info.encoding, errors=parameter_info.errors, @@ -1595,7 +1596,7 @@ def get_click_type( atomic=parameter_info.atomic, ) elif lenient_issubclass(annotation, FileBinaryWrite): - return click.File( + return types.File( mode=parameter_info.mode or "wb", encoding=parameter_info.encoding, errors=parameter_info.errors, @@ -1603,17 +1604,12 @@ def get_click_type( atomic=parameter_info.atomic, ) elif lenient_issubclass(annotation, Enum): - # The custom TyperChoice is only needed for Click < 8.2.0, to parse the - # command line values matching them to the enum values. Click 8.2.0 added - # support for enum values but reading enum names. - # Passing here the list of enum values (instead of just the enum) accounts for - # Click < 8.2.0. return TyperChoice( [item.value for item in annotation], case_sensitive=parameter_info.case_sensitive, ) elif is_literal_type(annotation): - return click.Choice( + return TyperChoice( literal_values(annotation), case_sensitive=parameter_info.case_sensitive, ) @@ -1626,7 +1622,7 @@ def lenient_issubclass(cls: Any, class_or_tuple: AnyType | tuple[AnyType, ...]) def get_click_param( param: ParamMeta, -) -> tuple[click.Argument | click.Option, Any]: +) -> tuple[TyperArgument | TyperOption, Any]: # First, find out what will be: # * ParamInfo (ArgumentInfo or OptionInfo) # * default_value @@ -1784,7 +1780,7 @@ def get_click_param( ), convertor, ) - raise AssertionError("A click.Parameter should be returned") # pragma: no cover + raise AssertionError("A _click.Parameter should be returned") # pragma: no cover def get_param_callback( @@ -1800,9 +1796,9 @@ def get_param_callback( value_name = None untyped_names: list[str] = [] for param_name, param_sig in parameters.items(): - if lenient_issubclass(param_sig.annotation, click.Context): + if lenient_issubclass(param_sig.annotation, _click.Context): ctx_name = param_name - elif lenient_issubclass(param_sig.annotation, click.Parameter): + elif lenient_issubclass(param_sig.annotation, _click.Parameter): click_param_name = param_name else: untyped_names.append(param_name) @@ -1817,11 +1813,11 @@ def get_param_callback( if untyped_names: click_param_name = untyped_names.pop(0) if untyped_names: - raise click.ClickException( + raise _click.ClickException( "Too many CLI parameter callback function parameters" ) - def wrapper(ctx: click.Context, param: click.Parameter, value: Any) -> Any: + def wrapper(ctx: _click.Context, param: _click.Parameter, value: Any) -> Any: use_params: dict[str, Any] = {} if ctx_name: use_params[ctx_name] = ctx @@ -1851,7 +1847,7 @@ def get_param_completion( unassigned_params = list(parameters.values()) for param_sig in unassigned_params[:]: origin = get_origin(param_sig.annotation) - if lenient_issubclass(param_sig.annotation, click.Context): + if lenient_issubclass(param_sig.annotation, _click.Context): ctx_name = param_sig.name unassigned_params.remove(param_sig) elif lenient_issubclass(origin, list): @@ -1874,11 +1870,11 @@ def get_param_completion( # Extract value param name first if unassigned_params: show_params = " ".join([param.name for param in unassigned_params]) - raise click.ClickException( + raise _click.ClickException( f"Invalid autocompletion callback parameters: {show_params}" ) - def wrapper(ctx: click.Context, args: list[str], incomplete: str | None) -> Any: + def wrapper(ctx: _click.Context, args: list[str], incomplete: str | None) -> Any: use_params: dict[str, Any] = {} if ctx_name: use_params[ctx_name] = ctx @@ -1947,7 +1943,6 @@ def launch( Doc( """ Wait for the program to exit before returning. This only works if the launched program blocks. - In particular, `xdg-open` on Linux does not block. """ ), ] = False, @@ -1999,9 +1994,12 @@ def launch( has_xdg_open = _is_linux_or_bsd() and shutil.which("xdg-open") is not None if has_xdg_open: - return subprocess.Popen( + process = subprocess.Popen( ["xdg-open", url], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT - ).wait() + ) + if wait: + return process.wait() + return 0 import webbrowser @@ -2010,4 +2008,4 @@ def launch( return 0 else: - return click.launch(url, wait=wait, locate=locate) + return _click.launch(url, wait=wait, locate=locate) diff --git a/contrib/python/typer/typer/models.py b/contrib/python/typer/typer/models.py index 3285a96a243..00385c38ce5 100644 --- a/contrib/python/typer/typer/models.py +++ b/contrib/python/typer/typer/models.py @@ -1,15 +1,20 @@ import inspect import io +import os +import stat from collections.abc import Callable, Sequence from typing import ( TYPE_CHECKING, Any, + ClassVar, Optional, TypeVar, + cast, ) -import click -import click.shell_completion +from . import _click +from ._click import types +from ._click.shell_completion import CompletionItem if TYPE_CHECKING: # pragma: no cover from .core import TyperCommand, TyperGroup @@ -23,7 +28,7 @@ AnyType = type[Any] Required = ... -class Context(click.Context): +class Context(_click.Context): """ The [`Context`](https://click.palletsprojects.com/en/stable/api/#click.Context) has some additional data about the current execution of your program. When declaring it in a [callback](https://typer.tiangolo.com/tutorial/options/callback-and-context/) function, @@ -153,7 +158,7 @@ class FileBinaryWrite(io.BufferedWriter): pass -class CallbackParam(click.Parameter): +class CallbackParam(_click.Parameter): """ In a callback function, you can declare a function parameter with type `CallbackParam` to access the specific Click [`Parameter`](https://click.palletsprojects.com/en/stable/api/#click.Parameter) object. @@ -286,15 +291,15 @@ class ParameterInfo: # Note that shell_complete is not fully supported and will be removed in future versions # TODO: Remove shell_complete in a future version (after 0.16.0) shell_complete: Callable[ - [click.Context, click.Parameter, str], - list["click.shell_completion.CompletionItem"] | list[str], + [_click.Context, _click.Parameter, str], + list["CompletionItem"] | list[str], ] | None = None, autocompletion: Callable[..., Any] | None = None, default_factory: Callable[[], Any] | None = None, # Custom type parser: Callable[[str], Any] | None = None, - click_type: click.ParamType | None = None, + click_type: types.ParamType | None = None, # TyperArgument show_default: bool | str = True, show_choices: bool = True, @@ -395,15 +400,15 @@ class OptionInfo(ParameterInfo): # Note that shell_complete is not fully supported and will be removed in future versions # TODO: Remove shell_complete in a future version (after 0.16.0) shell_complete: Callable[ - [click.Context, click.Parameter, str], - list["click.shell_completion.CompletionItem"] | list[str], + [_click.Context, _click.Parameter, str], + list["CompletionItem"] | list[str], ] | None = None, autocompletion: Callable[..., Any] | None = None, default_factory: Callable[[], Any] | None = None, # Custom type parser: Callable[[str], Any] | None = None, - click_type: click.ParamType | None = None, + click_type: types.ParamType | None = None, # Option show_default: bool | str = True, prompt: bool | str = False, @@ -523,15 +528,15 @@ class ArgumentInfo(ParameterInfo): # Note that shell_complete is not fully supported and will be removed in future versions # TODO: Remove shell_complete in a future version (after 0.16.0) shell_complete: Callable[ - [click.Context, click.Parameter, str], - list["click.shell_completion.CompletionItem"] | list[str], + [_click.Context, _click.Parameter, str], + list["CompletionItem"] | list[str], ] | None = None, autocompletion: Callable[..., Any] | None = None, default_factory: Callable[[], Any] | None = None, # Custom type parser: Callable[[str], Any] | None = None, - click_type: click.ParamType | None = None, + click_type: types.ParamType | None = None, # TyperArgument show_default: bool | str = True, show_choices: bool = True, @@ -640,11 +645,98 @@ class DeveloperExceptionConfig: self.pretty_exceptions_short = pretty_exceptions_short -class TyperPath(click.Path): - # Overwrite Click's behaviour to be compatible with Typer's autocompletion system +class TyperPath(types.ParamType): + # Based originally on code from Click 8.3.1 + # Partly rewritten and added an override for shell_complete + + envvar_list_splitter: ClassVar[str] = os.path.pathsep + + def __init__( + self, + exists: bool = False, + file_okay: bool = True, + dir_okay: bool = True, + writable: bool = False, + readable: bool = True, + resolve_path: bool = False, + allow_dash: bool = False, + path_type: type[Any] | None = None, + ): + self.exists = exists + self.file_okay = file_okay + self.dir_okay = dir_okay + self.readable = readable + self.writable = writable + self.resolve_path = resolve_path + self.allow_dash = allow_dash + self.type = path_type + + if self.file_okay and not self.dir_okay: + self.name = "file" + elif self.dir_okay and not self.file_okay: + self.name = "directory" + else: + self.name = "path" + + def coerce_path_result( + self, value: str | os.PathLike[str] + ) -> str | bytes | os.PathLike[str]: + if self.type is not None and not isinstance(value, self.type): + if ( + self.type is str + ): # pragma: no cover # TODO: perhaps this branch can't be hit and should be removed + return os.fsdecode(value) + elif self.type is bytes: + return os.fsencode(value) + else: + return cast("os.PathLike[str]", self.type(value)) + + return value + + def convert( # ty: ignore[invalid-method-override] + self, + value: str | os.PathLike[str], + param: _click.Parameter | None, + ctx: Context | None, # type: ignore[override] + ) -> str | bytes | os.PathLike[str]: + rv = value + + is_dash = self.file_okay and self.allow_dash and rv in (b"-", "-") + + if not is_dash: + if self.resolve_path: + rv = os.path.realpath(rv) + + try: + st = os.stat(rv) + except OSError: + if not self.exists: + return self.coerce_path_result(rv) + self.fail( + f"{self.name.title()} {_click.utils.format_filename(value)!r} does not exist.", + param, + ctx, + ) + + name = self.name.title() + loc = repr(_click.utils.format_filename(value)) + if not self.file_okay and stat.S_ISREG(st.st_mode): + self.fail(f"{name} {loc} is a file.", param, ctx) + + if not self.dir_okay and stat.S_ISDIR(st.st_mode): + self.fail(f"{name} {loc} is a directory.", param, ctx) + + if self.readable and not os.access(rv, os.R_OK): + self.fail(f"{name} {loc} is not readable.", param, ctx) + + if self.writable and not os.access(rv, os.W_OK): + self.fail(f"{name} {loc} is not writable.", param, ctx) + + return self.coerce_path_result(rv) + def shell_complete( - self, ctx: click.Context, param: click.Parameter, incomplete: str - ) -> list[click.shell_completion.CompletionItem]: + self, ctx: _click.Context, param: _click.Parameter, incomplete: str + ) -> list[CompletionItem]: """Return an empty list so that the autocompletion functionality will work properly from the commandline. """ diff --git a/contrib/python/typer/typer/params.py b/contrib/python/typer/typer/params.py index b325b273c43..833461fa786 100644 --- a/contrib/python/typer/typer/params.py +++ b/contrib/python/typer/typer/params.py @@ -1,13 +1,15 @@ from collections.abc import Callable from typing import TYPE_CHECKING, Annotated, Any, overload -import click from annotated_doc import Doc +from . import _click +from ._click import types +from ._click.shell_completion import CompletionItem from .models import ArgumentInfo, OptionInfo if TYPE_CHECKING: # pragma: no cover - import click.shell_completion + pass # Overload for Option created with custom type 'parser' @@ -24,8 +26,8 @@ def Option( # Note that shell_complete is not fully supported and will be removed in future versions # TODO: Remove shell_complete in a future version (after 0.16.0) shell_complete: Callable[ - [click.Context, click.Parameter, str], - list["click.shell_completion.CompletionItem"] | list[str], + [_click.Context, _click.Parameter, str], + list["CompletionItem"] | list[str], ] | None = None, autocompletion: Callable[..., Any] | None = None, @@ -89,14 +91,14 @@ def Option( # Note that shell_complete is not fully supported and will be removed in future versions # TODO: Remove shell_complete in a future version (after 0.16.0) shell_complete: Callable[ - [click.Context, click.Parameter, str], - list["click.shell_completion.CompletionItem"] | list[str], + [_click.Context, _click.Parameter, str], + list["CompletionItem"] | list[str], ] | None = None, autocompletion: Callable[..., Any] | None = None, default_factory: Callable[[], Any] | None = None, # Custom type - click_type: click.ParamType | None = None, + click_type: types.ParamType | None = None, # Option show_default: bool | str = True, prompt: bool | str = False, @@ -265,8 +267,8 @@ def Option( # TODO: Remove shell_complete in a future version (after 0.16.0) shell_complete: Annotated[ Callable[ - [click.Context, click.Parameter, str], - list["click.shell_completion.CompletionItem"] | list[str], + [_click.Context, _click.Parameter, str], + list["CompletionItem"] | list[str], ] | None, Doc( @@ -343,7 +345,7 @@ def Option( ), ] = None, click_type: Annotated[ - click.ParamType | None, + types.ParamType | None, Doc( """ Define this parameter to use a [custom Click type](https://click.palletsprojects.com/en/stable/parameters/#implementing-custom-types) in your Typer applications. @@ -1014,8 +1016,8 @@ def Argument( # Note that shell_complete is not fully supported and will be removed in future versions # TODO: Remove shell_complete in a future version (after 0.16.0) shell_complete: Callable[ - [click.Context, click.Parameter, str], - list["click.shell_completion.CompletionItem"] | list[str], + [_click.Context, _click.Parameter, str], + list["CompletionItem"] | list[str], ] | None = None, autocompletion: Callable[..., Any] | None = None, @@ -1070,14 +1072,14 @@ def Argument( # Note that shell_complete is not fully supported and will be removed in future versions # TODO: Remove shell_complete in a future version (after 0.16.0) shell_complete: Callable[ - [click.Context, click.Parameter, str], - list["click.shell_completion.CompletionItem"] | list[str], + [_click.Context, _click.Parameter, str], + list["CompletionItem"] | list[str], ] | None = None, autocompletion: Callable[..., Any] | None = None, default_factory: Callable[[], Any] | None = None, # Custom type - click_type: click.ParamType | None = None, + click_type: types.ParamType | None = None, # TyperArgument show_default: bool | str = True, show_choices: bool = True, @@ -1219,8 +1221,8 @@ def Argument( # TODO: Remove shell_complete in a future version (after 0.16.0) shell_complete: Annotated[ Callable[ - [click.Context, click.Parameter, str], - list["click.shell_completion.CompletionItem"] | list[str], + [_click.Context, _click.Parameter, str], + list["CompletionItem"] | list[str], ] | None, Doc( @@ -1297,7 +1299,7 @@ def Argument( ), ] = None, click_type: Annotated[ - click.ParamType | None, + types.ParamType | None, Doc( """ Define this parameter to use a [custom Click type](https://click.palletsprojects.com/en/stable/parameters/#implementing-custom-types) in your Typer applications. diff --git a/contrib/python/typer/typer/rich_utils.py b/contrib/python/typer/typer/rich_utils.py index 69be6312074..de68f606441 100644 --- a/contrib/python/typer/typer/rich_utils.py +++ b/contrib/python/typer/typer/rich_utils.py @@ -8,7 +8,6 @@ from gettext import gettext as _ from os import getenv from typing import Any, Literal -import click from rich import box from rich.align import Align from rich.columns import Columns @@ -25,13 +24,17 @@ from rich.theme import Theme from rich.traceback import Traceback from typer.models import DeveloperExceptionConfig +from . import _click +from ._click import types +from .core import TyperArgument, TyperGroup, TyperOption + # Default styles STYLE_OPTION = "bold cyan" STYLE_SWITCH = "bold green" STYLE_NEGATIVE_OPTION = "bold magenta" STYLE_NEGATIVE_SWITCH = "bold red" -STYLE_METAVAR = "bold yellow" -STYLE_METAVAR_SEPARATOR = "dim" +STYLE_TYPES = "bold yellow" +STYLE_TYPES_SEPARATOR = "dim" STYLE_USAGE = "yellow" STYLE_USAGE_COMMAND = "bold" STYLE_DEPRECATED = "red" @@ -109,7 +112,7 @@ class OptionHighlighter(RegexHighlighter): highlights = [ r"(^|\W)(?P<switch>\-\w+)(?![a-zA-Z0-9])", r"(^|\W)(?P<option>\-\-[\w\-]+)(?![a-zA-Z0-9])", - r"(?P<metavar>\<[^\>]+\>)", + r"(?P<types>\<[^\>]+\>)", r"(?P<usage>Usage: )", ] @@ -122,17 +125,17 @@ class NegativeOptionHighlighter(RegexHighlighter): # Highlighter to make [ | ] and <> dim -class MetavarHighlighter(RegexHighlighter): +class TypesHighlighter(RegexHighlighter): highlights = [ - r"^(?P<metavar_sep>(\[|<))", - r"(?P<metavar_sep>\|)", - r"(?P<metavar_sep>(\]|>))(\.\.\.)?$", + r"^(?P<types_sep>(\[|<))", + r"(?P<types_sep>\|)", + r"(?P<types_sep>(\]|>))(\.\.\.)?$", ] highlighter = OptionHighlighter() negative_highlighter = NegativeOptionHighlighter() -metavar_highlighter = MetavarHighlighter() +types_highlighter = TypesHighlighter() def _has_ansi_character(text: str) -> bool: @@ -147,8 +150,8 @@ def _get_rich_console(stderr: bool = False) -> Console: "switch": STYLE_SWITCH, "negative_option": STYLE_NEGATIVE_OPTION, "negative_switch": STYLE_NEGATIVE_SWITCH, - "metavar": STYLE_METAVAR, - "metavar_sep": STYLE_METAVAR_SEPARATOR, + "types": STYLE_TYPES, + "types_sep": STYLE_TYPES_SEPARATOR, "usage": STYLE_USAGE, }, ), @@ -184,7 +187,7 @@ def _make_rich_text( @group() def _get_help_text( *, - obj: click.Command | click.Group, + obj: _click.Command | TyperGroup, markup_mode: MarkupModeStrict, ) -> Iterable[Markdown | Text]: """Build primary help text for a click command or group. @@ -231,8 +234,8 @@ def _get_help_text( def _get_parameter_help( *, - param: click.Option | click.Argument | click.Parameter, - ctx: click.Context, + param: TyperOption | TyperArgument | _click.Parameter, + ctx: _click.Context, markup_mode: MarkupModeStrict, ) -> Columns: """Build primary help text for a click option or argument. @@ -348,8 +351,8 @@ def _make_command_help( def _print_options_panel( *, name: str, - params: list[click.Option] | list[click.Argument], - ctx: click.Context, + params: list[TyperOption] | list[TyperArgument], + ctx: _click.Context, markup_mode: MarkupModeStrict, console: Console, ) -> None: @@ -361,43 +364,57 @@ def _print_options_panel( opt_short_strs = [] secondary_opt_long_strs = [] secondary_opt_short_strs = [] + + # check whether argument has a metavar name or type set + metavar_name = None + metavar_type = None + metavar_str = param.make_metavar(ctx=ctx) + if isinstance(param, TyperArgument): + # TODO: revise this legacy behaviour of keeping argument names lowercased for Rich formatting + if param.metavar is None and param.name: + metavar_name = metavar_str.replace(param.name.upper(), param.name) + else: + metavar_name = metavar_str + if isinstance(param, TyperOption): + metavar_type = metavar_str + for opt_str in param.opts: if "--" in opt_str: opt_long_strs.append(opt_str) + elif metavar_name: + opt_short_strs.append(metavar_name) else: opt_short_strs.append(opt_str) for opt_str in param.secondary_opts: if "--" in opt_str: secondary_opt_long_strs.append(opt_str) + elif metavar_name: # pragma: no cover + secondary_opt_short_strs.append(metavar_name) else: secondary_opt_short_strs.append(opt_str) - # Column for a metavar, if we have one - metavar = Text(style=STYLE_METAVAR, overflow="fold") - metavar_str = param.make_metavar(ctx=ctx) - # Do it ourselves if this is a positional argument - if ( - isinstance(param, click.Argument) - and param.name - and metavar_str == param.name.upper() - ): - metavar_str = param.type.name.upper() + # Column for recording the type + types_data = Text(style=STYLE_TYPES, overflow="fold") - # Skip booleans and choices (handled above) - if metavar_str != "BOOLEAN": - metavar.append(metavar_str) + # Fetch type + if metavar_type and metavar_type != "BOOLEAN": + types_data.append(metavar_type) + else: + type_str = param.type.name.upper() + if type_str != "BOOLEAN": + types_data.append(type_str) # Range - from # https://github.com/pallets/click/blob/c63c70dabd3f86ca68678b4f00951f78f52d0270/src/click/core.py#L2698-L2706 # noqa: E501 # skip count with default range type if ( - isinstance(param.type, click.types._NumberRangeBase) - and isinstance(param, click.Option) + isinstance(param.type, types._NumberRangeBase) + and isinstance(param, TyperOption) and not (param.count and param.type.min == 0 and param.type.max is None) ): range_str = param.type._describe_range() if range_str: - metavar.append(RANGE_STRING.format(range_str)) + types_data.append(RANGE_STRING.format(range_str)) # Required asterisk required: str | Text = "" @@ -411,7 +428,7 @@ def _print_options_panel( highlighter(",".join(opt_short_strs)), negative_highlighter(",".join(secondary_opt_long_strs)), negative_highlighter(",".join(secondary_opt_short_strs)), - metavar_highlighter(metavar), + types_highlighter(types_data), _get_parameter_help( param=param, ctx=ctx, @@ -459,7 +476,7 @@ def _print_options_panel( def _print_commands_panel( *, name: str, - commands: list[click.Command], + commands: list[_click.Command], markup_mode: MarkupModeStrict, console: Console, cmd_len: int, @@ -534,8 +551,8 @@ def _print_commands_panel( def rich_format_help( *, - obj: click.Command | click.Group, - ctx: click.Context, + obj: _click.Command | TyperGroup, + ctx: _click.Context, markup_mode: MarkupModeStrict, ) -> None: """Print nicely formatted help text using rich. @@ -568,18 +585,18 @@ def rich_format_help( (0, 1, 1, 1), ) ) - panel_to_arguments: defaultdict[str, list[click.Argument]] = defaultdict(list) - panel_to_options: defaultdict[str, list[click.Option]] = defaultdict(list) + panel_to_arguments: defaultdict[str, list[TyperArgument]] = defaultdict(list) + panel_to_options: defaultdict[str, list[TyperOption]] = defaultdict(list) for param in obj.get_params(ctx): # Skip if option is hidden if getattr(param, "hidden", False): continue - if isinstance(param, click.Argument): + if isinstance(param, TyperArgument): panel_name = ( getattr(param, _RICH_HELP_PANEL_NAME, None) or ARGUMENTS_PANEL_TITLE ) panel_to_arguments[panel_name].append(param) - elif isinstance(param, click.Option): + elif isinstance(param, TyperOption): panel_name = ( getattr(param, _RICH_HELP_PANEL_NAME, None) or OPTIONS_PANEL_TITLE ) @@ -623,8 +640,8 @@ def rich_format_help( console=console, ) - if isinstance(obj, click.Group): - panel_to_commands: defaultdict[str, list[click.Command]] = defaultdict(list) + if isinstance(obj, TyperGroup): + panel_to_commands: defaultdict[str, list[_click.Command]] = defaultdict(list) for command_name in obj.list_commands(ctx): command = obj.get_command(ctx, command_name) if command and not command.hidden: @@ -674,18 +691,18 @@ def rich_format_help( console.print(Padding(Align(epilogue_text, pad=False), 1)) -def rich_format_error(self: click.ClickException) -> None: +def rich_format_error(self: _click.ClickException) -> None: """Print richly formatted click errors. Called by custom exception handler to print richly formatted click errors. - Mimics original click.ClickException.echo() function but with rich formatting. + Mimics original _click.ClickException.echo() function but with rich formatting. """ # Don't do anything when it's a NoArgsIsHelpError (without importing it, cf. #1278) if self.__class__.__name__ == "NoArgsIsHelpError": return console = _get_rich_console(stderr=True) - ctx: click.Context | None = getattr(self, "ctx", None) + ctx: _click.Context | None = getattr(self, "ctx", None) if ctx is not None: console.print(ctx.get_usage()) diff --git a/contrib/python/typer/typer/testing.py b/contrib/python/typer/typer/testing.py index 09711e66fd8..7ecc0e693c0 100644 --- a/contrib/python/typer/typer/testing.py +++ b/contrib/python/typer/typer/testing.py @@ -1,30 +1,342 @@ -from collections.abc import Mapping, Sequence -from typing import IO, Any +import contextlib +import io +import os +import shlex +import sys +from collections.abc import Iterator, Mapping, Sequence +from types import TracebackType +from typing import IO, TYPE_CHECKING, Any, BinaryIO, cast -from click.testing import CliRunner as ClickCliRunner # noqa -from click.testing import Result from typer.main import Typer from typer.main import get_command as _get_command +from . import _click +from ._click import _compat, formatting, termui, utils -class CliRunner(ClickCliRunner): - def invoke( # type: ignore +if TYPE_CHECKING: + from _typeshed import ReadableBuffer + + +def make_input_stream(input: str | bytes | None, charset: str) -> BinaryIO: + if input is None: + input = b"" + elif isinstance(input, str): + input = input.encode(charset) + + return io.BytesIO(input) + + +class BytesIOCopy(io.BytesIO): + """Patch ``io.BytesIO`` to let the written stream be copied to another.""" + + def __init__(self, copy_to: io.BytesIO) -> None: + super().__init__() + self.copy_to = copy_to + + def flush(self) -> None: + super().flush() + self.copy_to.flush() + + def write(self, b: "ReadableBuffer") -> int: + self.copy_to.write(b) + return super().write(b) + + +class StreamMixer: + """Mixes `<stdout>` and `<stderr>` streams. + + The result is available in the ``output`` attribute. + """ + + def __init__(self) -> None: + self.output: io.BytesIO = io.BytesIO() + self.stdout: io.BytesIO = BytesIOCopy(copy_to=self.output) + self.stderr: io.BytesIO = BytesIOCopy(copy_to=self.output) + + def __del__(self) -> None: + """Guarantee that file-like objects are closed in a predictable order""" + self.stderr.close() + self.stdout.close() + self.output.close() + + +class _NamedTextIOWrapper(io.TextIOWrapper): + def __init__(self, buffer: BinaryIO, name: str, mode: str, **kwargs: Any) -> None: + super().__init__(buffer, **kwargs) + self._name = name + self._mode = mode + + @property + def name(self) -> str: + return self._name # pragma: no cover + + @property + def mode(self) -> str: + return self._mode # pragma: no cover + + +class Result: + """Holds the captured result of an invoked CLI script.""" + + def __init__( + self, + runner: "CliRunner", + stdout_bytes: bytes, + stderr_bytes: bytes, + output_bytes: bytes, + return_value: Any, + exit_code: int, + exception: BaseException | None, + exc_info: tuple[type[BaseException], BaseException, TracebackType] + | None = None, + ): + self.runner = runner + self.stdout_bytes = stdout_bytes + self.stderr_bytes = stderr_bytes + self.output_bytes = output_bytes + self.return_value = return_value + self.exit_code = exit_code + self.exception = exception + self.exc_info = exc_info + + @property + def output(self) -> str: + """The terminal output as unicode string, as the user would see it.""" + return self.output_bytes.decode(self.runner.charset, "replace").replace( + "\r\n", "\n" + ) + + @property + def stdout(self) -> str: + """The standard output as unicode string.""" + return self.stdout_bytes.decode(self.runner.charset, "replace").replace( + "\r\n", "\n" + ) + + @property + def stderr(self) -> str: + """The standard error as unicode string.""" + return self.stderr_bytes.decode(self.runner.charset, "replace").replace( + "\r\n", "\n" + ) + + def __repr__(self) -> str: + exc_str = repr(self.exception) if self.exception else "okay" + return f"<{type(self).__name__} {exc_str}>" + + +class CliRunner: + """The CLI runner provides functionality to invoke a command line + script for unittesting purposes in an isolated environment. This only + works in single-threaded systems without any concurrency as it changes the + global interpreter state. Based on functionality from Click. + """ + + def __init__( + self, + charset: str = "utf-8", + env: Mapping[str, str | None] | None = None, + ) -> None: + self.charset = charset + self.env: Mapping[str, str | None] = env or {} + + def get_default_prog_name(self, cli: _click.Command) -> str: + """Return the default program name for a command. + The default is the `name` attribute or ``"root"`` if not set. + """ + return cli.name or "root" + + def make_env( + self, overrides: Mapping[str, str | None] | None = None + ) -> Mapping[str, str | None]: + """Returns the environment overrides for invoking a script.""" + rv = dict(self.env) + if overrides: + rv.update(overrides) + return rv + + @contextlib.contextmanager + def isolation( + self, + input: str | bytes | None = None, + env: Mapping[str, str | None] | None = None, + color: bool = False, + ) -> Iterator[tuple[io.BytesIO, io.BytesIO, io.BytesIO]]: + """A context manager that sets up the isolation for invoking of a + command line tool. This sets up `<stdin>` with the given input data + and `os.environ` with the overrides from the given dictionary. + """ + bytes_input = make_input_stream(input, self.charset) + + old_stdin = sys.stdin + old_stdout = sys.stdout + old_stderr = sys.stderr + old_forced_width = formatting.FORCED_WIDTH + formatting.FORCED_WIDTH = 80 + + env = self.make_env(env) + + stream_mixer = StreamMixer() + + sys.stdin = text_input = _NamedTextIOWrapper( + bytes_input, encoding=self.charset, name="<stdin>", mode="r" + ) + + sys.stdout = _NamedTextIOWrapper( + stream_mixer.stdout, encoding=self.charset, name="<stdout>", mode="w" + ) + + sys.stderr = _NamedTextIOWrapper( + stream_mixer.stderr, + encoding=self.charset, + name="<stderr>", + mode="w", + errors="backslashreplace", + ) + + def visible_input(prompt: str | None = None) -> str: + sys.stdout.write(prompt or "") + try: + val = next(text_input).rstrip("\r\n") + except StopIteration as e: # pragma: no cover + raise EOFError() from e + sys.stdout.write(f"{val}\n") + sys.stdout.flush() + return val + + def hidden_input(prompt: str | None = None) -> str: + sys.stdout.write(f"{prompt or ''}\n") + sys.stdout.flush() + try: + return next(text_input).rstrip("\r\n") + except StopIteration as e: # pragma: no cover + raise EOFError() from e + + def _getchar(echo: bool) -> str: + char = sys.stdin.read(1) + + if echo: + sys.stdout.write(char) + + sys.stdout.flush() + return char + + default_color = color + + def should_strip_ansi( + stream: IO[Any] | None = None, color: bool | None = None + ) -> bool: + if color is None: + return not default_color + return not color + + old_visible_prompt_func = termui.visible_prompt_func + old_hidden_prompt_func = termui.hidden_prompt_func + old__getchar_func = termui._getchar + old_should_strip_ansi = utils.should_strip_ansi # type: ignore[attr-defined] + old__compat_should_strip_ansi = _compat.should_strip_ansi + termui.visible_prompt_func = visible_input + termui.hidden_prompt_func = hidden_input # ty: ignore[invalid-assignment] + termui._getchar = _getchar + utils.should_strip_ansi = should_strip_ansi # type: ignore + _compat.should_strip_ansi = should_strip_ansi # ty: ignore[invalid-assignment] + + old_env = {} + try: + for key, value in env.items(): + old_env[key] = os.environ.get(key) + if value is None: + try: + del os.environ[key] + except Exception: # pragma: no cover + pass + else: + os.environ[key] = value + yield (stream_mixer.stdout, stream_mixer.stderr, stream_mixer.output) + finally: + for key, value in old_env.items(): + if value is None: + try: + del os.environ[key] + except Exception: # pragma: no cover + pass + else: + os.environ[key] = value + sys.stdout = old_stdout + sys.stderr = old_stderr + sys.stdin = old_stdin + termui.visible_prompt_func = old_visible_prompt_func + termui.hidden_prompt_func = old_hidden_prompt_func + termui._getchar = old__getchar_func + utils.should_strip_ansi = old_should_strip_ansi # type: ignore[attr-defined] + _compat.should_strip_ansi = old__compat_should_strip_ansi + formatting.FORCED_WIDTH = old_forced_width + + def invoke( self, app: Typer, args: str | Sequence[str] | None = None, - input: bytes | str | IO[Any] | None = None, + input: bytes | str | None = None, env: Mapping[str, str | None] | None = None, catch_exceptions: bool = True, color: bool = False, **extra: Any, ) -> Result: - use_cli = _get_command(app) - return super().invoke( - use_cli, - args=args, - input=input, - env=env, - catch_exceptions=catch_exceptions, - color=color, - **extra, + cli = _get_command(app) + exc_info = None + + with self.isolation(input=input, env=env, color=color) as outstreams: + return_value = None + exception: BaseException | None = None + exit_code = 0 + + if isinstance(args, str): + args = shlex.split(args) + + try: + prog_name = extra.pop("prog_name") + except KeyError: + prog_name = self.get_default_prog_name(cli) + + try: + return_value = cli.main(args=args or (), prog_name=prog_name, **extra) + except SystemExit as e: + exc_info = sys.exc_info() + e_code = cast("int | Any | None", e.code) + + if e_code is None: + e_code = 0 + + if e_code != 0: + exception = e + + if not isinstance(e_code, int): + sys.stdout.write(str(e_code)) + sys.stdout.write("\n") + e_code = 1 + + exit_code = e_code + + except Exception as e: + if not catch_exceptions: + raise + exception = e + exit_code = 1 + exc_info = sys.exc_info() + finally: + sys.stdout.flush() + sys.stderr.flush() + stdout = outstreams[0].getvalue() + stderr = outstreams[1].getvalue() + output = outstreams[2].getvalue() + + return Result( + runner=self, + stdout_bytes=stdout, + stderr_bytes=stderr, + output_bytes=output, + return_value=return_value, + exit_code=exit_code, + exception=exception, + exc_info=exc_info, # type: ignore ) diff --git a/contrib/python/typer/ya.make b/contrib/python/typer/ya.make index 9eea6e5fec9..cf221dfd26c 100644 --- a/contrib/python/typer/ya.make +++ b/contrib/python/typer/ya.make @@ -2,23 +2,41 @@ PY3_LIBRARY() -VERSION(0.25.1) +VERSION(0.26.8) LICENSE(MIT) PEERDIR( contrib/python/annotated-doc - contrib/python/click contrib/python/rich contrib/python/shellingham ) NO_LINT() +NO_CHECK_IMPORTS( + typer._click._winconsole +) + PY_SRCS( TOP_LEVEL typer/__init__.py typer/__main__.py + typer/_click/__init__.py + typer/_click/_compat.py + typer/_click/_termui_impl.py + typer/_click/_textwrap.py + typer/_click/_winconsole.py + typer/_click/core.py + typer/_click/decorators.py + typer/_click/exceptions.py + typer/_click/formatting.py + typer/_click/globals.py + typer/_click/parser.py + typer/_click/shell_completion.py + typer/_click/termui.py + typer/_click/types.py + typer/_click/utils.py typer/_completion_classes.py typer/_completion_shared.py typer/_types.py @@ -41,6 +59,8 @@ RESOURCE_FILES( .dist-info/entry_points.txt .dist-info/top_level.txt typer/.agents/skills/typer/SKILL.md + typer/_click/LICENSE.txt + typer/_click/py.typed typer/py.typed ) diff --git a/library/cpp/containers/paged_vector/README.md b/library/cpp/containers/paged_vector/README.md index 1f67a0d1f7a..ae49c0c6523 100644 --- a/library/cpp/containers/paged_vector/README.md +++ b/library/cpp/containers/paged_vector/README.md @@ -34,6 +34,7 @@ The interface mirrors a subset of `std::vector`: | Iterators | `begin()/end()`, `rbegin()/rend()` + const versions; random-access iterators | | Capacity | `size()`, `empty()`, `explicit operator bool()` (true when non-empty) | | Modifiers | `push_back()`, `emplace_back()` (returns a reference), `pop_back()`, `append(b, e)`, `erase(it)`, `erase(b, e)`, `resize()`, `clear()` | +| Iteration helpers | `ForEach(fn)`, `ForEachReverse(fn)` | | Comparison | `operator==`, `operator<` (lexicographical) | Notable differences from `std::vector`: @@ -48,6 +49,37 @@ Iterators are random-access and are implemented as an *(owner pointer, offset)* - Dereferencing goes through the vector, so an iterator is only valid while its source container is alive. - To get the current index of an element from an iterator, call `it.GetIndex()` — it returns the offset of the pointed-to element within the container (equivalent to `it - begin()`). +## Iteration helpers + +```cpp +template <class Function> +void ForEach(Function fn) const; + +template <class Function> +void ForEachReverse(Function fn) const; +``` + +`ForEach` applies `fn` to every element **from the first to the last**; `ForEachReverse` applies `fn` **from the last to the first**. + +These are faster than iterating with `begin()/end()` or `rbegin()/rend()`: they walk the pages directly through raw pointers, avoiding the two levels of indirection that the offset-based iterators go through on each dereference. This matters for containers with a large `PageSize` (the default is 1M elements per page), where the inner per-page loop is tight. + +```cpp +TPagedVector<int, 1024> v; +// ... fill v ... + +long long sum = 0; +v.ForEach([&](int x) { sum += x; }); + +// process elements back-to-front, e.g. for a stack-like traversal +v.ForEachReverse([&](int x) { + // ... +}); +``` + +Notes: + +- The order is well-defined and contiguous: `ForEach` visits element `0, 1, ..., size()-1`; `ForEachReverse` visits `size()-1, ..., 1, 0`. +- Both are O(n) and do not allocate. ## Complexity @@ -58,6 +90,7 @@ Iterators are random-access and are implemented as an *(owner pointer, offset)* | `pop_back` | O(1) | | `erase` | O(n) — shifts all following elements | | `clear` | O(n) for non-trivially destructible `T`, O(pages) otherwise | +| `ForEach` / `ForEachReverse` | O(n), no allocations | ## Notes diff --git a/library/cpp/containers/paged_vector/paged_vector.h b/library/cpp/containers/paged_vector/paged_vector.h index 5bb8d8eb521..43073852ab3 100644 --- a/library/cpp/containers/paged_vector/paged_vector.h +++ b/library/cpp/containers/paged_vector/paged_vector.h @@ -278,6 +278,51 @@ namespace NPagedVector { std::swap(CurrentPageSize_, v.CurrentPageSize_); } + // Fast iteration over all elements. + template <class Function> + void ForEach(Function fn) const { + if (Pages_.empty()) { + return; + } + + const auto currentPageIt = Pages_.end() - 1; + for (auto it = Pages_.begin(); it != currentPageIt; ++it) { + const TPage& page = **it; + for (size_t i = 0; i < PageSize; ++i) { + fn(page[i]); + } + } + + const TPage& currentPage = **currentPageIt; + + for (size_t i = 0; i < CurrentPageSize_; ++i) { + fn(currentPage[i]); + } + } + + // Fast iteration over all elements in reverse order. + template <class Function> + void ForEachReverse(Function fn) const { + if (Pages_.empty()) { + return; + } + + const TPage& currentPage = *Pages_.back(); + + for (size_t i = CurrentPageSize_; i > 0;) { + --i; + fn(currentPage[i]); + } + + for (auto it = Pages_.rbegin() + 1; it != Pages_.rend(); ++it) { + const TPage& page = **it; + for (size_t i = PageSize; i > 0;) { + --i; + fn(page[i]); + } + } + } + private: static size_t PageNumber(size_t idx) { return idx / PageSize; diff --git a/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp b/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp index 4863227e7f1..b0f39f748fe 100644 --- a/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp +++ b/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp @@ -29,6 +29,8 @@ class TPagedVectorTest: public TTestBase { UNIT_TEST(TestClear) UNIT_TEST(TestBack) UNIT_TEST(TestIterator) + UNIT_TEST(TestForEach) + UNIT_TEST(TestForEachReverse) UNIT_TEST_SUITE_END(); private: @@ -645,6 +647,224 @@ private: UNIT_ASSERT_VALUES_EQUAL(it.GetIndex(), 6); UNIT_ASSERT_VALUES_EQUAL(*it, "7"); } + + void TestForEach() { + using NPagedVector::TPagedVector; + + // Empty vector: the callback must not be invoked at all. + { + TPagedVector<int, 3> v; + size_t calls = 0; + v.ForEach([&](int) { + ++calls; + }); + UNIT_ASSERT_VALUES_EQUAL(calls, 0u); + } + + // Single element: the only element is visited once. + { + TPagedVector<int, 3> v; + v.push_back(42); + TVector<int> visited; + v.ForEach([&](int x) { + visited.push_back(x); + }); + TVector<int> expected{42}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Several elements within a single (partially filled) page. + { + TPagedVector<int, 3> v; + for (int i = 0; i < 2; ++i) { + v.push_back(i); + } + TVector<int> visited; + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector<int> expected{0, 1}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // A single exactly full page (3 elements): the visit order must be + // strictly forward. + { + TPagedVector<int, 3> v; + for (int i = 0; i < 3; ++i) { + v.push_back(i); + } + TVector<int> visited; + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector<int> expected{0, 1, 2}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Multiple pages with a partially filled last page: the visit order + // must be strictly forward (from the first element to the last). + { + TPagedVector<int, 3> v; + const int n = 10; // spans 4 pages of size 3: [0..2][3..5][6..8][9] + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector<int> visited; + visited.reserve(n); + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector<int> expected; + expected.reserve(n); + for (int i = 0; i < n; ++i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Exactly full pages (no partial tail): every element is visited, + // last page is completely filled. + { + TPagedVector<int, 3> v; + const int n = 9; // exactly 3 full pages of size 3 + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector<int> visited; + visited.reserve(n); + int expectedElement = 0; + v.ForEach([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + ++expectedElement; + visited.push_back(x); + }); + TVector<int> expected; + expected.reserve(n); + for (int i = 0; i < n; ++i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + } + + void TestForEachReverse() { + using NPagedVector::TPagedVector; + + // Empty vector: the callback must not be invoked at all. + { + TPagedVector<int, 3> v; + size_t calls = 0; + v.ForEachReverse([&](int) { + ++calls; + }); + UNIT_ASSERT_VALUES_EQUAL(calls, 0u); + } + + // Single element: the only element is visited once. + { + TPagedVector<int, 3> v; + v.push_back(42); + TVector<int> visited; + v.ForEachReverse([&](int x) { + visited.push_back(x); + }); + TVector<int> expected{42}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Several elements within a single (partially filled) page. + { + TPagedVector<int, 3> v; + for (int i = 0; i < 2; ++i) { + v.push_back(i); + } + TVector<int> visited; + int expectedElement = 1; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector<int> expected{1, 0}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // A single exactly full page (3 elements): the visit order must be + // strictly reverse. + { + TPagedVector<int, 3> v; + for (int i = 0; i < 3; ++i) { + v.push_back(i); + } + TVector<int> visited; + int expectedElement = 2; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector<int> expected{2, 1, 0}; + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Multiple pages with a partially filled last page: the visit order + // must be strictly reverse (from the last element to the first). + { + TPagedVector<int, 3> v; + const int n = 10; // spans 4 pages of size 3: [0..2][3..5][6..8][9] + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector<int> visited; + visited.reserve(n); + int expectedElement = n - 1; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector<int> expected; + expected.reserve(n); + for (int i = n - 1; i >= 0; --i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + + // Exactly full pages (no partial tail): every element is visited, + // last page is completely filled. + { + TPagedVector<int, 3> v; + const int n = 9; // exactly 3 full pages of size 3 + for (int i = 0; i < n; ++i) { + v.push_back(i); + } + TVector<int> visited; + visited.reserve(n); + int expectedElement = n - 1; + v.ForEachReverse([&](int x) { + UNIT_ASSERT_VALUES_EQUAL(x, expectedElement); + --expectedElement; + visited.push_back(x); + }); + TVector<int> expected; + expected.reserve(n); + for (int i = n - 1; i >= 0; --i) { + expected.push_back(i); + } + UNIT_ASSERT_VALUES_EQUAL(visited, expected); + } + } }; UNIT_TEST_SUITE_REGISTRATION(TPagedVectorTest); diff --git a/library/cpp/yt/error/error.h b/library/cpp/yt/error/error.h index 1e5c2147d82..73904315065 100644 --- a/library/cpp/yt/error/error.h +++ b/library/cpp/yt/error/error.h @@ -370,10 +370,8 @@ void ThrowErrorExceptionIfFailed(TErrorLike&& error); ::NYT::NDetail::ThrowErrorExceptionIfFailed((error) __VA_OPT__(,) __VA_ARGS__) \ #define THROW_ERROR_EXCEPTION_UNLESS(condition, head, ...) \ - if ((condition)) {\ - } else { \ - THROW_ERROR ::NYT::TError(head __VA_OPT__(,) __VA_ARGS__); \ - } + if ((condition)) {} else \ + THROW_ERROR ::NYT::TError(head __VA_OPT__(,) __VA_ARGS__) #define THROW_ERROR_EXCEPTION_IF(condition, head, ...) \ THROW_ERROR_EXCEPTION_UNLESS(!(condition), head, __VA_ARGS__) diff --git a/library/cpp/yt/error/unittests/error_ut.cpp b/library/cpp/yt/error/unittests/error_ut.cpp index e7cbbcd21d8..4cba6ca563f 100644 --- a/library/cpp/yt/error/unittests/error_ut.cpp +++ b/library/cpp/yt/error/unittests/error_ut.cpp @@ -395,6 +395,19 @@ TEST(TErrorTest, ThrowErrorExceptionIfFailedMacroExpression) } } +TEST(TErrorTest, ThrowErrorExceptionIfMacroAttributes) +{ + try { + THROW_ERROR_EXCEPTION_IF(true, "Condition holds") + << TErrorAttribute("attr", "attr_value"); + ADD_FAILURE() << "Expected the macro to throw."; + } catch (const std::exception& ex) { + TError error(ex); + EXPECT_EQ(error.GetMessage(), "Condition holds"); + EXPECT_EQ(error.Attributes().Get<std::string>("attr"), "attr_value"); + } +} + TEST(TErrorTest, ThrowErrorExceptionIfFailedMacroDontStealValue) { TErrorOr<TWidget> widget = TWidget(); diff --git a/ydb/core/base/auth.cpp b/ydb/core/base/auth.cpp index 118d1619d96..76b74b6c7b1 100644 --- a/ydb/core/base/auth.cpp +++ b/ydb/core/base/auth.cpp @@ -1,10 +1,10 @@ +#include "auth.h" +#include "appdata.h" + #include <ydb/core/protos/config.pb.h> #include <ydb/core/protos/flat_tx_scheme.pb.h> #include <ydb/library/aclib/aclib.h> -#include "auth.h" -#include "appdata.h" - namespace NKikimr { namespace { @@ -70,12 +70,41 @@ bool IsAdministrator(const TAppData* appData, const NACLib::TUserToken* userToke return IsTokenAllowed(userToken, appData->AdministrationAllowedSIDs); } -bool IsStrictDatabaseOnlyToken(const TAppData* appData, const TString& userTokenSerialized) { +EAccessLevel GetHighestAccessLevel(const TAppData* appData, const NACLib::TUserToken* userToken) { const auto& securityConfig = appData->DomainsConfig.GetSecurityConfig(); - return IsTokenAllowed(userTokenSerialized, securityConfig.GetDatabaseAllowedSIDs()) - && !IsTokenAllowed(userTokenSerialized, securityConfig.GetViewerAllowedSIDs()) - && !IsTokenAllowed(userTokenSerialized, securityConfig.GetMonitoringAllowedSIDs()) - && !IsTokenAllowed(userTokenSerialized, securityConfig.GetAdministrationAllowedSIDs()); + const bool isAdministrationAllowed = IsAdministrator(appData, userToken); + const bool isMonitoringAllowed = isAdministrationAllowed + || IsTokenAllowed(userToken, securityConfig.GetMonitoringAllowedSIDs()); + const bool isViewerAllowed = isMonitoringAllowed + || IsTokenAllowed(userToken, securityConfig.GetViewerAllowedSIDs()); + const bool isDatabaseAllowed = isViewerAllowed + || IsTokenAllowed(userToken, securityConfig.GetDatabaseAllowedSIDs()); + + // The order of checks is important: we want to return the highest level that is allowed. + // I.e. if a user is in any sids list, but the administration allowed sids list is empty, + // we need to return the EAccessLevel::Administration access level. + if (isAdministrationAllowed) { + return EAccessLevel::Administration; + } + if (isMonitoringAllowed) { + return EAccessLevel::Monitoring; + } + if (isViewerAllowed) { + return EAccessLevel::Viewer; + } + if (isDatabaseAllowed) { + return EAccessLevel::Database; + } + return EAccessLevel::None; +} + +EAccessLevel GetHighestAccessLevel(const TAppData* appData, const TString& userTokenSerialized) { + NACLib::TUserToken userToken = ParseUserToken(userTokenSerialized); + return GetHighestAccessLevel(appData, &userToken); +} + +bool IsStrictDatabaseOnlyToken(const TAppData* appData, const TString& userTokenSerialized) { + return GetHighestAccessLevel(appData, userTokenSerialized) == EAccessLevel::Database; } bool IsDatabaseAdministrator(const NACLib::TUserToken* userToken, const NACLib::TSID& databaseOwner) { diff --git a/ydb/core/base/auth.h b/ydb/core/base/auth.h index ebc888bdc17..5b7e30bdd98 100644 --- a/ydb/core/base/auth.h +++ b/ydb/core/base/auth.h @@ -19,8 +19,6 @@ bool IsTokenAllowed(const TString& userTokenSerialized, const NProtoBuf::Repeate bool IsAdministrator(const TAppData* appData, const TString& userTokenSerialized); bool IsAdministrator(const TAppData* appData, const NACLib::TUserToken* userToken); -bool IsStrictDatabaseOnlyToken(const TAppData* appData, const TString& userTokenSerialized); - // Check token against database owner bool IsDatabaseAdministrator(const NACLib::TUserToken* userToken, const NACLib::TSID& databaseOwner); @@ -33,4 +31,20 @@ bool IsDatabaseAdministrator(const NACLib::TUserToken* userToken, const NACLib:: TString ChooseAppropriateOwner(const NKikimrScheme::TEvModifySchemeTransaction& record, const TAppData* appData, const std::optional<NACLib::TUserToken>& userToken = std::nullopt); +enum class EAccessLevel { + None /* "none" */, + Database /* "database" */, + Viewer /* "viewer" */, + Monitoring /* "monitoring" */, + Administration /* "administration" */, +}; + +// EAccessLevel::None means that no access level was matched for the given token and security config. +// It is not the same as an anonymous request: a missing token may still resolve to any level +// when the corresponding allowed_sids list is empty. +EAccessLevel GetHighestAccessLevel(const TAppData* appData, const NACLib::TUserToken* userToken); +EAccessLevel GetHighestAccessLevel(const TAppData* appData, const TString& userTokenSerialized); + +bool IsStrictDatabaseOnlyToken(const TAppData* appData, const TString& userTokenSerialized); + } // namespace NKikimr diff --git a/ydb/core/base/auth_ut.cpp b/ydb/core/base/auth_ut.cpp index 374e841704c..09966f9b1d2 100644 --- a/ydb/core/base/auth_ut.cpp +++ b/ydb/core/base/auth_ut.cpp @@ -19,6 +19,7 @@ public: securityConfig.AddViewerAllowedSIDs("viewer"); securityConfig.AddMonitoringAllowedSIDs("monitoring"); securityConfig.AddAdministrationAllowedSIDs("admin"); + AppData.AdministrationAllowedSIDs = {"admin"}; } const TAppData* GetAppData() const { @@ -149,6 +150,81 @@ Y_UNIT_TEST_SUITE(AuthStrictDatabaseOnly) { } +Y_UNIT_TEST_SUITE(AuthAccessLevel) { + + Y_UNIT_TEST(AdministrationAllowedSid) { + TStrictDatabaseOnlyFixture fixture; + NACLib::TUserToken token({ .UserSID = "admin" }); + UNIT_ASSERT_EQUAL(GetHighestAccessLevel(fixture.GetAppData(), &token), EAccessLevel::Administration); + } + + Y_UNIT_TEST(ViewerAllowedSidOnly) { + TStrictDatabaseOnlyFixture fixture; + NACLib::TUserToken token({ .UserSID = "viewer" }); + UNIT_ASSERT_EQUAL(GetHighestAccessLevel(fixture.GetAppData(), &token), EAccessLevel::Viewer); + } + + Y_UNIT_TEST(NoToken) { + TStrictDatabaseOnlyFixture fixture; + UNIT_ASSERT_EQUAL(GetHighestAccessLevel(fixture.GetAppData(), nullptr), EAccessLevel::None); + } + + Y_UNIT_TEST(UnknownSid) { + TStrictDatabaseOnlyFixture fixture; + NACLib::TUserToken token({ .UserSID = "unknown" }); + UNIT_ASSERT_EQUAL(GetHighestAccessLevel(fixture.GetAppData(), &token), EAccessLevel::None); + } + + Y_UNIT_TEST(EmptyTokenWithEmptyDatabaseAllowedSids) { + TAppData appData(0, 0, 0, 0, TMap<TString, ui32>{}, nullptr, nullptr, nullptr, nullptr); + auto& securityConfig = *appData.DomainsConfig.MutableSecurityConfig(); + securityConfig.AddViewerAllowedSIDs("viewer"); + securityConfig.AddMonitoringAllowedSIDs("monitoring"); + securityConfig.AddAdministrationAllowedSIDs("admin"); + appData.AdministrationAllowedSIDs = {"admin"}; + + UNIT_ASSERT_EQUAL(GetHighestAccessLevel(&appData, nullptr), EAccessLevel::Database); + UNIT_ASSERT_EQUAL(GetHighestAccessLevel(&appData, ""), EAccessLevel::Database); + } + + Y_UNIT_TEST(EmptyTokenWithEmptyMonitoringAllowedSids) { + TAppData appData(0, 0, 0, 0, TMap<TString, ui32>{}, nullptr, nullptr, nullptr, nullptr); + auto& securityConfig = *appData.DomainsConfig.MutableSecurityConfig(); + securityConfig.AddDatabaseAllowedSIDs("database"); + securityConfig.AddViewerAllowedSIDs("viewer"); + securityConfig.AddAdministrationAllowedSIDs("admin"); + appData.AdministrationAllowedSIDs = {"admin"}; + + UNIT_ASSERT_EQUAL(GetHighestAccessLevel(&appData, nullptr), EAccessLevel::Monitoring); + } + + Y_UNIT_TEST(SameUserInAllSids) { + TAppData appData(0, 0, 0, 0, TMap<TString, ui32>{}, nullptr, nullptr, nullptr, nullptr); + auto& securityConfig = *appData.DomainsConfig.MutableSecurityConfig(); + securityConfig.AddDatabaseAllowedSIDs("admin"); + securityConfig.AddViewerAllowedSIDs("admin"); + securityConfig.AddMonitoringAllowedSIDs("admin"); + securityConfig.AddAdministrationAllowedSIDs("admin"); + appData.AdministrationAllowedSIDs = {"admin"}; + NACLib::TUserToken token({ .UserSID = "admin"}); + UNIT_ASSERT_EQUAL(GetHighestAccessLevel(&appData, &token), EAccessLevel::Administration); + UNIT_ASSERT_EQUAL(IsStrictDatabaseOnlyToken(&appData, token.SerializeAsString()), false); + } + + Y_UNIT_TEST(MatchToAllSids) { + TAppData appData(0, 0, 0, 0, TMap<TString, ui32>{}, nullptr, nullptr, nullptr, nullptr); + auto& securityConfig = *appData.DomainsConfig.MutableSecurityConfig(); + securityConfig.AddDatabaseAllowedSIDs("database"); + securityConfig.AddViewerAllowedSIDs("viewer"); + securityConfig.AddMonitoringAllowedSIDs("monitoring"); + securityConfig.AddAdministrationAllowedSIDs("admin"); + appData.AdministrationAllowedSIDs = {"admin"}; + NACLib::TUserToken token({ .UserSID = "user", .GroupSIDs = {"database", "viewer", "monitoring", "admin"} }); + UNIT_ASSERT_EQUAL(GetHighestAccessLevel(&appData, &token), EAccessLevel::Administration); + UNIT_ASSERT_EQUAL(IsStrictDatabaseOnlyToken(&appData, token.SerializeAsString()), false); + } +} + Y_UNIT_TEST_SUITE(AuthDatabaseAdmin) { // Empty owner forbids empty token (regardless of its kind) diff --git a/ydb/core/base/ya.make b/ydb/core/base/ya.make index 0c62c56f6c2..02584282bbc 100644 --- a/ydb/core/base/ya.make +++ b/ydb/core/base/ya.make @@ -141,6 +141,7 @@ ENDIF() GENERATE_ENUM_SERIALIZATION(boot_type.h) GENERATE_ENUM_SERIALIZATION(memory_controller_iface.h) +GENERATE_ENUM_SERIALIZATION(auth.h) END() diff --git a/ydb/core/blob_depot/s3_router.cpp b/ydb/core/blob_depot/s3_router.cpp index 8801f4058e0..2035e4e89ac 100644 --- a/ydb/core/blob_depot/s3_router.cpp +++ b/ydb/core/blob_depot/s3_router.cpp @@ -9,6 +9,7 @@ #include <ydb/library/actors/http/http_proxy.h> #include <library/cpp/random_provider/random_provider.h> +#include <util/string/cast.h> #include <util/string/strip.h> namespace NKikimr::NBlobDepot { @@ -76,6 +77,7 @@ namespace NKikimr::NBlobDepot { }; NKikimrBlobDepot::TS3BackendSettings Settings; + TString OriginalEndpoint; TString CurrentEndpoint; TActorId InnerWrapperId; TActorId HttpProxyId; @@ -100,22 +102,41 @@ namespace NKikimr::NBlobDepot { return TDuration::Seconds(sec); } - void BuildInnerWrapper(const TString& endpoint) { + ui16 BalancerProxyPort() const { + return Settings.GetBalancerProxyPort(); + } + + void RegisterInnerWrapper(NWrappers::IExternalStorageConfig::TPtr externalStorageConfig) { if (InnerWrapperId) { Send(InnerWrapperId, new TEvents::TEvPoison()); InnerWrapperId = {}; } - auto* mutableSettings = Settings.MutableSettings(); - mutableSettings->SetEndpoint(endpoint); - auto externalStorageConfig = NWrappers::IExternalStorageConfig::Construct( - AppData()->AwsClientConfig, *mutableSettings); + auto storageOperator = externalStorageConfig->ConstructStorageOperator(); storageOperator->InitReplyAdapter(std::make_shared<TRouterReplyAdapter>( TActivationContext::ActorSystem(), SelfId(), TEvPrivate::EvRefreshNow)); InnerWrapperId = Register(NWrappers::CreateStorageWrapper(std::move(storageOperator))); + } + + void BuildInnerWrapper(const TString& endpoint) { + auto* mutableSettings = Settings.MutableSettings(); + mutableSettings->SetEndpoint(endpoint); + RegisterInnerWrapper(NWrappers::IExternalStorageConfig::Construct( + AppData()->AwsClientConfig, *mutableSettings)); CurrentEndpoint = endpoint; } + void BuildInnerWrapperViaProxy(const TString& host, ui16 port) { + auto* mutableSettings = Settings.MutableSettings(); + mutableSettings->SetEndpoint(OriginalEndpoint); + mutableSettings->SetProxyHost(host); + mutableSettings->SetProxyPort(port); + mutableSettings->SetProxyScheme(Settings.GetBalancerProxyScheme()); + RegisterInnerWrapper(NWrappers::IExternalStorageConfig::Construct( + AppData()->AwsClientConfig, *mutableSettings)); + CurrentEndpoint = TStringBuilder() << host << ':' << port; + } + bool BalancerEnabled() const { return Settings.HasBalancerHost() && Settings.GetBalancerHost(); } @@ -127,7 +148,7 @@ namespace NKikimr::NBlobDepot { if (!HttpProxyId) { HttpProxyId = Register(NHttp::CreateHttpProxy()); } - const TString url = TStringBuilder() << "http://" << Settings.GetBalancerHost() << "/"; + const TString url = TStringBuilder() << "http://" << Settings.GetBalancerHost(); Send(HttpProxyId, new NHttp::TEvHttpProxy::TEvHttpOutgoingRequest( NHttp::THttpOutgoingRequest::CreateRequestGet(url), TDuration::Seconds(10))); @@ -157,9 +178,18 @@ namespace NKikimr::NBlobDepot { RefreshInFlight = false; const auto& msg = *ev->Get(); if (msg.Response && msg.Response->Status.StartsWith("2")) { - TString endpoint = TString(StripString(msg.Response->Body)); - if (!endpoint.empty() && endpoint != CurrentEndpoint) { - BuildInnerWrapper(endpoint); + TString host = TString(StripString(msg.Response->Body)); + if (!host.empty()) { + ui16 port = BalancerProxyPort(); + if (TStringBuf h, p; TStringBuf(host).TrySplit(':', h, p)) { + host = TString(h); + TryFromString(p, port); + } + + const TString endpoint = TStringBuilder() << host << ':' << port; + if (endpoint != CurrentEndpoint) { + BuildInnerWrapperViaProxy(host, port); + } } } ScheduleNextRefresh(); @@ -183,6 +213,7 @@ namespace NKikimr::NBlobDepot { void Bootstrap() { const TString& endpoint = Settings.GetSettings().GetEndpoint(); + OriginalEndpoint = endpoint; BuildInnerWrapper(endpoint); if (BalancerEnabled()) { IssueBalancerRequest(); diff --git a/ydb/core/blobstorage/ddisk/ddisk.h b/ydb/core/blobstorage/ddisk/ddisk.h index 3521d7ae841..f09e28ff506 100644 --- a/ydb/core/blobstorage/ddisk/ddisk.h +++ b/ydb/core/blobstorage/ddisk/ddisk.h @@ -327,6 +327,12 @@ struct TPersistentBufferFormat { ui32 DeallocateFreeSpaceThresholdPercent = 90; // Deallocate a chunk proactively when it has been freed for this many seconds. ui32 DeallocateThresholdSeconds = 30; + // TEvListPersistentBuffer must not observe a partially-applied write/erase for its tablet: the + // listing is deferred (queued and retried) while any disk operation is in flight for the + // requesting tablet. These parameters bound how long/how often we wait before giving up and + // replying with an OVERLOADED error to avoid returning a potentially-stale view. + ui32 ListPersistentBufferMaxRetries = 10; + ui32 ListPersistentBufferRetryPeriodMilliseconds = 20; }; #define DECLARE_DDISK_EVENT(NAME) \ diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor.cpp b/ydb/core/blobstorage/ddisk/ddisk_actor.cpp index b4571a4f566..25e63f5e9f0 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor.cpp +++ b/ydb/core/blobstorage/ddisk/ddisk_actor.cpp @@ -328,6 +328,7 @@ namespace { hFunc(TEvErasePersistentBuffer, Handle) hFunc(TEvBatchErasePersistentBuffer, Handle) hFunc(TEvListPersistentBuffer, Handle) + hFunc(TEvPrivate::TEvRetryListPersistentBuffer, Handle) hFunc(TEvGetPersistentBufferInfo, Handle) hFunc(TEvPrivate::TEvReadPersistentBufferPart, Handle) diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor.h b/ydb/core/blobstorage/ddisk/ddisk_actor.h index f3d0606c8b6..4b8334fce68 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor.h +++ b/ydb/core/blobstorage/ddisk/ddisk_actor.h @@ -254,6 +254,17 @@ namespace NKikimr::NDDisk { EvIssuePersistentBufferChunkAllocation, EvDeallocatePersistentBufferChunk, EvDeallocatePersistentBufferChunkResult, + EvRetryListPersistentBuffer, + }; + + struct TEvRetryListPersistentBuffer : TEventLocal<TEvRetryListPersistentBuffer, EvRetryListPersistentBuffer> { + TAutoPtr<TEventHandle<TEvListPersistentBuffer>> Ev; + ui32 RetriesLeft; + + TEvRetryListPersistentBuffer(TAutoPtr<TEventHandle<TEvListPersistentBuffer>> ev, ui32 retriesLeft) + : Ev(ev) + , RetriesLeft(retriesLeft) + {} }; struct TEvIssuePersistentBufferChunkAllocation : TEventLocal<TEvIssuePersistentBufferChunkAllocation, EvIssuePersistentBufferChunkAllocation> { @@ -847,6 +858,13 @@ namespace NKikimr::NDDisk { void Handle(TEvWriteResult::TPtr ev); void Handle(TEvents::TEvUndelivered::TPtr ev); void Handle(TEvListPersistentBuffer::TPtr ev); + void Handle(TEvPrivate::TEvRetryListPersistentBuffer::TPtr ev); + // Returns true if the given tablet currently has at least one persistent-buffer disk + // operation (write/erase/read) in flight. TEvListPersistentBuffer must not be answered + // while this holds, otherwise it could observe a partially-applied write or erase. + bool HasPersistentBufferInflightForTablet(ui64 tabletId) const; + void ProcessListPersistentBuffer(TAutoPtr<TEventHandle<TEvListPersistentBuffer>> ev, ui32 retriesLeft); + void ReplyListPersistentBuffer(TEventHandle<TEvListPersistentBuffer>& ev); void Handle(TEvPrivate::TEvIssuePersistentBufferChunkAllocation::TPtr ev); void Handle(TEvPrivate::TEvDeallocatePersistentBufferChunk::TPtr ev); void Handle(TEvPrivate::TEvDeallocatePersistentBufferChunkResult::TPtr ev); diff --git a/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp b/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp index 3660a33cf16..5805578bb4e 100644 --- a/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp +++ b/ydb/core/blobstorage/ddisk/ddisk_actor_persistent_buffer.cpp @@ -2076,6 +2076,80 @@ namespace NKikimr::NDDisk { Send(ev->Sender, std::move(reply), 0, ev->Cookie); } + bool TDDiskActor::HasPersistentBufferInflightForTablet(ui64 tabletId) const { + // A single scan over PersistentBufferDiskOperationInflight is sufficient: every in-flight + // persistent-buffer disk operation (write, batched write, erase, barrier/fast erase, read) + // registers exactly one entry here, and each entry's Records carry the owning TabletId. The + // map is bounded by the disk operations inflight limit, so this scan is cheap. + for (const auto& [_, inflight] : PersistentBufferDiskOperationInflight) { + for (const auto& record : inflight.Records) { + if (record.TabletId == tabletId) { + return true; + } + } + } + return false; + } + + void TDDiskActor::ReplyListPersistentBuffer(TEventHandle<TEvListPersistentBuffer>& ev) { + const auto& record = ev.Get()->Record; + const TQueryCredentials creds(record.GetCredentials()); + + Counters.Interface.ListPersistentBuffer.Request(0); + + auto reply = std::make_unique<TEvListPersistentBufferResult>(NKikimrBlobStorage::NDDisk::TReplyStatus::OK); + auto& rr = reply->Record; + rr.SetBarrierLsn(PersistentBufferBarriersManager.GetBarrier(creds.TabletId).Lsn); + for (auto it = PersistentBuffers.lower_bound({creds.TabletId, 0}); it != PersistentBuffers.end() && + it->first.TabletId == creds.TabletId; ++it) { + const TPersistentBuffer& buffer = it->second; + for (const auto& [lsn, pr] : buffer.Records) { + auto *pb = rr.AddRecords(); + auto *sel = pb->MutableSelector(); + sel->SetVChunkIndex(pr.VChunkIndex); + sel->SetOffsetInBytes(pr.OffsetInBytes); + sel->SetSize(pr.Size); + pb->SetGeneration(it->first.Generation); + pb->SetLsn(lsn); + } + } + + Counters.Interface.ListPersistentBuffer.Reply(true, 0); + SendReply(ev, std::move(reply)); + } + + void TDDiskActor::ProcessListPersistentBuffer(TAutoPtr<TEventHandle<TEvListPersistentBuffer>> ev, ui32 retriesLeft) { + const auto& record = ev->Get()->Record; + const TQueryCredentials creds(record.GetCredentials()); + + if (HasPersistentBufferInflightForTablet(creds.TabletId)) { + if (retriesLeft == 0) { + YDB_LOG_DEBUG_COMP(NKikimrServices::BS_PERSISTENT_BUFFER, "TDDiskActor::ProcessListPersistentBuffer retries exhausted, replying with error", + {"marker", "BSPB"}, + {"PBufferId", SelfId()}, + {"tabletId", creds.TabletId}); + Counters.Interface.ListPersistentBuffer.Request(0); + Counters.Interface.ListPersistentBuffer.Reply(false, 0); + SendReply(*ev, std::make_unique<TEvListPersistentBufferResult>( + NKikimrBlobStorage::NDDisk::TReplyStatus::OVERLOADED, + TStringBuilder() << "persistent buffer disk operation still in flight for tablet " + << creds.TabletId << " after " << PersistentBufferFormat.ListPersistentBufferMaxRetries + << " retries")); + return; + } + YDB_LOG_TRACE_COMP(NKikimrServices::BS_PERSISTENT_BUFFER, "TDDiskActor::ProcessListPersistentBuffer waiting for inflight to drain", + {"marker", "BSPB"}, + {"PBufferId", SelfId()}, + {"tabletId", creds.TabletId}, + {"retriesLeft", retriesLeft}); + Schedule(TDuration::MilliSeconds(PersistentBufferFormat.ListPersistentBufferRetryPeriodMilliseconds), + new TEvPrivate::TEvRetryListPersistentBuffer(ev, retriesLeft - 1)); + return; + } + + ReplyListPersistentBuffer(*ev); + } + void TDDiskActor::Handle(TEvListPersistentBuffer::TPtr ev) { if (!CheckQuery(*ev, &Counters.Interface.ListPersistentBuffer)) { return; @@ -2102,30 +2176,11 @@ namespace NKikimr::NDDisk { return; } - const auto& record = ev->Get()->Record; - const TQueryCredentials creds(record.GetCredentials()); - - Counters.Interface.ListPersistentBuffer.Request(0); - - auto reply = std::make_unique<TEvListPersistentBufferResult>(NKikimrBlobStorage::NDDisk::TReplyStatus::OK); - auto& rr = reply->Record; - rr.SetBarrierLsn(PersistentBufferBarriersManager.GetBarrier(creds.TabletId).Lsn); - for (auto it = PersistentBuffers.lower_bound({creds.TabletId, 0}); it != PersistentBuffers.end() && - it->first.TabletId == creds.TabletId; ++it) { - const TPersistentBuffer& buffer = it->second; - for (const auto& [lsn, pr] : buffer.Records) { - auto *pb = rr.AddRecords(); - auto *sel = pb->MutableSelector(); - sel->SetVChunkIndex(pr.VChunkIndex); - sel->SetOffsetInBytes(pr.OffsetInBytes); - sel->SetSize(pr.Size); - pb->SetGeneration(it->first.Generation); - pb->SetLsn(lsn); - } - } + ProcessListPersistentBuffer(ev.Release(), PersistentBufferFormat.ListPersistentBufferMaxRetries); + } - Counters.Interface.ListPersistentBuffer.Reply(true, 0); - SendReply(*ev, std::move(reply)); + void TDDiskActor::Handle(TEvPrivate::TEvRetryListPersistentBuffer::TPtr ev) { + ProcessListPersistentBuffer(ev->Get()->Ev, ev->Get()->RetriesLeft); } TString TDDiskActor::PersistentBufferToString() { diff --git a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_ut.cpp b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_ut.cpp index c9c5a9b69f3..968bac148b3 100644 --- a/ydb/core/blobstorage/ddisk/ut/ddisk_actor_ut.cpp +++ b/ydb/core/blobstorage/ddisk/ut/ddisk_actor_ut.cpp @@ -805,6 +805,85 @@ Y_UNIT_TEST_SUITE(TDDiskActorTest) { } + // TEvListPersistentBuffer must not observe a partially-applied write for its tablet: it has to + // wait for any in-flight persistent-buffer disk operation belonging to that tablet to finish + // before replying. Regression test for that ordering guarantee. + Y_UNIT_TEST(PersistentBufferListWaitsForInflightWrite) { + TTestContext ctx; + const TDiskHandle disk = ctx.CreateDDisk(6, 1); + NDDisk::TQueryCredentials creds = Connect(ctx, disk.PBServiceId, 40, 1); + + const ui64 lsn = 10; + const TString payload = MakeData('P', BlockSize); + const NDDisk::TBlockSelector selector{3, 0, BlockSize}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + SendToDDisk(ctx, disk.PBServiceId, write.release()); + + // The write's disk op is now in flight (not yet acked by PDisk). Issue the list request for + // the same tablet while it is still in flight: it must be deferred and only answered once the + // write completes, never with a stale/partial view. + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk); + UNIT_ASSERT(pbWriteRaw->Get()->Data.size() > 0); + + SendToDDisk(ctx, disk.PBServiceId, new NDDisk::TEvListPersistentBuffer(creds)); + + ctx.SendPDiskResponse(disk, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + + auto listResult = WaitFromDDisk<NDDisk::TEvListPersistentBufferResult>(ctx); + AssertStatus(listResult, TReplyStatus::OK); + // The list must reflect the completed write (i.e. it waited for the inflight to drain), + // not the state as it was before the write finished. + UNIT_ASSERT_VALUES_EQUAL(listResult->Get()->Record.RecordsSize(), 1); + const auto& record = listResult->Get()->Record.GetRecords(0); + UNIT_ASSERT_VALUES_EQUAL(record.GetLsn(), lsn); + } + + // Once retries are exhausted while the tablet's persistent-buffer disk operation is still in + // flight, TEvListPersistentBuffer must reply with an error (not hang, and not answer with a + // possibly-stale view). + Y_UNIT_TEST(PersistentBufferListRepliesErrorAfterRetriesExhausted) { + TTestContext ctx; + NDDisk::TPersistentBufferFormat fmt; + fmt.MaxChunks = 256; + fmt.InitChunks = PersistentBufferInitChunks; + fmt.MaxInMemoryCache = BlockSize * 128; + fmt.MaxChunkRestoreInflight = 8; + fmt.UpdateFreeSpaceInfoMilliseconds = 5000; + fmt.PerTabletStorageLimit = 512 * 1024; + fmt.ListPersistentBufferMaxRetries = 2; + fmt.ListPersistentBufferRetryPeriodMilliseconds = 5; + const TDiskHandle disk = ctx.CreateDDisk(6, 1, fmt); + NDDisk::TQueryCredentials creds = Connect(ctx, disk.PBServiceId, 40, 1); + + const ui64 lsn = 10; + const TString payload = MakeData('P', BlockSize); + const NDDisk::TBlockSelector selector{3, 0, BlockSize}; + + auto write = std::make_unique<NDDisk::TEvWritePersistentBuffer>(creds, selector, lsn, NDDisk::TWriteInstruction(0)); + write->AddPayloadThenChecksum(TRope(payload)); + SendToDDisk(ctx, disk.PBServiceId, write.release()); + + // Leave the write's disk op in flight (never ack it) and issue a list request for the same + // tablet: it must keep retrying, then give up and reply with an error once retries run out. + auto pbWriteRaw = ctx.WaitPDiskRequest<NPDisk::TEvChunkWriteRaw>(disk); + UNIT_ASSERT(pbWriteRaw->Get()->Data.size() > 0); + + SendToDDisk(ctx, disk.PBServiceId, new NDDisk::TEvListPersistentBuffer(creds)); + + auto listResult = WaitFromDDisk<NDDisk::TEvListPersistentBufferResult>(ctx); + AssertStatus(listResult, TReplyStatus::OVERLOADED); + + // Complete the write afterwards so the test tears down cleanly. + ctx.SendPDiskResponse(disk, *pbWriteRaw, new NPDisk::TEvChunkWriteRawResult(NKikimrProto::OK, "")); + auto writeResult = WaitFromDDisk<NDDisk::TEvWritePersistentBufferResult>(ctx); + AssertStatus(writeResult, TReplyStatus::OK); + } + Y_UNIT_TEST(PersistentBufferWriteTunnel) { TTestContext ctx; const TDiskHandle disk1 = ctx.CreateDDisk(6, 1); diff --git a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp index d6010da0a7d..3cfc6909113 100644 --- a/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp +++ b/ydb/core/blobstorage/nodewarden/node_warden_vdisk.cpp @@ -259,6 +259,12 @@ namespace NKikimr::NStorage { if (Cfg->PBufferConfig->HasMinFreeSectorsReserve()) { pbufferFormat.MinFreeSectorsReserve = Cfg->PBufferConfig->GetMinFreeSectorsReserve(); } + if (Cfg->PBufferConfig->HasListPersistentBufferMaxRetries()) { + pbufferFormat.ListPersistentBufferMaxRetries = Cfg->PBufferConfig->GetListPersistentBufferMaxRetries(); + } + if (Cfg->PBufferConfig->HasListPersistentBufferRetryPeriodMilliseconds()) { + pbufferFormat.ListPersistentBufferRetryPeriodMilliseconds = Cfg->PBufferConfig->GetListPersistentBufferRetryPeriodMilliseconds(); + } if (Cfg->PBufferConfig->HasPreallocateFreeSpaceThresholdPercent()) { auto newValue = Cfg->PBufferConfig->GetPreallocateFreeSpaceThresholdPercent(); if (newValue >= 100) { diff --git a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp index 3391632741e..22ed036907a 100644 --- a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp +++ b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.cpp @@ -58,6 +58,7 @@ struct TPDiskMockState::TImpl { ESpaceColorPolicy SpaceColorPolicy; std::shared_ptr<NPDisk::TQuotaRecord> ChunkSharedQuota; double Occupancy = 0; + bool ReportVDiskMetrics = false; struct TShredState { enum class EPhase : ui8 { @@ -336,6 +337,10 @@ struct TPDiskMockState::TImpl { StatusFlags = SpaceColorToStatusFlag(spaceColor); } + void SetReportVDiskMetrics(bool reportVDiskMetrics) { + ReportVDiskMetrics = reportVDiskMetrics; + } + void SetReadOnly(const TVDiskID& vDiskId, bool isReadOnly) { if (isReadOnly) { ReadOnlyVDisks.insert(vDiskId.GroupID.GetRawId()); @@ -398,6 +403,10 @@ void TPDiskMockState::SetReadOnly(const TVDiskID& vDiskId, bool isReadOnly) { Impl->SetReadOnly(vDiskId, isReadOnly); } +void TPDiskMockState::SetReportVDiskMetrics(bool reportVDiskMetrics) { + Impl->SetReportVDiskMetrics(reportVDiskMetrics); +} + bool TPDiskMockState::IsDiskReadOnly() const { return Impl->IsDiskReadOnly; } @@ -470,22 +479,24 @@ public: p->SetAvailableSize((ui64)(Impl.TotalChunks - usedChunks) * Impl.ChunkSize); p->SetTotalSize((ui64)Impl.TotalChunks * Impl.ChunkSize); p->SetState(NKikimrBlobStorage::TPDiskState::Normal); + // report a full performance metrics set (like a real PDisk does) so that BSC considers the PDisk complete p->SetMaxIOPS(1000); p->SetMaxReadThroughput(1'000'000'000); p->SetMaxWriteThroughput(1'000'000'000); - - // report per-VDisk metrics with normalized occupancy; deliberately do not touch status flags - for (const auto& [ownerId, owner] : Impl.Owners) { - auto *m = record.AddVDisksMetrics(); - VDiskIDFromVDiskID(owner.VDiskId, m->MutableVDiskId()); - auto *vslotId = m->MutableVSlotId(); - vslotId->SetNodeId(Impl.NodeId); - vslotId->SetPDiskId(Impl.PDiskId); - vslotId->SetVSlotId(owner.SlotId); - m->SetNormalizedOccupancy(GetOccupancy()); - m->SetAllocatedSize((ui64)owner.CommittedChunks.size() * Impl.ChunkSize); - m->SetAvailableSize(p->GetAvailableSize()); + if (Impl.ReportVDiskMetrics) { + // report per-VDisk metrics with normalized occupancy; deliberately do not touch status flags + for (const auto& [ownerId, owner] : Impl.Owners) { + auto *m = record.AddVDisksMetrics(); + VDiskIDFromVDiskID(owner.VDiskId, m->MutableVDiskId()); + auto *vslotId = m->MutableVSlotId(); + vslotId->SetNodeId(Impl.NodeId); + vslotId->SetPDiskId(Impl.PDiskId); + vslotId->SetVSlotId(owner.SlotId); + m->SetNormalizedOccupancy(GetOccupancy()); + m->SetAllocatedSize((ui64)owner.CommittedChunks.size() * Impl.ChunkSize); + m->SetAvailableSize(p->GetAvailableSize()); + } } Send(MakeBlobStorageNodeWardenID(SelfId().NodeId()), ev.release()); diff --git a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.h b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.h index 4a68914a41d..fb04d868f7c 100644 --- a/ydb/core/blobstorage/pdisk/mock/pdisk_mock.h +++ b/ydb/core/blobstorage/pdisk/mock/pdisk_mock.h @@ -49,6 +49,7 @@ namespace NKikimr { TPtr Snapshot(); // create a copy of PDisk whole state void SetReadOnly(const TVDiskID& vDiskId, bool isReadOnly); + void SetReportVDiskMetrics(bool reportVDiskMetrics); bool IsDiskReadOnly() const; }; diff --git a/ydb/core/blobstorage/ut_blobstorage/lib/env.h b/ydb/core/blobstorage/ut_blobstorage/lib/env.h index 2af17a1a489..277f633f072 100644 --- a/ydb/core/blobstorage/ut_blobstorage/lib/env.h +++ b/ydb/core/blobstorage/ut_blobstorage/lib/env.h @@ -67,6 +67,7 @@ struct TEnvironmentSetup { const ui64 PDiskSize = 10_TB; const ui64 PDiskChunkSize = 0; const bool TrackSharedQuotaInPDiskMock = false; + const bool ReportVDiskMetricsInPDiskMock = false; const bool SelfManagementConfig = false; const bool EnableDeepScrubbing = false; const ui32 NumPiles = 0; @@ -103,6 +104,7 @@ struct TEnvironmentSetup { : TPDiskMockState::ESpaceColorPolicy::None; state.Reset(new TPDiskMockState(nodeId, pdiskId, cfg->PDiskGuid, Env.Settings.PDiskSize, chunkSize, cfg->ReadOnly, Env.Settings.DiskType, spaceColorPolicy)); + state->SetReportVDiskMetrics(Env.Settings.ReportVDiskMetricsInPDiskMock); } const TActorId& actorId = ctx.Register(CreatePDiskMockActor(state), TMailboxType::HTSwap, poolId); const TActorId& serviceId = MakeBlobStoragePDiskID(nodeId, pdiskId); diff --git a/ydb/core/blobstorage/ut_blobstorage/select_groups.cpp b/ydb/core/blobstorage/ut_blobstorage/select_groups.cpp index 68b7270cece..c02336b1647 100644 --- a/ydb/core/blobstorage/ut_blobstorage/select_groups.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/select_groups.cpp @@ -9,7 +9,9 @@ Y_UNIT_TEST_SUITE(SelectGroups) { // VDisk status flags changed too -- which they don't for a fresh empty disk -- so the request hung // forever. Y_UNIT_TEST(BlockUntilAllResourcesAreComplete) { - TEnvironmentSetup env(false); + TEnvironmentSetup env(TEnvironmentSetup::TSettings{ + .ReportVDiskMetricsInPDiskMock = true, + }); // create a fresh group; at this point it has no VDisk metrics yet env.CreateBoxAndPool(); diff --git a/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp b/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp index 02e05754011..d0999bd9f1d 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp +++ b/ydb/core/formats/arrow/accessor/sub_columns/stats.cpp @@ -37,15 +37,15 @@ TDictStats TDictStats::SelectSeparatedColumns(const TSettings& settings, const u return columnsBuilder.Finish(); } -TDictStats TDictStats::Merge(const std::vector<const TDictStats*>& stats, const TSettings& settings, const ui32 recordsCount) { +TDictStats TDictStats::Merge(const std::vector<TDictStats>& stats, const TSettings& settings, const ui32 recordsCount) { std::map<std::string_view, TRTStats> resultMap; for (auto&& i : stats) { - for (ui32 idx = 0; idx < i->GetColumnsCount(); ++idx) { - auto it = resultMap.find(i->GetColumnName(idx)); + for (ui32 idx = 0; idx < i.GetColumnsCount(); ++idx) { + auto it = resultMap.find(i.GetColumnName(idx)); if (it == resultMap.end()) { - it = resultMap.emplace(i->GetColumnName(idx), TRTStats(i->GetColumnName(idx))).first; + it = resultMap.emplace(i.GetColumnName(idx), TRTStats(i.GetColumnName(idx))).first; } - it->second.Add(*i, idx); + it->second.Add(i, idx); } } auto builder = MakeBuilder(); diff --git a/ydb/core/formats/arrow/accessor/sub_columns/stats.h b/ydb/core/formats/arrow/accessor/sub_columns/stats.h index b065ce7290f..2e86d813dfa 100644 --- a/ydb/core/formats/arrow/accessor/sub_columns/stats.h +++ b/ydb/core/formats/arrow/accessor/sub_columns/stats.h @@ -152,7 +152,7 @@ public: } }; - static TDictStats Merge(const std::vector<const TDictStats*>& stats, const TSettings& settings, const ui32 recordsCount); + static TDictStats Merge(const std::vector<TDictStats>& stats, const TSettings& settings, const ui32 recordsCount); // Selects which keys become separated columns; // the rest fall into the Others store, whose stats are built by the caller. diff --git a/ydb/core/http_proxy/ut/ymq_ut.cpp b/ydb/core/http_proxy/ut/ymq_ut.cpp index 72d1b6d0a61..51947810463 100644 --- a/ydb/core/http_proxy/ut/ymq_ut.cpp +++ b/ydb/core/http_proxy/ut/ymq_ut.cpp @@ -3704,8 +3704,7 @@ Y_UNIT_TEST_SUITE(TestYmqHttpProxy) { DisableAuthorization(); // Send an XML API CreateQueue request without authorization header. - // This triggers the XML API auth path (TCloudAuthRequestProxy) which calls - // Callback_->OnIamAuthError(), setting SkipMetering on the response. + // This triggers the XML API auth path (TCloudAuthRequestProxy) setting SkipMetering on the response due to auth non-successful. // Auth failures are reported with HTTP code 400 and an IncompleteSignature error. auto json2 = CreateQueueXml({{"QueueName", "XmlAuthFailQueue"}}, 400); UNIT_ASSERT_STRING_CONTAINS(GetByPath<TString>(json2, "__type"), "IncompleteSignature"); diff --git a/ydb/core/http_proxy/ymq.cpp b/ydb/core/http_proxy/ymq.cpp index 31c80706a9c..9cd4aa062a1 100644 --- a/ydb/core/http_proxy/ymq.cpp +++ b/ydb/core/http_proxy/ymq.cpp @@ -195,7 +195,7 @@ namespace NKikimr::NHttpProxy { } void DoMetering(const THttpResponseData& data, THolder<THashMap<TString, TString>>&& queueTags, const TActorContext& ctx) { - if (IamAuthFailed_) { + if (!IamAuthenticated) { YDB_LOG_DEBUG_CTX(ctx, "Skip metering event due to IAM auth failure"); return; } @@ -314,6 +314,7 @@ namespace NKikimr::NHttpProxy { HttpContext.FolderId = FolderId = ev->Get()->FolderId; HttpContext.CloudId = CloudId = ev->Get()->CloudId; UserSid = ev->Get()->Sid; + IamAuthenticated = true; SendGrpcRequestNoDriver(ctx); } else { YDB_LOG_DEBUG_CTX(ctx, "Got cloud auth response", @@ -321,7 +322,7 @@ namespace NKikimr::NHttpProxy { {"httpStatusCode", ev->Get()->Error->HttpStatusCode}, {"errorCode", ev->Get()->Error->ErrorCode}, {"message", ev->Get()->Error->Message}); - IamAuthFailed_ = true; + ReplyWithError( ctx, ev->Get()->Error->HttpStatusCode, @@ -421,7 +422,7 @@ namespace NKikimr::NHttpProxy { TRetryCounter RetryCounter; TActorId AuthActor; bool InputCountersReported = false; - bool IamAuthFailed_ = false; + bool IamAuthenticated = false; TString FolderId; TString CloudId; TString ResourceId; diff --git a/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp index b33af01aa80..0fa46a2d006 100644 --- a/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_create_topics_actor.cpp @@ -175,7 +175,8 @@ void TKafkaCreateTopicsActor::Handle(const NKikimr::NPQ::NSchema::TEvSchemaRespo YDB_LOG_DEBUG("Create topics actor. Topic's response received", {LogPrefix()}, {"path", eventPtr->Path}, - {"status", std::to_string(eventPtr->Status)}); + {"status", std::to_string(eventPtr->Status)}, + {"errorMessage", eventPtr->ErrorMessage}); EKafkaErrors status; switch(eventPtr->Status) { diff --git a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp index 1f1d9f629d3..f77ec83c58b 100644 --- a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp +++ b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.cpp @@ -355,6 +355,35 @@ void TKafkaMetadataActor::AddBroker(ui64 nodeId, const TString& host, ui64 port) } } +void TKafkaMetadataActor::EnsureBrokersAndController() { + // Unknown topics used to return brokers=[]; AdminClient then cannot CreateTopics. + // NeedAllNodes also requires the full discovered broker set. + if (!WithProxy && (Response->Brokers.empty() || NeedAllNodes)) { + for (const auto& [id, nodeInfo] : Nodes) { + AddBroker(id, nodeInfo.Host, nodeInfo.Port); + } + } + + // ControllerId must be one of Brokers (SelfID may differ from discovery node ids). + if (Response->Brokers.empty()) { + return; + } + + for (const auto& broker : Response->Brokers) { + if (broker.NodeId == Response->ControllerId) { + return; + } + } + + // Prefer keeping ControllerId if that node is known from discovery. + if (auto it = Nodes.find(Response->ControllerId); it != Nodes.end()) { + AddBroker(it->first, it->second.Host, it->second.Port); + return; + } + + Response->ControllerId = Response->Brokers.front().NodeId; +} + void TKafkaMetadataActor::ApplyPendingTopicResponses() { while (!PendingTopicResponses.empty()) { auto& [index, ev] = *PendingTopicResponses.begin(); @@ -379,6 +408,7 @@ void TKafkaMetadataActor::ApplyPendingTopicResponses() { void TKafkaMetadataActor::RespondIfRequired(const TActorContext& ctx) { auto Respond = [&] { + EnsureBrokersAndController(); CancelRequestTimeout(); Send(Context->ConnectionId, new TEvKafka::TEvResponse(CorrelationId, Response, ErrorCode)); Die(ctx); @@ -397,12 +427,6 @@ void TKafkaMetadataActor::RespondIfRequired(const TActorContext& ctx) { } ApplyPendingTopicResponses(); - - if (NeedAllNodes) { - for (const auto& [id, nodeInfo] : Nodes) - AddBroker(id, nodeInfo.Host, nodeInfo.Port); - } - Respond(); } @@ -417,6 +441,7 @@ void TKafkaMetadataActor::HandleWakeup(TEvents::TEvWakeup::TPtr&, const TActorCo void TKafkaMetadataActor::RespondWithTimeout(const TActorContext& ctx) { ApplyPendingTopicResponses(); + EnsureBrokersAndController(); ErrorCode = EKafkaErrors::REQUEST_TIMED_OUT; for (auto& topic : Response->Topics) { diff --git a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h index 89d74574a0d..9212905c18d 100644 --- a/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h +++ b/ydb/core/kafka_proxy/actors/kafka_metadata_actor.h @@ -56,6 +56,7 @@ private: void RespondIfRequired(const NActors::TActorContext& ctx); void AddProxyNodeToBrokers(); void AddBroker(ui64 nodeId, const TString& host, ui64 port); + void EnsureBrokersAndController(); void RequestICNodeCache(); void ProcessTopicsFromRequest(); void SendDiscoveryRequest(); diff --git a/ydb/core/kafka_proxy/ut/metarequest_ut.cpp b/ydb/core/kafka_proxy/ut/metarequest_ut.cpp index 767bf9ebd7f..05827f7e52b 100644 --- a/ydb/core/kafka_proxy/ut/metarequest_ut.cpp +++ b/ydb/core/kafka_proxy/ut/metarequest_ut.cpp @@ -104,6 +104,17 @@ Y_UNIT_TEST_SUITE(TMetadataActorTests) { response = dynamic_cast<TMetadataResponseData*>(event->Response.get()); UNIT_ASSERT_VALUES_EQUAL(response->Topics.size(), 1); UNIT_ASSERT(response->Topics[0].ErrorCode == EKafkaErrors::UNKNOWN_TOPIC_OR_PARTITION); + UNIT_ASSERT(!response->Brokers.empty()); + { + bool controllerInBrokers = false; + for (const auto& broker : response->Brokers) { + if (broker.NodeId == response->ControllerId) { + controllerInBrokers = true; + break; + } + } + UNIT_ASSERT(controllerInBrokers); + } event = GetEvent(server, edgeId, {}); response = dynamic_cast<TMetadataResponseData*>(event->Response.get()); diff --git a/ydb/core/keyvalue/keyvalue_copy_blob.cpp b/ydb/core/keyvalue/keyvalue_copy_blob.cpp index 6bc02d1d225..84b08a75de6 100644 --- a/ydb/core/keyvalue/keyvalue_copy_blob.cpp +++ b/ydb/core/keyvalue/keyvalue_copy_blob.cpp @@ -47,6 +47,8 @@ public: } void Handle(TEvBlobStorage::TEvGetResult::TPtr& ev) { + // TODO: handle situation when blob was deleted before we started copying it + auto groupId = TabletInfo->GroupFor(BlobId.Channel(), BlobId.Generation()); if (ev->Get()->GroupId != groupId) { diff --git a/ydb/core/keyvalue/keyvalue_events.h b/ydb/core/keyvalue/keyvalue_events.h index b720ae2f488..4e81191c0e2 100644 --- a/ydb/core/keyvalue/keyvalue_events.h +++ b/ydb/core/keyvalue/keyvalue_events.h @@ -44,6 +44,7 @@ namespace TEvKeyValue { EvAdvanceMoveDataResult = EvResponse + 512, EvBlobCopied, + EvCheckTrash, EvEnd }; @@ -295,6 +296,8 @@ namespace TEvKeyValue { COPY_BLOB, YIELD, REPEAT, + CHECK_TRASH, + WAIT_FOR_GC, FINISH, }; EResult Result; @@ -321,6 +324,14 @@ namespace TEvKeyValue { return std::make_unique<TEvAdvanceMoveDataResult>(EResult::REPEAT); } + static std::unique_ptr<TEvAdvanceMoveDataResult> CheckTrash() { + return std::make_unique<TEvAdvanceMoveDataResult>(EResult::CHECK_TRASH); + } + + static std::unique_ptr<TEvAdvanceMoveDataResult> WaitForGC() { + return std::make_unique<TEvAdvanceMoveDataResult>(EResult::WAIT_FOR_GC); + } + static std::unique_ptr<TEvAdvanceMoveDataResult> Finish() { return std::make_unique<TEvAdvanceMoveDataResult>(EResult::FINISH); } @@ -335,6 +346,8 @@ namespace TEvKeyValue { , NewBlobId(newBlobId) {} }; + + struct TEvCheckTrash : public TEventLocal<TEvCheckTrash, EvCheckTrash> {}; } } // NKikimr diff --git a/ydb/core/keyvalue/keyvalue_flat_impl.h b/ydb/core/keyvalue/keyvalue_flat_impl.h index 581066dae41..162e8de5df7 100644 --- a/ydb/core/keyvalue/keyvalue_flat_impl.h +++ b/ydb/core/keyvalue/keyvalue_flat_impl.h @@ -374,18 +374,19 @@ protected: : Self(keyValueFlat) {} - bool Execute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) override { + bool Execute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override { YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "TTxAdvanceMoveData Execute", {"keyValue", txc.Tablet}); TSimpleDbFlat db(txc.DB, TrashBeingCommitted); - Result = Self->State.AdvanceMoveData(db, ctx); + Result = Self->State.AdvanceMoveData(db); return true; } void Complete(const TActorContext& ctx) override { YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "TTxAdvanceMoveData Complete", {"keyValue", Self->TabletID()}); + Self->State.PushTrashBeingCommitted(TrashBeingCommitted, ctx); ctx.Send(Self->Tablet(), Result.release()); } }; @@ -403,20 +404,21 @@ protected: , NewBlobId(newBlobId) {} - bool Execute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& ctx) override { + bool Execute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override { YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "TTxBlobCopied Execute", {"keyValue", txc.Tablet}, {"blobId", BlobId.ToString()}, {"newBlobId", NewBlobId.ToString()}); TSimpleDbFlat db(txc.DB, TrashBeingCommitted); - Result = Self->State.BlobCopied(BlobId, NewBlobId, db, ctx); + Result = Self->State.BlobCopied(BlobId, NewBlobId, db); return true; } void Complete(const TActorContext& ctx) override { YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "TTxBlobCopied Complete", {"keyValue", Self->TabletID()}); + Self->State.PushTrashBeingCommitted(TrashBeingCommitted, ctx); ctx.Send(Self->Tablet(), Result.release()); } }; @@ -662,8 +664,10 @@ protected: MoveDataRequestsQueue.push_back(ev); return; } - // TODO: fill group list from event TSet<ui32> moveDataGroups; + for (const auto& groupId : ev->Get()->Record.GetGroups()) { + moveDataGroups.insert(groupId); + } State.StartMoveData(std::move(moveDataGroups), ev->Sender); Execute(new TTxAdvanceMoveData(this)); } @@ -685,17 +689,12 @@ protected: Execute(new TTxAdvanceMoveData(this)); break; - case TEvKeyValue::TEvAdvanceMoveDataResult::EResult::FINISH: - if (!MoveDataRequestsQueue.empty()) { - auto ev = MoveDataRequestsQueue.front(); - // TODO: fill group list from event - TSet<ui32> moveDataGroups; - State.StartMoveData(std::move(moveDataGroups), ev->Sender); - MoveDataRequestsQueue.pop_front(); - - Execute(new TTxAdvanceMoveData(this)); - } + case TEvKeyValue::TEvAdvanceMoveDataResult::EResult::CHECK_TRASH: + Send(SelfId(), new TEvKeyValue::TEvCheckTrash); break; + + default: + Y_ABORT(); } } @@ -706,6 +705,29 @@ protected: Execute(new TTxBlobCopied(this, ev->Get()->BlobId, ev->Get()->NewBlobId)); } + void HandleCheckTrash() { + YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "Handle TEvCheckTrash", + {"keyValue", TabletID()}); + + auto result = State.CheckTrash(); + switch (result->Result) { + case TEvKeyValue::TEvAdvanceMoveDataResult::EResult::CHECK_TRASH: + Send(SelfId(), new TEvKeyValue::TEvCheckTrash); + break; + + case TEvKeyValue::TEvAdvanceMoveDataResult::EResult::WAIT_FOR_GC: + break; + + case TEvKeyValue::TEvAdvanceMoveDataResult::EResult::FINISH: + // now proceed with basic executor + Executor()->StartMoveDataVacuumFromOwner(); + break; + + default: + Y_ABORT(); + } + } + public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::KEYVALUE_ACTOR; @@ -754,6 +776,26 @@ public: Execute(new TTxCompleteVacuum(this, State.GetVacuumResetGeneration(), vacuumGeneration), ctx); } + void MoveDataCompleted(const TActorContext &ctx) override { + YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "MoveDataCompleted", + {"marker", "KV272"}, + {"tabletId", TabletID()}); + + State.FinishMoveData(ctx); + + if (!MoveDataRequestsQueue.empty()) { + TEvTablet::TEvMoveData::TPtr ev = MoveDataRequestsQueue.front(); + TSet<ui32> moveDataGroups; + for (const auto& groupId : ev->Get()->Record.GetGroups()) { + moveDataGroups.insert(groupId); + } + State.StartMoveData(std::move(moveDataGroups), ev->Sender); + MoveDataRequestsQueue.pop_front(); + + Execute(new TTxAdvanceMoveData(this)); + } + } + STFUNC(StateInit) { YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "StateInit flat event", {"keyValue", TabletID()}, @@ -788,6 +830,7 @@ public: //hFunc(TEvTablet::TEvMoveData, Handle); hFunc(TEvKeyValue::TEvAdvanceMoveDataResult, Handle); hFunc(TEvKeyValue::TEvBlobCopied, Handle); + sFunc(TEvKeyValue::TEvCheckTrash, HandleCheckTrash); default: if (!HandleDefaultEvents(ev, SelfId())) { diff --git a/ydb/core/keyvalue/keyvalue_state.cpp b/ydb/core/keyvalue/keyvalue_state.cpp index 069912437dc..716a8a71674 100644 --- a/ydb/core/keyvalue/keyvalue_state.cpp +++ b/ydb/core/keyvalue/keyvalue_state.cpp @@ -162,8 +162,11 @@ void TKeyValueState::Clear() { CompletedVacuumGeneration = 0; CompletedVacuumTrashGeneration = 0; + MoveDataIsInProgress = false; MoveDataGroups.clear(); - ClearMoveData(); + MoveDataRequestSender = {}; + ClearMoveDataBlobMovingStage(); + ClearMoveDataTrashCheckingStage(); Trash.clear(); TrashForVacuum.clear(); @@ -1893,7 +1896,7 @@ void TKeyValueState::UpdateKeyValue(const TString& key, const TIndexRecord& reco TString value = record.Serialize(); THelpers::DbUpdateUserKeyValue(key, value, db); - if (MoveDataIsInProgress) { + if (MoveDataBlobMovingIsInProgress) { if (MoveDataKey == key) { MoveDataRecordTouched = true; } @@ -1902,7 +1905,7 @@ void TKeyValueState::UpdateKeyValue(const TString& key, const TIndexRecord& reco continue; } if (NeedMoveBlob(item.LogoBlobId)) { - MoveDataNeedsAnotherPass = true; + MoveDataBlobMovingNeedsAnotherPass = true; } } } @@ -1911,7 +1914,7 @@ void TKeyValueState::UpdateKeyValue(const TString& key, const TIndexRecord& reco void TKeyValueState::EraseKey(const TString& key, ISimpleDb& db) { THelpers::DbEraseUserKey(key, db); - if (MoveDataIsInProgress) { + if (MoveDataBlobMovingIsInProgress) { if (MoveDataKey == key) { MoveDataRecordTouched = true; } diff --git a/ydb/core/keyvalue/keyvalue_state.h b/ydb/core/keyvalue/keyvalue_state.h index 6932f99ba09..b3304a391ab 100644 --- a/ydb/core/keyvalue/keyvalue_state.h +++ b/ydb/core/keyvalue/keyvalue_state.h @@ -255,19 +255,25 @@ protected: TMap<ui64, THashSet<TActorId>> VacuumGenerationToSender; ui64 VacuumResetGeneration = 0; // needs to distinguish between vacuum clanups of different resets - // move data state + // move data operation state static constexpr ui64 MaxMoveDataRecordsInOneTx = 16 << 10; - + static constexpr ui64 MaxMoveDataTrashCheckingBlobs = 128 << 10; + // current parameters + bool MoveDataIsInProgress = false; TSet<ui32> MoveDataGroups; TActorId MoveDataRequestSender; - - bool MoveDataIsInProgress = false; - bool MoveDataNeedsAnotherPass = false; + // blob moving stage + bool MoveDataBlobMovingIsInProgress = false; + bool MoveDataBlobMovingNeedsAnotherPass = false; TString MoveDataKey; ui32 MoveDataChainIndex = 0; bool MoveDataRecordTouched = false; TLogoBlobID MoveDataBlobId; THashMap<TLogoBlobID, TLogoBlobID> MoveDataBlobIdToNewBlobId; // for blobs with refcount > 1 + // trash checking stage + std::optional<ui64> MoveDataTrashCheckingVacuumGeneration = {}; // not set for Trash, set for TrashForVacuum + TLogoBlobID MoveDataTrashCheckingBlobId; + bool MoveDataTrashCheckingWaitingForGC = false; TMap<ui64, ui64> InFlightForStep; TMap<std::tuple<ui64, ui32>, ui32> RequestUidStepToCount; @@ -410,16 +416,19 @@ public: void OnEvCompleteGC(bool repeat); // move data methods - void ClearMoveData(); bool IsMoveDataInProgress() const { return MoveDataIsInProgress; } + + void ClearMoveDataBlobMovingStage(); + void ClearMoveDataTrashCheckingStage(); + void StartMoveData(TSet<ui32>&& moveDataGroups, const TActorId& moveDataRequestSender); - bool NeedMoveBlob(const TLogoBlobID& blobId) const; - std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> AdvanceMoveData( - ISimpleDb& db, const TActorContext& ctx); + bool NeedMoveBlob(const TLogoBlobID& blobId) const; + std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> AdvanceMoveData(ISimpleDb& db); std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> BlobCopied( - const TLogoBlobID& blobId, const TLogoBlobID& newBlobId, ISimpleDb& db, const TActorContext& ctx); - std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TryFinishMoveData( - const TActorContext& ctx); + const TLogoBlobID& blobId, const TLogoBlobID& newBlobId, ISimpleDb& db); + std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TryCheckTrash(); + std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> CheckTrash(); + void FinishMoveData(const TActorContext& ctx); void Reply(THolder<TIntermediate> &intermediate, const TActorContext &ctx, const TTabletStorageInfo *info); void ProcessCmd(TIntermediate::TRead &read, diff --git a/ydb/core/keyvalue/keyvalue_state_collect.cpp b/ydb/core/keyvalue/keyvalue_state_collect.cpp index 27b531744ae..46e4ca56a3d 100644 --- a/ydb/core/keyvalue/keyvalue_state_collect.cpp +++ b/ydb/core/keyvalue/keyvalue_state_collect.cpp @@ -148,6 +148,11 @@ void TKeyValueState::CompleteGCComplete(const TActorContext &ctx, const TTabletS {"trashCount", GetTrashCount()}); ProcessPostponedTrims(ctx, info); PrepareCollectIfNeeded(ctx); + + if (MoveDataTrashCheckingWaitingForGC) { + MoveDataTrashCheckingWaitingForGC = false; + ctx.Send(ctx.SelfID, new TEvKeyValue::TEvCheckTrash); + } } bool TKeyValueState::StartVacuum(ui64 generation, TActorId sender) { diff --git a/ydb/core/keyvalue/keyvalue_state_move_data.cpp b/ydb/core/keyvalue/keyvalue_state_move_data.cpp index 0542f126577..f211450c8fa 100644 --- a/ydb/core/keyvalue/keyvalue_state_move_data.cpp +++ b/ydb/core/keyvalue/keyvalue_state_move_data.cpp @@ -6,9 +6,9 @@ namespace NKikimr { namespace NKeyValue { -void TKeyValueState::ClearMoveData() { - MoveDataIsInProgress = false; - MoveDataNeedsAnotherPass = false; +void TKeyValueState::ClearMoveDataBlobMovingStage() { + MoveDataBlobMovingIsInProgress = false; + MoveDataBlobMovingNeedsAnotherPass = false; MoveDataKey.clear(); MoveDataBlobId = TLogoBlobID(); MoveDataChainIndex = 0; @@ -16,10 +16,19 @@ void TKeyValueState::ClearMoveData() { MoveDataBlobIdToNewBlobId.clear(); } +void TKeyValueState::ClearMoveDataTrashCheckingStage() { + MoveDataTrashCheckingVacuumGeneration = {}; + MoveDataTrashCheckingBlobId = TLogoBlobID(); + MoveDataTrashCheckingWaitingForGC = false; +} + void TKeyValueState::StartMoveData(TSet<ui32>&& moveDataGroups, const TActorId& moveDataRequestSender) { MoveDataGroups = std::move(moveDataGroups); MoveDataRequestSender = moveDataRequestSender; MoveDataIsInProgress = true; + + ClearMoveDataBlobMovingStage(); + MoveDataBlobMovingIsInProgress = true; } bool TKeyValueState::NeedMoveBlob(const TLogoBlobID& blobId) const { @@ -28,14 +37,15 @@ bool TKeyValueState::NeedMoveBlob(const TLogoBlobID& blobId) const { return MoveDataGroups.contains(groupId); } -std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TKeyValueState::AdvanceMoveData( - ISimpleDb& db, const TActorContext& ctx) { +std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TKeyValueState::AdvanceMoveData(ISimpleDb& db) { YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "AdvanceMoveData", {"keyValue", TabletId}, {"marker", "KV93"}); + Y_ABORT_UNLESS(MoveDataBlobMovingIsInProgress); + if (Index.empty()) { - return TryFinishMoveData(ctx); + return TryCheckTrash(); } if (MoveDataRecordTouched) { @@ -53,7 +63,7 @@ std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TKeyValueState::AdvanceMo itIndex = Index.upper_bound(MoveDataKey); MoveDataChainIndex = 0; if (itIndex == Index.end()) { - return TryFinishMoveData(ctx); + return TryCheckTrash(); } } } @@ -86,10 +96,17 @@ std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TKeyValueState::AdvanceMo if (itRefCounts->second > 1) { --itRefCounts->second; } else { - RefCounts.erase(itRefCounts); + Dereference(blobId, db, false); } auto newBlobId = MoveDataBlobIdToNewBlobId[blobId]; + if (RefCounts.find(newBlobId) == RefCounts.end()) { + // blob has been deleted since we copied it + // we need to copy it again + MoveDataBlobIdToNewBlobId.erase(blobId); + return TEvKeyValue::TEvAdvanceMoveDataResult::CopyBlob(newBlobId); + } + item.LogoBlobId = newBlobId; ++RefCounts[newBlobId]; UpdateKeyValue(MoveDataKey, record, db); @@ -98,24 +115,26 @@ std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TKeyValueState::AdvanceMo MoveDataChainIndex = 0; } - return TryFinishMoveData(ctx); + return TryCheckTrash(); } std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TKeyValueState::BlobCopied( - const TLogoBlobID& blobId, const TLogoBlobID& newBlobId, ISimpleDb& db, const TActorContext& ctx) { + const TLogoBlobID& blobId, const TLogoBlobID& newBlobId, ISimpleDb& db) { YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "OnBlobCopied", {"keyValue", TabletId}, {"marker", "KV94"}, {"blobId", blobId.ToString()}, {"newBlobId", newBlobId.ToString()}); + Y_ABORT_UNLESS(MoveDataBlobMovingIsInProgress); + Y_ABORT_UNLESS(!MoveDataKey.empty()); Y_ABORT_UNLESS(blobId == MoveDataBlobId); if (MoveDataRecordTouched) { MoveDataRecordTouched = false; MoveDataChainIndex = 0; - return AdvanceMoveData(db, ctx); + return AdvanceMoveData(db); } auto itIndex = Index.find(MoveDataKey); @@ -131,7 +150,7 @@ std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TKeyValueState::BlobCopie --itRefCounts->second; MoveDataBlobIdToNewBlobId[blobId] = newBlobId; } else { - RefCounts.erase(itRefCounts); + Dereference(blobId, db, false); } item.LogoBlobId = newBlobId; @@ -146,35 +165,100 @@ std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TKeyValueState::BlobCopie ++itIndex; if (itIndex != Index.end()) { MoveDataKey = itIndex->first; - return AdvanceMoveData(db, ctx); + return AdvanceMoveData(db); } else { - return TryFinishMoveData(ctx); + return TryCheckTrash(); } } - return AdvanceMoveData(db, ctx); + return AdvanceMoveData(db); } -std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TKeyValueState::TryFinishMoveData( - const TActorContext& ctx) { - YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "TryFinishMoveData", +std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TKeyValueState::TryCheckTrash() { + YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "TryCheckTrash", {"keyValue", TabletId}, {"marker", "KV95"}); - if (MoveDataNeedsAnotherPass) { - ClearMoveData(); - - MoveDataIsInProgress = true; + if (MoveDataBlobMovingNeedsAnotherPass) { + ClearMoveDataBlobMovingStage(); + MoveDataBlobMovingIsInProgress = true; return TEvKeyValue::TEvAdvanceMoveDataResult::Repeat(); } else { - ClearMoveData(); + MoveDataBlobMovingIsInProgress = false; + ClearMoveDataTrashCheckingStage(); + return TEvKeyValue::TEvAdvanceMoveDataResult::CheckTrash(); + } +} + +std::unique_ptr<TEvKeyValue::TEvAdvanceMoveDataResult> TKeyValueState::CheckTrash() { + YDB_LOG_DEBUG_COMP(NKikimrServices::KEYVALUE, "CheckTrash", + {"keyValue", TabletId}, + {"marker", "KV96"}); - ctx.Send(MoveDataRequestSender, new TEvTablet::TEvMoveDataResponse(TabletId)); + TSet<TLogoBlobID>& trashBin = Trash; - MoveDataGroups.clear(); - MoveDataRequestSender = {}; - return TEvKeyValue::TEvAdvanceMoveDataResult::Finish(); + TMap<ui64, TSet<TLogoBlobID>>::iterator itTrashBin; + bool finished = false; + + auto nextTrashBin = [&]() { + if (!MoveDataTrashCheckingVacuumGeneration) { + itTrashBin = TrashForVacuum.begin(); + } else { + itTrashBin = TrashForVacuum.upper_bound(*MoveDataTrashCheckingVacuumGeneration); + } + if (itTrashBin == TrashForVacuum.end()) { + finished = true; + return; + } + trashBin = itTrashBin->second; + MoveDataTrashCheckingVacuumGeneration = itTrashBin->first; + MoveDataTrashCheckingBlobId = TLogoBlobID(); + }; + + if (MoveDataTrashCheckingVacuumGeneration) { + itTrashBin = TrashForVacuum.find(*MoveDataTrashCheckingVacuumGeneration); + if (itTrashBin == TrashForVacuum.end()) { + nextTrashBin(); + if (finished) { + return TEvKeyValue::TEvAdvanceMoveDataResult::Finish(); + } + } else { + trashBin = itTrashBin->second; + } + } + + ui64 checkedBlobsCount = 0; + while (!finished) { + auto itTrash = trashBin.lower_bound(MoveDataTrashCheckingBlobId); + if (itTrash == trashBin.end()) { + nextTrashBin(); + if (finished) { + break; + } + itTrash = trashBin.begin(); + } + for (; itTrash != trashBin.end(); ++itTrash, ++checkedBlobsCount) { + MoveDataTrashCheckingBlobId = *itTrash; + if (checkedBlobsCount >= MaxMoveDataTrashCheckingBlobs) { + return TEvKeyValue::TEvAdvanceMoveDataResult::CheckTrash(); + } + if (NeedMoveBlob(MoveDataTrashCheckingBlobId)) { + MoveDataTrashCheckingWaitingForGC = true; + return TEvKeyValue::TEvAdvanceMoveDataResult::WaitForGC(); + } + } + nextTrashBin(); } + + return TEvKeyValue::TEvAdvanceMoveDataResult::Finish(); +} + +void TKeyValueState::FinishMoveData(const TActorContext& ctx) { + ctx.Send(MoveDataRequestSender, new TEvTablet::TEvMoveDataResponse(TabletId)); + + MoveDataIsInProgress = false; + MoveDataGroups.clear(); + MoveDataRequestSender = {}; } } // NKeyValue diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp index bd2267a62a3..4e79cae47cd 100644 --- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp @@ -32,6 +32,8 @@ #include <yql/essentials/public/issue/yql_issue_message.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + namespace NKikimr { namespace NKqp { @@ -107,8 +109,13 @@ public: sb << "CA " << shardId.first << ", "; } } - KQP_STLOG_D(KQPDATA, sb, - (trace_id, TraceId())); + YDB_LOG_DEBUG("Waiting for compute actors to finish", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"details", sb}, + {"traceId", TraceId()}); } return false; } @@ -209,9 +216,13 @@ public: } else if (Request.LocksOp == ELocksOp::Commit) { AFL_ENSURE(!ReadOnlyTx); Become(&TKqpDataExecuter::FinalizeState); - KQP_STLOG_D(KQPDATA, "Send Commit to BufferActor", - (buffer_actor_id, BufferActorId), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Send Commit to BufferActor", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"bufferActorId", BufferActorId}, + {"traceId", TraceId()}); auto event = std::make_unique<NKikimr::NKqp::TEvKqpBuffer::TEvCommit>(); event->ExecuterActorId = SelfId(); @@ -225,9 +236,13 @@ public: return; } else if (Request.LocksOp == ELocksOp::Rollback) { Become(&TKqpDataExecuter::FinalizeState); - KQP_STLOG_D(KQPDATA, "Send Rollback to BufferActor", - (buffer_actor_id, BufferActorId), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Send Rollback to BufferActor", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"bufferActorId", BufferActorId}, + {"traceId", TraceId()}); auto event = std::make_unique<NKikimr::NKqp::TEvKqpBuffer::TEvRollback>(); event->ExecuterActorId = SelfId(); @@ -240,9 +255,13 @@ public: return; } else if (Request.FlushEffects) { Become(&TKqpDataExecuter::FinalizeState); - KQP_STLOG_D(KQPDATA, "Send Flush to BufferActor", - (buffer_actor_id, BufferActorId), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Send Flush to BufferActor", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"bufferActorId", BufferActorId}, + {"traceId", TraceId()}); auto event = std::make_unique<NKikimr::NKqp::TEvKqpBuffer::TEvFlush>(); event->ExecuterActorId = SelfId(); @@ -291,9 +310,13 @@ public: if (IsCancelAfterAllowed(ev)) { TBase::HandleAbortExecution(ev); } else { - KQP_STLOG_D(KQPDATA, "Got TEvAbortExecution, but cancellation is not allowed", - (sender, ev->Sender), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Got TEvAbortExecution, but cancellation is not allowed", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"sender", ev->Sender}, + {"traceId", TraceId()}); } } @@ -307,9 +330,13 @@ public: void HandleFinalize(TEvents::TEvUndelivered::TPtr& ev) { AFL_ENSURE(ev->Sender == BufferActorId); - KQP_STLOG_W(KQPDATA, "Got Undelivered from BufferActor", - (sender, ev->Sender), - (trace_id, TraceId())); + YDB_LOG_WARN("Got Undelivered from BufferActor", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"sender", ev->Sender}, + {"traceId", TraceId()}); } void MakeResponseAndPassAway() { @@ -438,9 +465,13 @@ private: if (IsCancelAfterAllowed(ev)) { TBase::HandleAbortExecution(ev); } else { - KQP_STLOG_D(KQPDATA, "Got TEvAbortExecution, but cancellation is not allowed", - (sender, ev->Sender), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Got TEvAbortExecution, but cancellation is not allowed", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"sender", ev->Sender}, + {"traceId", TraceId()}); } } @@ -493,9 +524,13 @@ private: void HandleResolve(TEvPrivate::TEvResourcesSnapshot::TPtr& ev) { if (ev->Get()->Snapshot.empty()) { - KQP_STLOG_E(KQPDATA, "Can not find default state storage group for database", - (database, Database), - (trace_id, TraceId())); + YDB_LOG_ERROR("Can not find default state storage group for database", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"database", Database}, + {"traceId", TraceId()}); } ResourcesSnapshot = std::move(ev->Get()->Snapshot); ResourceSnapshotRequired = false; @@ -601,8 +636,13 @@ private: } if (error) { - KQP_STLOG_E(KQPDATA, *error, - (trace_id, TraceId())); + YDB_LOG_ERROR("Rejected operation on async index table", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"error", *error}, + {"traceId", TraceId()}); ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED, YqlIssue({}, NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED, *error)); return; @@ -612,8 +652,13 @@ private: if ((stageInfo.Meta.IsOlap() && HasDmlOperationOnOlap(tx.Body->GetType(), stage))) { auto error = TStringBuilder() << "Data manipulation queries with column-oriented tables are supported only by API QueryService."; - KQP_STLOG_E(KQPDATA, error, - (trace_id, TraceId())); + YDB_LOG_ERROR("DML on column-oriented table is not supported via this API", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"error", error}, + {"traceId", TraceId()}); ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED, YqlIssue({}, NYql::TIssuesIds::KIKIMR_PRECONDITION_FAILED, error)); return; @@ -652,9 +697,13 @@ private: } if (computeTasks.size() > Request.MaxComputeActors) { - KQP_STLOG_N(KQPDATA, "Too many compute actors", - (count, computeTasks.size()), - (trace_id, TraceId())); + YDB_LOG_NOTICE("Too many compute actors", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"count", computeTasks.size()}, + {"traceId", TraceId()}); ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED, YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() << "Requested too many execution units: " << computeTasks.size())); @@ -670,10 +719,14 @@ private: const size_t shards = sourceScanPartitionsCount; if (shardsLimit > 0 && shards > shardsLimit) { - KQP_STLOG_W(KQPDATA, "Too many affected shards", - (datashard_tasks, shards), - (limit, shardsLimit), - (trace_id, TraceId())); + YDB_LOG_WARN("Too many affected shards", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"datashardTasks", shards}, + {"limit", shardsLimit}, + {"traceId", TraceId()}); Counters->TxProxyMon->TxResultError->Inc(); ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED, YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() @@ -815,8 +868,12 @@ private: auto longTxService = NLongTxService::MakeLongTxServiceID(SelfId().NodeId()); Send(longTxService, new NLongTxService::TEvLongTxService::TEvAcquireReadSnapshot(Database, TableIdsForSnapshot)); - KQP_STLOG_T(KQPDATA, "Create temporary mvcc snapshot, become WaitSnapshotState", - (trace_id, TraceId())); + YDB_LOG_TRACE("Create temporary mvcc snapshot, become WaitSnapshotState", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); Become(&TKqpDataExecuter::WaitSnapshotState); ExecuterStateSpan = NWilson::TSpan(TWilsonKqp::DataExecuterAcquireSnapshot, ExecuterSpan.GetTraceId(), "WaitForSnapshot"); @@ -845,11 +902,15 @@ private: void Handle(NLongTxService::TEvLongTxService::TEvAcquireReadSnapshotResult::TPtr& ev) { auto* msg = ev->Get(); - KQP_STLOG_T(KQPDATA, "Read snapshot result", - (status, msg->Status), - (step, msg->Snapshot.Step), - (tx_id, msg->Snapshot.TxId), - (trace_id, TraceId())); + YDB_LOG_TRACE("Read snapshot result", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"status", msg->Status}, + {"step", msg->Snapshot.Step}, + {"snapshotTxId", msg->Snapshot.TxId}, + {"traceId", TraceId()}); if (msg->Status != Ydb::StatusIds::SUCCESS) { ExecuterStateSpan.EndError(TStringBuilder() << Ydb::StatusIds::StatusCode_Name(msg->Status)); @@ -879,10 +940,14 @@ private: } ExecuterStateSpan = NWilson::TSpan(TWilsonKqp::DataExecuterRunTasks, ExecuterSpan.GetTraceId(), "RunTasks", NWilson::EFlags::AUTO_END); - KQP_STLOG_D(KQPDATA, "become ExecuteState", - (current_state, CurrentStateFuncName()), - (immediate, true), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Become ExecuteState", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"currentState", CurrentStateFuncName()}, + {"immediate", true}, + {"traceId", TraceId()}); Become(&TKqpDataExecuter::ExecuteState); } @@ -903,17 +968,25 @@ private: return; } - KQP_STLOG_I(KQPDATA, "Total tasks", - (total_tasks, TasksGraph.GetTasks().size()), - (read_only, ReadOnlyTx), - (immediate, ImmediateTx), - (pending_compute_tasks, Planner ? Planner->GetPendingComputeTasks().size() : 0), - (use_followers, GetUseFollowers()), - (trace_id, TraceId())); + YDB_LOG_INFO("Total tasks", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"totalTasks", TasksGraph.GetTasks().size()}, + {"readOnly", ReadOnlyTx}, + {"immediate", ImmediateTx}, + {"pendingComputeTasks", Planner ? Planner->GetPendingComputeTasks().size() : 0}, + {"useFollowers", GetUseFollowers()}, + {"traceId", TraceId()}); // error - KQP_STLOG_T(KQPDATA, "Updating channels after the creation of compute actors", - (trace_id, TraceId())); + YDB_LOG_TRACE("Updating channels after the creation of compute actors", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); Y_ENSURE(Planner); THashMap<TActorId, THashSet<ui64>> updates; for (ui64 taskId : ComputeTasks) { @@ -928,15 +1001,23 @@ private: void Shutdown() override { if (Planner) { if (Planner->GetPendingComputeTasks().empty() && Planner->GetPendingComputeActors().empty()) { - KQP_STLOG_I(KQPDATA, "Shutdown immediately - nothing to wait", - (trace_id, TraceId())); + YDB_LOG_INFO("Shutdown immediately - nothing to wait", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); PassAway(); } else { this->Become(&TThis::WaitShutdownState); - KQP_STLOG_I(KQPDATA, "Waiting for shutdown", - (pending_tasks, Planner->GetPendingComputeTasks().size()), - (pending_compute_actors, Planner->GetPendingComputeActors().size()), - (trace_id, TraceId())); + YDB_LOG_INFO("Waiting for shutdown", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"pendingTasks", Planner->GetPendingComputeTasks().size()}, + {"pendingComputeActors", Planner->GetPendingComputeActors().size()}, + {"traceId", TraceId()}); TActivationContext::Schedule(WaitCAStatsTimeout, new IEventHandle(SelfId(), SelfId(), new TEvents::TEvPoison)); } } else { @@ -979,9 +1060,13 @@ private: hFunc(TEvents::TEvPoison, HandleShutdown); hFunc(TEvDq::TEvAbortExecution, HandleShutdown); default: - KQP_STLOG_E(KQPDATA, "Unexpected event while waiting for shutdown", - (event_type, ev->GetTypeName()), // ignore all other events - (trace_id, TraceId())); + YDB_LOG_ERROR("Unexpected event while waiting for shutdown", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"eventType", ev->GetTypeName()}, + {"traceId", TraceId()}); } } @@ -1001,9 +1086,13 @@ private: void HandleShutdown(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { const auto nodeId = ev->Get()->NodeId; - KQP_STLOG_N(KQPDATA, "Node has disconnected while shutdown", - (node_id, nodeId), - (trace_id, TraceId())); + YDB_LOG_NOTICE("Node has disconnected while shutdown", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"nodeId", nodeId}, + {"traceId", TraceId()}); YQL_ENSURE(Planner); @@ -1026,9 +1115,13 @@ private: void HandleShutdown(TEvents::TEvPoison::TPtr& ev) { // Self-poison means timeout - don't wait anymore. - KQP_STLOG_I(KQPDATA, "Timed out on waiting for Compute Actors to finish - forcing shutdown", - (Sender, ev->Sender), - (trace_id, TraceId())); + YDB_LOG_INFO("Timed out on waiting for Compute Actors to finish - forcing shutdown", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"sender", ev->Sender}, + {"traceId", TraceId()}); if (ev->Sender == SelfId()) { FillLocksFromExtraData(); @@ -1044,16 +1137,24 @@ private: // In case of external timeout the response is already sent to the client - no need to wait for stats. if (statusCode == Ydb::StatusIds::TIMEOUT) { FillLocksFromExtraData(); - KQP_STLOG_I(KQPDATA, "External timeout while waiting for Compute Actors to finish - forcing shutdown", - (Sender, ev->Sender), - (trace_id, TraceId())); + YDB_LOG_INFO("External timeout while waiting for Compute Actors to finish - forcing shutdown", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"sender", ev->Sender}, + {"traceId", TraceId()}); PassAway(); } } void Handle(NFq::TEvCheckpointCoordinator::TEvZeroCheckpointDone::TPtr& ev) { - KQP_STLOG_D(KQPDATA, "Coordinator saved zero checkpoint", - (trace_id, TraceId())); + YDB_LOG_DEBUG("Coordinator saved zero checkpoint", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); Send(CheckpointCoordinatorId, new NFq::TEvCheckpointCoordinator::TEvRunGraph()); if (const auto context = GetUserRequestContext()) { @@ -1062,9 +1163,13 @@ private: } void Handle(NFq::TEvCheckpointCoordinator::TEvRaiseTransientIssues::TPtr& ev) { - KQP_STLOG_N(KQPDATA, "TEvRaiseTransientIssues from checkpoint coordinator", - (TransientIssues, ev->Get()->TransientIssues.ToOneLineString()), - (trace_id, TraceId())); + YDB_LOG_NOTICE("TEvRaiseTransientIssues from checkpoint coordinator", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"transientIssues", ev->Get()->TransientIssues.ToOneLineString()}, + {"traceId", TraceId()}); } void StartCheckpointCoordinator() { @@ -1129,16 +1234,20 @@ private: graphParams, stateLoadMode, streamingDisposition).Release()); - KQP_STLOG_D(KQPDATA, "Created new CheckpointCoordinator", - (CheckpointCoordinatorId, CheckpointCoordinatorId), - (ExecutionId, context->CurrentExecutionId), - (CheckpointId, checkpointId), - (Generation, Generation), - (StateLoadMode, FederatedQuery::StateLoadMode_Name(stateLoadMode)), - (StreamingDisposition, streamingDisposition.ShortDebugString()), - (HasQueryPhysicalGraph, Request.QueryPhysicalGraph != nullptr), - (EnableWatermarks, Request.QueryPhysicalGraph && Request.QueryPhysicalGraph->GetPreparedQuery().GetPhysicalQuery().GetEnableWatermarks()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Created new CheckpointCoordinator", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"checkpointCoordinatorId", CheckpointCoordinatorId}, + {"executionId", context->CurrentExecutionId}, + {"checkpointId", checkpointId}, + {"generation", Generation}, + {"stateLoadMode", FederatedQuery::StateLoadMode_Name(stateLoadMode)}, + {"streamingDisposition", streamingDisposition.ShortDebugString()}, + {"hasQueryPhysicalGraph", Request.QueryPhysicalGraph != nullptr}, + {"enableWatermarks", Request.QueryPhysicalGraph && Request.QueryPhysicalGraph->GetPreparedQuery().GetPhysicalQuery().GetEnableWatermarks()}, + {"traceId", TraceId()}); } private: diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.h b/ydb/core/kqp/executer_actor/kqp_executer_impl.h index b60f081a998..2646a24f907 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_impl.h +++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.h @@ -58,13 +58,6 @@ LWTRACE_USING(KQP_PROVIDER); namespace NKikimr::NKqp { -#define KQP_STLOG_T(MARKER, MESSAGE, ...) STLOG(PRI_TRACE, NKikimrServices::KQP_EXECUTER, MARKER, "ActorId: " << SelfId() << " TxId: " << TxId << ". " << "Ctx: " << *GetUserRequestContext() << ". " << MESSAGE, __VA_ARGS__) -#define KQP_STLOG_D(MARKER, MESSAGE, ...) STLOG(PRI_DEBUG, NKikimrServices::KQP_EXECUTER, MARKER, "ActorId: " << SelfId() << " TxId: " << TxId << ". " << "Ctx: " << *GetUserRequestContext() << ". " << MESSAGE, __VA_ARGS__) -#define KQP_STLOG_I(MARKER, MESSAGE, ...) STLOG(PRI_INFO, NKikimrServices::KQP_EXECUTER, MARKER, "ActorId: " << SelfId() << " TxId: " << TxId << ". " << "Ctx: " << *GetUserRequestContext() << ". " << MESSAGE, __VA_ARGS__) -#define KQP_STLOG_N(MARKER, MESSAGE, ...) STLOG(PRI_NOTICE, NKikimrServices::KQP_EXECUTER, MARKER, "ActorId: " << SelfId() << " TxId: " << TxId << ". " << "Ctx: " << *GetUserRequestContext() << ". " << MESSAGE, __VA_ARGS__) -#define KQP_STLOG_W(MARKER, MESSAGE, ...) STLOG(PRI_WARN, NKikimrServices::KQP_EXECUTER, MARKER, "ActorId: " << SelfId() << " TxId: " << TxId << ". " << "Ctx: " << *GetUserRequestContext() << ". " << MESSAGE, __VA_ARGS__) -#define KQP_STLOG_E(MARKER, MESSAGE, ...) STLOG(PRI_ERROR, NKikimrServices::KQP_EXECUTER, MARKER, "ActorId: " << SelfId() << " TxId: " << TxId << ". " << "Ctx: " << *GetUserRequestContext() << ". " << MESSAGE, __VA_ARGS__) -#define KQP_STLOG_C(MARKER, MESSAGE, ...) STLOG(PRI_CRIT, NKikimrServices::KQP_EXECUTER, MARKER, "ActorId: " << SelfId() << " TxId: " << TxId << ". " << "Ctx: " << *GetUserRequestContext() << ". " << MESSAGE, __VA_ARGS__) using EExecType = TEvKqpExecuter::TEvTxResponse::EExecutionType; @@ -186,8 +179,12 @@ public: CancelAt = StartTime + *Request.CancelAfter; } - KQP_STLOG_T(KQPEX, "Bootstrap done, become ReadyState", - (trace_id, TraceId())); + YDB_LOG_TRACE_COMP(NKikimrServices::KQP_EXECUTER, "Bootstrap done, become ReadyState", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); } TActorId SelfId() const { @@ -322,9 +319,13 @@ protected: } if (shardIds.size() > 0) { - KQP_STLOG_D(KQPDATA, "Start resolving tablets nodes...", - (shard_ids_count, shardIds.size()), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Start resolving tablets nodes.", + {"marker", "KQPDATA"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"shardIdsCount", shardIds.size()}, + {"traceId", TraceId()}); ExecuterStateSpan = NWilson::TSpan(TWilsonKqp::ExecuterShardsResolve, ExecuterSpan.GetTraceId(), "WaitForShardsResolve", NWilson::EFlags::AUTO_END); auto kqpShardsResolver = CreateKqpShardsResolver(this->SelfId(), TxId, static_cast<TDerived*>(this)->GetSimplifiedUseFollowers(), std::move(shardIds)); @@ -351,10 +352,14 @@ protected: ExecuterStateSpan.EndError(Ydb::StatusIds_StatusCode_Name(reply.Status)); } - KQP_STLOG_W(KQPEX, "Shards nodes resolve failed", - (Status, Ydb::StatusIds_StatusCode_Name(reply.Status)), - (Issues, reply.Issues.ToString()), - (trace_id, TraceId())); + YDB_LOG_WARN_COMP(NKikimrServices::KQP_EXECUTER, "Shards nodes resolve failed", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"status", Ydb::StatusIds_StatusCode_Name(reply.Status)}, + {"issues", reply.Issues}, + {"traceId", TraceId()}); ReplyErrorAndDie(reply.Status, reply.Issues); return false; } @@ -362,10 +367,14 @@ protected: ExecuterStateSpan.EndOk(); } - KQP_STLOG_D(KQPEX, "Shards nodes resolved", - (SuccessNodes, reply.ShardsToNodes.size()), - (FailedNodes, reply.Unresolved), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Shards nodes resolved", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"successNodes", reply.ShardsToNodes.size()}, + {"failedNodes", reply.Unresolved}, + {"traceId", TraceId()}); for (const auto& [_, nodeId] : reply.ShardsToNodes) { TxManager->AddParticipantNode(nodeId); @@ -385,8 +394,13 @@ protected: << "(total " << pair.second.size() << ") " << Endl; } } - KQP_STLOG_D(KQPEX, sb, - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Shard distribution on nodes after resolve", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"details", sb}, + {"traceId", TraceId()}); } return true; @@ -436,10 +450,14 @@ protected: --PendingPartitionStatsRequests; if (record.GetStatus() != NKikimrScheme::StatusSuccess) { - KQP_STLOG_W(KQPEX, "DescribeScheme for partition stats returned non-success, using uniform distribution", - (Status, record.GetStatus()), - (Path, record.GetPath()), - (trace_id, TraceId())); + YDB_LOG_WARN_COMP(NKikimrServices::KQP_EXECUTER, "DescribeScheme for partition stats returned non-success, using uniform distribution", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"status", record.GetStatus()}, + {"path", record.GetPath()}, + {"traceId", TraceId()}); return; } @@ -531,11 +549,15 @@ protected: txResult.ColumnOrder, txResult.ColumnHints); // TODO: Calculate rows/bytes count for the arrow format of result set - KQP_STLOG_D(KQPEX, "Send TEvStreamData", - (Recipient, Target), - (SeqNo, streamEv->Record.GetSeqNo()), - (Rows, streamEv->Record.GetResultSet().rows().size()), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Send TEvStreamData", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"recipient", Target}, + {"seqNo", streamEv->Record.GetSeqNo()}, + {"rows", streamEv->Record.GetResultSet().rows().size()}, + {"traceId", TraceId()}); SentResultIndexes.insert(resultIndex); this->Send(Target, streamEv.Release()); @@ -679,11 +701,15 @@ protected: ui64 rowCount = batch.RowCount(); ResponseEv->TakeResult(channel.DstInputIndex, std::move(batch)); txResult.HasTrailingResult = true; - KQP_STLOG_D(KQPEX, "Staging TEvStreamData", - (Recipient, Target), - (SeqNo, computeData.Proto.GetSeqNo()), - (Rows, rowCount), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Staging TEvStreamData", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"recipient", Target}, + {"seqNo", computeData.Proto.GetSeqNo()}, + {"rows", rowCount}, + {"traceId", TraceId()}); } return; @@ -701,19 +727,27 @@ protected: Stats->ResultBytes += batch.Size(); Stats->ResultRows += batch.RowCount(); - KQP_STLOG_T(KQPEX, "Got result", - (ChannelId, channel.Id), - (InputIndex, channel.DstInputIndex), - (Sender, ev->Sender), - (Finished, channelData.GetFinished()), - (trace_id, TraceId())); + YDB_LOG_TRACE_COMP(NKikimrServices::KQP_EXECUTER, "Got result", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"channelId", channel.Id}, + {"inputIndex", channel.DstInputIndex}, + {"sender", ev->Sender}, + {"finished", channelData.GetFinished()}, + {"traceId", TraceId()}); ResponseEv->TakeResult(channel.DstInputIndex, std::move(batch)); - KQP_STLOG_T(KQPEX, "Send ack", - (ChannelId, channel.Id), - (SeqNo, record.GetSeqNo()), - (Recipient, ev->Sender), - (trace_id, TraceId())); + YDB_LOG_TRACE_COMP(NKikimrServices::KQP_EXECUTER, "Send ack", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"channelId", channel.Id}, + {"seqNo", record.GetSeqNo()}, + {"recipient", ev->Sender}, + {"traceId", TraceId()}); auto ackEv = MakeHolder<NYql::NDq::TEvDqCompute::TEvChannelDataAck>(); ackEv->Record.SetSeqNo(record.GetSeqNo()); @@ -750,13 +784,17 @@ protected: ui64 seqNo = ev->Get()->Record.GetSeqNo(); i64 freeSpace = ev->Get()->Record.GetFreeSpace(); - KQP_STLOG_D(KQPEX, "Send ack", - (ChannelId, channelId), - (SeqNo, seqNo), - (Enough, ev->Get()->Record.GetEnough()), - (FreeSpace, freeSpace), - (Recipient, channelComputeActorId), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Send ack", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"channelId", channelId}, + {"seqNo", seqNo}, + {"enough", ev->Get()->Record.GetEnough()}, + {"freeSpace", freeSpace}, + {"recipient", channelComputeActorId}, + {"traceId", TraceId()}); auto ackEv = MakeHolder<NYql::NDq::TEvDqCompute::TEvChannelDataAck>(); ackEv->Record.SetSeqNo(seqNo); @@ -771,13 +809,17 @@ protected: auto& state = ev->Get()->Record; ui64 taskId = state.GetTaskId(); - KQP_STLOG_D(KQPEX, "Got execution state from compute actor", - (ActorState, CurrentStateFuncName()), - (ComputeActor, computeActor), - (TaskId, taskId), - (State, NYql::NDqProto::EComputeState_Name((NYql::NDqProto::EComputeState) state.GetState())), - (Stats, state.GetStats()), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Got execution state from compute actor", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"actorState", CurrentStateFuncName()}, + {"computeActor", computeActor}, + {"taskId", taskId}, + {"state", NYql::NDqProto::EComputeState_Name((NYql::NDqProto::EComputeState) state.GetState())}, + {"stats", state.GetStats()}, + {"traceId", TraceId()}); YQL_ENSURE(Stats); @@ -1080,10 +1122,14 @@ protected: if (IsDebugLogEnabled()) { for (auto& tx : Request.Transactions) { - KQP_STLOG_D(KQPEX, "Executing physical tx", - (TxType, (ui32)tx.Body->GetType()), - (Stages, tx.Body->StagesSize()), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Executing physical tx", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"txType", (ui32)tx.Body->GetType()}, + {"stages", tx.Body->StagesSize()}, + {"traceId", TraceId()}); } } @@ -1097,8 +1143,12 @@ protected: auto kqpTableResolver = CreateKqpTableResolver(this->SelfId(), TxId, UserToken, TasksGraph, false); KqpTableResolverId = this->RegisterWithSameMailbox(kqpTableResolver); - KQP_STLOG_T(KQPEX, "Got request, become WaitResolveState", - (trace_id, TraceId())); + YDB_LOG_TRACE_COMP(NKikimrServices::KQP_EXECUTER, "Got request, become WaitResolveState", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); this->Become(&TDerived::WaitResolveState); auto now = TAppData::TimeProvider->Now(); @@ -1130,8 +1180,13 @@ protected: sb << "CA " << ca.first << ", "; } } - KQP_STLOG_D(KQPEX, sb, - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Waiting for pending compute tasks and actors", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"details", sb}, + {"traceId", TraceId()}); } return false; @@ -1151,10 +1206,14 @@ protected: case TEvKqpNode::TEvStartKqpTasksRequest::EventType: { switch (reason) { case TEvents::TEvUndelivered::EReason::ReasonActorUnknown: { - KQP_STLOG_D(KQPEX, "Schedule a retry by ActorUnknown reason", - (NodeId, ev->Sender.NodeId()), - (RequestId, ev->Cookie), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Schedule a retry by ActorUnknown reason", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"nodeId", ev->Sender.NodeId()}, + {"requestId", ev->Cookie}, + {"traceId", TraceId()}); this->Schedule(TDuration::MilliSeconds(Planner->GetCurrentRetryDelay(ev->Cookie)), new typename TEvPrivate::TEvRetry(ev->Cookie, ev->Sender)); return; } @@ -1172,10 +1231,14 @@ protected: } } default: { - KQP_STLOG_E(KQPEX, "Event lost", - (EventType, eventType), - (Reason, reason), - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "Event lost", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"eventType", eventType}, + {"reason", reason}, + {"traceId", TraceId()}); } } } @@ -1191,8 +1254,13 @@ protected: void HandleDisconnected(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { auto nodeId = ev->Get()->NodeId; - KQP_STLOG_N(KQPEX, "Disconnected node", (node_id, nodeId), - (trace_id, TraceId())); + YDB_LOG_NOTICE_COMP(NKikimrServices::KQP_EXECUTER, "Disconnected node", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"nodeId", nodeId}, + {"traceId", TraceId()}); if (Planner) { for (auto computeActor : Planner->GetPendingComputeActors()) { @@ -1220,10 +1288,14 @@ protected: auto reason = record.GetNotStartedTasks()[0].GetReason(); auto& message = record.GetNotStartedTasks()[0].GetMessage(); - KQP_STLOG_E(KQPEX, "Stop executing", - (Reason, NKikimrKqp::TEvStartKqpTasksResponse_ENotStartedTaskReason_Name(reason)), - (Message, message), - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "Stop executing", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"reason", NKikimrKqp::TEvStartKqpTasksResponse_ENotStartedTaskReason_Name(reason)}, + {"message", message}, + {"traceId", TraceId()}); switch (reason) { case NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_MEMORY: { @@ -1251,16 +1323,24 @@ protected: } case NKikimrKqp::TEvStartKqpTasksResponse::NODE_SHUTTING_DOWN: { if (!AppData()->FeatureFlags.GetEnableShuttingDownNodeState()) { - KQP_STLOG_D(KQPEX, "Received NODE_SHUTTING_DOWN but feature flag EnableShuttingDownNodeState is disabled", - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Received NODE_SHUTTING_DOWN but feature flag EnableShuttingDownNodeState is disabled", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, YqlIssue({}, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE, "Compute node is unavailable")); break; } - KQP_STLOG_D(KQPEX, "Received NODE_SHUTTING_DOWN, attempting run tasks locally", - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Received NODE_SHUTTING_DOWN, attempting run tasks locally", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); ui32 requestId = record.GetNotStartedTasks(0).GetRequestId(); auto localNode = MakeKqpNodeServiceID(SelfId().NodeId()); @@ -1289,10 +1369,14 @@ protected: auto& task = TasksGraph.GetTask(taskId); TActorId computeActorId = ActorIdFromProto(startedTask.GetActorId()); - KQP_STLOG_D(KQPEX, "Executing task", - (TaskId, taskId), - (ComputeActor, computeActorId), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Executing task", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"taskId", taskId}, + {"computeActor", computeActorId}, + {"traceId", TraceId()}); YQL_ENSURE(Planner); bool ack = Planner->AcknowledgeCA(taskId, computeActorId, nullptr); if (ack) { @@ -1324,10 +1408,14 @@ protected: NYql::NDqProto::StatusIds::StatusCode statusCode, const NYql::TIssues& issues, const bool isTargetSender) { - KQP_STLOG_D(KQPEX, "Got EvAbortExecution", - (Status, NYql::NDqProto::StatusIds_StatusCode_Name(statusCode)), - (Issues, issues.ToOneLineString()), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Got EvAbortExecution", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"status", NYql::NDqProto::StatusIds_StatusCode_Name(statusCode)}, + {"issues", issues.ToOneLineString()}, + {"traceId", TraceId()}); auto ydbStatusCode = NYql::NDq::DqStatusToYdbStatus(statusCode); if (ydbStatusCode == Ydb::StatusIds::INTERNAL_ERROR) { InternalError(issues); @@ -1368,19 +1456,27 @@ protected: if (Request.RlPath) { auto actorId = ReportToRl(ru, Database, UserToken->GetSerializedToken(), Request.RlPath.GetRef()); - KQP_STLOG_D(KQPEX, "Resource usage for last stat interval", - (Consumption, consumption), - (RequestUnits, ru), - (RlPath, Request.RlPath.GetRef()), - (RlActor, actorId), - (ForceFlag, force), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Resource usage for last stat interval", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"consumption", consumption}, + {"requestUnits", ru}, + {"rlPath", Request.RlPath.GetRef()}, + {"rlActor", actorId}, + {"forceFlag", force}, + {"traceId", TraceId()}); } else { - KQP_STLOG_D(KQPEX, "Resource usage for last stat interval, rate limiter was not found", - (Consumption, consumption), - (RequestUnits, ru), - (ForceFlag, force), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Resource usage for last stat interval, rate limiter was not found", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"consumption", consumption}, + {"requestUnits", ru}, + {"forceFlag", force}, + {"traceId", TraceId()}); } } @@ -1476,18 +1572,26 @@ protected: void TerminateComputeActors(Ydb::StatusIds::StatusCode code, const NYql::TIssues& issues) { for (const auto& task : this->TasksGraph.GetTasks()) { if (task.ComputeActorId && !task.Meta.Completed) { - KQP_STLOG_I(KQPEX, "Aborting compute actor execution", - (Issues, issues.ToOneLineString()), - (ComputeActor, task.ComputeActorId), - (TaskId, task.Id), - (trace_id, TraceId())); + YDB_LOG_INFO_COMP(NKikimrServices::KQP_EXECUTER, "Aborting compute actor execution", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"issues", issues.ToOneLineString()}, + {"computeActor", task.ComputeActorId}, + {"taskId", task.Id}, + {"traceId", TraceId()}); auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(NYql::NDq::YdbStatusToDqStatus(code), issues); this->Send(task.ComputeActorId, ev.Release()); } else { - KQP_STLOG_I(KQPEX, "Task does not have the CA id yet or is already complete", - (TaskId, task.Id), - (trace_id, TraceId())); + YDB_LOG_INFO_COMP(NKikimrServices::KQP_EXECUTER, "Task does not have the CA id yet or is already complete", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"taskId", task.Id}, + {"traceId", TraceId()}); } } } @@ -1505,24 +1609,37 @@ protected: } void UnexpectedEvent(const TString& state, ui32 eventType) { if (eventType == TEvents::TEvPoison::EventType) { - KQP_STLOG_D(KQPEX, "TKqpExecuter, TEvPoison event", - (State, state), - (SelfId, this->SelfId()), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "TKqpExecuter, TEvPoison event", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"state", state}, + {"selfId", this->SelfId()}, + {"traceId", TraceId()}); InternalError(TStringBuilder() << "TKqpExecuter got poisoned, state: " << state); } else { - KQP_STLOG_E(KQPEX, "TKqpExecuter, unexpected event", - (EventType, eventType), - (State, state), - (SelfId, this->SelfId()), - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "TKqpExecuter, unexpected event", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"eventType", eventType}, + {"state", state}, + {"selfId", this->SelfId()}, + {"traceId", TraceId()}); InternalError(TStringBuilder() << "Unexpected event at TKqpExecuter, state: " << state << ", event: " << eventType); } } void InternalError(const NYql::TIssues& issues) { - KQP_STLOG_E(KQPEX, issues.ToOneLineString(), - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "Internal error during transaction execution", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"issues", issues.ToOneLineString()}, + {"traceId", TraceId()}); auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::UNEXPECTED, "Internal error while executing transaction."); for (const NYql::TIssue& i : issues) { issue.AddSubIssue(MakeIntrusive<NYql::TIssue>(i)); @@ -1535,18 +1652,27 @@ protected: } void ReplyUnavailable(const TString& message) { - KQP_STLOG_E(KQPEX, "UNAVAILABLE: " << message, - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "Replying unavailable to client", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"message", message}, + {"traceId", TraceId()}); auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE); issue.AddSubIssue(new NYql::TIssue(message)); ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, issue); } void RuntimeError(Ydb::StatusIds::StatusCode code, const NYql::TIssues& issues) { - KQP_STLOG_E(KQPEX, "Runtime error", - (Status, Ydb::StatusIds_StatusCode_Name(code)), - (Issues, issues.ToOneLineString()), - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "Runtime error", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"status", Ydb::StatusIds_StatusCode_Name(code)}, + {"issues", issues.ToOneLineString()}, + {"traceId", TraceId()}); ReplyErrorAndDie(code, issues); } @@ -1564,9 +1690,13 @@ protected: void TimeoutError(bool isTargetSender, NYql::TIssues issues) { if (AlreadyReplied) { - KQP_STLOG_E(KQPEX, "Timeout when we already replied - not good", - (Backtrace, TBackTrace().PrintToString()), - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "Timeout when we already replied - not good", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"backtrace", TBackTrace().PrintToString()}, + {"traceId", TraceId()}); return; } @@ -1579,10 +1709,14 @@ protected: AlreadyReplied = true; - KQP_STLOG_E(KQPEX, "Abort execution", - (Status, NYql::NDqProto::StatusIds_StatusCode_Name(status)), - (Issues, issues.ToOneLineString()), - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "Abort execution", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"status", NYql::NDqProto::StatusIds_StatusCode_Name(status)}, + {"issues", issues.ToOneLineString()}, + {"traceId", TraceId()}); if (ExecuterSpan) { ExecuterSpan.EndError(TStringBuilder() << NYql::NDqProto::StatusIds_StatusCode_Name(status)); } @@ -1596,8 +1730,13 @@ protected: this->Send(Target, abortEv.Release()); } - KQP_STLOG_E(KQPEX, "Sending timeout response", (Recipient, Target), - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "Sending timeout response", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"recipient", Target}, + {"traceId", TraceId()}); // Pass away immediately, since we already sent response - don't wait for stats. this->PassAway(); @@ -1607,9 +1746,13 @@ protected: google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* issues) { if (AlreadyReplied) { - KQP_STLOG_E(KQPEX, "Error when we already replied - not good", - (Backtrace, TBackTrace().PrintToString()), - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "Error when we already replied - not good", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"backtrace", TBackTrace().PrintToString()}, + {"traceId", TraceId()}); return; } @@ -1623,10 +1766,14 @@ protected: response.MutableIssues()->Swap(issues); } - KQP_STLOG_T(KQPEX, "ReplyErrorAndDie", - (Response, response.DebugString()), - (TargetActor, Target), - (trace_id, TraceId())); + YDB_LOG_TRACE_COMP(NKikimrServices::KQP_EXECUTER, "ReplyErrorAndDie", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"response", response.DebugString()}, + {"targetActor", Target}, + {"traceId", TraceId()}); if constexpr (ExecType == EExecType::Data) { if (status != Ydb::StatusIds::SUCCESS) { @@ -1698,8 +1845,13 @@ protected: if (Stats->CollectStatsByLongTasks) { const auto& txPlansWithStats = response.GetResult().GetStats().GetTxPlansWithStats(); if (!txPlansWithStats.empty()) { - KQP_STLOG_I(KQPEX, "Full stats: " << response.GetResult().GetStats(), - (trace_id, TraceId())); + YDB_LOG_INFO_COMP(NKikimrServices::KQP_EXECUTER, "Collected full query stats for long task", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"stats", response.GetResult().GetStats()}, + {"traceId", TraceId()}); } } } @@ -1755,10 +1907,14 @@ protected: } for (auto channelPair: ResultChannelProxies) { - KQP_STLOG_D(KQPEX, "Terminate result channel", - (ChannelId, channelPair.first), - (ProxyActor, channelPair.second->SelfId()), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Terminate result channel", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"channelId", channelPair.first}, + {"proxyActor", channelPair.second->SelfId()}, + {"traceId", TraceId()}); TAutoPtr<IEventHandle> ev = new IEventHandle( channelPair.second->SelfId(), SelfId(), new TEvents::TEvPoison @@ -1766,8 +1922,12 @@ protected: channelPair.second->Receive(ev); } - KQP_STLOG_D(KQPEX, "Terminate execution", - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Terminate execution", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); if (KqpShardsResolverId) { this->Send(KqpShardsResolverId, new TEvents::TEvPoison); } @@ -1787,8 +1947,12 @@ protected: } this->Send(this->SelfId(), new TEvents::TEvPoison); - KQP_STLOG_T(KQPEX, "Terminate, become ZombieState", - (trace_id, TraceId())); + YDB_LOG_TRACE_COMP(NKikimrServices::KQP_EXECUTER, "Terminate, become ZombieState", + {"marker", "KQPEX"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); this->Become(&TKqpExecuterBase::ZombieState); } diff --git a/ydb/core/kqp/executer_actor/kqp_literal_executer.cpp b/ydb/core/kqp/executer_actor/kqp_literal_executer.cpp index f43bd1c55ad..27c9dcd54ea 100644 --- a/ydb/core/kqp/executer_actor/kqp_literal_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_literal_executer.cpp @@ -101,29 +101,46 @@ public: CancelAt = StartTime + *Request.CancelAfter; } - KQP_STLOG_D(KQPLIT, "Begin literal execution", - (operation_timeout, Request.Timeout), - (cancel_after, Request.CancelAfter), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Begin literal execution", + {"marker", "KQPLIT"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"operationTimeout", Request.Timeout}, + {"cancelAfter", Request.CancelAfter}, + {"traceId", TraceId()}); } std::unique_ptr<TEvKqpExecuter::TEvTxResponse> ExecuteLiteral() { try { ExecuteLiteralImpl(); } catch (const TMemoryLimitExceededException&) { - KQP_STLOG_W(KQPLIT, "TKqpLiteralExecuter, memory limit exceeded.", - (trace_id, TraceId())); + YDB_LOG_WARN_COMP(NKikimrServices::KQP_EXECUTER, "TKqpLiteralExecuter, memory limit exceeded", + {"marker", "KQPLIT"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); CreateErrorResponse(Ydb::StatusIds::PRECONDITION_FAILED, YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, "Memory limit exceeded")); } catch (const NMiniKQL::TKqpEnsureFail& e) { - KQP_STLOG_E(KQPLIT, "TKqpLiteralExecuter, TKqpEnsure failed.", - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "TKqpLiteralExecuter, TKqpEnsure failed", + {"marker", "KQPLIT"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); CreateErrorResponse(Ydb::StatusIds::PRECONDITION_FAILED, YqlIssue({}, EYqlIssueCode(e.GetCode()), e.GetMessage())); } catch (...) { auto msg = CurrentExceptionMessage(); - KQP_STLOG_C(KQPLIT, "TKqpLiteralExecuter, unexpected exception caught: " << msg, - (trace_id, TraceId())); + YDB_LOG_CRIT_COMP(NKikimrServices::KQP_EXECUTER, "TKqpLiteralExecuter, unexpected exception", + {"marker", "KQPLIT"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"caught", msg}, + {"traceId", TraceId()}); CreateErrorResponse(Ydb::StatusIds::PRECONDITION_FAILED, YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, msg)); } @@ -136,9 +153,13 @@ public: Stats->StartTs = TInstant::Now(); } - KQP_STLOG_D(KQPLIT, "Begin literal execution", - (transactions_count, Request.Transactions.size()), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Begin literal execution", + {"marker", "KQPLIT"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"transactionsCount", Request.Transactions.size()}, + {"traceId", TraceId()}); TasksGraph.BuildLiteralTasks(); @@ -205,8 +226,10 @@ public: } auto log = [as = TlsActivationContext->ActorSystem(), txId = TxId, taskId = task.Id](const TString& message) { - LOG_DEBUG_S(*as, NKikimrServices::KQP_TASKS_RUNNER, "TxId: " << txId << ", task: " << taskId << ". " - << message); + YDB_LOG_DEBUG_CTX_COMP(*as, NKikimrServices::KQP_TASKS_RUNNER, "", + {"txId", txId}, + {"task", taskId}, + {"message", message}); }; auto taskRunner = MakeDqTaskRunner(alloc, context, settings, log); @@ -280,9 +303,13 @@ public: LWTRACK(KqpLiteralExecuterFinalize, ResponseEv->Orbit, TxId); LiteralExecuterSpan.EndOk(); CleanupCtx(); - KQP_STLOG_D(KQPLIT, "Execution is complete", - (results_size, ResponseEv->ResultsSize()), - (trace_id, TraceId())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_EXECUTER, "Execution is complete", + {"marker", "KQPLIT"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"resultsSize", ResponseEv->ResultsSize()}, + {"traceId", TraceId()}); } private: @@ -306,8 +333,12 @@ private: auto now = AppData()->TimeProvider->Now(); if (Deadline && *Deadline <= now) { - KQP_STLOG_I(KQPLIT, "Timeout exceeded.", - (trace_id, TraceId())); + YDB_LOG_INFO_COMP(NKikimrServices::KQP_EXECUTER, "Timeout exceeded", + {"marker", "KQPLIT"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); CreateErrorResponse(Ydb::StatusIds::TIMEOUT, YqlIssue({}, TIssuesIds::KIKIMR_TIMEOUT, "Request timeout exceeded.")); @@ -315,8 +346,12 @@ private: } if (CancelAt && *CancelAt <= now) { - KQP_STLOG_I(KQPLIT, "CancelAt exceeded.", - (trace_id, TraceId())); + YDB_LOG_INFO_COMP(NKikimrServices::KQP_EXECUTER, "CancelAt exceeded", + {"marker", "KQPLIT"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"traceId", TraceId()}); CreateErrorResponse(Ydb::StatusIds::CANCELLED, YqlIssue({}, TIssuesIds::KIKIMR_OPERATION_CANCELLED, "Request timeout exceeded.")); @@ -332,8 +367,13 @@ private: } void InternalError(const TString& message) { - KQP_STLOG_E(KQPLIT, message, - (trace_id, TraceId())); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_EXECUTER, "", + {"marker", "KQPLIT"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"message", message}, + {"traceId", TraceId()}); auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::UNEXPECTED, "Internal error while executing transaction."); issue.AddSubIssue(MakeIntrusive<TIssue>(message)); CreateErrorResponse(Ydb::StatusIds::INTERNAL_ERROR, issue); diff --git a/ydb/core/kqp/executer_actor/kqp_partitioned_executer.cpp b/ydb/core/kqp/executer_actor/kqp_partitioned_executer.cpp index 9b4023c0a92..14a697d80fc 100644 --- a/ydb/core/kqp/executer_actor/kqp_partitioned_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_partitioned_executer.cpp @@ -18,19 +18,13 @@ #include <ydb/library/actors/core/actorid.h> #include <ydb/library/actors/core/actor_bootstrapped.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + namespace NKikimr { namespace NKqp { namespace { -#define PE_STLOG_T(MESSAGE, ...) STLOG(PRI_TRACE, NKikimrServices::KQP_EXECUTER, KQPPEA, LogPrefix() << MESSAGE << '.', ##__VA_ARGS__) -#define PE_STLOG_D(MESSAGE, ...) STLOG(PRI_DEBUG, NKikimrServices::KQP_EXECUTER, KQPPEA, LogPrefix() << MESSAGE << '.', ##__VA_ARGS__) -#define PE_STLOG_I(MESSAGE, ...) STLOG(PRI_INFO, NKikimrServices::KQP_EXECUTER, KQPPEA, LogPrefix() << MESSAGE << '.', ##__VA_ARGS__) -#define PE_STLOG_N(MESSAGE, ...) STLOG(PRI_NOTICE, NKikimrServices::KQP_EXECUTER, KQPPEA, LogPrefix() << MESSAGE << '.', ##__VA_ARGS__) -#define PE_STLOG_W(MESSAGE, ...) STLOG(PRI_WARN, NKikimrServices::KQP_EXECUTER, KQPPEA, LogPrefix() << MESSAGE << '.', ##__VA_ARGS__) -#define PE_STLOG_E(MESSAGE, ...) STLOG(PRI_ERROR, NKikimrServices::KQP_EXECUTER, KQPPEA, LogPrefix() << MESSAGE << '.', ##__VA_ARGS__) -#define PE_STLOG_C(MESSAGE, ...) STLOG(PRI_CRIT, NKikimrServices::KQP_EXECUTER, KQPPEA, LogPrefix() << MESSAGE << '.', ##__VA_ARGS__) - void FillRequestFrom(IKqpGateway::TExecPhysicalRequest& request, const IKqpGateway::TExecPhysicalRequest& from) { request.AllowTrailingResults = from.AllowTrailingResults; request.QueryType = from.QueryType; @@ -135,7 +129,9 @@ public: void Bootstrap() { Become(&TKqpPartitionedExecuter::PrepareState); - PE_STLOG_I("Start resolving table partitions"); + YDB_LOG_INFO("Start resolving table partitions", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}); Stats.StartTs = TInstant::Now(); FillTableMetaInfo(); @@ -187,8 +183,10 @@ public: TablePartitioning = result->Partitioning; - PE_STLOG_T("Partitions were resolved", - (PartitionsCount, result->Partitioning->Size())); + YDB_LOG_TRACE("Partitions were resolved", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionsCount", result->Partitioning->Size()}); CreateExecutersWithBuffers(); } @@ -197,11 +195,13 @@ public: const auto& msg = ev->Get()->Record; auto issues = ev->Get()->GetIssues(); - PE_STLOG_E("Got abort execution", - (Sender, ev->Sender), - (FromSessionActor, ev->Sender == SessionActorId), - (StatusCode, NYql::NDqProto::StatusIds_StatusCode_Name(msg.GetStatusCode())), - (Issues, issues.ToOneLineString())); + YDB_LOG_ERROR("Got abort execution", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"fromSessionActor", ev->Sender == SessionActorId}, + {"statusCode", NYql::NDqProto::StatusIds_StatusCode_Name(msg.GetStatusCode())}, + {"issues", issues.ToOneLineString()}); AbortWithError(NYql::NDq::DqStatusToYdbStatus(msg.GetStatusCode()), issues); } @@ -233,20 +233,24 @@ public: auto it = ExecuterToPartition.find(ev->Sender); if (it == ExecuterToPartition.end()) { - PE_STLOG_W("Got tx response from an unknown executer", - (Sender, ev->Sender), - (Status, response->GetStatus()), - (Issues, issues.ToOneLineString())); + YDB_LOG_WARN("Got tx response from an unknown executer", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", response->GetStatus()}, + {"issues", issues.ToOneLineString()}); return TryFinishExecution(); } auto [_, partInfo] = *it; - PE_STLOG_T("Got tx response", - (Sender, ev->Sender), - (PartitionIndex, partInfo->PartitionIndex), - (Status, response->GetStatus())); + YDB_LOG_TRACE("Got tx response", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"partitionIndex", partInfo->PartitionIndex}, + {"status", response->GetStatus()}); AbortBuffer(partInfo->BufferId); ForgetExecuterAndBuffer(partInfo); @@ -266,20 +270,24 @@ public: case Ydb::StatusIds::UNAVAILABLE: case Ydb::StatusIds::OVERLOADED: case Ydb::StatusIds::UNDETERMINED: - PE_STLOG_D("Executer retriable error, will retry", - (PartitionIndex, partInfo->PartitionIndex), - (Status, response->GetStatus()), - (Issues, issues.ToOneLineString())); + YDB_LOG_DEBUG("Executer retriable error, will retry", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", partInfo->PartitionIndex}, + {"status", response->GetStatus()}, + {"issues", issues.ToOneLineString()}); return ScheduleRetryWithNewLimit(partInfo); default: break; } - PE_STLOG_E("Executer unretriable error", - (PartitionIndex, partInfo->PartitionIndex), - (Status, response->GetStatus()), - (Issues, issues.ToOneLineString())); + YDB_LOG_ERROR("Executer unretriable error", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", partInfo->PartitionIndex}, + {"status", response->GetStatus()}, + {"issues", issues.ToOneLineString()}); ForgetPartition(partInfo); AbortWithError(response->GetStatus(), issues); @@ -288,8 +296,10 @@ public: void HandleExecute(TEvKqpExecuter::TEvTxDelayedExecution::TPtr& ev) { RequestCounters->Counters->BatchOperationRetries->Inc(); - PE_STLOG_D("Delayed execution timer fired", - (PartitionIndex, ev->Get()->PartitionIdx)); + YDB_LOG_DEBUG("Delayed execution timer fired", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", ev->Get()->PartitionIdx}); auto it = StartedPartitions.find(ev->Get()->PartitionIdx); if (it != StartedPartitions.end()) { @@ -302,20 +312,24 @@ public: auto it = BufferToPartition.find(ev->Sender); if (it == BufferToPartition.end()) { - PE_STLOG_W("Got error from an unknown buffer", - (Sender, ev->Sender), - (Status, NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode)), - (Issues, msg.Issues.ToOneLineString())); + YDB_LOG_WARN("Got error from an unknown buffer", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode)}, + {"issues", msg.Issues.ToOneLineString()}); return TryFinishExecution(); } auto [_, partInfo] = *it; - PE_STLOG_T("Got buffer error", - (Sender, ev->Sender), - (PartitionIndex, partInfo->PartitionIndex), - (Status, NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode))); + YDB_LOG_TRACE("Got buffer error", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"partitionIndex", partInfo->PartitionIndex}, + {"status", NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode)}); AbortExecuter(partInfo->ExecuterId, "got error from KqpBufferWriteActor"); ForgetExecuterAndBuffer(partInfo); @@ -330,20 +344,24 @@ public: case NYql::NDqProto::StatusIds::UNAVAILABLE: case NYql::NDqProto::StatusIds::OVERLOADED: case NYql::NDqProto::StatusIds::UNDETERMINED: - PE_STLOG_D("Buffer retriable error, will retry", - (PartitionIndex, partInfo->PartitionIndex), - (Status, NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode)), - (Issues, msg.Issues.ToOneLineString())); + YDB_LOG_DEBUG("Buffer retriable error, will retry", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", partInfo->PartitionIndex}, + {"status", NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode)}, + {"issues", msg.Issues.ToOneLineString()}); return ScheduleRetryWithNewLimit(partInfo); default: break; } - PE_STLOG_E("Buffer unretriable error", - (PartitionIndex, partInfo->PartitionIndex), - (Status, NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode)), - (Issues, msg.Issues.ToOneLineString())); + YDB_LOG_ERROR("Buffer unretriable error", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", partInfo->PartitionIndex}, + {"status", NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode)}, + {"issues", msg.Issues.ToOneLineString()}); ForgetPartition(partInfo); AbortWithError(NYql::NDq::DqStatusToYdbStatus(msg.StatusCode), msg.Issues); @@ -357,9 +375,11 @@ public: hFunc(TEvKqp::TEvAbortExecution, HandleAbort); hFunc(TEvKqpBuffer::TEvError, HandleAbort); default: - PE_STLOG_W("Got an unknown event", - (Sender, ev->Sender), - (EventType, ev->GetTypeRewrite())); + YDB_LOG_WARN("Got an unknown event", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"eventType", ev->GetTypeRewrite()}); return TryFinishExecution(); } @@ -377,21 +397,25 @@ public: auto it = ExecuterToPartition.find(ev->Sender); if (it == ExecuterToPartition.end()) { - PE_STLOG_W("Got tx response from an unknown executer", - (Sender, ev->Sender), - (Status, response->GetStatus()), - (Issues, issues.ToOneLineString())); + YDB_LOG_WARN("Got tx response from an unknown executer", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", response->GetStatus()}, + {"issues", issues.ToOneLineString()}); return TryFinishExecution(); } auto [_, partInfo] = *it; - PE_STLOG_T("Got tx response", - (Sender, ev->Sender), - (PartitionIndex, partInfo->PartitionIndex), - (Status, response->GetStatus()), - (Issues, issues.ToOneLineString())); + YDB_LOG_TRACE("Got tx response", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"partitionIndex", partInfo->PartitionIndex}, + {"status", response->GetStatus()}, + {"issues", issues.ToOneLineString()}); AbortBuffer(partInfo->BufferId); ForgetExecuterAndBuffer(partInfo); @@ -405,21 +429,25 @@ public: auto it = BufferToPartition.find(ev->Sender); if (it == BufferToPartition.end()) { - PE_STLOG_W("Got error from an unknown buffer", - (Sender, ev->Sender), - (Status, NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode)), - (Issues, msg.Issues.ToOneLineString())); + YDB_LOG_WARN("Got error from an unknown buffer", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode)}, + {"issues", msg.Issues.ToOneLineString()}); return TryFinishExecution(); } auto [_, partInfo] = *it; - PE_STLOG_E("Got buffer error", - (Sender, ev->Sender), - (PartitionIndex, partInfo->PartitionIndex), - (Status, NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode)), - (Issues, msg.Issues.ToOneLineString())); + YDB_LOG_ERROR("Got buffer error", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"partitionIndex", partInfo->PartitionIndex}, + {"status", NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode)}, + {"issues", msg.Issues.ToOneLineString()}); AbortExecuter(partInfo->ExecuterId, "got error from KqpBufferWriteActor"); ForgetExecuterAndBuffer(partInfo); @@ -509,9 +537,11 @@ private: KeyColumnTypes.emplace_back(typeInfoMod.TypeInfo); } - PE_STLOG_D("Filling table meta info", - (TableId, TableId), - (KeyColumnsCount, KeyColumnTypes.size())); + YDB_LOG_DEBUG("Filling table meta info", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"tableId", TableId}, + {"keyColumnsCount", KeyColumnTypes.size()}); YQL_ENSURE(!KeyIds.empty()); } @@ -522,9 +552,11 @@ private: YQL_ENSURE(range.IsFullRange(KeyIds.size())); - PE_STLOG_D("Resolving table partitioning", - (TableId, TableId), - (KeyColumnsCount, KeyIds.size())); + YDB_LOG_DEBUG("Resolving table partitioning", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"tableId", TableId}, + {"keyColumnsCount", KeyIds.size()}); auto keyRange = MakeHolder<TKeyDesc>(TableId, range, OperationType, KeyColumnTypes, TVector<TKeyDesc::TColumnOp>{}); @@ -543,9 +575,11 @@ private: YQL_ENSURE(TablePartitioning && !TablePartitioning->Empty(), "No partitions to execute"); auto partCount = std::min(Settings.PartitionExecutionLimit, TablePartitioning->Size()); - PE_STLOG_I("Starting execution, creating executers with buffers", - (PartitionsCount, TablePartitioning->Size()), - (InFlightPartitionsCount, partCount)); + YDB_LOG_INFO("Starting execution, creating executers with buffers", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionsCount", TablePartitioning->Size()}, + {"inFlightPartitionsCount", partCount}); while (NextPartitionIndex < partCount) { CreateExecuterWithBuffer(NextPartitionIndex++, /* isRetry */ false); @@ -569,12 +603,14 @@ private: partition->LimitSize = Settings.MaxBatchSize; partition->RetryDelayMs = Settings.StartRetryDelayMs; - PE_STLOG_D("Created partition", - (PartitionIndex, idx), - (HasBeginRange, partition->BeginRange.Defined()), - (HasEndRange, partition->EndRange.Defined()), - (InitialLimitSize, partition->LimitSize), - (InitialRetryDelayMs, partition->RetryDelayMs)); + YDB_LOG_DEBUG("Created partition", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", idx}, + {"hasBeginRange", partition->BeginRange.Defined()}, + {"hasEndRange", partition->EndRange.Defined()}, + {"initialLimitSize", partition->LimitSize}, + {"initialRetryDelayMs", partition->RetryDelayMs}); return partition; } @@ -604,9 +640,11 @@ private: alloc->SetLimit(WriteBufferInitialMemoryLimit); alloc->Ref().SetIncreaseMemoryLimitCallback([this, alloc=alloc.get()](ui64 currentLimit, ui64 required) { if (required < WriteBufferMemoryLimit) { - PE_STLOG_D("Increase memory limit", - (CurrentLimit, currentLimit), - (Required, required)); + YDB_LOG_DEBUG("Increase memory limit", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"currentLimit", currentLimit}, + {"required", required}); alloc->SetLimit(required); } }); @@ -646,21 +684,25 @@ private: partInfo->BufferId = bufferActorId; ExecuterToPartition[exId] = BufferToPartition[bufferActorId] = partInfo; - PE_STLOG_D("Created executer", - (PartitionIndex, partitionIndex), - (ExecuterId, partInfo->ExecuterId), - (BufferId, bufferActorId), - (LimitSize, partInfo->LimitSize), - (IsRetry, isRetry)); + YDB_LOG_DEBUG("Created executer", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", partitionIndex}, + {"executerId", partInfo->ExecuterId}, + {"bufferId", bufferActorId}, + {"limitSize", partInfo->LimitSize}, + {"isRetry", isRetry}); auto ev = std::make_unique<TEvTxUserProxy::TEvProposeKqpTransaction>(exId); Send(MakeTxProxyID(), ev.release()); } void Abort() { - PE_STLOG_I("Entering AbortState, trying to finish execution", - (ActivePartitionsCount, StartedPartitions.size()), - (ReturnStatus, Ydb::StatusIds_StatusCode_Name(ReturnStatus))); + YDB_LOG_INFO("Entering AbortState, trying to finish execution", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"activePartitionsCount", StartedPartitions.size()}, + {"returnStatus", Ydb::StatusIds_StatusCode_Name(ReturnStatus)}); Become(&TKqpPartitionedExecuter::AbortState); @@ -704,16 +746,20 @@ private: << "PartitionIndex = " << partInfo->PartitionIndex)})); } - PE_STLOG_D("Partition has more data, continue processing", - (PartitionIndex, partInfo->PartitionIndex), - (NextKeyCellsCount, minKey.GetCells().size())); + YDB_LOG_DEBUG("Partition has more data, continue processing", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", partInfo->PartitionIndex}, + {"nextKeyCellsCount", minKey.GetCells().size()}); partInfo->BeginRange = TKeyDesc::TPartitionRangeInfo(minKey, /* IsInclusive */ false, /* IsPoint */ false); return RetryPartExecution(partInfo); } - PE_STLOG_D("Partition finished completely", - (PartitionIndex, partInfo->PartitionIndex)); + YDB_LOG_DEBUG("Partition finished completely", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", partInfo->PartitionIndex}); ForgetPartition(partInfo); @@ -735,16 +781,20 @@ private: void RetryPartExecution(const TBatchPartitionInfo::TPtr& partInfo) { if (CurrentStateFunc() != &TKqpPartitionedExecuter::AbortState) { - PE_STLOG_D("Retrying partition", - (PartitionIndex, partInfo->PartitionIndex), - (LimitSize, partInfo->LimitSize), - (RetryDelayMs, partInfo->RetryDelayMs)); + YDB_LOG_DEBUG("Retrying partition", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", partInfo->PartitionIndex}, + {"limitSize", partInfo->LimitSize}, + {"retryDelayMs", partInfo->RetryDelayMs}); return CreateExecuterWithBuffer(partInfo->PartitionIndex, /* isRetry */ true); } - PE_STLOG_D("Partition retry cancelled due to AbortState", - (PartitionIndex, partInfo->PartitionIndex)); + YDB_LOG_DEBUG("Partition retry cancelled due to AbortState", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", partInfo->PartitionIndex}); ForgetPartition(partInfo); TryFinishExecution(); @@ -752,9 +802,11 @@ private: void ScheduleRetryWithNewLimit(TBatchPartitionInfo::TPtr& partInfo) { if (partInfo->RetryDelayMs == Settings.MaxRetryDelayMs) { - PE_STLOG_E("Partition reached maximum retry delay", - (PartitionIndex, partInfo->PartitionIndex), - (MaxRetryDelayMs, Settings.MaxRetryDelayMs)); + YDB_LOG_ERROR("Partition reached maximum retry delay", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", partInfo->PartitionIndex}, + {"maxRetryDelayMs", Settings.MaxRetryDelayMs}); ForgetPartition(partInfo); return AbortWithError(Ydb::StatusIds::UNAVAILABLE, NYql::TIssues({NYql::TIssue(TStringBuilder() @@ -769,12 +821,14 @@ private: partInfo->RetryDelayMs = newDelay; partInfo->LimitSize = std::max(partInfo->LimitSize / 2, Settings.MinBatchSize); - PE_STLOG_D("Scheduling retry for partition", - (PartitionIndex, partInfo->PartitionIndex), - (OldDelay, oldDelay), - (NewDelay, partInfo->RetryDelayMs), - (OldLimit, oldLimit), - (NewLimit, partInfo->LimitSize)); + YDB_LOG_DEBUG("Scheduling retry for partition", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"partitionIndex", partInfo->PartitionIndex}, + {"oldDelay", oldDelay}, + {"newDelay", partInfo->RetryDelayMs}, + {"oldLimit", oldLimit}, + {"newLimit", partInfo->LimitSize}); auto ev = std::make_unique<TEvKqpExecuter::TEvTxDelayedExecution>(partInfo->PartitionIndex); Schedule(TDuration::MilliSeconds(partInfo->RetryDelayMs), ev.release()); @@ -796,8 +850,10 @@ private: YQL_ENSURE(it != KeyColumnIdToPos.end()); if (it->second != i) { - PE_STLOG_D("Key columns need reorder to continue processing", - (KeyColumnsCount, KeyIds.size())); + YDB_LOG_DEBUG("Key columns need reorder to continue processing", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"keyColumnsCount", KeyIds.size()}); return true; } @@ -853,9 +909,11 @@ private: void TryFinishExecution() { if (CheckExecutersAreFinished()) { - PE_STLOG_I("All partitions processed, finish execution", - (Status, Ydb::StatusIds_StatusCode_Name(ReturnStatus)), - (Issues, ReturnIssues.ToOneLineString())); + YDB_LOG_INFO("All partitions processed, finish execution", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"status", Ydb::StatusIds_StatusCode_Name(ReturnStatus)}, + {"issues", ReturnIssues.ToOneLineString()}); Stats.FinishTs = TInstant::Now(); Stats.ExportExecStats(*ResponseEv->Record.MutableResponse()->MutableResult()->MutableStats()); @@ -867,23 +925,29 @@ private: return ReplySuccessAndDie(); } - PE_STLOG_D("Not all partitions have been processed, cannot finish execution", - (RemainingPartitionsCount, StartedPartitions.size()), - (TotalPartitions, TablePartitioning ? TablePartitioning->Size() : 0)); + YDB_LOG_DEBUG("Not all partitions have been processed, cannot finish execution", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"remainingPartitionsCount", StartedPartitions.size()}, + {"totalPartitions", TablePartitioning ? TablePartitioning->Size() : 0}); } void AbortWithError(Ydb::StatusIds::StatusCode code, const NYql::TIssues& issues) { if (CurrentStateFunc() == &TKqpPartitionedExecuter::AbortState) { - PE_STLOG_N("Ignoring error because already in AbortState", - (Status, Ydb::StatusIds_StatusCode_Name(code)), - (Issues, issues.ToOneLineString())); + YDB_LOG_NOTICE("Ignoring error because already in AbortState", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"status", Ydb::StatusIds_StatusCode_Name(code)}, + {"issues", issues.ToOneLineString()}); return TryFinishExecution(); } - PE_STLOG_E("First error occurred", - (Status, Ydb::StatusIds_StatusCode_Name(code)), - (Issues, issues.ToOneLineString())); + YDB_LOG_ERROR("First error occurred", + {"marker", "KQPPEA"}, + {"logPrefix", LogPrefix()}, + {"status", Ydb::StatusIds_StatusCode_Name(code)}, + {"issues", issues.ToOneLineString()}); ReturnStatus = code; ReturnIssues.AddIssues(issues); diff --git a/ydb/core/kqp/executer_actor/kqp_planner.cpp b/ydb/core/kqp/executer_actor/kqp_planner.cpp index d8f75423b86..1e8f3cf4e25 100644 --- a/ydb/core/kqp/executer_actor/kqp_planner.cpp +++ b/ydb/core/kqp/executer_actor/kqp_planner.cpp @@ -12,17 +12,12 @@ #include <ydb/core/kqp/common/control.h> #include <ydb/core/fq/libs/checkpointing/events/events.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + using namespace NActors; namespace NKikimr::NKqp { -#define LOG_T(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << "Ctx: " << *UserRequestContext << ". " << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << "Ctx: " << *UserRequestContext << ". " << stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << "Ctx: " << *UserRequestContext << ". " << stream) -#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << "Ctx: " << *UserRequestContext << ". " << stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << "Ctx: " << *UserRequestContext << ". " << stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << "Ctx: " << *UserRequestContext << ". " << stream) - static std::atomic<ui64> MaxTaskSize = 48_MB; void SetMaxTaskSize(ui64 size) { @@ -41,7 +36,12 @@ template <class TCollection> std::unique_ptr<TEvKqp::TEvAbortExecution> CheckTaskSize(ui64 TxId, const TIntrusivePtr<TUserRequestContext>& UserRequestContext, const TCollection& tasks) { for (const auto& task : tasks) { if (ui32 size = task.ByteSize(); size > GetMaxTaskSize()) { - LOG_E("Abort execution. Task #" << task.GetId() << " size is too big: " << size << " > " << GetMaxTaskSize()); + YDB_LOG_ERROR("Abort execution: task size exceeds limit", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"taskId", task.GetId()}, + {"taskSize", size}, + {"maxTaskSize", GetMaxTaskSize()}); return std::make_unique<TEvKqp::TEvAbortExecution>(NYql::NDqProto::StatusIds::LIMIT_EXCEEDED, TStringBuilder() << "Datashard program size limit exceeded (" << size << " > " << GetMaxTaskSize() << ")"); } @@ -176,7 +176,10 @@ bool TKqpPlanner::SendStartKqpTasksRequest(ui32 requestId, const TActorId& targe if (isShutdown) { requestData.RetryNumber = ExecuterRetriesConfig.GetMaxRetryNumber(); YQL_ENSURE(requestData.NodeId != target.NodeId()); - LOG_D("Try to retry after NODE_SHUTTING_DOWN, run tasks locally, requestId: " << requestId); + YDB_LOG_DEBUG("Try to retry after NODE_SHUTTING_DOWN, run tasks locally", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"requestId", requestId}); requestData.NodeId = target.NodeId(); TlsActivationContext->Send(std::make_unique<NActors::IEventHandle>(target, ExecuterId, ev.release(), CalcSendMessageFlagsForNode(target.NodeId()), requestId, nullptr, ExecuterSpan.GetTraceId())); @@ -184,7 +187,10 @@ bool TKqpPlanner::SendStartKqpTasksRequest(ui32 requestId, const TActorId& targe } if (requestData.RetryNumber == ExecuterRetriesConfig.GetMaxRetryNumber()) { - LOG_E("Retry failed by retries limit, requestId: " << requestId); + YDB_LOG_ERROR("Retry failed by retries limit", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"requestId", requestId}); TMaybe<ui32> targetNode; for (size_t i = 0; i < ResourcesSnapshot.size(); ++i) { if (!TrackingNodes.contains(ResourcesSnapshot[i].GetNodeId())) { @@ -193,7 +199,11 @@ bool TKqpPlanner::SendStartKqpTasksRequest(ui32 requestId, const TActorId& targe } } if (targetNode) { - LOG_D("Try to retry to another node, nodeId: " << *targetNode << ", requestId: " << requestId); + YDB_LOG_DEBUG("Try to retry to another node", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"nodeId", *targetNode}, + {"requestId", requestId}); auto anotherTarget = MakeKqpNodeServiceID(*targetNode); requestData.NodeId = *targetNode; TlsActivationContext->Send(std::make_unique<NActors::IEventHandle>(anotherTarget, ExecuterId, ev.release(), @@ -201,12 +211,19 @@ bool TKqpPlanner::SendStartKqpTasksRequest(ui32 requestId, const TActorId& targe requestData.RetryNumber++; return true; } - LOG_E("Retry failed because all nodes are busy, requestId: " << requestId); + YDB_LOG_ERROR("Retry failed because all nodes are busy", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"requestId", requestId}); return false; } if (requestData.RetryNumber >= 1) { - LOG_D("Try to retry by ActorUnknown reason, nodeId: " << target.NodeId() << ", requestId: " << requestId); + YDB_LOG_DEBUG("Try to retry by ActorUnknown reason", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"nodeId", target.NodeId()}, + {"requestId", requestId}); } requestData.RetryNumber++; @@ -389,7 +406,9 @@ std::unique_ptr<IEventHandle> TKqpPlanner::AssignTasksToNodes() { return nullptr; } - LOG_E("Not enough resources to execute query locally and no information about other nodes"); + YDB_LOG_ERROR("Not enough resources to execute query locally and no information about other nodes", + {"txId", TxId}, + {"ctx", *UserRequestContext}); auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(NYql::NDqProto::StatusIds::PRECONDITION_FAILED, "Not enough resources to execute query locally and no information about other nodes (estimation: " + ToString(LocalRunMemoryEst) + ";" + GetEstimationsInfo() + ")"); @@ -436,10 +455,16 @@ std::unique_ptr<IEventHandle> TKqpPlanner::AssignTasksToNodes() { auto ctx = TlsActivationContext->AsActorContext(); if (ctx.LoggerSettings() && ctx.LoggerSettings()->Satisfies(NActors::NLog::PRI_DEBUG, NKikimrServices::KQP_EXECUTER)) { - planner->SetLogFunc([TxId = TxId, &UserRequestContext = UserRequestContext](TStringBuf msg) { LOG_D(msg); }); + planner->SetLogFunc([TxId = TxId, &UserRequestContext = UserRequestContext](TStringBuf msg) { YDB_LOG_DEBUG("Planner debug message", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"msg", msg}); }); } - LogMemoryStatistics([TxId = TxId, &UserRequestContext = UserRequestContext](TStringBuf msg) { LOG_D(msg); }); + LogMemoryStatistics([TxId = TxId, &UserRequestContext = UserRequestContext](TStringBuf msg) { YDB_LOG_DEBUG("Memory statistics before task planning", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"msg", msg}); }); ui64 selfNodeId = ExecuterId.NodeId(); TString selfNodeDC; @@ -474,7 +499,10 @@ std::unique_ptr<IEventHandle> TKqpPlanner::AssignTasksToNodes() { } if (plan.empty()) { - LogMemoryStatistics([TxId = TxId, &UserRequestContext = UserRequestContext](TStringBuf msg) { LOG_E(msg); }); + LogMemoryStatistics([TxId = TxId, &UserRequestContext = UserRequestContext](TStringBuf msg) { YDB_LOG_ERROR("Memory statistics after failed task planning", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"msg", msg}); }); auto ev = MakeHolder<TEvKqp::TEvAbortExecution>(NYql::NDqProto::StatusIds::PRECONDITION_FAILED, TStringBuilder() << "Not enough resources to execute query. " << "TraceId: " << UserRequestContext->TraceId); @@ -595,11 +623,17 @@ std::unique_ptr<IEventHandle> TKqpPlanner::PlanExecution() { const auto scanTasksCount = TasksGraph.GetTasks().size() - UnassignedTasks.size(); - LOG_D("Total tasks: " << TasksGraph.GetTasks().size() << ", readonly: true" // TODO ??? - << ", " << scanTasksCount << " scan tasks on " << TasksPerNode.size() << " nodes" - << ", localComputeTasks: " << TasksGraph.GetMeta().LocalComputeTasks - << ", MayRunTasksLocally " << TasksGraph.GetMeta().MayRunTasksLocally - << ", snapshot: {" << GetSnapshot().TxId << ", " << GetSnapshot().Step << "}"); + YDB_LOG_DEBUG("Task placement snapshot after planning", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"tasks", TasksGraph.GetTasks().size()}, + {"readOnly", true}, + {"scanTasksCount", scanTasksCount}, + {"tasksPerNodeCount", TasksPerNode.size()}, + {"localComputeTasks", TasksGraph.GetMeta().LocalComputeTasks}, + {"mayRunTasksLocally", TasksGraph.GetMeta().MayRunTasksLocally}, + {"snapshotTxId", GetSnapshot().TxId}, + {"snapshotStep", GetSnapshot().Step}); PrepareCheckpoints(); @@ -679,7 +713,10 @@ void TKqpPlanner::PrepareCheckpoints() { break; } } - LOG_D("PrepareCheckpoints: has streaming ingress: " << hasStreamingIngress); + YDB_LOG_DEBUG("PrepareCheckpoints: checked streaming ingress", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"hasStreamingIngress", hasStreamingIngress}); if (!hasStreamingIngress) { CheckpointCoordinatorId = TActorId{}; return; @@ -748,7 +785,10 @@ bool TKqpPlanner::CompletedCA(ui64 taskId, TActorId computeActor) { LastStats.emplace_back(std::move(it->second)); PendingComputeActors.erase(it); - LOG_I("Compute actor has finished execution: " << computeActor.ToString()); + YDB_LOG_INFO("Compute actor has finished", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"execution", computeActor}); return true; } @@ -856,7 +896,11 @@ void TKqpPlanner::PropagateChannelsUpdates(const THashMap<TActorId, THashSet<ui6 TasksGraph.FillChannelDesc(*record.AddUpdate(), TasksGraph.GetChannel(channelId), TasksGraph.GetMeta().ChannelTransportVersion, false); } - LOG_T("Sending channels info to compute actor: " << computeActorId << ", channels: " << channelIds.size()); + YDB_LOG_DEBUG("Sending channels info to compute", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"actor", computeActorId}, + {"channels", channelIds.size()}); TlsActivationContext->Send(std::make_unique<NActors::IEventHandle>(computeActorId, ExecuterId, channelsInfoEv.Release())); } } @@ -864,7 +908,11 @@ void TKqpPlanner::PropagateChannelsUpdates(const THashMap<TActorId, THashSet<ui6 void TKqpPlanner::CollectTaskChannelsUpdates(const TKqpTasksGraph::TTaskType& task, THashMap<TActorId, THashSet<ui64>>& updates) { YQL_ENSURE(task.ComputeActorId); - LOG_T("Collect channels updates for task: " << task.Id << " at actor " << task.ComputeActorId); + YDB_LOG_DEBUG("Collecting channel updates for compute actor", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"taskId", task.Id}, + {"computeActorId", task.ComputeActorId}); auto& selfUpdates = updates[task.ComputeActorId]; @@ -880,8 +928,13 @@ void TKqpPlanner::CollectTaskChannelsUpdates(const TKqpTasksGraph::TTaskType& ta selfUpdates.emplace(channelId); } - LOG_T("Task: " << task.Id << ", input channelId: " << channelId << ", src task: " << channel.SrcTask - << ", at actor " << srcTask.ComputeActorId); + YDB_LOG_DEBUG("Collecting input channel update for compute actor", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"taskId", task.Id}, + {"channelId", channelId}, + {"srcTaskId", channel.SrcTask}, + {"srcComputeActorId", srcTask.ComputeActorId}); } } @@ -899,8 +952,13 @@ void TKqpPlanner::CollectTaskChannelsUpdates(const TKqpTasksGraph::TTaskType& ta updates[dstTask.ComputeActorId].emplace(channelId); } - LOG_T("Task: " << task.Id << ", output channelId: " << channelId << ", dst task: " << channel.DstTask - << ", at actor " << dstTask.ComputeActorId); + YDB_LOG_DEBUG("Collecting output channel update for compute actor", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"taskId", task.Id}, + {"channelId", channelId}, + {"dstTaskId", channel.DstTask}, + {"dstComputeActorId", dstTask.ComputeActorId}); } } } @@ -914,8 +972,10 @@ void TKqpPlanner::SendReadyStateToCheckpointCoordinator() { auto event = std::make_unique<NFq::TEvCheckpointCoordinator::TEvReadyState>(); for (const auto& dqTask : TasksGraph.GetTasks()) { if (!dqTask.ComputeActorId) { - LOG_W("Skip sending TEvReadyState to checkpoint coordinator" - << ": task " << dqTask.Id << " has no ComputeActorId (node disconnected / task not started)"); + YDB_LOG_WARN("Skip sending TEvReadyState to checkpoint coordinator: task has no compute actor id (node disconnected or task not started)", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"taskId", dqTask.Id}); return; } auto* taskDesc = TasksGraph.ArenaSerializeTaskToProto(dqTask, true); @@ -932,7 +992,10 @@ void TKqpPlanner::SendReadyStateToCheckpointCoordinator() { }; event->Tasks.emplace_back(std::move(task)); } - LOG_I("Sending TEvReadyState to checkpoint coordinator (" << CheckpointCoordinatorId << ")"); + YDB_LOG_INFO("Sending TEvReadyState to checkpoint coordinator", + {"txId", TxId}, + {"ctx", *UserRequestContext}, + {"checkpointCoordinatorId", CheckpointCoordinatorId}); TlsActivationContext->Send(std::make_unique<NActors::IEventHandle>(CheckpointCoordinatorId, ExecuterId, event.release())); CheckpointsReadyStateSent = true; } diff --git a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp index 1856535632b..ee04909e778 100644 --- a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp @@ -24,6 +24,8 @@ #include <ydb/library/actors/core/interconnect.h> #include <ydb/library/actors/core/log.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + namespace NKikimr { namespace NKqp { @@ -153,9 +155,13 @@ private: void HandleResolve(TEvPrivate::TEvResourcesSnapshot::TPtr& ev) { if (ev->Get()->Snapshot.empty()) { - KQP_STLOG_E(KQPSCAN, "Can not find default state storage group for database", - (database, Database), - (trace_id, TraceId())); + YDB_LOG_ERROR("Can not find default state storage group for database", + {"marker", "KQPSCAN"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"database", Database}, + {"traceId", TraceId()}); } ResourcesSnapshot = std::move(ev->Get()->Snapshot); @@ -213,18 +219,26 @@ private: if (TasksGraph.GetTasks().size() > Request.MaxComputeActors) { // LOG_N("Too many compute actors: computeTasks=" << computeTasks.size() << ", scanTasks=" << nScanTasks); - KQP_STLOG_N(KQPSCAN, "Too many compute actors", - (total_tasks, TasksGraph.GetTasks().size()), - (trace_id, TraceId())); + YDB_LOG_NOTICE("Too many compute actors", + {"marker", "KQPSCAN"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"totalTasks", TasksGraph.GetTasks().size()}, + {"traceId", TraceId()}); TBase::ReplyErrorAndDie(Ydb::StatusIds::PRECONDITION_FAILED, YqlIssue({}, TIssuesIds::KIKIMR_PRECONDITION_FAILED, TStringBuilder() << "Requested too many execution units: " << TasksGraph.GetTasks().size())); return; } - KQP_STLOG_D(KQPSCAN, "TotalShardScans", - (count, nShardScans), - (trace_id, TraceId())); + YDB_LOG_DEBUG("TotalShardScans", + {"marker", "KQPSCAN"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"count", nShardScans}, + {"traceId", TraceId()}); ExecuterStateSpan = NWilson::TSpan(TWilsonKqp::ScanExecuterRunTasks, ExecuterSpan.GetTraceId(), "RunTasks", NWilson::EFlags::AUTO_END); ExecuteScanTx(); @@ -269,9 +283,13 @@ private: { if (Planner) { if (!Planner->GetPendingComputeTasks().empty()) { - KQP_STLOG_D(KQPSCAN, "terminate pending resources request", - (status, Ydb::StatusIds::StatusCode_Name(status)), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Terminate pending resources request", + {"marker", "KQPSCAN"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"status", Ydb::StatusIds::StatusCode_Name(status)}, + {"traceId", TraceId()}); auto ev = MakeHolder<TEvKqpNode::TEvCancelKqpTasksRequest>(); ev->Record.SetTxId(TxId); diff --git a/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp b/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp index 52555e07776..5aeb9bc2759 100644 --- a/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_scheme_executer.cpp @@ -21,6 +21,8 @@ #include <functional> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + namespace NKikimr::NKqp { @@ -278,7 +280,12 @@ public: const auto errText = TStringBuilder() << "Cannot resolve working dir." << " path# " << JoinPath(dirPath); - KQP_STLOG_D(KQPSCHEME, errText); + YDB_LOG_DEBUG("Cannot resolve working dir for CTAS move table", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"errText", errText}); const auto issue = MakeIssue(NKikimrIssues::TIssuesIds::RESOLVE_LOOKUP_ERROR, errText); return ReplyErrorAndDie(Ydb::StatusIds::BAD_REQUEST, issue); @@ -958,8 +965,12 @@ public: } void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { - KQP_STLOG_D(KQPSCHEME, "Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult", - (error_count, ev->Get()->Request.Get()->ErrorCount)); + YDB_LOG_DEBUG("Handle TEvTxProxySchemeCache::TEvNavigateKeySetResult", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"errorCount", ev->Get()->Request.Get()->ErrorCount}); NSchemeCache::TSchemeCacheNavigate* resp = ev->Get()->Request.Get(); @@ -982,20 +993,33 @@ public: } TString error(builder); - KQP_STLOG_E(KQPSCHEME, error); + YDB_LOG_ERROR("Unable to navigate scheme paths", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"error", error}); return ReplyErrorAndDie(Ydb::StatusIds::SCHEME_ERROR, NYql::TIssue(error)); } AFL_ENSURE(resp->ResultSet.size() <= 2); if (UserToken && !UserToken->GetSerializedToken().empty() && !CheckAlterAccess(*UserToken, resp)) { - KQP_STLOG_E(KQPSCHEME, "Access check failed"); + YDB_LOG_ERROR("Access check failed", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}); return ReplyErrorAndDie(Ydb::StatusIds::UNAUTHORIZED, NYql::TIssue("Unauthorized")); } auto domainInfo = resp->ResultSet.front().DomainInfo; if (!domainInfo) { - KQP_STLOG_E(KQPSCHEME, "Got empty domain info"); + YDB_LOG_ERROR("Got empty domain info", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}); return ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, NYql::TIssue("empty domain info")); } @@ -1054,8 +1078,12 @@ public: const auto status = response.GetStatus(); auto issuesProto = response.GetIssues(); - KQP_STLOG_D(KQPSCHEME, "Handle TEvIndexBuilder::TEvCreateResponse", - (response, response.ShortUtf8DebugString())); + YDB_LOG_DEBUG("Handle TEvIndexBuilder::TEvCreateResponse", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"response", response.ShortUtf8DebugString()}); if (status == Ydb::StatusIds::SUCCESS) { if (response.HasSchemeStatus() && response.GetSchemeStatus() == NKikimrScheme::EStatus::StatusAlreadyExists) { @@ -1073,8 +1101,12 @@ public: const auto status = response.GetStatus(); auto issuesProto = response.GetIssues(); - KQP_STLOG_D(KQPSCHEME, "Handle TEvForcedCompaction::TEvCreateResponse", - (response, response.ShortUtf8DebugString())); + YDB_LOG_DEBUG("Handle TEvForcedCompaction::TEvCreateResponse", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"response", response.ShortUtf8DebugString()}); if (status == Ydb::StatusIds::SUCCESS) { DoSubscribe(); @@ -1088,8 +1120,12 @@ public: const auto status = response.GetStatus(); auto issuesProto = response.GetIssues(); - KQP_STLOG_D(KQPSCHEME, "Handle TEvSetColumnConstraint::TEvCreateResponse", - (response, response.ShortUtf8DebugString())); + YDB_LOG_DEBUG("Handle TEvSetColumnConstraint::TEvCreateResponse", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"response", response.ShortUtf8DebugString()}); if (status == Ydb::StatusIds::SUCCESS) { DoSubscribe(); @@ -1159,8 +1195,12 @@ public: void Handle(NSchemeShard::TEvIndexBuilder::TEvGetResponse::TPtr& ev) { auto& record = ev->Get()->Record; - KQP_STLOG_D(KQPSCHEME, "Handle TEvIndexBuilder::TEvGetResponse", - (record, record.ShortDebugString())); + YDB_LOG_DEBUG("Handle TEvIndexBuilder::TEvGetResponse", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"record", record.ShortDebugString()}); if (record.GetStatus() != Ydb::StatusIds::SUCCESS) { // Internal error: we made incorrect request to get status of index build operation NYql::TIssues responseIssues; @@ -1184,8 +1224,12 @@ public: void Handle(NSchemeShard::TEvForcedCompaction::TEvGetResponse::TPtr& ev) { auto& record = ev->Get()->Record; - KQP_STLOG_D(KQPSCHEME, "Handle TEvForcedCompaction::TEvGetResponse", - (record, record.ShortDebugString())); + YDB_LOG_DEBUG("Handle TEvForcedCompaction::TEvGetResponse", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"record", record.ShortDebugString()}); if (record.GetStatus() != Ydb::StatusIds::SUCCESS) { // Internal error: we made incorrect request to get status of compaction operation NYql::TIssues responseIssues; @@ -1212,8 +1256,12 @@ public: void Handle(NSchemeShard::TEvSetColumnConstraint::TEvGetResponse::TPtr& ev) { auto& record = ev->Get()->Record; - KQP_STLOG_D(KQPSCHEME, "Handle TEvSetColumnConstraint::TEvGetResponse", - (record, record.ShortDebugString())); + YDB_LOG_DEBUG("Handle TEvSetColumnConstraint::TEvGetResponse", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"record", record.ShortDebugString()}); if (record.GetStatus() != Ydb::StatusIds::SUCCESS) { NYql::TIssues responseIssues; NYql::IssuesFromMessage(record.GetIssues(), responseIssues); @@ -1300,9 +1348,13 @@ public: void HandleAbortExecution(TEvKqp::TEvAbortExecution::TPtr& ev) { auto& msg = ev->Get()->Record; NYql::TIssues issues = ev->Get()->GetIssues(); - KQP_STLOG_D(KQPSCHEME, "Got EvAbortExecution", - (status, NYql::NDqProto::StatusIds_StatusCode_Name(msg.GetStatusCode())), - (issues, issues.ToOneLineString())); + YDB_LOG_DEBUG("Got EvAbortExecution", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"status", NYql::NDqProto::StatusIds_StatusCode_Name(msg.GetStatusCode())}, + {"issues", issues.ToOneLineString()}); if (AnalyzeActorId) { auto abortEv = MakeHolder<TEvKqp::TEvAbortExecution>(msg.GetStatusCode(), issues); @@ -1333,10 +1385,14 @@ private: } void UnexpectedEvent(const TString& state, ui32 eventType) { - KQP_STLOG_C(KQPSCHEME, "TKqpSchemeExecuter, unexpected event", - (event_type, eventType), - (state, state), - (self_id, SelfId())); + YDB_LOG_CRIT("TKqpSchemeExecuter, unexpected event", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"eventType", eventType}, + {"state", state}, + {"selfId", SelfId()}); InternalError(TStringBuilder() << "Unexpected event at TKqpSchemeExecuter, state: " << state << ", event: " << eventType); @@ -1356,7 +1412,12 @@ private: } void InternalError(const NYql::TIssues& issues) { - KQP_STLOG_E(KQPSCHEME, issues.ToOneLineString()); + YDB_LOG_ERROR("Internal error during scheme operation", + {"marker", "KQPSCHEME"}, + {"actorId", SelfId()}, + {"txId", TxId}, + {"ctx", *GetUserRequestContext()}, + {"issues", issues.ToOneLineString()}); auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::UNEXPECTED, "Internal error while executing scheme operation."); diff --git a/ydb/core/kqp/executer_actor/kqp_table_resolver.cpp b/ydb/core/kqp/executer_actor/kqp_table_resolver.cpp index b61d70457ac..253002e9cdd 100644 --- a/ydb/core/kqp/executer_actor/kqp_table_resolver.cpp +++ b/ydb/core/kqp/executer_actor/kqp_table_resolver.cpp @@ -6,6 +6,8 @@ #include <ydb/core/kqp/executer_actor/kqp_executer.h> #include <ydb/library/actors/core/actor_bootstrapped.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + namespace NKikimr::NKqp { using namespace NActors; @@ -14,11 +16,6 @@ using namespace NYql::NDq; namespace { -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream) -#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, "TxId: " << TxId << ". " << stream) - class TKqpTableResolver : public TActorBootstrapped<TKqpTableResolver> { public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { @@ -45,7 +42,9 @@ private: hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleResolveNames); hFunc(TEvents::TEvPoison, HandleResolveNames); default: { - LOG_C("ResolveKeysState: unexpected event " << ev->GetTypeRewrite()); + YDB_LOG_CRIT("ResolveNamesState: unexpected event", + {"txId", TxId}, + {"eventType", ev->GetTypeRewrite()}); GotUnexpectedEvent = ev->GetTypeRewrite(); } } @@ -57,7 +56,9 @@ private: hFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, HandleResolveKeys); hFunc(TEvents::TEvPoison, HandleResolveKeys); default: { - LOG_C("ResolveKeysState: unexpected event " << ev->GetTypeRewrite()); + YDB_LOG_CRIT("ResolveKeysState: unexpected event", + {"txId", TxId}, + {"eventType", ev->GetTypeRewrite()}); GotUnexpectedEvent = ev->GetTypeRewrite(); } } @@ -73,7 +74,9 @@ private: ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, TIssue(TStringBuilder() << "navigation problems for tables")); return; } - LOG_D("Navigated key sets: " << results.size()); + YDB_LOG_DEBUG("Navigated key", + {"txId", TxId}, + {"sets", results.size()}); for (auto& entry : results) { if (entry.Status != NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { ReplyErrorAndDie(Ydb::StatusIds::SCHEME_ERROR, @@ -235,7 +238,9 @@ private: ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, TIssue(TStringBuilder() << "navigation problems for tables")); return; } - LOG_D("Navigated key sets: " << results.size()); + YDB_LOG_DEBUG("Navigated key", + {"txId", TxId}, + {"sets", results.size()}); for (auto& entry : results) { if (entry.Status != NSchemeCache::TSchemeCacheNavigate::EStatus::Ok) { ReplyErrorAndDie(Ydb::StatusIds::SCHEME_ERROR, @@ -280,11 +285,15 @@ private: auto timer = std::make_unique<NCpuTime::TCpuTimer>(CpuTime); auto& results = ev->Get()->Request->ResultSet; - LOG_D("Resolved key sets: " << results.size()); + YDB_LOG_DEBUG("Resolved key", + {"txId", TxId}, + {"sets", results.size()}); for (auto& entry : results) { if (entry.Status != NSchemeCache::TSchemeCacheRequest::EStatus::OkData) { - LOG_E("Error resolving keys for entry: " << entry.ToString(*AppData()->TypeRegistry)); + YDB_LOG_ERROR("Error resolving keys", + {"txId", TxId}, + {"entry", entry.ToString(*AppData()->TypeRegistry)}); TStringBuilder path; if (auto it = TablePathsById.find(entry.KeyDescription->TableId); it != TablePathsById.end()) { @@ -304,7 +313,9 @@ private: AFL_ENSURE(partition.Range); } - LOG_D("Resolved key: " << entry.ToString(*AppData()->TypeRegistry)); + YDB_LOG_DEBUG("Resolved", + {"txId", TxId}, + {"key", entry.ToString(*AppData()->TypeRegistry)}); auto& stageInfo = DecodeStageInfo(entry.UserData); @@ -494,7 +505,11 @@ private: private: void UnexpectedEvent(const TString& state, ui32 eventType) { - LOG_C("TKqpTableResolver, unexpected event: " << eventType << ", at state:" << state << ", self: " << SelfId()); + YDB_LOG_CRIT("TKqpTableResolver received unexpected event", + {"txId", TxId}, + {"eventType", eventType}, + {"state", state}, + {"selfId", SelfId()}); auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::UNEXPECTED, "Internal error while executing transaction."); ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, std::move(issue)); } diff --git a/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp b/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp index f890f1bd886..46c9c5a9cab 100644 --- a/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp +++ b/ydb/core/kqp/executer_actor/kqp_tasks_graph.cpp @@ -28,13 +28,7 @@ #include <algorithm> -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, stream) -#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_EXECUTER, stream) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER namespace NKikimr::NKqp { @@ -443,13 +437,14 @@ void FillTaskMeta(const TStageInfo& stageInfo, const TTask& task, NYql::NDqProto void AppendMKQLValueToToken(TString& token, NKikimr::NMiniKQL::TType* type, NUdf::TUnboxedValue value) { if (type->GetKind() != NKikimr::NMiniKQL::TType::EKind::Data) { - LOG_W("Cannot append parameter value to token, unexpected type: " << static_cast<int>(type->GetKind())); + YDB_LOG_WARN("Cannot append parameter value to token: unexpected type kind", + {"typeKind", static_cast<int>(type->GetKind())}); return; } auto dataSlot = static_cast<NKikimr::NMiniKQL::TDataType*>(type)->GetDataSlot(); if (!dataSlot) { - LOG_W("Cannot append parameter value to token: no data slot"); + YDB_LOG_WARN("Cannot append parameter value to token: no data slot"); return; } @@ -533,7 +528,8 @@ void AppendMKQLValueToToken(TString& token, NKikimr::NMiniKQL::TType* type, NUdf } default: - LOG_W("Cannot append parameter value to token, unexpected data slot: " << static_cast<int>(*dataSlot)); + YDB_LOG_WARN("Cannot append parameter value to token, unexpected data", + {"slot", static_cast<int>(*dataSlot)}); } } @@ -545,7 +541,8 @@ TString ResolveFullTextQueryToken(const NKqpProto::TKqpFullTextSource::TKqpQuery auto* paramPtr = stageInfo.Meta.Tx.Params->GetParameterUnboxedValuePtr(token.GetParamName()); if (!paramPtr) { - LOG_W("Failed to get parameter value for token: " << token.GetParamName()); + YDB_LOG_WARN("Failed to get parameter value for full-text query token", + {"paramName", token.GetParamName()}); return fullToken; } @@ -563,7 +560,8 @@ TVector<TString> ResolveFullTextQueryTokenExpanded( auto* paramPtr = stageInfo.Meta.Tx.Params->GetParameterUnboxedValuePtr(token.GetParamName()); if (!paramPtr) { - LOG_W("Failed to get parameter value for token: " << token.GetParamName()); + YDB_LOG_WARN("Failed to get parameter value for full-text query token", + {"paramName", token.GetParamName()}); return { baseToken }; } @@ -778,7 +776,8 @@ void TKqpTasksGraph::FillStages() { } auto& stageInfo = GetStageInfo(stageId); - LOG_D(stageInfo.DebugString()); + YDB_LOG_DEBUG("Built stage info for shard scan", + {"stageInfo", stageInfo.DebugString()}); THashSet<TTableId> tables; for (const auto& op : stage.GetTableOps()) { @@ -842,7 +841,10 @@ void TKqpTasksGraph::BuildResultChannels(const TKqpPhyTxHolder::TConstPtr& tx, u taskOutput.Type = TTaskOutputType::Map; taskOutput.Channels.push_back(channel.Id); - LOG_D("Create result channelId: " << channel.Id << " from task: " << originTaskId << " with index: " << outputIdx); + YDB_LOG_DEBUG("Created result channel from task output", + {"channelId", channel.Id}, + {"taskId", originTaskId}, + {"outputIndex", outputIdx}); } } @@ -1164,10 +1166,14 @@ void TKqpTasksGraph::BuildKqpStageChannels(TStageInfo& stageInfo, ui64 txId, boo } auto log = [&stageInfo, txId](ui64 channel, ui64 from, ui64 to, TStringBuf type, bool spilling) { - LOG_T( "TxId: " << txId << ". " - << "Stage " << stageInfo.Id << " create channelId: " << channel - << " from task: " << from << " to task: " << to << " of type " << type - << (spilling ? " with spilling" : " without spilling")); + YDB_LOG_TRACE("Created stage channel between tasks", + {"txId", txId}, + {"stageId", stageInfo.Id}, + {"channelId", channel}, + {"fromTask", from}, + {"toTask", to}, + {"type", type}, + {"enableSpilling", spilling}); }; bool hasMap = false; @@ -1180,12 +1186,13 @@ void TKqpTasksGraph::BuildKqpStageChannels(TStageInfo& stageInfo, ui64 txId, boo ui32 outputIdx = input.GetOutputIndex(); columnShardHashV1Params = originStageInfo.Meta.GetColumnShardHashV1Params(outputIdx); if (input.GetTypeCase() == NKqpProto::TKqpPhyConnection::kMap || inputIndex == stage.InputsSize() - 1) { // this branch is only for logging purposes - LOG_D( "Chose " - << "[" << originStageInfo.Id.TxId << ":" << originStageInfo.Id.StageId << "]" - << " outputIdx: " << outputIdx << " to propagate through inputs stages of the stage " - << "[" << stageInfo.Id.TxId << ":" << stageInfo.Id.StageId << "]" << ": " - << columnShardHashV1Params.KeyTypesToString(); - ); + YDB_LOG_DEBUG("Propagating column shard hash params from input stage", + {"originStageTxId", originStageInfo.Id.TxId}, + {"originStageId", originStageInfo.Id.StageId}, + {"outputIdx", outputIdx}, + {"stageTxId", stageInfo.Id.TxId}, + {"stageId", stageInfo.Id.StageId}, + {"columnShardHashKeyTypes", columnShardHashV1Params.KeyTypesToString()}); } if (input.GetTypeCase() == NKqpProto::TKqpPhyConnection::kMap) { // We want to enforce sourceShardCount from map connection, cause it can be at most one map connection @@ -1256,12 +1263,13 @@ void TKqpTasksGraph::BuildKqpStageChannels(TStageInfo& stageInfo, ui64 txId, boo case NKqpProto::TKqpPhyCnHashShuffle::kColumnShardHashV1: { Y_ENSURE(enableShuffleElimination, "OptShuffleElimination wasn't turned on, but ColumnShardHashV1 detected!"); - LOG_D( "Propagating columnhashv1 params to stage" - << "[" << inputStageInfo.Id.TxId << ":" << inputStageInfo.Id.StageId << "]" << " which is input of stage " - << "[" << stageInfo.Id.TxId << ":" << stageInfo.Id.StageId << "]" << ": " - << columnShardHashV1Params.KeyTypesToString() << " " - << "[" << JoinSeq(",", input.GetHashShuffle().GetKeyColumns()) << "]"; - ); + YDB_LOG_DEBUG("Propagating column shard hash v1 params to input stage", + {"inputStageTxId", inputStageInfo.Id.TxId}, + {"inputStageId", inputStageInfo.Id.StageId}, + {"stageTxId", stageInfo.Id.TxId}, + {"stageId", stageInfo.Id.StageId}, + {"columnShardHashKeyTypes", columnShardHashV1Params.KeyTypesToString()}, + {"keyColumns", JoinSeq(",", input.GetHashShuffle().GetKeyColumns())}); Y_ENSURE( columnShardHashV1Params.SourceTableKeyColumnTypes->size() == input.GetHashShuffle().KeyColumnsSize(), @@ -1420,11 +1428,11 @@ void TKqpTasksGraph::FillOutputDesc(NYql::NDqProto::TTaskOutput& outputDesc, con } case ColumnShardHashV1: { const auto& columnShardHashV1Params = stageInfo.Meta.GetColumnShardHashV1Params(outputIdx); - LOG_D( "Filling columnshardhashv1 params for sending it to runtime " - << "[" << stageInfo.Id.TxId << ":" << stageInfo.Id.StageId << "]" - << ": " << columnShardHashV1Params.KeyTypesToString() - << " for the columns: " << "[" << JoinSeq(",", output.KeyColumns) << "]" - ); + YDB_LOG_DEBUG("Filling column shard hash v1 params for runtime output", + {"stageTxId", stageInfo.Id.TxId}, + {"stageId", stageInfo.Id.StageId}, + {"columnShardHashKeyTypes", columnShardHashV1Params.KeyTypesToString()}, + {"keyColumns", JoinSeq(",", output.KeyColumns)}); Y_ENSURE(columnShardHashV1Params.SourceShardCount != 0, "ShardCount for ColumnShardHashV1 Shuffle can't be equal to 0"); Y_ENSURE(columnShardHashV1Params.TaskIndexByHash != nullptr, "TaskIndexByHash for ColumnShardHashV1 wasn't propagated to this stage"); Y_ENSURE(columnShardHashV1Params.SourceTableKeyColumnTypes != nullptr, "SourceTableKeyColumnTypes for ColumnShardHashV1 wasn't propagated to this stage"); diff --git a/ydb/core/kqp/executer_actor/kqp_tasks_graph_ut.cpp b/ydb/core/kqp/executer_actor/kqp_tasks_graph_ut.cpp index 849e02048f8..6213088074a 100644 --- a/ydb/core/kqp/executer_actor/kqp_tasks_graph_ut.cpp +++ b/ydb/core/kqp/executer_actor/kqp_tasks_graph_ut.cpp @@ -63,6 +63,8 @@ #include <ydb/public/lib/ydb_cli/common/format.h> #include <yql/essentials/core/pg_settings/guc_settings.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + namespace NKikimr::NKqp { using namespace NYql::NDq; @@ -332,8 +334,8 @@ public: // per-node distribution below is read uniformly from Meta.ExpectedNodeId. RunPlannerPlacement(snapshot); - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_EXECUTER, - "Tasks graph after BuildAllTasks:\n" << Graph->DumpToString()); + YDB_LOG_DEBUG("Tasks graph after BuildAllTasks", + {"tasksGraphDump", Graph->DumpToString()}); auto reply = MakeHolder<TEvBuildTasksDone>(); for (const auto& [stageId, stageInfo] : Graph->GetStagesInfo()) { diff --git a/ydb/core/kqp/executer_actor/shards_resolver/kqp_shards_resolver.cpp b/ydb/core/kqp/executer_actor/shards_resolver/kqp_shards_resolver.cpp index 2e57263c2e6..2de5825972f 100644 --- a/ydb/core/kqp/executer_actor/shards_resolver/kqp_shards_resolver.cpp +++ b/ydb/core/kqp/executer_actor/shards_resolver/kqp_shards_resolver.cpp @@ -9,6 +9,8 @@ #include <util/generic/set.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + namespace NKikimr::NKqp { @@ -17,15 +19,6 @@ using namespace NYql; namespace { -#define LOG__(prio, stream) LOG_LOG_S(*TlsActivationContext, prio, NKikimrServices::KQP_EXECUTER, \ - "[ShardsResolver] TxId: " << TxId << ". " << stream) - -#define LOG_T(stream) LOG__(NActors::NLog::PRI_TRACE, stream) -#define LOG_D(stream) LOG__(NActors::NLog::PRI_DEBUG, stream) -#define LOG_W(stream) LOG__(NActors::NLog::PRI_WARN, stream) -#define LOG_E(stream) LOG__(NActors::NLog::PRI_ERROR, stream) -#define LOG_C(stream) LOG__(NActors::NLog::PRI_CRIT, stream) - constexpr ui32 MAX_RETRIES_COUNT = 3; class TKqpShardsResolver : public TActorBootstrapped<TKqpShardsResolver> { @@ -46,7 +39,9 @@ public: Y_ASSERT(ShardIds.size() > 0); for (ui64 tabletId : ShardIds) { - LOG_T("Send request about tabletId: " << tabletId); + YDB_LOG_TRACE("[ShardsResolver] Send request about", + {"txId", TxId}, + {"tabletId", tabletId}); bool sent = Send(TabletResolver, new TEvPipeCache::TEvGetTabletNode(tabletId)); Y_DEBUG_ABORT_UNLESS(sent); } @@ -60,7 +55,9 @@ private: hFunc(TEvPipeCache::TEvGetTabletNodeResult, HandleResolve); cFunc(TEvents::TSystem::Poison, PassAway); default: { - LOG_C("Unexpected event: " << ev->GetTypeRewrite()); + YDB_LOG_CRIT("[ShardsResolver] Unexpected", + {"txId", TxId}, + {"event", ev->GetTypeRewrite()}); ReplyErrorAndDie(Ydb::StatusIds::INTERNAL_ERROR, "Unexpected event while resolving shards"); } } @@ -68,11 +65,16 @@ private: void HandleResolve(TEvPipeCache::TEvGetTabletNodeResult::TPtr& ev) { auto* msg = ev->Get(); - LOG_T("Got resolve event for tabletId: " << msg->TabletId << ", nodeId: " << msg->NodeId); + YDB_LOG_TRACE("[ShardsResolver] Got resolve event", + {"txId", TxId}, + {"tabletId", msg->TabletId}, + {"nodeId", msg->NodeId}); if (msg->NodeId != 0) { Result[msg->TabletId] = msg->NodeId; if (Result.size() == ShardIds.size()) { - LOG_D("Shard resolve complete, resolved shards: " << Result.size()); + YDB_LOG_DEBUG("[ShardsResolver] Shard resolve complete, resolved", + {"txId", TxId}, + {"shards", Result.size()}); return ReplyAndDie(); } @@ -82,7 +84,9 @@ private: ui32& retryCount = RetryCount[msg->TabletId]; if (retryCount > MAX_RETRIES_COUNT) { TString reply = TStringBuilder() << "Failed to resolve tablet: " << msg->TabletId << " after several retries."; - LOG_W(reply); + YDB_LOG_WARN("[ShardsResolver]", + {"txId", TxId}, + {"reply", reply}); ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, std::move(reply)); return; } diff --git a/ydb/core/kqp/federated_query/actors/kqp_federated_query_actors.cpp b/ydb/core/kqp/federated_query/actors/kqp_federated_query_actors.cpp index e047d1abbc1..b926303c192 100644 --- a/ydb/core/kqp/federated_query/actors/kqp_federated_query_actors.cpp +++ b/ydb/core/kqp/federated_query/actors/kqp_federated_query_actors.cpp @@ -9,6 +9,8 @@ #include <ydb/services/scheme_secret/resolver.h> #include <ydb/library/actors/core/log.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_GATEWAY + namespace NKikimr::NKqp { void RegisterDescribeSecretsActor( @@ -196,17 +198,21 @@ private: .Database(state->Database) .SslCredentials(NYdb::TSslCredentials(state->Ssl, state->CaCert)) .AuthToken(state->Token); - LOG_DEBUG_S(*actorSystem, NKikimrServices::KQP_GATEWAY, - "DescribeResourceId: SelfId=" << selfId << " DescribeTable " << state->Database << " at " << state->Endpoint << (state->Ssl ? " (Ssl)" : "")); + YDB_LOG_DEBUG_CTX(*actorSystem, "DescribeResourceId: describing table", + {"selfId", selfId}, + {"database", state->Database}, + {"endpoint", state->Endpoint}, + {"sslEnabled", state->Ssl}); NYdb::NTable::TTableClient tableClient(*Driver, settings); tableClient.GetSession().Subscribe([actorSystem, selfId, state](const NYdb::NTable::TAsyncCreateSessionResult& future) mutable { try { auto& result = future.GetValue(); if (!result.IsSuccess()) { - LOG_WARN_S(*actorSystem, NKikimrServices::KQP_GATEWAY, "DescribeResourceId: SelfId=" << selfId << " GetSession failed" - << ", status# " << result.GetStatus() - << ", issues# " << result.GetIssues().ToOneLineString() - << ", iteration# " << state->Backoff.GetIteration()); + YDB_LOG_WARN_CTX(*actorSystem, "DescribeResourceId: GetSession failed", + {"selfId", selfId}, + {"status", result.GetStatus()}, + {"issues", result.GetIssues().ToOneLineString()}, + {"iteration", state->Backoff.GetIteration()}); if (IsRetryableError(result) && state->Backoff.HasMore()) { auto delay = state->Backoff.Next(); actorSystem->Schedule(delay, @@ -224,10 +230,11 @@ private: try { const auto& result = future.GetValue(); if (!result.IsSuccess()) { - LOG_WARN_S(*actorSystem, NKikimrServices::KQP_GATEWAY, "DescribeResourceId: SelfId=" << selfId << " DescribeTable failed" - << ", status# " << result.GetStatus() - << ", issues# " << result.GetIssues().ToOneLineString() - << ", iteration# " << state->Backoff.GetIteration()); + YDB_LOG_WARN_CTX(*actorSystem, "DescribeResourceId: DescribeTable failed", + {"selfId", selfId}, + {"status", result.GetStatus()}, + {"issues", result.GetIssues().ToOneLineString()}, + {"iteration", state->Backoff.GetIteration()}); if (IsRetryableError(result) && state->Backoff.HasMore()) { auto delay = state->Backoff.Next(); @@ -239,32 +246,43 @@ private: } return; } - LOG_DEBUG_S(*actorSystem, NKikimrServices::KQP_GATEWAY, - "DescribeResourceId: SelfId=" << selfId << " Succeed"); + YDB_LOG_DEBUG_CTX(*actorSystem, "DescribeResourceId: Succeed", + {"selfId", selfId}); for (const auto& [k, v] : result.GetTableDescription().GetAttributes()) { - LOG_TRACE_S(*actorSystem, NKikimrServices::KQP_GATEWAY, - "DescribeResourceId: SelfId=" << selfId << " key=" << k << " value=" << v); + YDB_LOG_TRACE_CTX(*actorSystem, "DescribeResourceId", + {"selfId", selfId}, + {"key", k}, + {"value", v}); if (k == "cloud_id") { - LOG_DEBUG_S(*actorSystem, NKikimrServices::KQP_GATEWAY, "DescribeResourceId: SelfId=" << selfId << " Resolved ResourceId=" << v); + YDB_LOG_DEBUG_CTX(*actorSystem, "DescribeResourceId: Resolved", + {"selfId", selfId}, + {"resourceId", v}); state->Promise.SetValue(TEvDescribeResourceIdResponse::TDescription(TString(v))); return; } } - LOG_WARN_S(*actorSystem, NKikimrServices::KQP_GATEWAY, "DescribeResourceId: SelfId=" << selfId << " cloud_id not found"); + YDB_LOG_WARN_CTX(*actorSystem, "DescribeResourceId: cloud_id not found", + {"selfId", selfId}); state->Promise.SetValue(TEvDescribeResourceIdResponse::TDescription("")); } catch(const std::exception& ex) { - LOG_WARN_S(*actorSystem, NKikimrServices::KQP_GATEWAY, "DescribeResourceId: SelfId=" << selfId << " got exception: " << ex.what()); + YDB_LOG_WARN_CTX(*actorSystem, "DescribeResourceId: got", + {"selfId", selfId}, + {"exception", ex.what()}); state->Promise.SetException(std::current_exception()); } }); } catch(const std::exception& ex) { - LOG_WARN_S(*actorSystem, NKikimrServices::KQP_GATEWAY, "DescribeResourceId: SelfId=" << selfId << " got exception: " << ex.what()); + YDB_LOG_WARN_CTX(*actorSystem, "DescribeResourceId: got", + {"selfId", selfId}, + {"exception", ex.what()}); state->Promise.SetException(std::current_exception()); } }); } catch(const std::exception& ex) { - LOG_WARN_S(*actorSystem, NKikimrServices::KQP_GATEWAY, "DescribeResourceId: SelfId=" << selfId << " got exception: " << ex.what()); + YDB_LOG_WARN_CTX(*actorSystem, "DescribeResourceId: got", + {"selfId", selfId}, + {"exception", ex.what()}); state->Promise.SetException(std::current_exception()); } } diff --git a/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp b/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp index 376885a8958..33f5a8a505e 100644 --- a/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp +++ b/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp @@ -32,6 +32,8 @@ #include <util/system/file.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_GATEWAY + namespace NKikimr::NKqp { namespace { @@ -53,14 +55,9 @@ namespace { const TString& endpoint, const TString& database, bool useTls, - const TString& structuredTokenJson, + std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory, const TString& path, bool addRoot) { - if (!federatedQuerySetup || !federatedQuerySetup->Driver || !endpoint || !database) { - LOG_NOTICE_S(*NActors::TActivationContext::ActorSystem(), NKikimrServices::KQP_GATEWAY, "Skipped describe for path '" << path << "' in external YDB database '" << database << "' with endpoint '" << endpoint << "'"); - return NThreading::MakeFuture<TGetSchemeEntryResult>(TGetSchemeEntryResult{.EntryType = NYdb::NScheme::ESchemeEntryType::Table}); - } - std::shared_ptr<NYdb::ICredentialsProviderFactory> credentialsProviderFactory = federatedQuerySetup->CredentialsFactory->Create(structuredTokenJson); auto driver = federatedQuerySetup->Driver; NYdb::TCommonClientSettings opts; @@ -73,15 +70,16 @@ namespace { auto schemeClient = std::make_shared<NYdb::NScheme::TSchemeClient>(*driver, opts); return schemeClient->DescribePath(addRoot ? "/Root" + path : path) - .Apply([actorSystem, p = path, sc = schemeClient, database, endpoint, f = federatedQuerySetup, useTls, structuredTokenJson, addRoot](const NThreading::TFuture<NYdb::NScheme::TDescribePathResult>& result) { + .Apply([actorSystem, p = path, sc = schemeClient, database, endpoint, f = federatedQuerySetup, useTls, credentialsProviderFactory, addRoot](const NThreading::TFuture<NYdb::NScheme::TDescribePathResult>& result) { auto describePathResult = result.GetValue(); TGetSchemeEntryResult res; if (!describePathResult.IsSuccess()) { if (describePathResult.GetStatus() == NYdb::EStatus::CLIENT_UNAUTHENTICATED && !addRoot) { - return GetSchemeEntryTypeImpl(actorSystem, f, endpoint, database, useTls, structuredTokenJson, p, true); + return GetSchemeEntryTypeImpl(actorSystem, f, endpoint, database, useTls, credentialsProviderFactory, p, true); } TString message = TStringBuilder() << "Describe path '" << p << "' in external YDB database '" << database << "' with endpoint '" << endpoint << "' failed."; - LOG_WARN_S(*actorSystem, NKikimrServices::KQP_GATEWAY, message + describePathResult.GetIssues().ToString()); + YDB_LOG_WARN_CTX(*actorSystem, message, + {"issues", describePathResult.GetIssues()}); auto rootIssue = NYql::TIssue(message); for (const auto& issue : describePathResult.GetIssues()) { rootIssue.AddSubIssue(MakeIntrusive<NYql::TIssue>(NYdb::NAdapters::ToYqlIssue(issue))); @@ -412,7 +410,22 @@ namespace { bool useTls, const TString& structuredTokenJson, const TString& path) { - return GetSchemeEntryTypeImpl(NActors::TActivationContext::ActorSystem(), federatedQuerySetup, endpoint, NKikimr::CanonizePath(database), useTls, structuredTokenJson, path, false); + if (!federatedQuerySetup || !federatedQuerySetup->Driver || !endpoint || !database) { + YDB_LOG_NOTICE_CTX(*NActors::TActivationContext::ActorSystem(), "Skipped describe for path in external YDB database with endpoint", + {"path", path}, + {"database", database}, + {"endpoint", endpoint}); + return NThreading::MakeFuture<TGetSchemeEntryResult>(TGetSchemeEntryResult{.EntryType = NYdb::NScheme::ESchemeEntryType::Table}); + } + return GetSchemeEntryTypeImpl( + NActors::TActivationContext::ActorSystem(), + federatedQuerySetup, + endpoint, + NKikimr::CanonizePath(database), + useTls, + federatedQuerySetup->CredentialsFactory->Create(structuredTokenJson), + path, + false); }; std::vector<NKqpProto::TKqpExternalSink> FilterExternalSinksWithEffects(const std::vector<NKqpProto::TKqpExternalSink>& sinks) { diff --git a/ydb/core/kqp/finalize_script_service/kqp_check_script_lease_actor.cpp b/ydb/core/kqp/finalize_script_service/kqp_check_script_lease_actor.cpp index 29b1ab4b905..1e53578f88b 100644 --- a/ydb/core/kqp/finalize_script_service/kqp_check_script_lease_actor.cpp +++ b/ydb/core/kqp/finalize_script_service/kqp_check_script_lease_actor.cpp @@ -6,6 +6,8 @@ #include <ydb/library/actors/core/interconnect.h> #include <ydb/library/actors/interconnect/interconnect.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_PROXY + namespace NKikimr::NKqp { namespace { @@ -30,11 +32,14 @@ public: {} void Bootstrap() { - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Bootstrap"); + YDB_LOG_DEBUG("Script execution lease check actor bootstrapped", + {"logPrefix", LogPrefix()}); Become(&TScriptExecutionLeaseCheckActor::MainState); const auto& creatorId = Register(CreateScriptExecutionsTablesCreator(AppData()->FeatureFlags.GetEnableSecureScriptExecutions())); - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Start script executions tables creator: " << creatorId); + YDB_LOG_DEBUG("Starting script execution tables creation", + {"logPrefix", LogPrefix()}, + {"creator", creatorId}); } STRICT_STFUNC(MainState, @@ -54,12 +59,16 @@ public: break; case EWakeup::RefreshScriptExecutions: { const auto& checkerId = Register(CreateRefreshScriptExecutionLeasesActor(SelfId(), QueryServiceConfig, Counters)); - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Start lease checker: " << checkerId); + YDB_LOG_DEBUG("Starting script execution lease refresh", + {"logPrefix", LogPrefix()}, + {"checker", checkerId}); break; } case EWakeup::CreateTables: { const auto& creatorId = Register(CreateScriptExecutionsTablesCreator(AppData()->FeatureFlags.GetEnableSecureScriptExecutions())); - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Start script executions tables creator: " << creatorId); + YDB_LOG_DEBUG("Starting script execution tables creation", + {"logPrefix", LogPrefix()}, + {"creator", creatorId}); break; } } @@ -69,7 +78,8 @@ public: WaitRefreshNodes = false; if (!ev->Get()->Success) { - LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Failed to discover tenant nodes"); + YDB_LOG_WARN("Failed to discover tenant nodes", + {"logPrefix", LogPrefix()}); return; } @@ -77,7 +87,10 @@ public: RefreshLeasePeriod = std::max(nodesCount, static_cast<size_t>(1)) * CHECK_PERIOD; HasNodesInfo = true; - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Handle discover tenant nodes result, number of nodes #" << nodesCount << ", new RefreshLeasePeriod: " << RefreshLeasePeriod); + YDB_LOG_DEBUG("Handle discover tenant nodes result, number of nodes new", + {"logPrefix", LogPrefix()}, + {"nodesCount", nodesCount}, + {"refreshLeasePeriod", RefreshLeasePeriod}); } void Handle(TEvRefreshScriptExecutionLeasesResponse::TPtr& ev) { @@ -86,27 +99,40 @@ public: const auto expiredLeasesCount = ev->Get()->ExpiredLeasesCount; if (!ev->Get()->Success) { - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Refresh " << ev->Sender << " failed, found expired leases: " << expiredLeasesCount << ", issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_ERROR("Script execution lease refresh failed because expired leases were found", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"leases", expiredLeasesCount}, + {"issues", ev->Get()->Issues.ToOneLineString()}); } else { - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Refresh " << ev->Sender << " successfully completed, found expired leases: " << expiredLeasesCount); + YDB_LOG_DEBUG("Refresh successfully completed, found expired", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"leases", expiredLeasesCount}); } } void Handle(TEvScriptExecutionsTablesCreationFinished::TPtr& ev) { if (!ev->Get()->Success) { - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Script executions tables creation failed with issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_ERROR("Script executions tables creation failed with", + {"logPrefix", LogPrefix()}, + {"issues", ev->Get()->Issues.ToOneLineString()}); Schedule(CREATE_TABLES_PERIOD, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::CreateTables))); return; } - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Script executions tables creation finished, start lease checks"); + YDB_LOG_DEBUG("Script executions tables creation finished, start lease checks", + {"logPrefix", LogPrefix()}); RefreshNodesInfo(); Schedule(StartupTimeout, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::ScheduleRefreshScriptExecutions))); } private: void RefreshNodesInfo() { - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Do RefreshNodesInfo (WaitRefreshNodes: " << WaitRefreshNodes << "), next refresh after " << REFRESH_NODES_PERIOD); + YDB_LOG_DEBUG("Scheduled next RefreshNodesInfo", + {"logPrefix", LogPrefix()}, + {"waitRefreshNodes", WaitRefreshNodes}, + {"refreshNodesPeriod", REFRESH_NODES_PERIOD}); Schedule(REFRESH_NODES_PERIOD, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::RefreshNodesInfo))); if (!WaitRefreshNodes) { @@ -116,11 +142,15 @@ private: } void ScheduleRefreshScriptExecutions() { - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Do ScheduleRefreshScriptExecutions (WaitRefreshScriptExecutions: " << WaitRefreshScriptExecutions << "), next refresh after " << RefreshLeasePeriod); + YDB_LOG_DEBUG("Scheduled next script execution refresh", + {"logPrefix", LogPrefix()}, + {"waitRefreshScriptExecutions", WaitRefreshScriptExecutions}, + {"refreshLeasePeriod", RefreshLeasePeriod}); Schedule(RefreshLeasePeriod, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::ScheduleRefreshScriptExecutions))); if (!HasNodesInfo) { - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Skip ScheduleRefreshScriptExecutions, node info is not arrived"); + YDB_LOG_DEBUG("Skip ScheduleRefreshScriptExecutions, node info is not arrived", + {"logPrefix", LogPrefix()}); return; } @@ -131,7 +161,9 @@ private: // to reduce the number of tli const auto leaseCheckTime = RefreshLeasePeriod * RandomNumber<double>(); Schedule(leaseCheckTime, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::RefreshScriptExecutions))); - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Schedule lease check after " << leaseCheckTime); + YDB_LOG_DEBUG("Scheduled script execution lease check", + {"logPrefix", LogPrefix()}, + {"leaseCheckTime", leaseCheckTime}); } } diff --git a/ydb/core/kqp/finalize_script_service/kqp_finalize_script_service.cpp b/ydb/core/kqp/finalize_script_service/kqp_finalize_script_service.cpp index 143ddfe247d..f8260d64477 100644 --- a/ydb/core/kqp/finalize_script_service/kqp_finalize_script_service.cpp +++ b/ydb/core/kqp/finalize_script_service/kqp_finalize_script_service.cpp @@ -10,6 +10,8 @@ #include <queue> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_PROXY + namespace NKikimr::NKqp { namespace { @@ -116,7 +118,9 @@ private: } void Handle(TEvents::TEvUndelivered::TPtr& ev) { - LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Failed to check script execution tables existence, got undelivered to scheme cache: " << ev->Get()->Reason); + YDB_LOG_WARN("Failed to check script execution tables existence, got undelivered to scheme", + {"logPrefix", LogPrefix()}, + {"cache", ev->Get()->Reason}); Retry(); } @@ -128,7 +132,10 @@ private: for (const auto& result : request.ResultSet) { if (result.Status != EStatus::Ok) { - LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Failed to check script execution tables existence, scheme status: " << result.Status << ", path: " << JoinPath(result.Path)); + YDB_LOG_WARN("Failed to check script execution tables existence, scheme", + {"logPrefix", LogPrefix()}, + {"status", result.Status}, + {"path", JoinPath(result.Path)}); } switch (result.Status) { @@ -139,7 +146,9 @@ private: case EStatus::RedirectLookupError: case EStatus::RootUnknown: case EStatus::PathErrorUnknown: - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Script execution table " << JoinPath(result.Path) << " not found"); + YDB_LOG_DEBUG("Script execution table not found", + {"logPrefix", LogPrefix()}, + {"tablePath", JoinPath(result.Path)}); return; case EStatus::LookupError: case EStatus::TableCreationNotComplete: @@ -150,7 +159,8 @@ private: } } - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Start script execution background checks"); + YDB_LOG_DEBUG("Start script execution background checks", + {"logPrefix", LogPrefix()}); StartScriptExecutionBackgroundChecks(); } @@ -171,7 +181,8 @@ private: if (const auto delay = RetryState->GetNextRetryDelay(longDelay)) { Schedule(*delay, new NActors::TEvents::TEvWakeup()); } else { - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, LogPrefix() << "Failed to check script execution tables existence, retry limit exceeded"); + YDB_LOG_ERROR("Failed to check script execution tables existence, retry limit exceeded", + {"logPrefix", LogPrefix()}); } } diff --git a/ydb/core/kqp/gateway/actors/analyze_actor.cpp b/ydb/core/kqp/gateway/actors/analyze_actor.cpp index d170e987e8f..6035cd1e8da 100644 --- a/ydb/core/kqp/gateway/actors/analyze_actor.cpp +++ b/ydb/core/kqp/gateway/actors/analyze_actor.cpp @@ -5,6 +5,8 @@ #include <ydb/library/actors/core/log.h> #include <ydb/library/services/services.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_GATEWAY + namespace NKikimr::NKqp { @@ -47,14 +49,14 @@ void TAnalyzeActor::Handle(NStat::TEvStatistics::TEvAnalyzeResponse::TPtr& ev, c NYql::IKikimrGateway::TGenericResult result; if (operationId != OperationId) { - ALOG_CRIT(NKikimrServices::KQP_GATEWAY, - "TAnalyzeActor, TEvAnalyzeResponse has operationId=" << operationId - << " , but expected " << OperationId); + YDB_LOG_CRIT("TAnalyzeActor received unexpected operation id in TEvAnalyzeResponse", + {"operationId", operationId}, + {"expectedOperationId", OperationId}); result.SetStatus(NYql::TIssuesIds::KIKIMR_INTERNAL_ERROR); result.AddIssue(NYql::TIssue("ANALYZE failed: OperationId mismatch")); } else if (status != NKikimrStat::TEvAnalyzeResponse::STATUS_SUCCESS) { - ALOG_CRIT(NKikimrServices::KQP_GATEWAY, - "TAnalyzeActor, TEvAnalyzeResponse has status=" << status); + YDB_LOG_CRIT("TAnalyzeActor, TEvAnalyzeResponse has", + {"status", status}); result.SetStatus(NYql::TIssuesIds::KIKIMR_INTERNAL_ERROR); NYql::TIssue error("Executing ANALYZE"); for (const auto& issue : record.GetIssues()) { @@ -232,9 +234,8 @@ void TAnalyzeActor::SendStatisticsAggregatorAnalyze(const TNavigate::TEntry& ent } void TAnalyzeActor::Handle(TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) { - ALOG_NOTICE( - NKikimrServices::KQP_GATEWAY, - "got TEvAbortExecution, issues: " << ev->Get()->GetIssues().ToOneLineString()); + YDB_LOG_NOTICE("Got TEvAbortExecution", + {"issues", ev->Get()->GetIssues().ToOneLineString()}); // ANALYZE is a long-running operation: tying its lifetime to the calling query // would mean a session timeout silently cancels work the user may want to keep. @@ -248,9 +249,8 @@ void TAnalyzeActor::Handle(TEvKqp::TEvAbortExecution::TPtr& ev, const TActorCont } void TAnalyzeActor::HandleUnexpectedEvent(ui32 typeRewrite) { - ALOG_CRIT( - NKikimrServices::KQP_GATEWAY, - "TAnalyzeActor, unexpected event, request type: " << typeRewrite); + YDB_LOG_CRIT("TAnalyzeActor, unexpected event, request", + {"type", typeRewrite}); Promise.SetValue( NYql::NCommon::ResultFromError<NYql::IKikimrGateway::TGenericResult>( diff --git a/ydb/core/kqp/gateway/actors/kqp_ic_gateway_actors.h b/ydb/core/kqp/gateway/actors/kqp_ic_gateway_actors.h index 8e7eef287ca..a19023bdbce 100644 --- a/ydb/core/kqp/gateway/actors/kqp_ic_gateway_actors.h +++ b/ydb/core/kqp/gateway/actors/kqp_ic_gateway_actors.h @@ -38,8 +38,9 @@ public: } void HandleUnexpectedEvent(const TString& requestType, ui32 eventType) { - ALOG_CRIT(NKikimrServices::KQP_GATEWAY, "TRequestHandlerBase, unexpected event, request type: " - << requestType << ", event type: " << eventType); + YDB_LOG_CRIT_COMP(NKikimrServices::KQP_GATEWAY, "TRequestHandlerBase, unexpected event", + {"requestType", requestType}, + {"eventType", eventType}); Promise.SetValue(NYql::NCommon::ResultFromError<TResult>(YqlIssue({}, NYql::TIssuesIds::UNEXPECTED, TStringBuilder() << "Unexpected event in " << requestType << ": " << eventType))); diff --git a/ydb/core/kqp/gateway/actors/scheme.h b/ydb/core/kqp/gateway/actors/scheme.h index 1078af9d00a..23b0ebbc630 100644 --- a/ydb/core/kqp/gateway/actors/scheme.h +++ b/ydb/core/kqp/gateway/actors/scheme.h @@ -47,10 +47,10 @@ public: auto& response = ev->Get()->Record; auto status = static_cast<TEvTxUserProxy::TEvProposeTransactionStatus::EStatus>(response.GetStatus()); - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Received TEvProposeTransactionStatus for scheme request" - << ", TxId: " << response.GetTxId() - << ", status: " << status - << ", scheme shard status: " << response.GetSchemeShardStatus()); + YDB_LOG_DEBUG_CTX_COMP(ctx, NKikimrServices::KQP_GATEWAY, "Received TEvProposeTransactionStatus for scheme request scheme", + {"txId", response.GetTxId()}, + {"status", status}, + {"shardStatus", response.GetSchemeShardStatus()}); switch (status) { case TEvTxUserProxy::TResultStatus::ExecInProgress: { @@ -66,15 +66,15 @@ public: } NTabletPipe::SendData(ctx, ShemePipeActorId, request.Release()); - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Sent TEvNotifyTxCompletion request" - << ", TxId: " << response.GetTxId()); + YDB_LOG_DEBUG_CTX_COMP(ctx, NKikimrServices::KQP_GATEWAY, "Sent TEvNotifyTxCompletion request", + {"txId", response.GetTxId()}); return; } case TEvTxUserProxy::TResultStatus::AccessDenied: { - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Access denied for scheme request" - << ", TxId: " << response.GetTxId()); + YDB_LOG_DEBUG_CTX_COMP(ctx, NKikimrServices::KQP_GATEWAY, "Access denied for scheme request", + {"txId", response.GetTxId()}); NYql::TIssue issue(NYql::TPosition(), "Access denied."); Promise.SetValue(NYql::NCommon::ResultFromIssues<TResult>(NYql::TIssuesIds::KIKIMR_ACCESS_DENIED, @@ -87,8 +87,8 @@ public: if (response.GetSchemeShardStatus() == NKikimrScheme::EStatus::StatusSuccess || (!FailedOnAlreadyExists && response.GetSchemeShardStatus() == NKikimrScheme::EStatus::StatusAlreadyExists)) { - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Successful completion of scheme request" - << ", TxId: " << response.GetTxId()); + YDB_LOG_DEBUG_CTX_COMP(ctx, NKikimrServices::KQP_GATEWAY, "Successful completion of scheme request", + {"txId", response.GetTxId()}); if (!response.GetIssues().empty()) { NYql::TIssues issues; @@ -123,8 +123,8 @@ public: case TEvTxUserProxy::TResultStatus::ResolveError: { if (response.GetSchemeShardStatus() == NKikimrScheme::EStatus::StatusPathDoesNotExist && SuccessOnNotExist) { - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Successful completion of scheme request: path does not exist," - << "SuccessOnNotExist: true, TxId: " << response.GetTxId()); + YDB_LOG_DEBUG_CTX_COMP(ctx, NKikimrServices::KQP_GATEWAY, "Successful completion of scheme request: path does not exist, SuccessOnNotExist: true", + {"txId", response.GetTxId()}); TResult result; result.SetSuccess(); Promise.SetValue(std::move(result)); @@ -206,10 +206,10 @@ public: break; } - LOG_ERROR_S(ctx, NKikimrServices::KQP_GATEWAY, "Unexpected error on scheme request" - << ", TxId: " << response.GetTxId() - << ", ProxyStatus: " << status - << ", SchemeShardReason: " << response.GetSchemeShardReason()); + YDB_LOG_ERROR_CTX_COMP(ctx, NKikimrServices::KQP_GATEWAY, "Unexpected error on scheme request", + {"txId", response.GetTxId()}, + {"proxyStatus", status}, + {"schemeShardReason", response.GetSchemeShardReason()}); TStringBuilder message; message << "Scheme operation failed, status: " << status; @@ -224,11 +224,11 @@ public: void Handle(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev, const TActorContext& ctx) { auto& response = ev->Get()->Record; - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Received TEvNotifyTxCompletionResult for scheme request" - << ", TxId: " << response.GetTxId()); + YDB_LOG_DEBUG_CTX_COMP(ctx, NKikimrServices::KQP_GATEWAY, "Received TEvNotifyTxCompletionResult for scheme request", + {"txId", response.GetTxId()}); - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, "Successful completion of scheme request" - << ", TxId: " << response.GetTxId()); + YDB_LOG_DEBUG_CTX_COMP(ctx, NKikimrServices::KQP_GATEWAY, "Successful completion of scheme request", + {"txId", response.GetTxId()}); TResult result; result.SetSuccess(); diff --git a/ydb/core/kqp/gateway/behaviour/streaming_query/queries.cpp b/ydb/core/kqp/gateway/behaviour/streaming_query/queries.cpp index 40592a4612c..7613dd98975 100644 --- a/ydb/core/kqp/gateway/behaviour/streaming_query/queries.cpp +++ b/ydb/core/kqp/gateway/behaviour/streaming_query/queries.cpp @@ -27,18 +27,12 @@ #include <fmt/format.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_PROXY + namespace NKikimr::NKqp { namespace { -#define LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[StreamingQueries] " << LogPrefix() << stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[StreamingQueries] " << LogPrefix() << stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[StreamingQueries] " << LogPrefix() << stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[StreamingQueries] " << LogPrefix() << stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[StreamingQueries] " << LogPrefix() << stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[StreamingQueries] " << LogPrefix() << stream) -#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[StreamingQueries] " << LogPrefix() << stream) - using namespace fmt::literals; using TExternalContext = NMetadata::NModifications::IOperationsManager::TExternalModificationContext; using TStatus = NKikimr::TYQLConclusionSpecialStatus<Ydb::StatusIds::StatusCode, Ydb::StatusIds::SUCCESS, Ydb::StatusIds::INTERNAL_ERROR>; @@ -359,12 +353,20 @@ protected: bool HandleResult(TEvPtr& ev, const TString& message) { const auto status = ev->Get()->Status; if (status == Ydb::StatusIds::SUCCESS) { - LOG_D(message << " " << ev->Sender << " success"); + YDB_LOG_DEBUG("[StreamingQueries] Operation succeeded", + {"logPrefix", LogPrefix()}, + {"message", message}, + {"sender", ev->Sender}); return false; } const auto& issues = ev->Get()->Issues; - LOG_W(message << " " << ev->Sender << " failed " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("[StreamingQueries] Operation failed", + {"logPrefix", LogPrefix()}, + {"message", message}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", issues.ToOneLineString()}); FatalError(status, AddRootIssue(TStringBuilder() << message << " failed", issues)); return true; @@ -372,14 +374,20 @@ protected: void Finish(Ydb::StatusIds::StatusCode status) { if (BeforeFinish(status)) { - LOG_D("Do action before finish with status " << status); + YDB_LOG_DEBUG("[StreamingQueries] Deferring finish to run action before completion", + {"logPrefix", LogPrefix()}, + {"status", status}); return; } if (status == Ydb::StatusIds::SUCCESS) { - LOG_D("Successfully finished"); + YDB_LOG_DEBUG("[StreamingQueries] Successfully finished", + {"logPrefix", LogPrefix()}); } else { - LOG_W("Failed " << status << ", with issues: " << Issues.ToOneLineString()); + YDB_LOG_WARN("[StreamingQueries] Operation failed with errors", + {"logPrefix", LogPrefix()}, + {"status", status}, + {"issues", Issues.ToOneLineString()}); } OnFinish(status); @@ -429,7 +437,9 @@ public: {} void Bootstrap() { - LOG_D("Bootstrap. Database: " << Database); + YDB_LOG_DEBUG("[StreamingQueries] Bootstrapping scheme actor", + {"logPrefix", LogPrefix()}, + {"database", Database}); StartRequest(); TBase::Become(&TDerived::StateFunc); @@ -445,7 +455,8 @@ public: return; } - LOG_E("Scheme service is unavailable"); + YDB_LOG_ERROR("[StreamingQueries] Scheme service is unavailable", + {"logPrefix", LogPrefix()}); TBase::FatalError(Ydb::StatusIds::UNAVAILABLE, "Scheme service is unavailable"); } @@ -468,7 +479,10 @@ protected: } if (const auto delay = RetryState->GetNextRetryDelay(longDelay)) { - LOG_W("Schedule retry for error: " << issues.ToOneLineString() << " in " << *delay); + YDB_LOG_WARN("[StreamingQueries] Scheduling retry after scheme error", + {"logPrefix", LogPrefix()}, + {"error", issues.ToOneLineString()}, + {"retryDelay", *delay}); TBase::Issues.AddIssues(std::move(issues)); TBase::Schedule(*delay, new TEvents::TEvWakeup()); return true; @@ -516,7 +530,9 @@ public: } const auto& result = results[0]; - LOG_D("Got scheme cache response: " << result.Status); + YDB_LOG_DEBUG("[StreamingQueries] Received scheme cache response", + {"logPrefix", LogPrefix()}, + {"response", result.Status}); switch (result.Status) { case EStatus::Unknown: @@ -564,7 +580,9 @@ public: protected: void StartRequest() final { - LOG_D("Describe streaming query in database: " << Database); + YDB_LOG_DEBUG("[StreamingQueries] Describing streaming query in scheme cache", + {"logPrefix", LogPrefix()}, + {"database", Database}); auto request = std::make_unique<NSchemeCache::TSchemeCacheNavigate>(); request->DatabaseName = Database; @@ -623,11 +641,13 @@ public: TxId = response.GetTxId(); SchemeShardTabletId = response.GetSchemeShardTabletId(); - LOG_D("Got propose transaction " << NKikimrSchemeOp::EOperationType_Name(SchemeTx.GetOperationType()) << " response" - << ", Status: " << status - << ", SchemeShardStatus: " << NKikimrScheme::EStatus_Name(ssStatus) - << ", TxId: " << TxId - << ", SchemeShardTabletId: " << SchemeShardTabletId); + YDB_LOG_DEBUG("[StreamingQueries] Received propose transaction response", + {"logPrefix", LogPrefix()}, + {"operationType", NKikimrSchemeOp::EOperationType_Name(SchemeTx.GetOperationType())}, + {"status", status}, + {"schemeShardStatus", NKikimrScheme::EStatus_Name(ssStatus)}, + {"txId", TxId}, + {"schemeShardTabletId", SchemeShardTabletId}); if (ssStatus == NKikimrScheme::EStatus::StatusPathDoesNotExist && IsIn({NTxProxy::TResultStatus::ResolveError, NTxProxy::TResultStatus::ExecError}, status) && AllowNotFoundAfterRetry && RetriesCount) { // After retry previous transaction may continue working, finish DROP operation if path was deleted (path existence already validated before and path was externally locked) @@ -660,7 +680,11 @@ public: case NTxProxy::TResultStatus::ProxyNotReady: case NTxProxy::TResultStatus::ProxyShardTryLater: case NTxProxy::TResultStatus::ProxyShardNotAvailable: { - LOG_W("Retry scheme transaction error: " << status << ", tablet id: " << SchemeShardTabletId << ", tx id: " << TxId); + YDB_LOG_WARN("[StreamingQueries] Retrying scheme transaction after proxy shard error", + {"logPrefix", LogPrefix()}, + {"error", status}, + {"tabletId", SchemeShardTabletId}, + {"txId", TxId}); ScheduleRetry(response, TStringBuilder() << "proxy shard not available " << status); break; } @@ -700,7 +724,10 @@ public: switch (static_cast<NKikimrScheme::EStatus>(ssStatus)) { case NKikimrScheme::StatusMultipleModifications: { // All operations are expected to be retriable in case of scheme shard temporary unavailable - LOG_W("Retry scheme transaction, previous tx execution is not finished, tablet id: " << SchemeShardTabletId << ", failed tx id: " << TxId); + YDB_LOG_WARN("[StreamingQueries] Retrying scheme transaction because previous execution has not finished", + {"logPrefix", LogPrefix()}, + {"tabletId", SchemeShardTabletId}, + {"txId", TxId}); ScheduleRetry(response, "multiple modifications"); break; } @@ -766,7 +793,8 @@ public: return; } - LOG_T("Tablet pipe successfully connected"); + YDB_LOG_TRACE("[StreamingQueries] Tablet pipe successfully connected", + {"logPrefix", LogPrefix()}); } void HandleWaitCompletion(TEvTabletPipe::TEvClientDestroyed::TPtr& ev) { @@ -785,7 +813,9 @@ public: void HandleWaitCompletion(NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletionResult::TPtr& ev) { const auto completedTxId = ev->Get()->Record.GetTxId(); Y_VALIDATE(completedTxId == TxId, "Unexpected completed tx id: " << completedTxId << ", expected tx: " << TxId); - LOG_D("Scheme transaction " << completedTxId << " successfully finished"); + YDB_LOG_DEBUG("[StreamingQueries] Scheme transaction successfully finished", + {"logPrefix", LogPrefix()}, + {"completedTxId", completedTxId}); Finish(Ydb::StatusIds::SUCCESS); } @@ -796,7 +826,10 @@ public: protected: void StartRequest() final { - LOG_D("Start scheme transaction " << NKikimrSchemeOp::EOperationType_Name(SchemeTx.GetOperationType()) << " in database: " << Database); + YDB_LOG_DEBUG("[StreamingQueries] Starting scheme transaction", + {"logPrefix", LogPrefix()}, + {"operationType", NKikimrSchemeOp::EOperationType_Name(SchemeTx.GetOperationType())}, + {"database", Database}); auto event = std::make_unique<TEvTxUserProxy::TEvProposeTransaction>(); *event->Record.MutableTransaction()->MutableModifyScheme() = SchemeTx; @@ -825,7 +858,11 @@ private: Y_VALIDATE(TxId, "Can not subscribe on completion without tx id"); NTabletPipe::SendData(SelfId(), SchemePipeActorId, new NSchemeShard::TEvSchemeShard::TEvNotifyTxCompletion(TxId)); - LOG_D("Subscribe on scheme tx: " << TxId << " on scheme shard: " << SchemeShardTabletId << ", pipe id: " << SchemePipeActorId); + YDB_LOG_DEBUG("[StreamingQueries] Subscribing to scheme transaction completion on scheme pipe", + {"logPrefix", LogPrefix()}, + {"tx", TxId}, + {"shard", SchemeShardTabletId}, + {"id", SchemePipeActorId}); } void ClosePipeClient() { @@ -1013,13 +1050,14 @@ public: {} void OnRunQuery() final { - LOG_D("Locking streaming query" - << ", OperationName: " << Settings.OperationName - << ", OperationStartedAt: " << Settings.OperationStartedAt - << ", OperationOwner: " << Settings.OperationOwner - << ", PreviousOperationOwner: " << Settings.PreviousOperationOwner.value_or(TActorId()) - << ", CreateLockIfNotExists: " << Settings.CreateLockIfNotExists - << ", DefaultQueryStatus: " << NKikimrKqp::TStreamingQueryState::EStatus_Name(Settings.DefaultQueryStatus)); + YDB_LOG_DEBUG("[StreamingQueries] Locking streaming query", + {"logPrefix", LogPrefix()}, + {"operationName", Settings.OperationName}, + {"operationStartedAt", Settings.OperationStartedAt}, + {"operationOwner", Settings.OperationOwner}, + {"previousOperationOwner", Settings.PreviousOperationOwner.value_or(TActorId())}, + {"createLockIfNotExists", Settings.CreateLockIfNotExists}, + {"defaultQueryStatus", NKikimrKqp::TStreamingQueryState::EStatus_Name(Settings.DefaultQueryStatus)}); SetQueryResultHandler(&TLockStreamingQueryRequestActor::OnGetQueryInfo, "Get query info"); ReadQueryInfo(TTxControl::BeginTx()); @@ -1028,7 +1066,9 @@ public: void OnGetQueryInfo() { auto result = ParseQueryInfo(); if (result.GetStatus() == Ydb::StatusIds::NOT_FOUND) { - LOG_D("Streaming query not found, CreateLockIfNotExists: " << Settings.CreateLockIfNotExists); + YDB_LOG_DEBUG("[StreamingQueries] Streaming query not found", + {"logPrefix", LogPrefix()}, + {"createLockIfNotExists", Settings.CreateLockIfNotExists}); if (Settings.CreateLockIfNotExists) { State.SetStatus(Settings.DefaultQueryStatus); @@ -1048,7 +1088,8 @@ public: QueryExists = true; State = result.DetachResult(); if (!State.HasOperationActorId()) { - LOG_D("Streaming query has no locks, creating new lock"); + YDB_LOG_DEBUG("[StreamingQueries] Streaming query has no locks, creating new lock", + {"logPrefix", LogPrefix()}); LockQuery(); return; } @@ -1060,13 +1101,19 @@ public: PreviousOperationStartedAt = NProtoInterop::CastFromProto(State.GetOperationStartedAt()); PreviousOperationName = State.GetOperationName(); - LOG_D("Streaming query under lock from " << PreviousOperationOwner << " started at " << PreviousOperationStartedAt << ", with operation name " << PreviousOperationName); + YDB_LOG_DEBUG("[StreamingQueries] Streaming query is locked by a previous operation", + {"logPrefix", LogPrefix()}, + {"previousOperationOwner", PreviousOperationOwner}, + {"previousOperationStartedAt", PreviousOperationStartedAt}, + {"previousOperationName", PreviousOperationName}); if (!Settings.PreviousOperationOwner) { if (Settings.OperationStartedAt - PreviousOperationStartedAt <= LOCK_TIMEOUT) { FinishUnderOperation(); } else { - LOG_I("Streaming query lock " << PreviousOperationOwner << " expired, start check"); + YDB_LOG_INFO("[StreamingQueries] Streaming query lock expired, starting owner check", + {"logPrefix", LogPrefix()}, + {"previousOperationOwner", PreviousOperationOwner}); CheckLockOwner = true; Finish(); } @@ -1075,12 +1122,16 @@ public: } if (PreviousOperationOwner != *Settings.PreviousOperationOwner) { - LOG_I("Streaming query was locked by " << PreviousOperationOwner << " during lock check"); + YDB_LOG_INFO("[StreamingQueries] Streaming query was locked by another actor during lock check", + {"logPrefix", LogPrefix()}, + {"previousOperationOwner", PreviousOperationOwner}); FinishUnderOperation(); return; } - LOG_I("Remove expired lock from " << PreviousOperationOwner); + YDB_LOG_INFO("[StreamingQueries] Remove expired lock", + {"logPrefix", LogPrefix()}, + {"previousOperationOwner", PreviousOperationOwner}); LockQuery(); } @@ -1158,7 +1209,11 @@ public: {} void Bootstrap() { - LOG_D("Bootstrap"); + YDB_LOG_DEBUG("[StreamingQueries] Bootstrapping lock streaming query table actor", + {"logPrefix", LogPrefix()}, + {"databaseId", DatabaseId}, + {"queryPath", QueryPath}, + {"operationName", Settings.OperationName}); StartLockStreamingQueryRequestActor(); Become(&TLockStreamingQueryTableActor::StateFunc); @@ -1181,14 +1236,15 @@ public: return; } - LOG_D("Lock streaming query finished" - << ", State: " << LogQueryState(Info.State) - << ", PreviousOwner: " << Info.PreviousOwner - << ", PreviousOperationStartedAt: " << Info.PreviousOperationStartedAt - << ", PreviousOperationName: " << Info.PreviousOperationName - << ", QueryExists: " << Info.QueryExists - << ", LockCreated: " << Info.LockCreated - << ", CheckLockOwner: " << Info.CheckLockOwner); + YDB_LOG_DEBUG("[StreamingQueries] Lock streaming query finished", + {"logPrefix", LogPrefix()}, + {"state", LogQueryState(Info.State)}, + {"previousOwner", Info.PreviousOwner}, + {"previousOperationStartedAt", Info.PreviousOperationStartedAt}, + {"previousOperationName", Info.PreviousOperationName}, + {"queryExists", Info.QueryExists}, + {"lockCreated", Info.LockCreated}, + {"checkLockOwner", Info.CheckLockOwner}); if (!Info.CheckLockOwner) { Finish(Ydb::StatusIds::SUCCESS); @@ -1201,7 +1257,9 @@ public: SubscribedOnSession = Info.PreviousOwner.NodeId(); } - LOG_D("Start check alive for " << Info.PreviousOwner); + YDB_LOG_DEBUG("[StreamingQueries] Starting check-alive request to previous owner", + {"logPrefix", LogPrefix()}, + {"previousOwner", Info.PreviousOwner}); Send(Info.PreviousOwner, new TEvPrivate::TEvCheckAliveRequest(), CheckAliveFlags); Schedule(CHECK_ALIVE_REQUEST_SOFT_TIMEOUT, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::CheckAliveSoftTimeout))); Schedule(CHECK_ALIVE_REQUEST_HARD_TIMEOUT, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::CheckAliveHardTimeout))); @@ -1209,9 +1267,13 @@ public: void Handle(TEvPrivate::TEvCheckAliveResponse::TPtr& ev) { if (WaitLock) { - LOG_W("Streaming query " << ev->Sender << " owner was verified after started lock"); + YDB_LOG_WARN("[StreamingQueries] Streaming query owner was verified after started lock", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); } else { - LOG_I("Previous query owner " << ev->Sender << " is alive"); + YDB_LOG_INFO("[StreamingQueries] Previous query owner is alive", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); FatalError(Ydb::StatusIds::ABORTED, {NYql::TIssue(TStringBuilder() << "Streaming query already under operation " << Info.PreviousOperationName << " started at " << Info.PreviousOperationStartedAt << ", try repeat request later")}); } } @@ -1221,21 +1283,28 @@ public: case EWakeup::RetryCheckAlive: { WaitRetryCheckAlive = false; if (WaitLock) { - LOG_N("Skipped retry check alive, already waiting query lock"); + YDB_LOG_NOTICE("[StreamingQueries] Skipped retry check alive, already waiting query lock", + {"logPrefix", LogPrefix()}); } else { - LOG_D("Retry check alive request for " << Info.PreviousOwner); + YDB_LOG_DEBUG("[StreamingQueries] Retrying check-alive request", + {"logPrefix", LogPrefix()}, + {"previousOwner", Info.PreviousOwner}); Send(Info.PreviousOwner, new TEvPrivate::TEvCheckAliveRequest(), CheckAliveFlags); Schedule(CHECK_ALIVE_REQUEST_SOFT_TIMEOUT, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::CheckAliveSoftTimeout))); } break; } case EWakeup::CheckAliveSoftTimeout: { - LOG_W("Deliver streaming query owner " << Info.PreviousOwner << " check alive request timed out, retry check alive"); + YDB_LOG_WARN("[StreamingQueries] Deliver streaming query owner check alive request timed out, retry check alive", + {"logPrefix", LogPrefix()}, + {"previousOwner", Info.PreviousOwner}); RetryCheckAlive(/* longDelay */ false); break; } case EWakeup::CheckAliveHardTimeout: { - LOG_W("Deliver streaming query owner " << Info.PreviousOwner << " check alive request timed out, start lock"); + YDB_LOG_WARN("[StreamingQueries] Deliver streaming query owner check alive request timed out, start lock", + {"logPrefix", LogPrefix()}, + {"previousOwner", Info.PreviousOwner}); StartLockStreamingQueryRequestActor(Info.PreviousOwner); break; } @@ -1245,16 +1314,23 @@ public: void Handle(TEvents::TEvUndelivered::TPtr& ev) { const auto reason = ev->Get()->Reason; if (reason == TEvents::TEvUndelivered::ReasonActorUnknown) { - LOG_W("Streaming query operation owner " << ev->Sender << " not found, start lock"); + YDB_LOG_WARN("[StreamingQueries] Streaming query operation owner not found, start lock", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); StartLockStreamingQueryRequestActor(Info.PreviousOwner); } else { - LOG_W("Got delivery problem to " << ev->Sender << ", node with owner unavailable, reason: " << reason); + YDB_LOG_WARN("[StreamingQueries] Failed to deliver check-alive request to the query owner node", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"reason", reason}); RetryCheckAlive(/* longDelay */ true); } } void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { - LOG_W("Node " << ev->Get()->NodeId << " with streaming query operation owner was disconnected, retry check alive"); + YDB_LOG_WARN("[StreamingQueries] Node with streaming query operation owner was disconnected, retry check alive", + {"logPrefix", LogPrefix()}, + {"nodeId", ev->Get()->NodeId}); RetryCheckAlive(/* longDelay */ true); } @@ -1285,7 +1361,9 @@ private: .CreateLockIfNotExists = Settings.CreateLockIfNotExists, .DefaultQueryStatus = Settings.DefaultQueryStatus, })); - LOG_D("Start TLockStreamingQueryRequestActor " << lockActorId); + YDB_LOG_DEBUG("[StreamingQueries] Start TLockStreamingQueryRequestActor", + {"logPrefix", LogPrefix()}, + {"lockActorId", lockActorId}); } void RetryCheckAlive(bool longDelay) { @@ -1306,11 +1384,15 @@ private: } if (const auto delay = CheckAliveRetryState->GetNextRetryDelay(longDelay)) { - LOG_D("Schedule retry check alive in " << *delay); + YDB_LOG_DEBUG("[StreamingQueries] Scheduling check-alive retry", + {"logPrefix", LogPrefix()}, + {"retryDelay", *delay}); Schedule(*delay, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::RetryCheckAlive))); WaitRetryCheckAlive = true; } else { - LOG_W("Retry limit " << MAX_CHECK_ALIVE_RETRIES << " exceeded for streaming query operation owner check alive, start lock"); + YDB_LOG_WARN("[StreamingQueries] Retry limit exceeded for streaming query operation owner check alive, start lock", + {"logPrefix", LogPrefix()}, + {"maxCheckAliveRetries", MAX_CHECK_ALIVE_RETRIES}); StartLockStreamingQueryRequestActor(Info.PreviousOwner); } } @@ -1334,7 +1416,9 @@ public: {} void OnRunQuery() final { - LOG_D("Unlocking streaming query, OperationOwner: " << OperationOwner); + YDB_LOG_DEBUG("[StreamingQueries] Unlocking streaming query", + {"logPrefix", LogPrefix()}, + {"operationOwner", OperationOwner}); SetQueryResultHandler(&TUnlockStreamingQueryRequestActor::OnGetQueryInfo, "Get query info"); ReadQueryInfo(TTxControl::BeginTx()); } @@ -1356,21 +1440,28 @@ public: } if (OperationOwner != currentOperationOwner) { - LOG_E("Streaming query was locked by " << currentOperationOwner << " during operation (expected owner: " << OperationOwner << ")"); + YDB_LOG_ERROR("[StreamingQueries] Streaming query lock owner changed during operation", + {"logPrefix", LogPrefix()}, + {"currentOperationOwner", currentOperationOwner}, + {"owner", OperationOwner}); Finish(Ydb::StatusIds::INTERNAL_ERROR, "Streaming query was changed during operation"); return; } } else { - LOG_E("Streaming query lock was lost"); + YDB_LOG_ERROR("[StreamingQueries] Streaming query lock was lost", + {"logPrefix", LogPrefix()}); Finish(Ydb::StatusIds::INTERNAL_ERROR, "Streaming query was changed during operation"); return; } if (State.GetStatus() == NKikimrKqp::TStreamingQueryState::STATUS_UNSPECIFIED) { - LOG_D("Delete streaming query from table"); + YDB_LOG_DEBUG("[StreamingQueries] Delete streaming query from table", + {"logPrefix", LogPrefix()}); RemoveQuery(); } else { - LOG_D("Remove streaming query lock " << OperationOwner); + YDB_LOG_DEBUG("[StreamingQueries] Remove streaming query lock", + {"logPrefix", LogPrefix()}, + {"operationOwner", OperationOwner}); UnlockQuery(); } } @@ -1432,7 +1523,9 @@ public: {} void OnRunQuery() final { - LOG_D("Updating streaming query state to " << LogQueryState(State)); + YDB_LOG_DEBUG("[StreamingQueries] Updating streaming query state", + {"logPrefix", LogPrefix()}, + {"queryState", LogQueryState(State)}); SetQueryResultHandler(&TUpdateStreamingQueryStateRequestActor::OnGetQueryInfo, "Get query info"); ReadQueryInfo(TTxControl::BeginTx()); } @@ -1447,7 +1540,10 @@ public: const auto previousOwner = State.GetOperationActorId(); const auto currentOwner = result.GetResult().GetOperationActorId(); if (currentOwner != previousOwner) { - LOG_E("Streaming query was locked by " << currentOwner << " during operation (expected owner: " << previousOwner << ")"); + YDB_LOG_ERROR("[StreamingQueries] Streaming query lock owner changed during operation", + {"logPrefix", LogPrefix()}, + {"currentOwner", currentOwner}, + {"owner", previousOwner}); Finish(Ydb::StatusIds::INTERNAL_ERROR, "Streaming query was changed during operation"); return; } @@ -1487,7 +1583,9 @@ public: {} void Bootstrap() { - LOG_D("Bootstrap"); + YDB_LOG_DEBUG("[StreamingQueries] Bootstrapping cleanup streaming query state table actor", + {"logPrefix", LogPrefix()}, + {"queryPath", QueryPath}); Become(&TCleanupStreamingQueryStateTableActor::StateFunc); ClearStreamingQueryExecutions(); } @@ -1513,7 +1611,11 @@ public: } const auto entryExists = ev->Get()->ExecutionEntryExists; - LOG_D("Cancel streaming query execution " << ev->Sender << " finished, entry exists: " << entryExists << ", execution id: " << executionId); + YDB_LOG_DEBUG("[StreamingQueries] Cancel streaming query execution finished", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"exists", entryExists}, + {"id", executionId}); if (entryExists) { State.AddPreviousExecutionIds(executionId); @@ -1532,7 +1634,12 @@ public: } --OperationsToForget; - LOG_D("Forget streaming query execution #" << ev->Cookie << " " << ev->Sender << " finished, execution id: " << executionId << ", remains: " << OperationsToForget); + YDB_LOG_DEBUG("[StreamingQueries] Forget streaming query execution finished", + {"logPrefix", LogPrefix()}, + {"cookie", ev->Cookie}, + {"sender", ev->Sender}, + {"id", executionId}, + {"remains", OperationsToForget}); if (OperationsToForget == 0) { State.ClearPreviousExecutionIds(); @@ -1548,7 +1655,10 @@ protected: private: void StartUpdateState(const TString& info) const { const auto& updaterId = Register(TUpdateStreamingQueryStateRequestActor::MakeRetry(SelfId(), Context.GetDatabaseId(), QueryPath, State)); - LOG_D("Start TUpdateStreamingQueryStateRequestActor " << updaterId << " (" << info << ")"); + YDB_LOG_DEBUG("[StreamingQueries] Start TUpdateStreamingQueryStateRequestActor", + {"logPrefix", LogPrefix()}, + {"updaterId", updaterId}, + {"info", info}); } void ClearStreamingQueryExecutions() { @@ -1560,7 +1670,9 @@ private: if (State.HasCurrentExecutionId()) { const auto& executionId = State.GetCurrentExecutionId(); - LOG_D("Cancel streaming query execution " << executionId); + YDB_LOG_DEBUG("[StreamingQueries] Cancel streaming query execution", + {"logPrefix", LogPrefix()}, + {"executionId", executionId}); SendToKqpProxy(std::make_unique<TEvCancelScriptExecutionOperation>(Context.GetDatabase(), OperationIdFromExecutionId(executionId), BUILTIN_ACL_METADATA, TEvCancelScriptExecutionOperation::TSettings{ .FailOnNotFound = false, .FailOnAlreadyStopped = false, @@ -1569,14 +1681,19 @@ private: } if (State.PreviousExecutionIdsSize() > 0) { - LOG_D("Cleanup #" << State.PreviousExecutionIdsSize() << " previous executions"); + YDB_LOG_DEBUG("[StreamingQueries] Cleanup previous executions", + {"logPrefix", LogPrefix()}, + {"previousExecutionIdsCount", State.PreviousExecutionIdsSize()}); for (const auto& executionId : State.GetPreviousExecutionIds()) { SendToKqpProxy(std::make_unique<TEvForgetScriptExecutionOperation>(Context.GetDatabase(), OperationIdFromExecutionId(executionId), BUILTIN_ACL_METADATA, TEvForgetScriptExecutionOperation::TSettings{ .FailOnNotFound = false, .CancelIfRunning = true, }), OperationsToForget++); - LOG_D("Forget streaming query execution #" << OperationsToForget << " " << executionId); + YDB_LOG_DEBUG("[StreamingQueries] Forget streaming query execution", + {"logPrefix", LogPrefix()}, + {"operationsToForget", OperationsToForget}, + {"executionId", executionId}); } return; } @@ -1617,7 +1734,11 @@ public: {} void Bootstrap() { - LOG_D("Bootstrap. SS text revision: " << Settings.QueryTextRevision << ", last query execution revision: " << State.GetQueryTextRevision() << ", start new query: " << State.GetQueryText()); + YDB_LOG_DEBUG("[StreamingQueries] Bootstrap: starting a new streaming query execution", + {"logPrefix", LogPrefix()}, + {"textRevision", Settings.QueryTextRevision}, + {"executionRevision", State.GetQueryTextRevision()}, + {"query", State.GetQueryText()}); if (State.HasCurrentExecutionId()) { FatalError(Ydb::StatusIds::INTERNAL_ERROR, TStringBuilder() << "Can not start query, already started: " << State.GetCurrentExecutionId()); @@ -1651,12 +1772,17 @@ public: } if (!ev->Get()->ExecutionEntryExists) { - LOG_W("Previous script execution not found, lease generation and graph was reset"); + YDB_LOG_WARN("[StreamingQueries] Previous script execution not found, lease generation and graph was reset", + {"logPrefix", LogPrefix()}); State.ClearCheckpointId(); // We don't know previous generation, so must start from fresh checkpoint } PreviousGeneration = ev->Get()->Generation; - LOG_D("Load previous query execution state " << ev->Sender << " finished, generation: " << PreviousGeneration << ", has saved state: " << PreviousPhysicalGraph.has_value()); + YDB_LOG_DEBUG("[StreamingQueries] Finished loading previous query execution state", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"generation", PreviousGeneration}, + {"state", PreviousPhysicalGraph.has_value()}); PrepareToStart(); } @@ -1671,7 +1797,12 @@ public: } --OperationsToForget; - LOG_D("Forget streaming query execution #" << ev->Cookie << " " << ev->Sender << " finished, execution id: " << executionId << ", remains: " << OperationsToForget); + YDB_LOG_DEBUG("[StreamingQueries] Forget streaming query execution finished", + {"logPrefix", LogPrefix()}, + {"cookie", ev->Cookie}, + {"sender", ev->Sender}, + {"id", executionId}, + {"remains", OperationsToForget}); if (OperationsToForget == 0) { auto& executionIds = *State.MutablePreviousExecutionIds(); @@ -1701,20 +1832,27 @@ public: } RequestStarted = true; - LOG_D("Script execution created: " << ev->Get()->ExecutionId << ", wait for saving query state"); + YDB_LOG_DEBUG("[StreamingQueries] Script execution started, waiting for query state to be saved", + {"logPrefix", LogPrefix()}, + {"created", ev->Get()->ExecutionId}); GetScriptExecutionOperation(); } void HandleStartQuery(TEvents::TEvWakeup::TPtr&) { const auto& executionId = State.GetCurrentExecutionId(); - LOG_D("Get streaming query execution " << executionId); + YDB_LOG_DEBUG("[StreamingQueries] Fetching streaming query execution", + {"logPrefix", LogPrefix()}, + {"executionId", executionId}); SendToKqpProxy(std::make_unique<TEvGetScriptExecutionOperation>(Context.GetDatabase(), OperationIdFromExecutionId(executionId), BUILTIN_ACL_METADATA, /* failOnNotFound */ true)); } void HandleStartQuery(TEvGetScriptExecutionOperationResponse::TPtr& ev) { const auto& info = *ev->Get(); - LOG_D("Got script execution info, state saved: " << info.StateSaved << ", ready: " << info.Ready); + YDB_LOG_DEBUG("[StreamingQueries] Received script execution info", + {"logPrefix", LogPrefix()}, + {"saved", info.StateSaved}, + {"ready", info.Ready}); if (HandleResult(ev, "Query compilation / planing")) { return; @@ -1781,7 +1919,10 @@ protected: private: void UpdateQueryState(const TString& info) const { const auto& updaterId = Register(TUpdateStreamingQueryStateRequestActor::MakeRetry(SelfId(), Context.GetDatabaseId(), QueryPath, State)); - LOG_D("Start TUpdateStreamingQueryStateRequestActor " << updaterId << " (" << info << ")"); + YDB_LOG_DEBUG("[StreamingQueries] Start TUpdateStreamingQueryStateRequestActor", + {"logPrefix", LogPrefix()}, + {"updaterId", updaterId}, + {"info", info}); } void PrepareToStart() { @@ -1797,13 +1938,18 @@ private: StateLoaded = true; const auto& executionId = *State.GetPreviousExecutionIds().rbegin(); SendToKqpProxy(std::make_unique<TEvGetScriptExecutionPhysicalGraph>(Context.GetDatabase(), executionId)); - LOG_D("Load previous query state from execution: " << executionId); + YDB_LOG_DEBUG("[StreamingQueries] Load previous query state", + {"logPrefix", LogPrefix()}, + {"execution", executionId}); return; } if (State.PreviousExecutionIdsSize() > MAX_QUERY_EXECUTIONS) { const auto toCleanup = State.PreviousExecutionIdsSize() - MAX_QUERY_EXECUTIONS; - LOG_D("Cleanup #" << toCleanup << " previous executions (max executions: " << MAX_QUERY_EXECUTIONS << ")"); + YDB_LOG_DEBUG("[StreamingQueries] Cleaning up previous executions that exceed the limit", + {"logPrefix", LogPrefix()}, + {"toCleanup", toCleanup}, + {"executions", MAX_QUERY_EXECUTIONS}); for (ui64 i = 0; i < toCleanup; ++i) { const auto& executionId = State.GetPreviousExecutionIds(i); @@ -1811,7 +1957,10 @@ private: .FailOnNotFound = false, .CancelIfRunning = true }), OperationsToForget++); - LOG_D("Forget streaming query execution #" << OperationsToForget << " " << executionId); + YDB_LOG_DEBUG("[StreamingQueries] Forget streaming query execution", + {"logPrefix", LogPrefix()}, + {"operationsToForget", OperationsToForget}, + {"executionId", executionId}); } return; } @@ -1872,7 +2021,9 @@ private: request.SetQuery(State.GetQueryText()); request.SetTimeoutMs(TDuration::Max().MilliSeconds()); - LOG_D("Send start streaming query request, execution id: " << State.GetCurrentExecutionId()); + YDB_LOG_DEBUG("[StreamingQueries] Sending start streaming query request", + {"logPrefix", LogPrefix()}, + {"id", State.GetCurrentExecutionId()}); SendToKqpProxy(std::move(ev)); } @@ -1893,7 +2044,9 @@ private: const auto delay = GetOperationRetryState->GetNextRetryDelay(); Y_VALIDATE(delay, "Retries unexpectedly finished"); - LOG_D("Schedule get script execution operation in " << *delay); + YDB_LOG_DEBUG("[StreamingQueries] Scheduling get script execution operation", + {"logPrefix", LogPrefix()}, + {"retryDelay", *delay}); Schedule(*delay, new TEvents::TEvWakeup()); } @@ -1958,10 +2111,11 @@ public: {} void Bootstrap() { - LOG_D("Bootstrap" - << ". Has in SS: " << ExistsInSS - << ", SS info: " << (Settings.SchemeInfo ? Settings.SchemeInfo->DebugString() : "null") - << ", initial status: " << NKikimrKqp::TStreamingQueryState::EStatus_Name(State.GetStatus())); + YDB_LOG_DEBUG("[StreamingQueries] Bootstrap: syncing streaming query with scheme shard", + {"logPrefix", LogPrefix()}, + {"existsInSchemeShard", ExistsInSS}, + {"info", (Settings.SchemeInfo ? Settings.SchemeInfo->DebugString() : "null")}, + {"status", NKikimrKqp::TStreamingQueryState::EStatus_Name(State.GetStatus())}); if (!Settings.SchemeInfo || State.GetStatus() == NKikimrKqp::TStreamingQueryState::STATUS_DELETING) { RemoveQuery(); @@ -2025,7 +2179,11 @@ public: } const auto entryExists = ev->Get()->ExecutionEntryExists; - LOG_D("Cancel streaming query execution " << ev->Sender << " finished, execution id: " << executionId << ", entry exists: " << entryExists); + YDB_LOG_DEBUG("[StreamingQueries] Cancel streaming query execution finished", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"id", executionId}, + {"exists", entryExists}); if (entryExists) { State.AddPreviousExecutionIds(executionId); @@ -2086,7 +2244,10 @@ protected: private: void UpdateQueryState(const TString& info) const { const auto& updaterId = Register(TUpdateStreamingQueryStateRequestActor::MakeRetry(SelfId(), Context.GetDatabaseId(), QueryPath, State)); - LOG_D("Start TUpdateStreamingQueryStateRequestActor " << updaterId << " (" << info << ")"); + YDB_LOG_DEBUG("[StreamingQueries] Start TUpdateStreamingQueryStateRequestActor", + {"logPrefix", LogPrefix()}, + {"updaterId", updaterId}, + {"info", info}); } void RemoveQuery() { @@ -2094,7 +2255,9 @@ private: if (State.HasCurrentExecutionId() || State.PreviousExecutionIdsSize() > 0) { const auto& cleanupActorId = Register(new TCleanupStreamingQueryStateTableActor(Context, QueryPath, State)); - LOG_D("Start TCleanupStreamingQueryStateTableActor " << cleanupActorId << " (remove query)"); + YDB_LOG_DEBUG("[StreamingQueries] Start TCleanupStreamingQueryStateTableActor (remove query)", + {"logPrefix", LogPrefix()}, + {"cleanupActorId", cleanupActorId}); return; } @@ -2115,7 +2278,9 @@ private: .UserToken = NACLib::TUserToken(BUILTIN_ACL_METADATA, TVector<NACLib::TSID>{}), .AllowNotFoundAfterRetry = true, })); - LOG_D("Start TExecuteTransactionSchemeActor " << executerId << " (drop streaming query)"); + YDB_LOG_DEBUG("[StreamingQueries] Start TExecuteTransactionSchemeActor (drop streaming query)", + {"logPrefix", LogPrefix()}, + {"executerId", executerId}); return; } @@ -2139,7 +2304,10 @@ private: if (State.HasCurrentExecutionId()) { const auto& executionId = State.GetCurrentExecutionId(); - LOG_D("Cancel streaming query execution " << executionId << " (" << info << ")"); + YDB_LOG_DEBUG("[StreamingQueries] Cancel streaming query execution", + {"logPrefix", LogPrefix()}, + {"executionId", executionId}, + {"info", info}); SendToKqpProxy(std::make_unique<TEvCancelScriptExecutionOperation>(Context.GetDatabase(), OperationIdFromExecutionId(executionId), BUILTIN_ACL_METADATA, TEvCancelScriptExecutionOperation::TSettings{ .FailOnNotFound = false, .FailOnAlreadyStopped = false, @@ -2177,7 +2345,9 @@ private: .WatermarkLateEventsPolicy = QuerySettings.WatermarkLateEventsPolicy, .StreamingDisposition = QuerySettings.StreamingDisposition, })); - LOG_D("Start TStartStreamingQueryTableActor " << startActorId); + YDB_LOG_DEBUG("[StreamingQueries] Start TStartStreamingQueryTableActor", + {"logPrefix", LogPrefix()}, + {"startActorId", startActorId}); } void SyncQuery() { @@ -2247,7 +2417,8 @@ public: {} void Bootstrap() { - LOG_D("Bootstrap"); + YDB_LOG_DEBUG("[StreamingQueries] Bootstrapping request handler", + {"logPrefix", LogPrefix()}); TBase::Become(&TDerived::StateFunc); DescribeQuery("start handling"); @@ -2268,7 +2439,10 @@ public: SchemeInfo = ev->Get()->Info; if (Context.GetUserToken() && Context.GetUserToken()->GetSerializedToken() && SchemeInfo && SchemeInfo->SecurityObject) { if (const auto& securityObject = *SchemeInfo->SecurityObject; !securityObject.CheckAccess(Access, *Context.GetUserToken())) { - LOG_W("Access denied for " << Context.GetUserToken()->GetUserSID() << ", access: " << Access); + YDB_LOG_WARN("[StreamingQueries] Access denied", + {"logPrefix", LogPrefix()}, + {"userSid", Context.GetUserToken()->GetUserSID()}, + {"access", Access}); if (!securityObject.CheckAccess(NACLib::DescribeSchema, *Context.GetUserToken())) { TBase::FatalError(Ydb::StatusIds::NOT_FOUND, TStringBuilder() << "Streaming query " << TBase::QueryPath << " not found or you don't have access permissions"); @@ -2280,7 +2454,9 @@ public: } } - LOG_D("Describe streaming query success, SchemeInfo: " << (SchemeInfo ? SchemeInfo->DebugString() : "null")); + YDB_LOG_DEBUG("[StreamingQueries] Describe streaming query success", + {"logPrefix", LogPrefix()}, + {"schemeInfo", (SchemeInfo ? SchemeInfo->DebugString() : "null")}); OnQueryDescribed(); } @@ -2293,10 +2469,12 @@ public: const auto& info = ev->Get()->Info; IsLockCreated = info.LockCreated; QueryState = info.State; - LOG_D("Lock streaming query " << ev->Sender << " success" - << ", IsLockCreated: " << IsLockCreated - << ", QueryExists: " << info.QueryExists - << ", QueryState: " << LogQueryState(QueryState)); + YDB_LOG_DEBUG("[StreamingQueries] Lock streaming query success", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"isLockCreated", IsLockCreated}, + {"queryExists", info.QueryExists}, + {"queryState", LogQueryState(QueryState)}); OnQueryLocked(info.QueryExists); } @@ -2312,7 +2490,9 @@ public: } void Handle(TEvPrivate::TEvCheckAliveRequest::TPtr& ev) { - LOG_N("Got check alive request from " << ev->Sender); + YDB_LOG_NOTICE("[StreamingQueries] Received check-alive request", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); TBase::Send(ev->Sender, new TEvPrivate::TEvCheckAliveResponse()); } @@ -2330,7 +2510,9 @@ protected: .CreateLockIfNotExists = createLockIfNotExists, .DefaultQueryStatus = defaultQueryStatus, })); - LOG_D("Start TLockStreamingQueryTableActor " << lockActorId); + YDB_LOG_DEBUG("[StreamingQueries] Start TLockStreamingQueryTableActor", + {"logPrefix", LogPrefix()}, + {"lockActorId", lockActorId}); } bool BeforeFinish(Ydb::StatusIds::StatusCode status) final { @@ -2340,7 +2522,9 @@ protected: TBase::Become(&TDerived::StateFunc); const auto& unlockActorId = TBase::Register(TUnlockStreamingQueryRequestActor::MakeRetry(TBase::SelfId(), Context.GetDatabaseId(), TBase::QueryPath, TBase::SelfId())); - LOG_D("Start TUnlockStreamingQueryRequestActor " << unlockActorId); + YDB_LOG_DEBUG("[StreamingQueries] Start TUnlockStreamingQueryRequestActor", + {"logPrefix", LogPrefix()}, + {"unlockActorId", unlockActorId}); FinalStatus = status; return true; @@ -2357,7 +2541,10 @@ protected: void DescribeQuery(const TString& info) { // Access by user token will be checked during scheme transaction execution const auto& describerId = TBase::Register(new TDescribeStreamingQuerySchemeActor(Context.GetDatabase(), TBase::QueryPath, NACLib::TUserToken(BUILTIN_ACL_METADATA, TVector<NACLib::TSID>{}))); - LOG_D("Start TDescribeStreamingQuerySchemeActor " << describerId << " (" << info << ")"); + YDB_LOG_DEBUG("[StreamingQueries] Start TDescribeStreamingQuerySchemeActor", + {"logPrefix", LogPrefix()}, + {"describerId", describerId}, + {"info", info}); } private: @@ -2436,13 +2623,17 @@ public: } TBase::SchemeInfo = ev->Get()->Info; - LOG_D("Describe streaming query success, scheme info: " << (TBase::SchemeInfo ? TBase::SchemeInfo->DebugString() : "null")); + YDB_LOG_DEBUG("[StreamingQueries] Describe streaming query success, scheme", + {"logPrefix", LogPrefix()}, + {"info", (TBase::SchemeInfo ? TBase::SchemeInfo->DebugString() : "null")}); const auto& syncActorId = TBase::Register(new TSyncStreamingQueryTableActor(TBase::Context, TBase::QueryPath, { .InitialState = TBase::QueryState, .SchemeInfo = TBase::SchemeInfo, })); - LOG_D("Start TSyncStreamingQueryTableActor " << syncActorId << " (sync previous state)"); + YDB_LOG_DEBUG("[StreamingQueries] Start TSyncStreamingQueryTableActor (sync previous state)", + {"logPrefix", LogPrefix()}, + {"syncActorId", syncActorId}); } void HandleSync(TEvPrivate::TEvSyncStreamingQueryResult::TPtr& ev) { @@ -2456,9 +2647,10 @@ public: TBase::SchemeInfo = std::nullopt; } - LOG_D("Sync query with scheme shard success" - << ", QueryState: " << LogQueryState(TBase::QueryState) - << ", query exists in SS: " << TBase::SchemeInfo.has_value()); + YDB_LOG_DEBUG("[StreamingQueries] Sync with scheme shard succeeded", + {"logPrefix", LogPrefix()}, + {"queryState", LogQueryState(TBase::QueryState)}, + {"existsInSchemeShard", TBase::SchemeInfo.has_value()}); TBase::Become(&TDerived::StateFunc); OnQuerySynced(); @@ -2532,7 +2724,9 @@ public: SchemeInfo = std::nullopt; } - LOG_D("Sync query with scheme shard success, State: " << LogQueryState(QueryState)); + YDB_LOG_DEBUG("[StreamingQueries] Sync with scheme shard succeeded", + {"logPrefix", LogPrefix()}, + {"state", LogQueryState(QueryState)}); Finish(Ydb::StatusIds::SUCCESS); } @@ -2554,7 +2748,9 @@ protected: .InitialState = QueryState, .SchemeInfo = *SchemeInfo, })); - LOG_D("Start TSyncStreamingQueryTableActor " << syncActorId << " to finish creation"); + YDB_LOG_DEBUG("[StreamingQueries] Start TSyncStreamingQueryTableActor to finish creation", + {"logPrefix", LogPrefix()}, + {"syncActorId", syncActorId}); } } @@ -2587,7 +2783,9 @@ protected: const auto& executerId = Register(new TExecuteTransactionSchemeActor(Context.GetDatabase(), QueryPath, SchemeTx, { .UserToken = Context.GetUserToken(), })); - LOG_D("Start TExecuteTransactionSchemeActor " << executerId); + YDB_LOG_DEBUG("[StreamingQueries] Start TExecuteTransactionSchemeActor", + {"logPrefix", LogPrefix()}, + {"executerId", executerId}); } private: @@ -2661,7 +2859,9 @@ public: .PathId = SchemeInfo->PathId, }, })); - LOG_D("Start TSyncStreamingQueryTableActor " << syncActorId << " to finish alter"); + YDB_LOG_DEBUG("[StreamingQueries] Start TSyncStreamingQueryTableActor to finish alter", + {"logPrefix", LogPrefix()}, + {"syncActorId", syncActorId}); } } @@ -2676,7 +2876,9 @@ public: SchemeInfo = std::nullopt; } - LOG_D("Sync query with scheme shard success, State: " << LogQueryState(QueryState)); + YDB_LOG_DEBUG("[StreamingQueries] Sync with scheme shard succeeded", + {"logPrefix", LogPrefix()}, + {"state", LogQueryState(QueryState)}); Finish(Ydb::StatusIds::SUCCESS); } @@ -2707,7 +2909,9 @@ protected: const auto& executerId = Register(new TExecuteTransactionSchemeActor(Context.GetDatabase(), QueryPath, SchemeTx, { .UserToken = Context.GetUserToken(), })); - LOG_D("Start TExecuteTransactionSchemeActor " << executerId); + YDB_LOG_DEBUG("[StreamingQueries] Start TExecuteTransactionSchemeActor", + {"logPrefix", LogPrefix()}, + {"executerId", executerId}); } private: @@ -2831,7 +3035,9 @@ private: if (QueryExistsInTable) { // Clear query state const auto& cleanupActorId = Register(new TCleanupStreamingQueryStateTableActor(Context, QueryPath, QueryState)); - LOG_D("Start TCleanupStreamingQueryStateTableActor " << cleanupActorId); + YDB_LOG_DEBUG("[StreamingQueries] Start TCleanupStreamingQueryStateTableActor", + {"logPrefix", LogPrefix()}, + {"cleanupActorId", cleanupActorId}); return; } @@ -2841,7 +3047,9 @@ private: .UserToken = Context.GetUserToken(), .AllowNotFoundAfterRetry = true, })); - LOG_D("Start TExecuteTransactionSchemeActor " << executerId); + YDB_LOG_DEBUG("[StreamingQueries] Start TExecuteTransactionSchemeActor", + {"logPrefix", LogPrefix()}, + {"executerId", executerId}); return; } @@ -2851,7 +3059,9 @@ private: QueryState.ClearSchemeInfo(); const auto& updaterId = Register(TUpdateStreamingQueryStateRequestActor::MakeRetry(SelfId(), Context.GetDatabaseId(), QueryPath, QueryState)); - LOG_D("Start TUpdateStreamingQueryStateRequestActor " << updaterId); + YDB_LOG_DEBUG("[StreamingQueries] Start TUpdateStreamingQueryStateRequestActor", + {"logPrefix", LogPrefix()}, + {"updaterId", updaterId}); return; } diff --git a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp index 9eddefebe5f..1224eac0fa4 100644 --- a/ydb/core/kqp/gateway/kqp_ic_gateway.cpp +++ b/ydb/core/kqp/gateway/kqp_ic_gateway.cpp @@ -42,6 +42,8 @@ #include <ydb/core/protos/auth.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_GATEWAY + namespace NKikimr { namespace NKqp { @@ -302,8 +304,9 @@ public: void Handle(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) { const TString msg = ev->Get()->GetIssues().ToOneLineString(); - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, SelfId() - << "Received abort execution event for scan query: " << msg); + YDB_LOG_DEBUG_CTX(ctx, "Received abort execution event for scan", + {"selfId", SelfId()}, + {"query", msg}); TBase::HandleError(msg, ctx); } @@ -399,8 +402,9 @@ public: void Handle(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) { const TString msg = ev->Get()->GetIssues().ToOneLineString(); - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, this->SelfId() - << "Received abort execution event for data query: " << msg); + YDB_LOG_DEBUG_CTX(ctx, "Received abort execution event for data query", + {"selfId", this->SelfId()}, + {"query", msg}); TBase::HandleError(msg, ctx); } @@ -465,8 +469,9 @@ public: void Handle(NKqp::TEvKqp::TEvAbortExecution::TPtr& ev, const TActorContext& ctx) { const TString msg = ev->Get()->GetIssues().ToOneLineString(); - LOG_DEBUG_S(ctx, NKikimrServices::KQP_GATEWAY, SelfId() - << "Received abort execution event for query: " << msg); + YDB_LOG_DEBUG_CTX(ctx, "Received abort execution event", + {"selfId", SelfId()}, + {"query", msg}); TBase::HandleError(msg, ctx); } diff --git a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp index bccfbf4731d..c755a2718d4 100644 --- a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp +++ b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp @@ -17,6 +17,8 @@ #include <yql/essentials/providers/common/structured_token/yql_token_builder.h> #include <ydb/library/yql/providers/common/token_accessor/client/factory.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_GATEWAY + namespace NKikimr::NKqp { namespace { @@ -981,18 +983,19 @@ NThreading::TFuture<TTableMetadataResult> TKqpTableMetadataLoader::LoadIndexMeta const auto implTablePaths = NSchemeHelpers::CreateIndexTablePath(tableName, index); for (const auto& implTablePath : implTablePaths) { if (!index.SchemaVersion) { - LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_GATEWAY, "Load index metadata without schema version check index: " << index.Name); + YDB_LOG_DEBUG_CTX(*ActorSystem, "Load index metadata without schema version check", + {"index", index.Name}); children.push_back( LoadTableMetadata(cluster, implTablePath, TLoadTableMetadataSettings().WithPrivateTables(true), database, userToken) ); } else { - LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_GATEWAY, "Load index metadata with schema version check" - << "index: " << index.Name - << "pathId: " << index.LocalPathId - << "ownerId: " << index.PathOwnerId - << "schemaVersion: " << index.SchemaVersion - << "tableOwnerId: " << tableOwnerId); + YDB_LOG_DEBUG_CTX(*ActorSystem, "Load index metadata with schema version check", + {"index", index.Name}, + {"pathId", index.LocalPathId}, + {"ownerId", index.PathOwnerId}, + {"schemaVersion", index.SchemaVersion}, + {"tableOwnerId", tableOwnerId}); auto ownerId = index.PathOwnerId ? index.PathOwnerId : tableOwnerId; //for compat with 20-2 children.push_back( LoadIndexMetadataByPathId(cluster, @@ -1172,7 +1175,8 @@ NThreading::TFuture<TTableMetadataResult> TKqpTableMetadataLoader::LoadTableMeta const auto externalEntry = resolveEntityInsideDataSource ? std::optional<NavigateEntryResult>{} : externalEntryItem; const ui64 expectedSchemaVersion = GetExpectedVersion(entityName); - LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_GATEWAY, "Load table metadata from cache by path, request" << GetDebugString(entityName)); + YDB_LOG_DEBUG_CTX(*ActorSystem, "Loading table metadata from cache", + {"entityName", GetDebugString(entityName)}); auto navigate = MakeHolder<TNavigate>(); navigate->ResultSet.emplace_back(entry); diff --git a/ydb/core/kqp/node_service/kqp_node_service.cpp b/ydb/core/kqp/node_service/kqp_node_service.cpp index c5f052edc1d..284fe95332b 100644 --- a/ydb/core/kqp/node_service/kqp_node_service.cpp +++ b/ydb/core/kqp/node_service/kqp_node_service.cpp @@ -29,6 +29,8 @@ #include <util/string/join.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_NODE + namespace NKikimr { namespace NKqp { @@ -76,8 +78,9 @@ public: } void Bootstrap() { - STLOG_I("Starting KQP Node service", - (node_id, SelfId().NodeId())); + YDB_LOG_INFO("Starting KQP Node service", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}); State_ = std::make_shared<TNodeState>(); @@ -132,10 +135,11 @@ private: IgnoreFunc(NConsole::TEvConsole::TEvConfigNotificationRequest); default: { - STLOG_W("Ignoring unexpected event 0x%x (" << ev->GetTypeName() - << ") during graceful shutdown", - (node_id, SelfId().NodeId()), - (sender, ev->Sender)); + YDB_LOG_WARN("Ignoring unexpected event during graceful shutdown", + {"marker", "KQPNS"}, + {"eventType", ev->GetTypeName()}, + {"nodeId", SelfId().NodeId()}, + {"sender", ev->Sender}); } } } @@ -169,10 +173,11 @@ private: const auto executerId = ev->Sender; auto& reason = ev->Get()->Record.GetReason(); - STLOG_W("Terminate transaction", - (node_id, SelfId().NodeId()), - (tx_id, txId), - (reason, reason)); + YDB_LOG_WARN("Terminate transaction", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}, + {"txId", txId}, + {"reason", reason}); TerminateTx(txId, executerId, reason); Counters->NodeServiceProcessCancelTime->Collect(timer.Passed() * SecToUsec); @@ -182,9 +187,11 @@ private: State_->MarkRequestAsCancelled(executerId); if (auto tasksToAbort = State_->GetTasksByExecuterId(executerId); !tasksToAbort.empty()) { - STLOG_E("Node service cancelled the task, because it " << reason, - (node_id, SelfId().NodeId()), - (tx_id, txId)); + YDB_LOG_ERROR("Node service cancelled the task, because it", + {"marker", "KQPNS"}, + {"reason", reason}, + {"nodeId", SelfId().NodeId()}, + {"txId", txId}); for (const auto& [taskId, computeActorId]: tasksToAbort) { auto abortEv = std::make_unique<TEvKqp::TEvAbortExecution>(status, reason); Send(computeActorId, abortEv.release()); @@ -202,12 +209,14 @@ private: void HandleWork(TEvKqp::TEvInitiateShutdownRequest::TPtr& ev) { if (!AppData()->FeatureFlags.GetEnableShuttingDownNodeState()) { - STLOG_I("Feature flag EnableShuttingDownNodeState is disabled, ignoring shutdown request", - (node_id, SelfId().NodeId())); + YDB_LOG_INFO("Feature flag EnableShuttingDownNodeState is disabled, ignoring shutdown request", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}); return; } - STLOG_I("Prepare to shutdown: do not accept any messages from this time", - (node_id, SelfId().NodeId())); + YDB_LOG_INFO("Prepare to shutdown: do not accept any messages from this time", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}); ShutdownState_.Reset(ev->Get()->ShutdownState.Get()); Become(&TKqpNodeService::ShuttingDownState); } @@ -217,14 +226,16 @@ private: // continue to process tasks that are already started before shutdown auto& msg = ev->Get()->Record; if (ev->Sender.NodeId() == SelfId().NodeId()) { - STLOG_D("Accepting local StartRequest during shutdown", - (node_id, SelfId().NodeId()), - (tx_id, msg.GetTxId())); + YDB_LOG_DEBUG("Accepting local StartRequest during shutdown", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}, + {"txId", msg.GetTxId()}); HandleWork(ev); } else if (msg.HasSupportShuttingDown() && msg.GetSupportShuttingDown()) { - STLOG_D("Rejecting remote StartRequest in ShuttingDown State", - (node_id, SelfId().NodeId()), - (tx_id, msg.GetTxId())); + YDB_LOG_DEBUG("Rejecting remote StartRequest in ShuttingDown State", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}, + {"txId", msg.GetTxId()}); ReplyError(ev->Sender, msg, NKikimrKqp::TEvStartKqpTasksResponse::NODE_SHUTTING_DOWN, ev->Cookie); } else { HandleWork(ev); @@ -258,7 +269,8 @@ private: } private: static void HandleWork(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) { - STLOG_D("Subscribed for config changes"); + YDB_LOG_DEBUG("Subscribed for config changes", + {"marker", "KQPNS"}); } void HandleWork(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) { @@ -283,9 +295,10 @@ private: CaFactory_->ApplyConfig(Config); CaFactory_->AccountDefaultPoolInScheduler.store(event.GetConfig().GetTableServiceConfig().GetComputeSchedulerSettings().GetAccountDefaultPool()); - STLOG_I("Updated table service RM config", - (node_id, SelfId().NodeId()), - (config, Config.DebugString())); + YDB_LOG_INFO("Updated table service RM config", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}, + {"config", Config.DebugString()}); } EnableChannelMemoryTracking = event.GetConfig().GetTableServiceConfig().GetEnableChannelMemoryTracking(); @@ -360,19 +373,22 @@ private: } case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest: - STLOG_C("Failed to deliver subscription request to config dispatcher", - (node_id, SelfId().NodeId())); + YDB_LOG_CRIT("Failed to deliver subscription request to config dispatcher", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}); break; case NConsole::TEvConsole::EvConfigNotificationResponse: - STLOG_E("Failed to deliver config notification response", - (node_id, SelfId().NodeId())); + YDB_LOG_ERROR("Failed to deliver config notification response", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}); break; default: - STLOG_E("Undelivered event with unexpected source type", - (node_id, SelfId().NodeId()), - (source_type, ev->Get()->SourceType)); + YDB_LOG_ERROR("Undelivered event with unexpected source type", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}, + {"sourceType", ev->Get()->SourceType}); break; } } diff --git a/ydb/core/kqp/node_service/kqp_node_service.h b/ydb/core/kqp/node_service/kqp_node_service.h index 00b1b3880d8..0e24206cbe4 100644 --- a/ydb/core/kqp/node_service/kqp_node_service.h +++ b/ydb/core/kqp/node_service/kqp_node_service.h @@ -24,14 +24,6 @@ namespace NYql::NDq { namespace NKikimr::NKqp { -#define STLOG_C(MESSAGE, ...) STLOG(PRI_CRIT, NKikimrServices::KQP_NODE, KQPNS, MESSAGE, __VA_ARGS__) -#define STLOG_E(MESSAGE, ...) STLOG(PRI_ERROR, NKikimrServices::KQP_NODE, KQPNS, MESSAGE, __VA_ARGS__) -#define STLOG_W(MESSAGE, ...) STLOG(PRI_WARN, NKikimrServices::KQP_NODE, KQPNS, MESSAGE, __VA_ARGS__) -#define STLOG_N(MESSAGE, ...) STLOG(PRI_NOTICE, NKikimrServices::KQP_NODE, KQPNS, MESSAGE, __VA_ARGS__) -#define STLOG_I(MESSAGE, ...) STLOG(PRI_INFO, NKikimrServices::KQP_NODE, KQPNS, MESSAGE, __VA_ARGS__) -#define STLOG_D(MESSAGE, ...) STLOG(PRI_DEBUG, NKikimrServices::KQP_NODE, KQPNS, MESSAGE, __VA_ARGS__) -#define STLOG_T(MESSAGE, ...) STLOG(PRI_TRACE, NKikimrServices::KQP_NODE, KQPNS, MESSAGE, __VA_ARGS__) - static constexpr double SecToUsec = 1e6; struct TKqpNodeEvents { diff --git a/ydb/core/kqp/node_service/kqp_query_control_plane.cpp b/ydb/core/kqp/node_service/kqp_query_control_plane.cpp index d1852404e1e..d61f8036dda 100644 --- a/ydb/core/kqp/node_service/kqp_query_control_plane.cpp +++ b/ydb/core/kqp/node_service/kqp_query_control_plane.cpp @@ -39,11 +39,11 @@ struct TMemoryQuotaManager : public NYql::NDq::TGuaranteeQuotaManager { NRm::TKqpResourcesRequest{.Memory = extraSize}); if (!result) { - AFL_WARN(NKikimrServices::KQP_COMPUTE) - ("problem", "cannot_allocate_memory") - ("tx_id", Tx->TxId) - ("task_id", TaskId) - ("memory", extraSize); + YDB_LOG_WARN_COMP(NKikimrServices::KQP_COMPUTE, "", + {"problem", "cannot_allocate_memory"}, + {"txId", Tx->TxId}, + {"taskId", TaskId}, + {"memory", extraSize}); return false; } @@ -108,11 +108,11 @@ struct TChannelQuotaManager : public NYql::NDq::IMemoryQuotaManager { AvailableQuota.fetch_add(memoryRequired); Limit.fetch_add(memoryRequired); } else { - AFL_WARN(NKikimrServices::KQP_COMPUTE) - ("problem", "cannot_allocate_memory") - ("tx_id", Tx->TxId) - ("task_id", 0) - ("memory", memoryRequired); + YDB_LOG_WARN_COMP(NKikimrServices::KQP_COMPUTE, "", + {"problem", "cannot_allocate_memory"}, + {"txId", Tx->TxId}, + {"taskId", 0}, + {"memory", memoryRequired}); if (memoryRequired >= AllocationStep * 10) { AvailableQuota.fetch_add(memorySize); return false; @@ -235,13 +235,14 @@ public: ui32 lockNodeId = msg.GetLockNodeId(); TMaybe<NKikimrDataEvents::ELockMode> lockMode = msg.HasLockMode() ? TMaybe<NKikimrDataEvents::ELockMode>(msg.GetLockMode()) : Nothing(); - STLOG_D("HandleStartKqpTasksRequest", - (node_id, SelfId().NodeId()), - (tx_id, txId), - (requester, executerId), - (tasks_count, msg.GetTasks().size()), - (task_ids, TasksIdsStr(msg.GetTasks())), - (trace_id, ev->TraceId.GetHexTraceIdLowerCase())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_NODE, "HandleStartKqpTasksRequest", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}, + {"txId", txId}, + {"requester", executerId}, + {"tasksCount", msg.GetTasks().size()}, + {"taskIds", TasksIdsStr(msg.GetTasks())}, + {"traceId", ev->TraceId.GetHexTraceIdLowerCase()}); const auto& poolId = msg.GetPoolId().empty() ? NResourcePool::DEFAULT_POOL_ID : msg.GetPoolId(); const auto& databaseId = msg.GetDatabaseId(); @@ -289,12 +290,13 @@ public: ev->Cookie, "Request was cancelled"); } - STLOG_D(((tasks.size() == taskCount) ? "Created new request" : "Added tasks to existing request"), - (node_id, SelfId().NodeId()), - (tx_id, txId), - (tasks_count, tasks.size()), - (executer, executerId), - (trace_id, ev->TraceId.GetHexTraceIdLowerCase())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_NODE, ((tasks.size() == taskCount) ? "Created new request" : "Added tasks to existing request"), + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}, + {"txId", txId}, + {"tasksCount", tasks.size()}, + {"executer", executerId}, + {"traceId", ev->TraceId.GetHexTraceIdLowerCase()}); auto reply = MakeHolder<TEvKqpNode::TEvStartKqpTasksResponse>(); reply->Record.SetTxId(txId); @@ -334,9 +336,12 @@ public: State_->MarkRequestAsCancelled(executerId); if (auto tasksToAbort = State_->GetTasksByExecuterId(executerId); !tasksToAbort.empty()) { - STLOG_E("Node service unable to allocate " << tasksCount << " tasks, reason: " << rmResult.GetFailReason(), - (node_id, SelfId().NodeId()), - (tx_id, txId)); + YDB_LOG_ERROR_COMP(NKikimrServices::KQP_NODE, "Node service unable to allocate tasks", + {"marker", "KQPNS"}, + {"tasksCount", tasksCount}, + {"reason", rmResult.GetFailReason()}, + {"nodeId", SelfId().NodeId()}, + {"txId", txId}); for (const auto& [taskId, computeActorId]: tasksToAbort) { auto abortEv = std::make_unique<TEvKqp::TEvAbortExecution>(NYql::NDqProto::StatusIds::UNSPECIFIED, rmResult.GetFailReason()); Send(computeActorId, abortEv.release()); @@ -393,19 +398,21 @@ public: startedTask->SetTaskId(taskId); ActorIdToProto(actorId, startedTask->MutableActorId()); if (State_->OnTaskStarted(executerId, taskId, actorId)) { - STLOG_D("Executing task", - (node_id, SelfId().NodeId()), - (tx_id, txId), - (task_id, taskId), - (compute_actor_id, actorId), - (trace_id, ev->TraceId.GetHexTraceIdLowerCase())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_NODE, "Executing task", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}, + {"txId", txId}, + {"taskId", taskId}, + {"computeActorId", actorId}, + {"traceId", ev->TraceId.GetHexTraceIdLowerCase()}); } else { - STLOG_D("Task finished in an instant", - (node_id, SelfId().NodeId()), - (tx_id, txId), - (task_id, taskId), - (compute_actor_id, actorId), - (trace_id, ev->TraceId.GetHexTraceIdLowerCase())); + YDB_LOG_DEBUG_COMP(NKikimrServices::KQP_NODE, "Task finished in an instant", + {"marker", "KQPNS"}, + {"nodeId", SelfId().NodeId()}, + {"txId", txId}, + {"taskId", taskId}, + {"computeActorId", actorId}, + {"traceId", ev->TraceId.GetHexTraceIdLowerCase()}); } } diff --git a/ydb/core/kqp/opt/rbo/kqp_rewrite_select.cpp b/ydb/core/kqp/opt/rbo/kqp_rewrite_select.cpp index 54168404428..a9516ce1729 100644 --- a/ydb/core/kqp/opt/rbo/kqp_rewrite_select.cpp +++ b/ydb/core/kqp/opt/rbo/kqp_rewrite_select.cpp @@ -898,7 +898,7 @@ void ProcessAggregationsInResultItems(TExprNode::TPtr result, THashSet<TString>& // For each result item, we want to process result lambda to extract aggregations and pre/post expressions. for (ui32 i = 0, e = result->Child(1)->ChildrenSize(); i < e; ++i) { auto resultItem = result->Child(1)->ChildPtr(i); - ProcessAggregations(resultItem->ChildPtr(2), TString(resultItem->Child(0)->Content()), aggregationUniqueColNames, expressionsMapPreAgg, + ProcessAggregations(resultItem->TailPtr(), TString(resultItem->Child(0)->Content()), aggregationUniqueColNames, expressionsMapPreAgg, groupByKeysExpressionsMap, aggTraits, distinctAggregationTraitsPostAggregate, expressionsMapPostAgg, uniqueAggColumnId, distinctAll, ctx, pos); } @@ -998,7 +998,7 @@ TExprNode::TPtr RewriteSublinks(TExprNode::TPtr& node, TExprContext& ctx, const if (sublinks.empty()) { return node; } - + while (auto currentSize = sublinks.size()) { auto& sublink = sublinks[currentSize-1]; @@ -1044,7 +1044,7 @@ TExprNode::TPtr RewriteSublinks(TExprNode::TPtr& node, TExprContext& ctx, const TExprNode::TPtr RewriteSelect(const TExprNode::TPtr& input, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, const TKqpOptimizeContext& kqpCtx, ui64& uniqueSourceIdCounter, THashMap<const TExprNode*, TExprNode::TPtr>& translated, bool generateRoot) { - + if(translated.contains(input.Get())) { return translated.at(input.Get()); } @@ -1160,7 +1160,7 @@ TExprNode::TPtr RewriteSelect(const TExprNode::TPtr& input, TExprContext& ctx, c TVector<TExprNode::TPtr> joinPredicates; TExprNode::TPtr joinLambda; Y_ENSURE(join->ChildrenSize() > 1 && join->Child(1)->ChildrenSize() > 1); - + auto yqlWhere = join->ChildPtr(1); Y_ENSURE(yqlWhere->IsCallable("YqlWhere"), yqlWhere->Content()); Y_ENSURE(yqlWhere->ChildPtr(1)->IsLambda(), "YqlWhere invalid child type."); @@ -1544,7 +1544,7 @@ TExprNode::TPtr RewriteSelect(const TExprNode::TPtr& input, TExprContext& ctx, c .ForceOptional().Value("False").Build() .Done().Ptr()); // clang-format on - + finalProjection.push_back(columnName); }; @@ -1554,7 +1554,7 @@ TExprNode::TPtr RewriteSelect(const TExprNode::TPtr& input, TExprContext& ctx, c // We can have a single column or mutlitple columns in the item if (maybeColumn->IsAtom()) { - processResultColumn(maybeColumn, resultItem->Child(2)); + processResultColumn(maybeColumn, resultItem->TailPtr()); } // In case of a list of columns, we have different cases: // - Each column can be a list of input/output column names @@ -1574,13 +1574,13 @@ TExprNode::TPtr RewriteSelect(const TExprNode::TPtr& input, TExprContext& ctx, c } else { outputColumn = columnSpec; - auto starLambda = resultItem->Child(2); + auto starLambda = resultItem->TailPtr(); Y_ENSURE(starLambda->IsLambda()); // Output column can be found in the struct inside lambda if (starLambda->Child(1)->IsCallable("AsStruct")) { auto member = starLambda->Child(1)->Child(i); inputColumn = member->Child(1)->Child(1); - } + } // Input is the same as output else { inputColumn = outputColumn; diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index bd59a2837a2..cfc69d14f9f 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -1806,12 +1806,6 @@ public: } if (auto maybeTruncateTable = TMaybeNode<TKiTruncateTable>(input)) { - if (!SessionCtx->Config().FeatureFlags.GetEnableTruncateTable()) { - ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), - TStringBuilder() << "TRUNCATE TABLE statement is disabled. Please contact your system administrator to enable it")); - return SyncError(); - } - auto requireStatus = RequireChild(*input, TKiExecDataQuery::idx_World); if (requireStatus.Level != TStatus::Ok) { return SyncStatus(requireStatus); diff --git a/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp b/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp index ad76fb9cb92..1942839d0d2 100644 --- a/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp +++ b/ydb/core/kqp/proxy_service/kqp_proxy_service.cpp @@ -72,19 +72,12 @@ #include <util/folder/dirut.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_PROXY + namespace NKikimr::NKqp { namespace { -#define KQP_PROXY_LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, stream) -#define KQP_PROXY_LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, stream) -#define KQP_PROXY_LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, stream) -#define KQP_PROXY_LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, stream) -#define KQP_PROXY_LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, stream) -#define KQP_PROXY_LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, stream) -#define KQP_PROXY_LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, stream) - - static constexpr TDuration DEFAULT_KEEP_ALIVE_TIMEOUT = TDuration::MilliSeconds(5000); static constexpr TDuration DEFAULT_EXTRA_TIMEOUT_WAIT = TDuration::MilliSeconds(50); static constexpr TDuration DEFAULT_CREATE_SESSION_TIMEOUT = TDuration::MilliSeconds(5000); @@ -269,7 +262,8 @@ public: TStringStream errorStream; ModuleResolverState->ExprCtx.IssueManager.GetIssues().PrintTo(errorStream); - KQP_PROXY_LOG_E("Failed to load default YQL libraries: " << errorStream.Str()); + YDB_LOG_ERROR("Failed to load default YQL", + {"libraries", errorStream.Str()}); PassAway(); } @@ -534,14 +528,14 @@ public: } void Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) { - KQP_PROXY_LOG_D("Subscribed for config changes."); + YDB_LOG_DEBUG("Subscribed for config changes"); } void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) { auto &event = ev->Get()->Record; TableServiceConfig.Swap(event.MutableConfig()->MutableTableServiceConfig()); - KQP_PROXY_LOG_D("Updated table service config."); + YDB_LOG_DEBUG("Updated table service config"); ExecuterConfig->ApplyFromTableServiceConfig(TableServiceConfig); RebuildKqpConfig(); @@ -581,26 +575,28 @@ public: void Handle(TEvents::TEvUndelivered::TPtr& ev) { switch (ev->Get()->SourceType) { case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest: - KQP_PROXY_LOG_C("Failed to deliver subscription request to config dispatcher."); + YDB_LOG_CRIT("Failed to deliver subscription request to config dispatcher"); break; case NConsole::TEvConsole::EvConfigNotificationResponse: - KQP_PROXY_LOG_E("Failed to deliver config notification response."); + YDB_LOG_ERROR("Failed to deliver config notification response"); break; case NNodeWhiteboard::TEvWhiteboard::EvSystemStateRequest: - KQP_PROXY_LOG_D("Failed to get system details"); + YDB_LOG_DEBUG("Failed to get system details"); break; case TKqpEvents::EvCreateSessionRequest: { - KQP_PROXY_LOG_D("Remote create session request failed"); + YDB_LOG_DEBUG("Remote create session request failed"); ReplyProcessError(Ydb::StatusIds::UNAVAILABLE, "Session not found.", ev->Cookie); break; } case TKqpEvents::EvQueryRequest: case TKqpEvents::EvPingSessionRequest: { - KQP_PROXY_LOG_D("Session not found, targetId: " << ev->Sender << " requestId: " << ev->Cookie); + YDB_LOG_DEBUG("Session not found", + {"targetId", ev->Sender}, + {"requestId", ev->Cookie}); ReplyProcessError(Ydb::StatusIds::BAD_SESSION, "Session not found.", ev->Cookie); RemoveSession("", ev->Sender); @@ -608,13 +604,14 @@ public: } default: - KQP_PROXY_LOG_E("Undelivered event with unexpected source type: " << ev->Get()->SourceType); + YDB_LOG_ERROR("Undelivered event with unexpected source", + {"type", ev->Get()->SourceType}); break; } } void Handle(TEvKqp::TEvInitiateShutdownRequest::TPtr& ev) { - KQP_PROXY_LOG_N("KQP proxy shutdown requested."); + YDB_LOG_NOTICE("KQP proxy shutdown requested"); ShutdownRequested = true; ShutdownState.Reset(ev->Get()->ShutdownState.Get()); ShutdownState->Update(LocalSessions->size()); @@ -696,7 +693,8 @@ public: } Counters->ReportCreateSession(dbCounters, request.ByteSize()); - KQP_PROXY_LOG_D("Received create session request, trace_id: " << event.GetTraceId()); + YDB_LOG_DEBUG("Received create session request", + {"traceId", event.GetTraceId()}); responseEv->Record.SetResourceExhausted(result.ResourceExhausted); responseEv->Record.SetYdbStatus(result.YdbStatus); @@ -827,9 +825,13 @@ public: if (cancelAfter) { timerDuration = Min(timerDuration, cancelAfter); } - KQP_PROXY_LOG_D("Ctx: " << *ev->Get()->GetUserRequestContext() << ". TEvQueryRequest, set timer for: " << timerDuration - << " timeout: " << timeout << " cancelAfter: " << cancelAfter - << ". " << "Send request to target, requestId: " << requestId << ", targetId: " << targetId); + YDB_LOG_DEBUG("TEvQueryRequest, set timer Send request to target", + {"ctx", *ev->Get()->GetUserRequestContext()}, + {"for", timerDuration}, + {"timeout", timeout}, + {"cancelAfter", cancelAfter}, + {"requestId", requestId}, + {"targetId", targetId}); auto status = timerDuration == cancelAfter ? NYql::NDqProto::StatusIds::CANCELLED : NYql::NDqProto::StatusIds::TIMEOUT; StartQueryTimeout(requestId, timerDuration, status); Send(targetId, ev->Release().Release(), IEventHandle::FlagTrackDelivery, requestId, std::move(ev->TraceId)); @@ -894,10 +896,11 @@ public: // Local session if (sessionInfo) { const bool sameNode = ev->Sender.NodeId() == SelfId().NodeId(); - KQP_PROXY_LOG_D("Received ping session request, has local session: " << sessionId - << ", rpc ctrl: " << ctrlActor - << ", sameNode: " << sameNode - << ", trace_id: " << traceId); + YDB_LOG_DEBUG("Received ping session request, has local rpc", + {"session", sessionId}, + {"ctrl", ctrlActor}, + {"sameNode", sameNode}, + {"traceId", traceId}); const bool isIdle = LocalSessions->IsSessionIdle(sessionInfo); if (isIdle) { @@ -918,14 +921,18 @@ public: //TODO: fix ui32 flags = IEventHandle::FlagTrackDelivery; if (sameNode) { - KQP_PROXY_LOG_T("Attach local session: " << sessionInfo->WorkerId - << " to rpc: " << ctrlActor << " on same node"); + YDB_LOG_TRACE("Attach local to on same node", + {"session", sessionInfo->WorkerId}, + {"rpc", ctrlActor}); LocalSessions->AttachSession(sessionInfo, 0, ctrlActor); } else { const TNodeId nodeId = ev->Sender.NodeId(); - KQP_PROXY_LOG_T("Subscribe local session: " << sessionInfo->WorkerId - << " to remote: " << ev->Sender << " , nodeId: " << nodeId << ", with rpc: " << ctrlActor); + YDB_LOG_TRACE("Subscribe local to with", + {"session", sessionInfo->WorkerId}, + {"remote", ev->Sender}, + {"nodeId", nodeId}, + {"rpc", ctrlActor}); LocalSessions->AttachSession(sessionInfo, nodeId, ctrlActor); @@ -941,9 +948,10 @@ public: // Forward request to another proxy ui64 requestId = PendingRequests.RegisterRequest(ev->Sender, ev->Cookie, traceId, TKqpEvents::EvPingSessionRequest); - KQP_PROXY_LOG_D("Received ping session request, request_id: " << requestId - << ", sender: " << ev->Sender - << ", trace_id: " << traceId); + YDB_LOG_DEBUG("Received ping session request", + {"requestId", requestId}, + {"sender", ev->Sender}, + {"traceId", traceId}); const TActorId targetId = TryGetSessionTargetActor(sessionId, requestInfo, requestId); if (!targetId) { @@ -970,7 +978,9 @@ public: ui64 requestId = PendingRequests.RegisterRequest(ev->Sender, ev->Cookie, traceId, TKqpEvents::EvCancelQueryRequest); const TKqpSessionInfo* sessionInfo = LocalSessions->FindPtr(sessionId); auto dbCounters = sessionInfo ? sessionInfo->DbCounters : nullptr; - KQP_PROXY_LOG_D("Received cancel query request, request_id: " << requestId << ", trace_id: " << traceId); + YDB_LOG_DEBUG("Received cancel query request", + {"requestId", requestId}, + {"traceId", traceId}); Counters->ReportCancelQuery(dbCounters, request.ByteSize()); PendingRequests.SetSessionId(requestId, sessionId, dbCounters); @@ -987,8 +997,10 @@ public: } Send(targetId, ev->Release().Release(), IEventHandle::FlagTrackDelivery, requestId); - KQP_PROXY_LOG_D("Sent request to target, requestId: " << requestId - << ", targetId: " << targetId << ", sessionId: " << sessionId); + YDB_LOG_DEBUG("Sent request to target", + {"requestId", requestId}, + {"targetId", targetId}, + {"sessionId", sessionId}); } template<typename TEvent> @@ -998,7 +1010,8 @@ public: StopQueryTimeout(requestId); auto proxyRequest = PendingRequests.FindPtr(requestId); if (!proxyRequest) { - KQP_PROXY_LOG_E("Unknown sender for proxy response, requestId: " << requestId); + YDB_LOG_ERROR("Unknown sender for proxy response", + {"requestId", requestId}); return; } @@ -1014,8 +1027,12 @@ public: } TKqpRequestInfo requestInfo(proxyRequest->TraceId); - KQP_PROXY_LOG_D(requestInfo << "Forwarded response to sender actor, requestId: " << requestId - << ", sender: " << proxyRequest->Sender << ", selfId: " << SelfId() << ", source: " << ev->Sender); + YDB_LOG_DEBUG("Forwarded response to sender actor", + {"requestInfo", requestInfo}, + {"requestId", requestId}, + {"sender", proxyRequest->Sender}, + {"selfId", SelfId()}, + {"source", ev->Sender}); PendingRequests.Erase(requestId); } @@ -1025,15 +1042,20 @@ public: auto proxyRequest = PendingRequests.FindPtr(requestId); if (!proxyRequest) { - KQP_PROXY_LOG_E("Unknown sender for proxy response, requestId: " << requestId); + YDB_LOG_ERROR("Unknown sender for proxy response", + {"requestId", requestId}); return; } Send(proxyRequest->Sender, ev->Release().Release(), 0, proxyRequest->SenderCookie); TKqpRequestInfo requestInfo(proxyRequest->TraceId); - KQP_PROXY_LOG_D(requestInfo << "Forwarded response to sender actor, requestId: " << requestId - << ", sender: " << proxyRequest->Sender << ", selfId: " << SelfId() << ", source: " << ev->Sender); + YDB_LOG_DEBUG("Forwarded response to sender actor", + {"requestInfo", requestInfo}, + {"requestId", requestId}, + {"sender", proxyRequest->Sender}, + {"selfId", SelfId()}, + {"source", ev->Sender}); } void Handle(TEvPrivate::TEvCollectPeerProxyData::TPtr&) { @@ -1088,8 +1110,8 @@ public: if (proxyResources.empty()) { PeerProxyNodeResources.clear(); - KQP_PROXY_LOG_D("Received unexpected data from rm for database " << - AppData()->TenantName); + YDB_LOG_DEBUG("Received unexpected resource manager data for tenant", + {"tenantName", AppData()->TenantName}); return; } @@ -1118,8 +1140,8 @@ public: for (const auto& resource : PeerProxyNodeResources) { nodeIds.push_back(resource.GetNodeId()); } - KQP_PROXY_LOG_I("Discovered " << PeerProxyNodeResources.size() - << " proxy nodes, starting warmup"); + YDB_LOG_INFO("Discovered proxy nodes, starting warmup", + {"peerProxyNodesCount", PeerProxyNodeResources.size()}); Send(MakeKqpWarmupActorId(SelfId().NodeId()), new TEvStartWarmup(PeerProxyNodeResources.size(), std::move(nodeIds))); } } @@ -1131,7 +1153,7 @@ public: void Handle(TEvPrivate::TEvWarmupGateFallback::TPtr&) { if (!WarmupGateOpen) { - KQP_PROXY_LOG_W("Warmup gate fallback fired: opening gate (no TEvKqpWarmupComplete received, warmup actor likely died)"); + YDB_LOG_WARN("Warmup gate fallback fired: opening gate (no TEvKqpWarmupComplete received, warmup actor likely died)"); WarmupGateOpen = true; } } @@ -1219,7 +1241,8 @@ public: return; const auto& sbs = TableServiceConfig.GetSessionBalancerSettings(); - KQP_PROXY_LOG_D("Started grace shutdown of session, session id: " << sessionInfo->SessionId); + YDB_LOG_DEBUG("Started grace shutdown of session, session", + {"id", sessionInfo->SessionId}); ui32 hardTimeout = sbs.GetHardSessionShutdownTimeoutMs(); ui32 softTimeout = sbs.GetSoftSessionShutdownTimeoutMs(); Counters->ReportSessionShutdownRequest(sessionInfo->DbCounters); @@ -1335,7 +1358,10 @@ public: new IEventHandle(SelfId(), SelfId(), new TEvPrivate::TEvOnRequestTimeout{requestId, timeout, status, 0}) ); - KQP_PROXY_LOG_D("Scheduled timeout timer for requestId: " << requestId << " timeout: " << timeout << " actor id: " << timeoutTimer); + YDB_LOG_DEBUG("Scheduled timeout timer for actor", + {"requestId", requestId}, + {"timeout", timeout}, + {"id", timeoutTimer}); if (timeoutTimer) { TimeoutTimers.emplace(requestId, timeoutTimer); } @@ -1354,15 +1380,20 @@ public: ui64 requestId = ev->Get()->RequestId; TimeoutTimers.erase(requestId); - KQP_PROXY_LOG_D("Handle TEvPrivate::TEvOnRequestTimeout(" << requestId << ")"); + YDB_LOG_DEBUG("Handle TEvPrivate::TEvOnRequestTimeout(", + {"requestId", requestId}); const TKqpProxyRequest* reqInfo = PendingRequests.FindPtr(requestId); if (!reqInfo) { - KQP_PROXY_LOG_D("Invalid request info while on request timeout handle. RequestId: " << requestId); + YDB_LOG_DEBUG("Invalid request info while on request timeout handle", + {"requestId", requestId}); return; } - KQP_PROXY_LOG_D("Reply timeout: requestId " << requestId << " sessionId: " << reqInfo->SessionId - << " status: " << NYql::NDq::DqStatusToYdbStatus(msg->Status) << " round: " << msg->Round); + YDB_LOG_DEBUG("Reply timeout: requestId", + {"requestId", requestId}, + {"sessionId", reqInfo->SessionId}, + {"status", NYql::NDq::DqStatusToYdbStatus(msg->Status)}, + {"round", msg->Round}); const TKqpSessionInfo* info = LocalSessions->FindPtr(reqInfo->SessionId); if (msg->Round == 0 && info) { @@ -1393,8 +1424,10 @@ public: RemoveSession(sessionId, workerId); - KQP_PROXY_LOG_D("Session closed, sessionId: " << event.GetResponse().GetSessionId() - << ", workerId: " << workerId << ", local sessions count: " << LocalSessions->size()); + YDB_LOG_DEBUG("Session closed, local sessions", + {"sessionId", event.GetResponse().GetSessionId()}, + {"workerId", workerId}, + {"count", LocalSessions->size()}); } } @@ -1474,7 +1507,10 @@ private: return true; } - KQP_PROXY_LOG_W("Reply process error for request " << static_cast<ui64>(request->EventType) << ", status: " << ydbStatus << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("Replying with process error for request", + {"eventType", static_cast<ui64>(request->EventType)}, + {"status", ydbStatus}, + {"issues", issues.ToOneLineString()}); if (request->EventType == TKqpEvents::EvPingSessionRequest) { auto response = std::make_unique<TEvKqp::TEvPingSessionResponse>(); @@ -1503,14 +1539,18 @@ private: auto now = TInstant::Now(); if (now >= deadline) { TString error = TStringBuilder() << "Request deadline has expired for " << now - deadline << " seconds"; - KQP_PROXY_LOG_E(requestInfo << error); + YDB_LOG_ERROR("Request deadline has expired", + {"requestInfo", requestInfo}, + {"error", error}); // In theory client should not see this status due to internal grpc deadline accounting. result.YdbStatus = Ydb::StatusIds::TIMEOUT; result.Error = error; return false; } else { - KQP_PROXY_LOG_D(requestInfo << "Request has " << deadline - now << " seconds to be completed"); + YDB_LOG_DEBUG("Request deadline approaching", + {"requestInfo", requestInfo}, + {"timeUntilDeadline", deadline - now}); return true; } } @@ -1527,7 +1567,9 @@ private: if (!database.empty() && AppData()->TenantName.empty()) { TString error = TStringBuilder() << "Node isn't ready to serve database requests."; - KQP_PROXY_LOG_E(requestInfo << error); + YDB_LOG_ERROR("Node is not ready to serve database requests", + {"requestInfo", requestInfo}, + {"error", error}); result.YdbStatus = Ydb::StatusIds::UNAVAILABLE; result.Error = error; @@ -1537,7 +1579,9 @@ private: if (ShutdownRequested) { TString error = TStringBuilder() << "Cannot create session: system shutdown requested."; - KQP_PROXY_LOG_N(requestInfo << error); + YDB_LOG_NOTICE("Cannot create session: system shutdown requested", + {"requestInfo", requestInfo}, + {"error", error}); result.ResourceExhausted = true; result.YdbStatus = Ydb::StatusIds::OVERLOADED; @@ -1549,7 +1593,9 @@ private: if (sessionsLimitPerNode && !LocalSessions->CheckDatabaseLimits(database, sessionsLimitPerNode)) { TString error = TStringBuilder() << "Active sessions limit exceeded, maximum allowed: " << sessionsLimitPerNode; - KQP_PROXY_LOG_W(requestInfo << error); + YDB_LOG_WARN("Active sessions limit exceeded", + {"requestInfo", requestInfo}, + {"error", error}); result.YdbStatus = Ydb::StatusIds::OVERLOADED; result.Error = error; @@ -1579,12 +1625,13 @@ private: sessionInfo->ClientPID = clientPid; sessionInfo->ClientApplicationName = clientApplicationName; - KQP_PROXY_LOG_D(requestInfo << "Created new session" - << ", sessionId: " << sessionInfo->SessionId - << ", workerId: " << sessionInfo->WorkerId - << ", database: " << sessionInfo->Database - << ", longSession: " << longSession - << ", local sessions count: " << LocalSessions->size()); + YDB_LOG_DEBUG("Created new session local sessions", + {"requestInfo", requestInfo}, + {"sessionId", sessionInfo->SessionId}, + {"workerId", sessionInfo->WorkerId}, + {"database", sessionInfo->Database}, + {"longSession", longSession}, + {"count", LocalSessions->size()}); result.YdbStatus = Ydb::StatusIds::SUCCESS; result.Error.clear(); @@ -1597,14 +1644,18 @@ private: auto nodeId = TryDecodeYdbSessionId(sessionId); if (!nodeId) { TString error = TStringBuilder() << "Failed to parse session id: " << sessionId; - KQP_PROXY_LOG_W(requestInfo << error); + YDB_LOG_WARN("Failed to parse session id", + {"requestInfo", requestInfo}, + {"error", error}); ReplyProcessError(Ydb::StatusIds::BAD_REQUEST, error, requestId); return TActorId(); } if (*nodeId == SelfId().NodeId()) { TString error = TStringBuilder() << "Session not found: " << sessionId; - KQP_PROXY_LOG_N(requestInfo << error); + YDB_LOG_NOTICE("Session not found on this node", + {"requestInfo", requestInfo}, + {"error", error}); ReplyProcessError(Ydb::StatusIds::BAD_SESSION, error, requestId); return TActorId(); } @@ -1680,7 +1731,8 @@ private: if (entry.GetComponent() == kqpYqlName && entry.HasLevel()) { auto yqlPriority = static_cast<NActors::NLog::EPriority>(entry.GetLevel()); NYql::NDq::SetYqlLogLevels(yqlPriority); - KQP_PROXY_LOG_D("Updated YQL logs priority: " << (ui32)yqlPriority); + YDB_LOG_DEBUG("Updated YQL logs", + {"priority", (ui32)yqlPriority}); return; } } @@ -1689,7 +1741,8 @@ private: ui8 currentLevel = TlsActivationContext->LoggerSettings()->GetComponentSettings(NKikimrServices::KQP_YQL).Raw.X.Level; auto yqlPriority = static_cast<NActors::NLog::EPriority>(currentLevel); - KQP_PROXY_LOG_D("Updated YQL logs priority to current level: " << (ui32)yqlPriority); + YDB_LOG_DEBUG("Updated YQL logs priority to current", + {"level", (ui32)yqlPriority}); NYql::NDq::SetYqlLogLevels(yqlPriority); } @@ -1839,17 +1892,21 @@ private: TNodeId nodeId = ev->Get()->NodeId; auto sessions = LocalSessions->FindSessions(nodeId); if (sessions) { - KQP_PROXY_LOG_T("Got TEvNodeConnected event from node: " << nodeId - << ", has " << sessions.size() << " sessions"); + YDB_LOG_TRACE("Received node connected event with local sessions", + {"node", nodeId}, + {"sessionsCount", sessions.size()}); } else { - KQP_PROXY_LOG_E("Got TEvNodeConnected event from node without sessions: " << nodeId); + YDB_LOG_ERROR("Got TEvNodeConnected event from node without", + {"sessions", nodeId}); } } void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { TNodeId nodeId = ev->Get()->NodeId; auto sessions = LocalSessions->FindSessions(nodeId); - KQP_PROXY_LOG_D("Node: " << nodeId << " disconnected, had " << sessions.size() << " sessions."); + YDB_LOG_DEBUG("Node disconnected with active local sessions", + {"node", nodeId}, + {"sessionsCount", sessions.size()}); const static auto IdleDurationAfterDisconnect = TDuration::Seconds(1); // Just start standard idle check with small timeout // It allows to use common code to close and delete expired session @@ -1859,7 +1916,9 @@ private: } void Handle(TEvKqp::TEvListSessionsRequest::TPtr& ev) { - KQP_PROXY_LOG_D("incoming list sessions request " << ev->Get()->Record.ShortUtf8DebugString() << ", local sessions #" << LocalSessions->size()); + YDB_LOG_DEBUG("Received list sessions request", + {"request", ev->Get()->Record.ShortUtf8DebugString()}, + {"localSessionsCount", LocalSessions->size()}); auto result = std::make_unique<TEvKqp::TEvListSessionsResponse>(); diff --git a/ydb/core/kqp/proxy_service/kqp_script_executions.cpp b/ydb/core/kqp/proxy_service/kqp_script_executions.cpp index 359789c61d7..a768357ac13 100644 --- a/ydb/core/kqp/proxy_service/kqp_script_executions.cpp +++ b/ydb/core/kqp/proxy_service/kqp_script_executions.cpp @@ -60,20 +60,14 @@ #include <memory> #include <vector> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_PROXY + namespace NKikimr::NKqp { using namespace NKikimr::NKqp::NPrivate; namespace { -#define KQP_PROXY_LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[ScriptExecutions] " << LogPrefix() << stream) -#define KQP_PROXY_LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[ScriptExecutions] " << LogPrefix() << stream) -#define KQP_PROXY_LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[ScriptExecutions] " << LogPrefix() << stream) -#define KQP_PROXY_LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[ScriptExecutions] " << LogPrefix() << stream) -#define KQP_PROXY_LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[ScriptExecutions] " << LogPrefix() << stream) -#define KQP_PROXY_LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[ScriptExecutions] " << LogPrefix() << stream) -#define KQP_PROXY_LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_PROXY, "[ScriptExecutions] " << LogPrefix() << stream) - constexpr TDuration LEASE_DURATION = TDuration::Seconds(30); Ydb::Query::ExecMode GetExecModeFromAction(const NKikimrKqp::EQueryAction action) { @@ -295,7 +289,9 @@ public: protected: bool ValidateUserSID() { if (const auto& error = CheckScriptExecutionAccess(UserSID)) { - KQP_PROXY_LOG_W("Token validation failed: " << error); + YDB_LOG_WARN("[ScriptExecutions] Token validation failed", + {"logPrefix", LogPrefix()}, + {"error", error}); TBase::Finish(Ydb::StatusIds::UNAUTHORIZED, error); return false; } @@ -309,7 +305,9 @@ protected: } if (const auto& ownerUser = result.ColumnParser("user_token").GetOptionalUtf8(); ownerUser && *ownerUser != *UserSID) { - KQP_PROXY_LOG_W("Access denied for user " << *UserSID); + YDB_LOG_WARN("[ScriptExecutions] Access denied for user", + {"logPrefix", LogPrefix()}, + {"userSid", *UserSID}); TBase::Finish(Ydb::StatusIds::UNAUTHORIZED, "Access denied. User is not owner of script execution operation."); return false; } @@ -501,7 +499,9 @@ public: private: void OnRunQuery() override { const auto metaTtl = std::min(MaxRunTime.MicroSeconds(), NYql::NUdf::MAX_TIMESTAMP - 1); - KQP_PROXY_LOG_D("Creating query in database, meta ttl: " << TDuration::MicroSeconds(metaTtl)); + YDB_LOG_DEBUG("[ScriptExecutions] Creating script execution query in database", + {"logPrefix", LogPrefix()}, + {"metaTtl", TDuration::MicroSeconds(metaTtl)}); constexpr char sql[] = R"( -- TCreateScriptOperationQuery::OnRunQuery @@ -622,11 +622,12 @@ private: } void OnFinish(const Ydb::StatusIds::StatusCode status, NYql::TIssues&& issues) override { - KQP_PROXY_LOG_D("Create script execution operation" - << ", RetryState: " << RetryState.ShortDebugString() - << ", has PhysicalGraph: " << PhysicalGraph.has_value() - << ", Result: " << status - << ", Issues: " << issues.ToOneLineString()); + YDB_LOG_DEBUG("[ScriptExecutions] Create script execution operation finished", + {"logPrefix", LogPrefix()}, + {"retryState", RetryState.ShortDebugString()}, + {"physicalGraph", PhysicalGraph.has_value()}, + {"status", status}, + {"issues", issues.ToOneLineString()}); if (status == Ydb::StatusIds::SUCCESS) { Send(Owner, new TEvPrivate::TEvCreateScriptOperationResponse(ExecutionId, std::move(issues))); @@ -721,17 +722,27 @@ public: } const auto& creatorId = Register(TCreateScriptOperationQuery::MakeRetry(SelfId(), ExecutionId, RunScriptActorId, ev.Record, std::move(meta), MaxRunTime, GetRetryState(), ev.QueryPhysicalGraph, QueryServiceConfig, std::move(disposition), ev.Generation)); - KQP_PROXY_LOG_D("Bootstrap. Start TCreateScriptOperationQuery " << creatorId << ", RunScriptActorId: " << RunScriptActorId); + YDB_LOG_DEBUG("[ScriptExecutions] Bootstrap: starting TCreateScriptOperationQuery", + {"logPrefix", LogPrefix()}, + {"creatorId", creatorId}, + {"runScriptActorId", RunScriptActorId}); } void Handle(TEvPrivate::TEvCreateScriptOperationResponse::TPtr& ev) { if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { - KQP_PROXY_LOG_W("Create script operation " << ev->Sender << " failed " << status << ", Issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Create script operation failed", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", ev->Get()->Issues.ToOneLineString()}); SendFailResponse(status, AddRootIssue("Internal error. Failed to save meta information about new script execution", ev->Get()->Issues)); return; } - KQP_PROXY_LOG_D("Create script operation " << ev->Sender << " succeeded, RunScriptActorId: " << RunScriptActorId); + YDB_LOG_DEBUG("[ScriptExecutions] Create script operation succeeded", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"runScriptActorId", RunScriptActorId}); Send(Event->Sender, new TEvKqp::TEvScriptResponse( ScriptExecutionOperationFromExecutionId(ev->Get()->ExecutionId), @@ -744,7 +755,9 @@ public: } bool OnUnhandledException(const std::exception& ex) final { - KQP_PROXY_LOG_E("Got unexpected exception: " << ex.what()); + YDB_LOG_ERROR("[ScriptExecutions] Got unexpected exception", + {"logPrefix", LogPrefix()}, + {"exception", ex.what()}); SendFailResponse(Ydb::StatusIds::INTERNAL_ERROR, TStringBuilder() << "Got unexpected exception: " << ex.what()); return true; } @@ -840,7 +853,9 @@ public: private: void OnRunQuery() override { - KQP_PROXY_LOG_D("Update lease on duration: " << LeaseDuration); + YDB_LOG_DEBUG("[ScriptExecutions] Updating script execution lease", + {"logPrefix", LogPrefix()}, + {"leaseDuration", LeaseDuration}); constexpr char sql[] = R"( -- TScriptLeaseUpdaterQuery::OnRunQuery @@ -876,7 +891,7 @@ private: void UpdateLease() { // Updating the lease in the table can take a long time, - // so the query uses CurrentUtcTimestamp(), + // so the query uses CurrentUtcTimestamp(), // but for the next update, LeaseDeadline is used, // which corresponds to a strictly shorter time. LeaseDeadline = TInstant::Now() + LeaseDuration; @@ -1141,7 +1156,10 @@ private: .StreamingDisposition = streamingDisposition, }, QueryServiceConfig)); - KQP_PROXY_LOG_D("Restart with RunScriptActorId: " << RunScriptActorId << ", has PhysicalGraph: " << hasPhysicalGraph); + YDB_LOG_DEBUG("[ScriptExecutions] Restarting script execution operation", + {"logPrefix", LogPrefix()}, + {"runScriptActorId", RunScriptActorId}, + {"hasPhysicalGraph", hasPhysicalGraph}); RestartScriptExecution(); } @@ -1258,7 +1276,9 @@ public: private: void OnRunQuery() override { - KQP_PROXY_LOG_D("Start, Cookie: " << Cookie); + YDB_LOG_DEBUG("[ScriptExecutions] Starting lease status check", + {"logPrefix", LogPrefix()}, + {"cookie", Cookie}); constexpr char sql[] = R"( -- TCheckLeaseStatusQueryActor::OnRunQuery @@ -1468,7 +1488,10 @@ public: void Bootstrap() { const auto& checkerId = Register(TCheckLeaseStatusQueryActor::MakeRetry(SelfId(), Database, ExecutionId, TCheckLeaseStatusQueryActor::TSettings{})); - KQP_PROXY_LOG_D("Bootstrap, Cookie: " << Settings.Cookie << ". Start TCheckLeaseStatusQueryActor " << checkerId); + YDB_LOG_DEBUG("[ScriptExecutions] Bootstrap: starting TCheckLeaseStatusQueryActor", + {"logPrefix", LogPrefix()}, + {"cookie", Settings.Cookie}, + {"checkerId", checkerId}); Become(&TThis::StateFunc); } @@ -1495,30 +1518,41 @@ private: void Handle(TEvPrivate::TEvLeaseCheckResult::TPtr& ev) { auto& event = *ev->Get(); if (event.Status != Ydb::StatusIds::SUCCESS) { - KQP_PROXY_LOG_W("Failed to check lease " << ev->Sender << " status, Status: " << event.Status << ", Issues: " << event.Issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Failed to check lease status", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", event.Status}, + {"issues", event.Issues.ToOneLineString()}); return Reply(event.Status, std::move(event.Issues)); } if (!event.EntryExists) { - KQP_PROXY_LOG_W("Lease check " << ev->Sender << " finished with not found, Status: " << event.Status << ", Issues: " << event.Issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Lease check finished: script execution operation not found", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", event.Status}, + {"issues", event.Issues.ToOneLineString()}); return ReplyNotFound(); } RunScriptActorId = event.RunScriptActorId; - KQP_PROXY_LOG_D("Extracted script execution operation " << ev->Sender - << ", Status: " << event.Status - << ", Issues: " << event.Issues.ToOneLineString() - << ", LeaseExpired: " << event.LeaseExpired - << ", RetryRequired: " << event.RetryRequired - << (event.OperationStatus ? ", OperationStatus: " + Ydb::StatusIds::StatusCode_Name(*event.OperationStatus) : "") - << (event.ExecutionStatus ? ", ExecutionStatus: " + Ydb::Query::ExecStatus_Name(*event.ExecutionStatus) : "") - << ", OperationIssues: " << event.OperationIssues.ToOneLineString() - << (event.FinalizationStatus ? (TStringBuilder() << ", FinalizationStatus: " << static_cast<ui64>(*event.FinalizationStatus)) : TStringBuilder()) - << ", RunScriptActorId: " << RunScriptActorId - << ", LeaseGeneration: " << event.LeaseGeneration); + YDB_LOG_DEBUG("[ScriptExecutions] Extracted script execution operation from lease check", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", event.Status}, + {"issues", event.Issues.ToOneLineString()}, + {"leaseExpired", event.LeaseExpired}, + {"retryRequired", event.RetryRequired}, + {"operationStatus", (event.OperationStatus ? Ydb::StatusIds::StatusCode_Name(*event.OperationStatus) : "<null>")}, + {"executionStatus", (event.ExecutionStatus ? Ydb::Query::ExecStatus_Name(*event.ExecutionStatus) : "<null>")}, + {"operationIssues", event.OperationIssues.ToOneLineString()}, + {"finalizationStatus", (event.FinalizationStatus ? ToString(static_cast<ui64>(*event.FinalizationStatus)) : "<null>")}, + {"runScriptActorId", RunScriptActorId}, + {"leaseGeneration", event.LeaseGeneration}); if (!event.LeaseExpired) { - KQP_PROXY_LOG_N("Lease finalization skipped, lease not expired"); + YDB_LOG_NOTICE("[ScriptExecutions] Lease finalization skipped, lease not expired", + {"logPrefix", LogPrefix()}); LeaseVerified = true; return Reply(); } @@ -1526,9 +1560,13 @@ private: if (event.RetryRequired) { if (Settings.AllowRestart) { const auto& restartActorId = Register(TRestartScriptOperationQuery::MakeRetry(SelfId(), Database, ExecutionId, event.LeaseGeneration, QueryServiceConfig, Counters)); - KQP_PROXY_LOG_N("Restarting script execution " << restartActorId << ", lease generation: " << event.LeaseGeneration); + YDB_LOG_NOTICE("[ScriptExecutions] Restarting script execution lease", + {"logPrefix", LogPrefix()}, + {"restartActorId", restartActorId}, + {"generation", event.LeaseGeneration}); } else { - KQP_PROXY_LOG_N("Lease finalization skipped, script execution wait retry"); + YDB_LOG_NOTICE("[ScriptExecutions] Lease finalization skipped because script execution is waiting for retry", + {"logPrefix", LogPrefix()}); Reply(); } return; @@ -1549,12 +1587,13 @@ private: execStatus = *event.ExecutionStatus; } - KQP_PROXY_LOG_W("Script execution lease is expired" - << ", FinalizationStatus: " << finalizationStatus - << ", Status: " << status - << ", ExecStatus: " << Ydb::Query::ExecStatus_Name(execStatus) - << ", Issues: " << issues.ToOneLineString() - << ", LeaseGeneration: " << event.LeaseGeneration); + YDB_LOG_WARN("[ScriptExecutions] Script execution lease is expired", + {"logPrefix", LogPrefix()}, + {"finalizationStatus", finalizationStatus}, + {"status", status}, + {"execStatus", Ydb::Query::ExecStatus_Name(execStatus)}, + {"issues", issues.ToOneLineString()}, + {"leaseGeneration", event.LeaseGeneration}); ScriptFinalizeRequest = std::make_unique<TEvScriptFinalizeRequest>( finalizationStatus, @@ -1582,9 +1621,13 @@ private: void Handle(TEvCheckAliveResponse::TPtr& ev) { if (WaitFinishQuery) { - KQP_PROXY_LOG_W("Script execution " << ev->Sender << " lease was verified after started finalization"); + YDB_LOG_WARN("[ScriptExecutions] Script execution lease was verified after started finalization", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); } else { - KQP_PROXY_LOG_N("Script execution " << ev->Sender << " lease successfully verified"); + YDB_LOG_NOTICE("[ScriptExecutions] Script execution lease successfully verified", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); LeaseVerified = true; Reply(); } @@ -1595,20 +1638,25 @@ private: case EWakeup::RetryCheckAlive: WaitRetryCheckAlive = false; if (WaitFinishQuery) { - KQP_PROXY_LOG_N("Skipped retry check alive query, already waiting finish query"); + YDB_LOG_NOTICE("[ScriptExecutions] Skipped check alive retry because finalization is already in progress", + {"logPrefix", LogPrefix()}); } else { CheckAliveRetries++; - KQP_PROXY_LOG_D("Start check alive request #" << CheckAliveRetries + 1); + YDB_LOG_DEBUG("[ScriptExecutions] Starting check alive request", + {"logPrefix", LogPrefix()}, + {"checkAliveAttempt", CheckAliveRetries + 1}); Send(RunScriptActorId, new TEvCheckAliveRequest(), CheckAliveFlags); Schedule(CHECK_ALIVE_REQUEST_SOFT_TIMEOUT, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::CheckAliveSoftTimeout))); } break; case EWakeup::CheckAliveSoftTimeout: - KQP_PROXY_LOG_W("Deliver TRunScriptActor check alive request soft timeout, retry check alive"); + YDB_LOG_WARN("[ScriptExecutions] Check alive request to TRunScriptActor timed out, retrying", + {"logPrefix", LogPrefix()}); RetryCheckAlive(/* longDelay */ false); break; case EWakeup::CheckAliveHardTimeout: - KQP_PROXY_LOG_W("Deliver TRunScriptActor check alive request hard timeout, start finalization"); + YDB_LOG_WARN("[ScriptExecutions] Check alive request to TRunScriptActor hard timeout exceeded, starting finalization", + {"logPrefix", LogPrefix()}); RunScriptFinalizeRequest(); break; } @@ -1622,16 +1670,23 @@ private: // 2. Script execution may be already under finalization // 3. Script execution may be retried // We will start finalization only in first and second cases - KQP_PROXY_LOG_W("Got delivery problem to " << ev->Sender << " TRunScriptActor not found, start finalization"); + YDB_LOG_WARN("[ScriptExecutions] TRunScriptActor not found, starting finalization", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); RunScriptFinalizeRequest(); } else { - KQP_PROXY_LOG_W("Got delivery problem to " << ev->Sender << ", node with TRunScriptActor unavailable, reason: " << reason); + YDB_LOG_WARN("[ScriptExecutions] Failed to deliver check alive request to node with TRunScriptActor", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"reason", reason}); RetryCheckAlive(/* longDelay */ true); } } void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { - KQP_PROXY_LOG_W("Node " << ev->Get()->NodeId << " with TRunScriptActor was disconnected, retry check alive"); + YDB_LOG_WARN("[ScriptExecutions] Node with TRunScriptActor was disconnected, retrying check alive", + {"logPrefix", LogPrefix()}, + {"nodeId", ev->Get()->NodeId}); RetryCheckAlive(/* longDelay */ false); } @@ -1639,9 +1694,15 @@ private: const auto status = ev->Get()->Status; const auto& issues = ev->Get()->Issues; if (status != Ydb::StatusIds::SUCCESS) { - KQP_PROXY_LOG_W("Failed to restart " << ev->Sender << " script execution operation, status: " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Failed to restart script execution operation", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", issues.ToOneLineString()}); } else { - KQP_PROXY_LOG_D("Successfully restarted " << ev->Sender << " script execution operation"); + YDB_LOG_DEBUG("[ScriptExecutions] Successfully restarted script execution operation", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); } Reply(status, AddRootIssue("Restart script execution operation", issues, /* addEmptyRoot */ false)); @@ -1650,12 +1711,19 @@ private: void HandleFinalized(TEvScriptExecutionFinished::TPtr& ev) { if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { const auto& issues = ev->Get()->Issues; - KQP_PROXY_LOG_W("Failed to finalize " << ev->Sender << " script execution operation, status: " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Failed to finalize script execution operation", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", issues.ToOneLineString()}); return Reply(status, AddRootIssue("Finish script execution operation failed", issues)); } const auto& info = ev->Get()->Info; - KQP_PROXY_LOG_D("Successfully finalized " << ev->Sender << " script execution operation, entry exists: " << info.ExecutionEntryExists); + YDB_LOG_DEBUG("[ScriptExecutions] Successfully finalized script execution operation", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"executionEntryExists", info.ExecutionEntryExists}); if (info.ExecutionEntryExists) { Reply(); @@ -1673,12 +1741,13 @@ private: Y_VALIDATE(ScriptFinalizeRequest, "Script finalize request is not set"); const auto& description = ScriptFinalizeRequest->Description; - KQP_PROXY_LOG_D("Run script finalization request" - << ", FinalizationStatus: " << static_cast<i32>(description.FinalizationStatus) - << ", OperationStatus: " << description.OperationStatus - << ", ExecStatus: " << Ydb::Query::ExecStatus_Name(description.ExecStatus) - << ", Issues: " << description.Issues.ToOneLineString() - << ", LeaseGeneration: " << description.LeaseGeneration); + YDB_LOG_DEBUG("[ScriptExecutions] Run script finalization request", + {"logPrefix", LogPrefix()}, + {"finalizationStatus", static_cast<i32>(description.FinalizationStatus)}, + {"operationStatus", description.OperationStatus}, + {"execStatus", Ydb::Query::ExecStatus_Name(description.ExecStatus)}, + {"issues", description.Issues.ToOneLineString()}, + {"leaseGeneration", description.LeaseGeneration}); Send(MakeKqpFinalizeScriptServiceId(SelfId().NodeId()), ScriptFinalizeRequest.release()); } @@ -1702,23 +1771,29 @@ private: } if (const auto delay = CheckAliveRetryState->GetNextRetryDelay(longDelay)) { - KQP_PROXY_LOG_D("Schedule retry check alive in " << *delay); + YDB_LOG_DEBUG("[ScriptExecutions] Scheduling check alive retry", + {"logPrefix", LogPrefix()}, + {"delay", *delay}); Schedule(*delay, new TEvents::TEvWakeup(static_cast<ui64>(EWakeup::RetryCheckAlive))); WaitRetryCheckAlive = true; } else { - KQP_PROXY_LOG_E("Retry limit " << MAX_CHECK_ALIVE_RETRIES << " exceeded for TRunScriptActor check alive, start finalization"); + YDB_LOG_ERROR("[ScriptExecutions] Retry limit exceeded for TRunScriptActor check alive, starting finalization", + {"logPrefix", LogPrefix()}, + {"maxCheckAliveRetries", MAX_CHECK_ALIVE_RETRIES}); RunScriptFinalizeRequest(); } } void ReplyNotFound() { - KQP_PROXY_LOG_W("Script execution operation not found"); + YDB_LOG_WARN("[ScriptExecutions] Script execution operation not found", + {"logPrefix", LogPrefix()}); EntryExists = false; return Reply(Settings.FailOnNotFound ? Ydb::StatusIds::NOT_FOUND : Ydb::StatusIds::SUCCESS, "Script execution operation not found"); } void Reply() { - KQP_PROXY_LOG_D("Reply success"); + YDB_LOG_DEBUG("[ScriptExecutions] Reply success", + {"logPrefix", LogPrefix()}); Send(ReplyActorId, new TEvPrivate::TEvFinalizeScriptLeaseResult(Ydb::StatusIds::SUCCESS, { .LeaseVerified = LeaseVerified, .ExecutionEntryExists = EntryExists, @@ -1731,7 +1806,10 @@ private: } void Reply(const Ydb::StatusIds::StatusCode status, NYql::TIssues issues) { - KQP_PROXY_LOG_W("Reply " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Reply failed", + {"logPrefix", LogPrefix()}, + {"status", status}, + {"issues", issues.ToOneLineString()}); Send(ReplyActorId, new TEvPrivate::TEvFinalizeScriptLeaseResult(status, { .LeaseVerified = LeaseVerified, .ExecutionEntryExists = EntryExists, @@ -1869,7 +1947,9 @@ private: } void DeleteScriptResults() { - KQP_PROXY_LOG_D("Do deleting of script results, amount result sets: " << ResultSets.size()); + YDB_LOG_DEBUG("[ScriptExecutions] Deleting script results", + {"logPrefix", LogPrefix()}, + {"resultSetCount", ResultSets.size()}); if (ResultSets.empty()) { Finish(); @@ -1877,7 +1957,11 @@ private: } auto& resultSet = ResultSets.back(); - KQP_PROXY_LOG_D("Deleting rows from result set #" << resultSet.Id << ", remains rows range [" << resultSet.MinRowId << "; " << resultSet.MaxRowId << "]"); + YDB_LOG_DEBUG("[ScriptExecutions] Deleting rows from result set", + {"logPrefix", LogPrefix()}, + {"resultSetId", resultSet.Id}, + {"minRowId", resultSet.MinRowId}, + {"maxRowId", resultSet.MaxRowId}); constexpr char sql[] = R"( -- TForgetScriptExecutionOperationQueryActor::DeleteScriptResults @@ -1922,7 +2006,10 @@ private: auto& resultSet = ResultSets.back(); resultSet.MaxRowId -= NUMBER_ROWS_IN_BATCH + 1; if (resultSet.MaxRowId < resultSet.MinRowId) { - KQP_PROXY_LOG_D("Deleting of script result set #" << resultSet.Id << " is finished, remains result sets: " << ResultSets.size() - 1); + YDB_LOG_DEBUG("[ScriptExecutions] Finished deleting script result set", + {"logPrefix", LogPrefix()}, + {"resultSetId", resultSet.Id}, + {"remainingResultSetCount", ResultSets.size() - 1}); ResultSets.pop_back(); } @@ -1970,7 +2057,9 @@ public: ExecutionId = std::move(*executionId); const auto& checkerId = Register(TCheckLeaseStatusQueryActor::MakeRetry(SelfId(), Database, ExecutionId, TCheckLeaseStatusQueryActor::TSettings{.UserSID = userSID})); - KQP_PROXY_LOG_D("Bootstrap. Start TCheckLeaseStatusQueryActor " << checkerId); + YDB_LOG_DEBUG("[ScriptExecutions] Bootstrap: starting TCheckLeaseStatusQueryActor", + {"logPrefix", LogPrefix()}, + {"checkerId", checkerId}); Become(&TThis::StateFunc); } @@ -1985,7 +2074,11 @@ private: void Handle(TEvPrivate::TEvLeaseCheckResult::TPtr& ev) { if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { const auto& issues = ev->Get()->Issues; - KQP_PROXY_LOG_W("Lease check " << ev->Sender << " failed " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Lease check failed", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", issues.ToOneLineString()}); Reply(status, AddRootIssue("Check lease status", issues)); return; } @@ -1995,44 +2088,64 @@ private: if (ExecutionEntryExists && (!ev->Get()->OperationStatus || ev->Get()->FinalizationStatus || ev->Get()->WaitRetry)) { if (Request->Get()->Settings.CancelIfRunning) { - KQP_PROXY_LOG_W("Lease check " << ev->Sender << " finished, but operation is still running. Cancel it"); + YDB_LOG_WARN("[ScriptExecutions] Lease check finished, but operation is still running. Cancel it", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); Send(MakeKqpProxyID(SelfId().NodeId()), new TEvCancelScriptExecutionOperation(Database, Request->Get()->OperationId, Request->Get()->UserSID, { .FailOnNotFound = false, .FailOnAlreadyStopped = false, })); } else { - KQP_PROXY_LOG_I("Lease check " << ev->Sender << " finished, but operation is still running"); + YDB_LOG_INFO("[ScriptExecutions] Lease check finished, but operation is still running", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); Reply(Ydb::StatusIds::PRECONDITION_FAILED, "Operation is still running"); } return; } - KQP_PROXY_LOG_D("Lease check " << ev->Sender << " success, execution entry exists: " << ExecutionEntryExists); + YDB_LOG_DEBUG("[ScriptExecutions] Lease check succeeded", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"executionEntryExists", ExecutionEntryExists}); StartForgetOperation(); } void Handle(TEvCancelScriptExecutionOperationResponse::TPtr& ev) { if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { const auto& issues = ev->Get()->Issues; - KQP_PROXY_LOG_W("Cancel operation " << ev->Sender << " failed " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Cancel operation failed", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", issues.ToOneLineString()}); Reply(status, AddRootIssue("Cancel script execution operation", issues)); return; } ExecutionEntryExists = ev->Get()->ExecutionEntryExists; - KQP_PROXY_LOG_D("Cancel script execution operation " << ev->Sender << " finished, execution entry exists: " << ExecutionEntryExists); + YDB_LOG_DEBUG("[ScriptExecutions] Cancel script execution operation finished", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"executionEntryExists", ExecutionEntryExists}); StartForgetOperation(); } void Handle(TEvForgetScriptExecutionOperationResponse::TPtr& ev) { - KQP_PROXY_LOG_D("Forget operation " << ev->Sender << " finished " << ev->Get()->Status << ", issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_DEBUG("[ScriptExecutions] Forget operation finished", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", ev->Get()->Status}, + {"issues", ev->Get()->Issues.ToOneLineString()}); Reply(ev->Get()->Status, AddRootIssue("Forget script execution operation", ev->Get()->Issues, /* addEmptyRoot */ false)); } void StartForgetOperation() { const auto& forgetId = Register(TForgetScriptExecutionOperationQueryActor::MakeRetry(SelfId(), ExecutionId, Database, std::move(ResultSetMetas))); - KQP_PROXY_LOG_D("Start TForgetOperationRetryActor " << forgetId); + YDB_LOG_DEBUG("[ScriptExecutions] Starting TForgetOperationRetryActor", + {"logPrefix", LogPrefix()}, + {"forgetId", forgetId}); } void Reply(Ydb::StatusIds::StatusCode status, NYql::TIssues issues) { @@ -2044,9 +2157,13 @@ private: } if (status == Ydb::StatusIds::SUCCESS) { - KQP_PROXY_LOG_D("Reply success"); + YDB_LOG_DEBUG("[ScriptExecutions] Reply success", + {"logPrefix", LogPrefix()}); } else { - KQP_PROXY_LOG_W("Reply " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Reply failed", + {"logPrefix", LogPrefix()}, + {"status", status}, + {"issues", issues.ToOneLineString()}); } Send(Request->Sender, new TEvForgetScriptExecutionOperationResponse(status, std::move(issues)), /* flags */ 0, Request->Cookie); @@ -2241,9 +2358,10 @@ private: } void OnFinish(const Ydb::StatusIds::StatusCode status, NYql::TIssues&& issues) override { - KQP_PROXY_LOG_D("Finish" - << ", OperationStatus: " << (OperationStatus ? Ydb::StatusIds::StatusCode_Name(*OperationStatus) : "null") - << ", LeaseStatus: " << (LeaseStatus ? static_cast<i64>(*LeaseStatus) : -1)); + YDB_LOG_DEBUG("[ScriptExecutions] Finished getting script execution operation", + {"logPrefix", LogPrefix()}, + {"operationStatus", (OperationStatus ? Ydb::StatusIds::StatusCode_Name(*OperationStatus) : "<null>")}, + {"leaseStatus", (LeaseStatus ? static_cast<i64>(*LeaseStatus) : -1)}); bool ready = !!OperationStatus; if (LeaseStatus && OperationStatus) { @@ -2331,7 +2449,10 @@ private: } void OnRunQuery() override { - KQP_PROXY_LOG_D("List with PageToken: " << PageToken << ", PageSize: " << PageSize); + YDB_LOG_DEBUG("[ScriptExecutions] Listing script execution operations", + {"logPrefix", LogPrefix()}, + {"pageToken", PageToken}, + {"pageSize", PageSize}); if (!ValidateUserSID()) { return; @@ -2533,13 +2654,15 @@ private: operationStatus = result.ColumnParser("operation_status").GetOptionalInt32(); if (!operationStatus || !executionStatus) { OperationStillRunning = true; - KQP_PROXY_LOG_N("Can not reset retry state then operation is running now"); + YDB_LOG_NOTICE("[ScriptExecutions] Cannot reset retry state while operation is running", + {"logPrefix", LogPrefix()}); return Finish(); } if (result.ColumnParser("finalization_status").GetOptionalInt32()) { OperationStillRunning = true; - KQP_PROXY_LOG_N("Can not reset retry state then operation is finalizing now"); + YDB_LOG_NOTICE("[ScriptExecutions] Cannot reset retry state while operation is finalizing", + {"logPrefix", LogPrefix()}); return Finish(); } @@ -2556,7 +2679,9 @@ private: if (const auto leaseState = result.ColumnParser("lease_state").GetOptionalInt32()) { if (static_cast<ELeaseState>(*leaseState) != ELeaseState::WaitRetry) { OperationStillRunning = true; - KQP_PROXY_LOG_N("Can not reset retry state then operation is running or not finalized, lease state: " << static_cast<ELeaseState>(*leaseState)); + YDB_LOG_NOTICE("[ScriptExecutions] Cannot reset retry state while operation is running or not waiting for retry", + {"logPrefix", LogPrefix()}, + {"leaseState", static_cast<ELeaseState>(*leaseState)}); return Finish(); } @@ -2659,7 +2784,8 @@ public: void Bootstrap() { Become(&TThis::StateFunc); - KQP_PROXY_LOG_D("Bootstrap"); + YDB_LOG_DEBUG("[ScriptExecutions] Bootstrap: starting cancel script execution operation", + {"logPrefix", LogPrefix()}); UserSID = Request->Get()->UserSID; if (const auto& error = CheckScriptExecutionAccess(UserSID)) { @@ -2696,7 +2822,11 @@ private: void Handle(TEvPrivate::TEvLeaseCheckResult::TPtr& ev) { if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { const auto& issues = ev->Get()->Issues; - KQP_PROXY_LOG_W("Check lease " << ev->Sender << " failed " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Check lease failed", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", issues.ToOneLineString()}); return Reply(status, AddRootIssue("Fetch script execution info failed", issues)); } @@ -2709,11 +2839,13 @@ private: const auto& operationStatus = ev->Get()->OperationStatus; const auto& finalizationStatus = ev->Get()->FinalizationStatus; const auto leaseExpired = ev->Get()->LeaseExpired; - KQP_PROXY_LOG_D("Check lease " << ev->Sender << " success" - << ", operation status: " << (operationStatus ? Ydb::StatusIds::StatusCode_Name(*operationStatus) : "<null>") - << ", finalization status: " << (finalizationStatus ? ToString(*finalizationStatus) : "<null>") - << ", has retries: " << HasRetryPolicy - << ", lease expired: " << leaseExpired); + YDB_LOG_DEBUG("[ScriptExecutions] Lease check succeeded", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"operationStatus", (operationStatus ? Ydb::StatusIds::StatusCode_Name(*operationStatus) : "<null>")}, + {"finalizationStatus", (finalizationStatus ? ToString(*finalizationStatus) : "<null>")}, + {"hasRetryPolicy", HasRetryPolicy}, + {"leaseExpired", leaseExpired}); // Possible script execution states: // 1. Is running now @@ -2744,17 +2876,24 @@ private: // In case of TLI (status ABORTED) retry cancel operation const auto status = ev->Get()->Status; if (status != Ydb::StatusIds::SUCCESS && status != Ydb::StatusIds::ABORTED) { - KQP_PROXY_LOG_W("Reset retry state " << ev->Sender << " failed " << status << ", issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Reset retry state failed", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", ev->Get()->Issues.ToOneLineString()}); return Reply(status, AddRootIssue("Reset retry state failed", ev->Get()->Issues)); } const auto& issues = ev->Get()->Issues; const auto& info = ev->Get()->Info; - KQP_PROXY_LOG_D("Reset retry state " << ev->Sender << " finished " << status - << ", execution entry exists: " << info.ExecutionEntryExists - << ", operation still running: " << info.OperationStillRunning - << ", already stopped: " << info.AlreadyStopped - << ", issues: " << issues.ToOneLineString()); + YDB_LOG_DEBUG("[ScriptExecutions] Reset retry state finished", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"executionEntryExists", info.ExecutionEntryExists}, + {"operationStillRunning", info.OperationStillRunning}, + {"alreadyStopped", info.AlreadyStopped}, + {"issues", issues.ToOneLineString()}); if (status == Ydb::StatusIds::SUCCESS) { if (!info.ExecutionEntryExists) { @@ -2781,16 +2920,23 @@ private: // In case of TLI (status ABORTED) retry cancel operation const auto status = ev->Get()->Status; if (status != Ydb::StatusIds::SUCCESS && status != Ydb::StatusIds::ABORTED) { - KQP_PROXY_LOG_W("Finalize script execution " << ev->Sender << " failed " << status << ", issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Finalize script execution failed", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", ev->Get()->Issues.ToOneLineString()}); return Reply(status, AddRootIssue("Finalize script execution failed", ev->Get()->Issues)); } const auto& issues = ev->Get()->Issues; const auto& info = ev->Get()->Info; - KQP_PROXY_LOG_D("Finalize script execution " << ev->Sender << " finished " << status - << ", lease verified: " << info.LeaseVerified - << ", execution entry exists: " << info.ExecutionEntryExists - << ", issues: " << issues.ToOneLineString()); + YDB_LOG_DEBUG("[ScriptExecutions] Script execution finalization finished", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"leaseVerified", info.LeaseVerified}, + {"executionEntryExists", info.ExecutionEntryExists}, + {"issues", issues.ToOneLineString()}); if (status == Ydb::StatusIds::SUCCESS) { if (!info.ExecutionEntryExists) { @@ -2822,15 +2968,23 @@ private: // In case of TLI (status ABORTED) retry cancel operation const auto status = ev->Get()->Record.GetStatus(); if (status != Ydb::StatusIds::SUCCESS && status != Ydb::StatusIds::ABORTED) { - KQP_PROXY_LOG_W("Script execution cancel failed " << status << " from RunScriptActor: " << ev->Sender << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Script execution cancel failed", + {"logPrefix", LogPrefix()}, + {"status", status}, + {"runScriptActor", ev->Sender}, + {"issues", issues.ToOneLineString()}); return Reply(status, AddRootIssue("Cancel script execution failed", issues)); } const auto alreadyFinished = ev->Get()->Record.GetAlreadyFinished(); const auto executionEntryExists = ev->Get()->Record.GetExecutionEntryExists(); - KQP_PROXY_LOG_D("Got cancel response from RunScriptActor: " << ev->Sender << ", status: " << status - << ", already finished: " << alreadyFinished - << ", issues: " << issues.ToOneLineString()); + YDB_LOG_DEBUG("[ScriptExecutions] Received cancel response from RunScriptActor", + {"logPrefix", LogPrefix()}, + {"runScriptActor", ev->Sender}, + {"status", status}, + {"alreadyFinished", alreadyFinished}, + {"executionEntryExists", executionEntryExists}, + {"issues", issues.ToOneLineString()}); if (status == Ydb::StatusIds::SUCCESS) { if (!executionEntryExists) { @@ -2849,7 +3003,10 @@ private: void Handle(TEvents::TEvUndelivered::TPtr& ev) { const auto reason = ev->Get()->Reason; - KQP_PROXY_LOG_W("Delivery failed " << reason << " to RunScriptActor: " << ev->Sender); + YDB_LOG_WARN("[ScriptExecutions] Delivery failed", + {"logPrefix", LogPrefix()}, + {"reason", reason}, + {"runScriptActor", ev->Sender}); if (ev->Cookie != CancellationCookie || !RunScriptActor) { return; @@ -2862,7 +3019,9 @@ private: void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { const auto nodeId = ev->Get()->NodeId; - KQP_PROXY_LOG_W("Delivery failed to RunScriptActor, node " << nodeId << " disconnected"); + YDB_LOG_WARN("[ScriptExecutions] Delivery failed to RunScriptActor, node disconnected", + {"logPrefix", LogPrefix()}, + {"nodeId", nodeId}); if (!RunScriptActor || nodeId != RunScriptActor.NodeId()) { return; @@ -2882,12 +3041,16 @@ private: void GetScriptExecutionOperationStatus() { WaitRetry = false; const auto& checkerId = Register(TCheckLeaseStatusQueryActor::MakeRetry(SelfId(), Request->Get()->Database, ExecutionId, TCheckLeaseStatusQueryActor::TSettings{.UserSID = UserSID})); - KQP_PROXY_LOG_D("Start TCheckLeaseStatusQueryActor " << checkerId); + YDB_LOG_DEBUG("[ScriptExecutions] Starting TCheckLeaseStatusQueryActor", + {"logPrefix", LogPrefix()}, + {"checkerId", checkerId}); } void ResetRetryPolicy() { const auto& resetActorId = Register(TResetScriptExecutionRetriesQueryActor::MakeRetry(SelfId(), Request->Get()->Database, ExecutionId)); - KQP_PROXY_LOG_D("Start TResetRetryStateRetryActor " << resetActorId); + YDB_LOG_DEBUG("[ScriptExecutions] Starting TResetRetryStateRetryActor", + {"logPrefix", LogPrefix()}, + {"resetActorId", resetActorId}); } void FinalizeScriptLease() { @@ -2895,12 +3058,16 @@ private: .AllowRestart = false, .FailOnNotFound = false, })); - KQP_PROXY_LOG_D("Start TFinalizeScriptLeaseActor " << finalizeActorId); + YDB_LOG_DEBUG("[ScriptExecutions] Starting TFinalizeScriptLeaseActor", + {"logPrefix", LogPrefix()}, + {"finalizeActorId", finalizeActorId}); } void SendCancelToRunScriptActor() { CancellationCookie++; - KQP_PROXY_LOG_D("Send cancel request to RunScriptActor, CancellationCookie: " << CancellationCookie); + YDB_LOG_DEBUG("[ScriptExecutions] Send cancel request to RunScriptActor", + {"logPrefix", LogPrefix()}, + {"cancellationCookie", CancellationCookie}); ResetSessionSubscribe(); ui64 flags = IEventHandle::FlagTrackDelivery; @@ -2938,7 +3105,10 @@ private: delay = TDuration::Zero(); } - KQP_PROXY_LOG_W("Schedule retry for error: " << issues.ToOneLineString() << " in " << *delay); + YDB_LOG_WARN("[ScriptExecutions] Scheduling cancel retry", + {"logPrefix", LogPrefix()}, + {"error", issues.ToOneLineString()}, + {"delay", *delay}); Issues.AddIssues(issues); Schedule(*delay, new TEvents::TEvWakeup()); WaitRetry = true; @@ -2953,13 +3123,15 @@ private: } void ReplyNotFound() { - KQP_PROXY_LOG_W("Script execution operation not found"); + YDB_LOG_WARN("[ScriptExecutions] Script execution operation not found", + {"logPrefix", LogPrefix()}); EntryExists = false; return Reply(Request->Get()->Settings.FailOnNotFound ? Ydb::StatusIds::NOT_FOUND : Ydb::StatusIds::SUCCESS, "Script execution operation not found"); } void ReplyAlreadyStopped() { - KQP_PROXY_LOG_W("Script execution operation is already finished"); + YDB_LOG_WARN("[ScriptExecutions] Script execution operation is already finished", + {"logPrefix", LogPrefix()}); return Reply(Request->Get()->Settings.FailOnAlreadyStopped ? Ydb::StatusIds::PRECONDITION_FAILED : Ydb::StatusIds::SUCCESS, "Script execution operation is already finished"); } @@ -2967,9 +3139,14 @@ private: Issues.AddIssues(issues); if (status == Ydb::StatusIds::SUCCESS) { - KQP_PROXY_LOG_D("Reply success, issues: " << Issues.ToOneLineString()); + YDB_LOG_DEBUG("[ScriptExecutions] Reply success", + {"logPrefix", LogPrefix()}, + {"issues", Issues.ToOneLineString()}); } else { - KQP_PROXY_LOG_W("Reply failed, status: " << status << ", issues: " << Issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Reply failed", + {"logPrefix", LogPrefix()}, + {"status", status}, + {"issues", Issues.ToOneLineString()}); } Send(Request->Sender, new TEvCancelScriptExecutionOperationResponse(status, EntryExists, std::move(Issues))); @@ -3108,7 +3285,13 @@ private: .EndList() .Build(); - KQP_PROXY_LOG_D("Save result #" << ResultSetId << ", FirstRow: " << FirstRow << ", AccumulatedSize: " << AccumulatedSize << ", rows to save: " << ResultSet.rows_size() << ", size to save: " << SavedSize); + YDB_LOG_DEBUG("[ScriptExecutions] Saving result rows", + {"logPrefix", LogPrefix()}, + {"resultSetId", ResultSetId}, + {"firstRow", FirstRow}, + {"accumulatedSize", AccumulatedSize}, + {"rowCount", ResultSet.rows_size()}, + {"savedSize", SavedSize}); RunDataQuery(sql, ¶ms); } @@ -3151,14 +3334,22 @@ public: i64 numberRows = ResultSets.back().rows_size(); const auto& saverId = Register(TSaveScriptExecutionResultQuery::MakeRetry(SelfId(), Database, ExecutionId, ResultSetId, ExpireAt, FirstRow, AccumulatedSize, ResultSets.back())); - KQP_PROXY_LOG_D("Start saving rows range [" << FirstRow << "; " << FirstRow + numberRows << "), remains parts: " << ResultSets.size() << ", saver actor: " << saverId); + YDB_LOG_DEBUG("[ScriptExecutions] Starting to save result rows batch", + {"logPrefix", LogPrefix()}, + {"firstRow", FirstRow}, + {"lastRow", FirstRow + numberRows}, + {"remainingParts", ResultSets.size()}, + {"saverId", saverId}); FirstRow += numberRows; ResultSets.pop_back(); } void Bootstrap() { - KQP_PROXY_LOG_D("Bootstrap. FirstRow: " << FirstRow << ", AccumulatedSize: " << AccumulatedSize); + YDB_LOG_DEBUG("[ScriptExecutions] Bootstrap: starting save script execution result", + {"logPrefix", LogPrefix()}, + {"firstRow", FirstRow}, + {"accumulatedSize", AccumulatedSize}); NFq::TSplittedResultSets splittedResultSets = RowsSplitter.Split(); if (!splittedResultSets.Success) { @@ -3180,13 +3371,18 @@ public: private: void Handle(TEvSaveScriptResultPartFinished::TPtr& ev) { if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { - KQP_PROXY_LOG_W("Failed to save result part, saver actor: " << ev->Sender); + YDB_LOG_WARN("[ScriptExecutions] Failed to save result part", + {"logPrefix", LogPrefix()}, + {"saverId", ev->Sender}); Reply(status, std::move(ev->Get()->Issues)); return; } AccumulatedSize += ev->Get()->SavedSize; - KQP_PROXY_LOG_D("Result part successfully saved, AccumulatedSize: " << AccumulatedSize << ", saver actor: " << ev->Sender); + YDB_LOG_DEBUG("[ScriptExecutions] Result part successfully saved", + {"logPrefix", LogPrefix()}, + {"accumulatedSize", AccumulatedSize}, + {"saverId", ev->Sender}); StartSaveResultQuery(); } @@ -3196,7 +3392,10 @@ private: } void Reply(const Ydb::StatusIds::StatusCode status, NYql::TIssues issues = {}) { - KQP_PROXY_LOG_D("Reply " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_DEBUG("[ScriptExecutions] Sending save result reply", + {"logPrefix", LogPrefix()}, + {"status", status}, + {"issues", issues.ToOneLineString()}); Send(ReplyActorId, new TEvSaveScriptResultFinished(status, ResultSetId, std::move(issues))); PassAway(); } @@ -3233,7 +3432,10 @@ private: } if (RowsLimit < 0 || SizeLimit < 0) { - KQP_PROXY_LOG_W("Invalid fetch result limits, RowsLimit: " << RowsLimit << ", SizeLimit: " << SizeLimit); + YDB_LOG_WARN("[ScriptExecutions] Invalid fetch result limits", + {"logPrefix", LogPrefix()}, + {"rowsLimit", RowsLimit}, + {"sizeLimit", SizeLimit}); return Finish(Ydb::StatusIds::BAD_REQUEST, "Result rows limit and size limit should not be negative"); } @@ -3359,7 +3561,12 @@ private: } void FetchScriptResults() { - KQP_PROXY_LOG_D("Fetch results #" << ResultSetIndex << " with offset: " << Offset << ", limit: " << RowsLimit << ", saved rows: " << NumberOfSavedRows); + YDB_LOG_DEBUG("[ScriptExecutions] Fetching script results", + {"logPrefix", LogPrefix()}, + {"resultSetIndex", ResultSetIndex}, + {"offset", Offset}, + {"rowsLimit", RowsLimit}, + {"savedRowCount", NumberOfSavedRows}); constexpr char sql[] = R"( -- TGetScriptExecutionResultQuery::FetchScriptResults @@ -3422,14 +3629,19 @@ private: const i64 rowSize = serializedRow->size(); if (SizeLimit && ResultSet.rows_size()) { if (const auto newSize = ResultSetSize + rowSize + AdditionalRowSize; newSize > SizeLimit) { - KQP_PROXY_LOG_D("Finish by SizeLimit: " << SizeLimit << ", new result size: " << newSize); + YDB_LOG_DEBUG("[ScriptExecutions] Stopping fetch because size limit would be exceeded", + {"logPrefix", LogPrefix()}, + {"sizeLimit", SizeLimit}, + {"newSize", newSize}); CancelFetchQuery(); return; } } if (RowsLimit && ResultSet.rows_size() >= RowsLimit) { - KQP_PROXY_LOG_D("Finish by RowsLimit: " << RowsLimit); + YDB_LOG_DEBUG("[ScriptExecutions] Stopping fetch because rows limit reached", + {"logPrefix", LogPrefix()}, + {"rowsLimit", RowsLimit}); CancelFetchQuery(); return; } @@ -3448,14 +3660,18 @@ private: } if (TInstant::Now() + TDuration::Seconds(5) + GetAverageTime() >= Deadline) { - KQP_PROXY_LOG_D("Finish by operation deadline: " << Deadline); + YDB_LOG_DEBUG("[ScriptExecutions] Stopping fetch because deadline is approaching", + {"logPrefix", LogPrefix()}, + {"deadline", Deadline}); CancelFetchQuery(); } } void OnFinish(const Ydb::StatusIds::StatusCode status, NYql::TIssues&& issues) override { if (status == Ydb::StatusIds::SUCCESS) { - KQP_PROXY_LOG_D("Successfully fetched " << ResultSet.rows_size() << " rows"); + YDB_LOG_DEBUG("[ScriptExecutions] Successfully fetched rows", + {"logPrefix", LogPrefix()}, + {"rowCount", ResultSet.rows_size()}); Send(Owner, new TEvFetchScriptResultsResponse(status, std::move(ResultSet), HasMoreResults, std::move(issues))); } else { Send(Owner, new TEvFetchScriptResultsResponse(status, std::nullopt, true, std::move(issues))); @@ -3561,7 +3777,11 @@ private: } void SaveExternalEffect() { - KQP_PROXY_LOG_D("Save #" << Request.Sinks.size() << " sinks, #" << Request.SecretNames.size() << " secret names, CustomerSuppliedId: " << Request.CustomerSuppliedId); + YDB_LOG_DEBUG("[ScriptExecutions] Saving external effect sinks and secret names", + {"logPrefix", LogPrefix()}, + {"sinksCount", Request.Sinks.size()}, + {"secretNamesCount", Request.SecretNames.size()}, + {"customerSuppliedId", Request.CustomerSuppliedId}); constexpr char sql[] = R"( -- TSaveScriptExternalEffectActor::OnRunQuery @@ -3928,13 +4148,16 @@ private: } const auto operationTtl = std::min(OperationTtl.MicroSeconds(), NYql::NUdf::MAX_TIMESTAMP - 1); - KQP_PROXY_LOG_D("Do finalization with status " << Request.OperationStatus - << ", exec status: " << Ydb::Query::ExecStatus_Name(Request.ExecStatus) - << ", finalization status (applicate effect: " << Response->ApplicateScriptExternalEffectRequired << "): " << Request.FinalizationStatus - << ", issues: " << Request.Issues.ToOneLineString() - << ", retry deadline: " << retryDeadline - << ", lease state: " << leaseState - << ", operation_ttl: " << TDuration::MicroSeconds(operationTtl)); + YDB_LOG_DEBUG("[ScriptExecutions] Running script execution finalization phase one", + {"logPrefix", LogPrefix()}, + {"operationStatus", Request.OperationStatus}, + {"executionStatus", Ydb::Query::ExecStatus_Name(Request.ExecStatus)}, + {"applicateExternalEffect", Response->ApplicateScriptExternalEffectRequired}, + {"finalizationStatus", Request.FinalizationStatus}, + {"issues", Request.Issues.ToOneLineString()}, + {"retryDeadline", retryDeadline}, + {"leaseState", leaseState}, + {"operationTtl", TDuration::MicroSeconds(operationTtl)}); auto params = CreateParams(); params @@ -3996,9 +4219,10 @@ private: void OnFinish(const Ydb::StatusIds::StatusCode status, NYql::TIssues&& issues) override { if (!Response->FinalStatusAlreadySaved) { - KQP_PROXY_LOG_D("Finish script execution operation" - << ". Status: " << Ydb::StatusIds::StatusCode_Name(Request.OperationStatus) - << ". Issues: " << Request.Issues.ToOneLineString()); + YDB_LOG_DEBUG("[ScriptExecutions] Finished saving script execution operation status", + {"logPrefix", LogPrefix()}, + {"status", Ydb::StatusIds::StatusCode_Name(Request.OperationStatus)}, + {"issues", Request.Issues.ToOneLineString()}); } Response->Status = status; @@ -4031,7 +4255,10 @@ public: private: void OnRunQuery() override { - KQP_PROXY_LOG_D("Start" << (OperationStatus ? " with status " + Ydb::StatusIds::StatusCode_Name(*OperationStatus) : "") << ", issues: " << OperationIssues.ToOneLineString()); + YDB_LOG_DEBUG("[ScriptExecutions] Starting script finalization finisher", + {"logPrefix", LogPrefix()}, + {"operationStatus", (OperationStatus ? Ydb::StatusIds::StatusCode_Name(*OperationStatus) : "<null>")}, + {"issues", OperationIssues.ToOneLineString()}); constexpr char sql[] = R"( -- TScriptFinalizationFinisherActor::OnRunQuery @@ -4179,11 +4406,13 @@ private: sql << leaseInfo.Sql; retryDeadline += leaseInfo.Backoff; - KQP_PROXY_LOG_D("Do finalization with status " << *OperationStatus - << ", exec status: " << Ydb::Query::ExecStatus_Name(ExecutionStatus) - << ", issues: " << OperationIssues.ToOneLineString() - << ", retry deadline: " << retryDeadline - << ", lease state: " << static_cast<i32>(leaseInfo.NewLeaseState)); + YDB_LOG_DEBUG("[ScriptExecutions] Running script execution finalization phase two", + {"logPrefix", LogPrefix()}, + {"operationStatus", Ydb::StatusIds::StatusCode_Name(*OperationStatus)}, + {"executionStatus", Ydb::Query::ExecStatus_Name(ExecutionStatus)}, + {"issues", OperationIssues.ToOneLineString()}, + {"retryDeadline", retryDeadline}, + {"leaseState", static_cast<i32>(leaseInfo.NewLeaseState)}); auto params = CreateParams(); params @@ -4248,7 +4477,9 @@ private: void OnRunQuery() override { const auto& artifacts = CompressScriptArtifacts(QueryAst, QueryPlan, Compressor); if (artifacts.Issues) { - KQP_PROXY_LOG_N("Compress script artifacts finished with issues: " << artifacts.Issues.ToOneLineString()); + YDB_LOG_NOTICE("[ScriptExecutions] Compressing script artifacts finished with issues", + {"logPrefix", LogPrefix()}, + {"issues", artifacts.Issues.ToOneLineString()}); } auto parameters = TStringBuilder() << R"( @@ -4386,20 +4617,26 @@ private: while (result.TryNextRow()) { std::optional<TString> database = result.ColumnParser("database").GetOptionalUtf8(); if (!database) { - KQP_PROXY_LOG_E("Database field is null for script execution lease"); + YDB_LOG_ERROR("[ScriptExecutions] Database field is null for script execution lease", + {"logPrefix", LogPrefix()}); continue; } std::optional<TString> executionId = result.ColumnParser("execution_id").GetOptionalUtf8(); if (!executionId) { - KQP_PROXY_LOG_E("Execution id field is null for script execution lease in database " << *database); + YDB_LOG_ERROR("[ScriptExecutions] Execution id field is null for script execution lease", + {"logPrefix", LogPrefix()}, + {"database", *database}); continue; } Leases.emplace_back(TEvListExpiredLeasesResponse::TLeaseInfo{std::move(*database), std::move(*executionId)}); } - KQP_PROXY_LOG_D("Found " << Leases.size() << " expired leases (fetched rows " << rowsCount << ")"); + YDB_LOG_DEBUG("[ScriptExecutions] Found expired script execution leases", + {"logPrefix", LogPrefix()}, + {"leasesCount", Leases.size()}, + {"fetchedRowCount", rowsCount}); Finish(); } @@ -4421,7 +4658,9 @@ public: void Bootstrap() { const auto& listerId = Register(new TListExpiredLeasesQueryActor()); - KQP_PROXY_LOG_D("Bootstrap. Started TListExpiredLeasesQueryActor: " << listerId); + YDB_LOG_DEBUG("[ScriptExecutions] Bootstrap: starting TListExpiredLeasesQueryActor", + {"logPrefix", LogPrefix()}, + {"listerId", listerId}); Become(&TThis::StateFunc); } @@ -4435,22 +4674,34 @@ private: void Handle(TEvListExpiredLeasesResponse::TPtr& ev) { const auto& leases = ev->Get()->Leases; ExpiredLeasesCount = leases.size(); - KQP_PROXY_LOG_D("Got list expired leases response " << ev->Sender << ", found " << ExpiredLeasesCount << " expired leases"); + YDB_LOG_DEBUG("[ScriptExecutions] Received list of expired leases", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"expiredLeasesCount", ExpiredLeasesCount}); for (const auto& lease : leases) { const auto& checkerId = Register(new TFinalizeScriptLeaseActor(SelfId(), lease.Database, lease.ExecutionId, QueryServiceConfig, Counters, {.Cookie = CookieId++})); - KQP_PROXY_LOG_D("Database: " << lease.Database << "ExecutionId: " << lease.ExecutionId << ", start TFinalizeScriptLeaseActor #" << CookieId << " " << checkerId); + YDB_LOG_DEBUG("[ScriptExecutions] Starting TFinalizeScriptLeaseActor", + {"logPrefix", LogPrefix()}, + {"database", lease.Database}, + {"executionId", lease.ExecutionId}, + {"cookieId", CookieId}, + {"checkerId", checkerId}); ++OperationsToCheck; } if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { const auto& issues = ev->Get()->Issues; - KQP_PROXY_LOG_W("List expired leases failed with status " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN("[ScriptExecutions] Failed to list expired leases", + {"logPrefix", LogPrefix()}, + {"status", status}, + {"issues", issues.ToOneLineString()}); Success = false; Issues.AddIssues(AddRootIssue(TStringBuilder() << "Failed to list expired leases (" << status << ")", issues, true)); } else { - KQP_PROXY_LOG_D("List expired leases successfully completed"); + YDB_LOG_DEBUG("[ScriptExecutions] List expired leases successfully completed", + {"logPrefix", LogPrefix()}); } MaybeFinish(); @@ -4463,12 +4714,22 @@ private: if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { const auto& issues = ev->Get()->Issues; - KQP_PROXY_LOG_W("Lease check #" << ev->Cookie << " " << ev->Sender << " failed, status: " << status << ", issues: " << issues.ToOneLineString() << ", OperationsToCheck: " << OperationsToCheck); + YDB_LOG_WARN("[ScriptExecutions] Lease check failed", + {"logPrefix", LogPrefix()}, + {"cookie", ev->Cookie}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", issues.ToOneLineString()}, + {"operationsToCheck", OperationsToCheck}); Success = false; Issues.AddIssues(AddRootIssue(TStringBuilder() << "Lease check failed #" << ev->Cookie << " (" << status << ")", issues, true)); } else { - KQP_PROXY_LOG_D("Lease check #" << ev->Cookie << " " << ev->Sender << " successfully completed, OperationsToCheck: " << OperationsToCheck); + YDB_LOG_DEBUG("[ScriptExecutions] Lease check successfully completed", + {"logPrefix", LogPrefix()}, + {"cookie", ev->Cookie}, + {"sender", ev->Sender}, + {"operationsToCheck", OperationsToCheck}); } MaybeFinish(); @@ -4479,7 +4740,10 @@ private: return; } - KQP_PROXY_LOG_D("Finish, success: " << Success << ", issues: " << Issues.ToOneLineString()); + YDB_LOG_DEBUG("[ScriptExecutions] Finished refreshing script execution leases", + {"logPrefix", LogPrefix()}, + {"success", Success}, + {"issues", Issues.ToOneLineString()}); Send(ReplyActorId, new TEvRefreshScriptExecutionLeasesResponse(Success, ExpiredLeasesCount, std::move(Issues))); PassAway(); } diff --git a/ydb/core/kqp/query_data/kqp_predictor.cpp b/ydb/core/kqp/query_data/kqp_predictor.cpp index a89fb8c9baa..4a4c8322819 100644 --- a/ydb/core/kqp/query_data/kqp_predictor.cpp +++ b/ydb/core/kqp/query_data/kqp_predictor.cpp @@ -15,6 +15,8 @@ #include <cmath> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + namespace NKikimr::NKqp { using namespace NActors; @@ -151,7 +153,7 @@ ui32 TStagePredictor::GetUsableThreads() { userPoolSize = TlsActivationContext->ActorSystem()->GetPoolThreadsCount(AppData()->UserPoolId); } if (!userPoolSize) { - ALS_INFO(NKikimrServices::KQP_EXECUTER) << "user pool is undefined for executer tasks construction"; + YDB_LOG_INFO("User pool is undefined for executer tasks construction"); userPoolSize = NSystemInfo::NumberOfCpus(); } return Max<ui32>(1, *userPoolSize); @@ -173,7 +175,7 @@ ui32 TStagePredictor::GetPossibleMaxLimitThreads() { ui32 TStagePredictor::CalcTasksOptimalCount(const ui32 availableThreadsCount, const std::optional<ui32> previousStageTasksCount) const { ui32 result = 0; if (!LevelDataPrediction || *LevelDataPrediction == 0) { - ALS_ERROR(NKikimrServices::KQP_EXECUTER) << "level difficulty not defined for correct calculation"; + YDB_LOG_ERROR("Level difficulty not defined for correct calculation"); result = availableThreadsCount; } else { result = (availableThreadsCount - previousStageTasksCount.value_or(0) * 0.25) * (InputDataPrediction / *LevelDataPrediction); diff --git a/ydb/core/kqp/query_data/kqp_prepared_query.cpp b/ydb/core/kqp/query_data/kqp_prepared_query.cpp index 6251cfc04e0..ad536216859 100644 --- a/ydb/core/kqp/query_data/kqp_prepared_query.cpp +++ b/ydb/core/kqp/query_data/kqp_prepared_query.cpp @@ -16,6 +16,8 @@ #include <ydb/library/actors/core/log.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + namespace NKikimr::NKqp { using namespace NKikimr::NMiniKQL; @@ -68,7 +70,7 @@ TKqpPhyTxHolder::TKqpPhyTxHolder(const std::shared_ptr<const NKikimrKqp::TPrepar for (auto&& i : Proto->GetStages()) { TStagePredictor predictor; if (!predictor.DeserializeFromKqpSettings(i.GetProgram().GetSettings())) { - ALS_ERROR(NKikimrServices::KQP_EXECUTER) << "cannot parse program settings for data prediction"; + YDB_LOG_ERROR("Cannot parse program settings for data prediction"); Predictors.emplace_back(); } else { Predictors.emplace_back(std::move(predictor)); diff --git a/ydb/core/kqp/rm_service/kqp_resource_info_exchanger.cpp b/ydb/core/kqp/rm_service/kqp_resource_info_exchanger.cpp index 82781135d37..00a107d7242 100644 --- a/ydb/core/kqp/rm_service/kqp_resource_info_exchanger.cpp +++ b/ydb/core/kqp/rm_service/kqp_resource_info_exchanger.cpp @@ -14,17 +14,12 @@ #include <ydb/core/util/ulid.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_RESOURCE_MANAGER + namespace NKikimr { namespace NKqp { namespace NRm { -#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) - class TKqpResourceInfoExchangerActor : public TActorBootstrapped<TKqpResourceInfoExchangerActor> { using TBase = TActorBootstrapped<TKqpResourceInfoExchangerActor>; @@ -118,7 +113,8 @@ public: } void Bootstrap() { - LOG_D("Start KqpResourceInfoExchangerActor at " << SelfId()); + YDB_LOG_DEBUG("Start KqpResourceInfoExchangerActor", + {"selfId", SelfId()}); ui32 tableServiceConfigKind = (ui32) NKikimrConsole::TConfigItem::TableServiceConfigItem; @@ -163,8 +159,9 @@ private: } if (BoardState.Subscriber) { - LOG_I("Kill previous info exchanger subscriber for '" << BoardState.Path - << "' at " << BoardState.Subscriber << ", reason: tenant updated"); + YDB_LOG_INFO("Kill previous info exchanger subscriber for at reason: tenant updated", + {"boardPath", BoardState.Path}, + {"subscriber", BoardState.Subscriber}); Send(BoardState.Subscriber, new TEvents::TEvPoison); } BoardState.Subscriber = TActorId(); @@ -179,8 +176,9 @@ private: void CreatePublisher() { if (BoardState.Publisher) { - LOG_I("Kill previous info exchanger publisher for '" << BoardState.Path - << "' at " << BoardState.Publisher << ", reason: tenant updated"); + YDB_LOG_INFO("Kill previous info exchanger publisher for at reason: tenant updated", + {"boardPath", BoardState.Path}, + {"publisher", BoardState.Publisher}); Send(BoardState.Publisher, new TEvents::TEvPoison); } BoardState.Publisher = TActorId(); @@ -369,7 +367,7 @@ private: private: void Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) { - LOG_D("Subscribed for config changes."); + YDB_LOG_DEBUG("Subscribed for config changes"); } void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) { @@ -378,7 +376,7 @@ private: NKikimrConfig::TTableServiceConfig tableServiceConfig; tableServiceConfig.Swap(event.MutableConfig()->MutableTableServiceConfig()); - LOG_D("Updated table service config."); + YDB_LOG_DEBUG("Updated table service config"); const auto& infoExchangerSettings = tableServiceConfig.GetResourceManager().GetInfoExchangerSettings(); const auto& publisherSettings = infoExchangerSettings.GetPublisherSettings(); @@ -407,11 +405,11 @@ private: void Handle(TEvents::TEvUndelivered::TPtr& ev) { switch (ev->Get()->SourceType) { case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest: - LOG_C("Failed to deliver subscription request to config dispatcher."); + YDB_LOG_CRIT("Failed to deliver subscription request to config dispatcher"); break; case NConsole::TEvConsole::EvConfigNotificationResponse: - LOG_E("Failed to deliver config notification response."); + YDB_LOG_ERROR("Failed to deliver config notification response"); break; default: @@ -426,7 +424,8 @@ private: if (tenant.empty()) { tenant = slot.GetAssignedTenant(); } else { - LOG_E("Multiple tenants are served by the node: " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_ERROR("Multiple tenants are served by the", + {"node", ev->Get()->Record.ShortDebugString()}); } } } @@ -435,7 +434,8 @@ private: BoardState.Path = MakeKqpInfoExchangerBoardPath(tenant); if (auto *domain = AppData()->DomainsInfo->GetDomain(); domain->Name != ExtractDomain(tenant)) { - LOG_E("Can not find default state storage group for database " << BoardState.Tenant); + YDB_LOG_ERROR("Cannot find default state storage group for database", + {"tenant", BoardState.Tenant}); return; } @@ -458,22 +458,25 @@ private: CreatePublisher(); CreateSubscriber(); - LOG_I("Received tenant pool status for exchanger, serving tenant: " << BoardState.Tenant - << ", board: " << BoardState.Path); + YDB_LOG_INFO("Received tenant pool status for exchanger, serving", + {"tenant", BoardState.Tenant}, + {"board", BoardState.Path}); } void Handle(TEvStateStorage::TEvBoardInfo::TPtr& ev) { if (ev->Get()->Status == TEvStateStorage::TEvBoardInfo::EStatus::NotAvailable) { - LOG_I("Subcriber is not available for info exchanger, serving tenant: " << BoardState.Tenant - << ", board: " << BoardState.Path); + YDB_LOG_INFO("Subcriber is not available for info exchanger, serving", + {"tenant", BoardState.Tenant}, + {"board", BoardState.Path}); CreateSubscriber(); return; } - LOG_D("Get board info from subscriber, serving tenant: " << BoardState.Tenant - << ", board: " << BoardState.Path - << ", with size: " << ev->Get()->InfoEntries.size()); + YDB_LOG_DEBUG("Get board info from subscriber, serving with", + {"tenant", BoardState.Tenant}, + {"board", BoardState.Path}, + {"size", ev->Get()->InfoEntries.size()}); auto [nodeIds, isChanged] = UpdateBoardInfo(ev->Get()->InfoEntries); @@ -502,14 +505,16 @@ private: void Handle(TEvStateStorage::TEvBoardInfoUpdate::TPtr& ev) { if (ev->Get()->Status == TEvStateStorage::TEvBoardInfo::EStatus::NotAvailable) { - LOG_I("Subcriber is not available for info exchanger, serving tenant: " << BoardState.Tenant - << ", board: " << BoardState.Path); + YDB_LOG_INFO("Subcriber is not available for info exchanger, serving", + {"tenant", BoardState.Tenant}, + {"board", BoardState.Path}); CreateSubscriber(); return; } - LOG_D("Get board info update from subscriber, serving tenant: " << BoardState.Tenant - << ", board: " << BoardState.Path - << ", with size: " << ev->Get()->Updates.size()); + YDB_LOG_DEBUG("Get board info update from subscriber, serving with", + {"tenant", BoardState.Tenant}, + {"board", BoardState.Path}, + {"size", ev->Get()->Updates.size()}); auto [nodeIds, isChanged] = UpdateBoardInfo(ev->Get()->Updates); @@ -534,7 +539,8 @@ private: void Handle(TEvKqpResourceInfoExchanger::TEvSendResources::TPtr& ev) { auto nodeId = ev->Sender.NodeId(); - LOG_D("Get resources info from node: " << nodeId); + YDB_LOG_DEBUG("Get resources info", + {"node", nodeId}); const TVector<NKikimrKqp::TResourceExchangeNodeData> resourceInfos( ev->Get()->Record.GetSnapshot().begin(), ev->Get()->Record.GetSnapshot().end()); diff --git a/ydb/core/kqp/rm_service/kqp_resource_tracker.cpp b/ydb/core/kqp/rm_service/kqp_resource_tracker.cpp index 5103c173a44..0ce0824a0ef 100644 --- a/ydb/core/kqp/rm_service/kqp_resource_tracker.cpp +++ b/ydb/core/kqp/rm_service/kqp_resource_tracker.cpp @@ -15,18 +15,13 @@ #include <yql/essentials/utils/yql_panic.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_RESOURCE_MANAGER + namespace NKikimr::NKqp::NRm { namespace { -#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) - using namespace NKikimr; using namespace NActors; @@ -50,8 +45,9 @@ public: hFunc(TEvStateStorage::TEvBoardInfo, HandleWait); cFunc(TEvents::TSystem::Poison, PassAway); default: - LOG_C("Unexpected event type: " << ev->GetTypeRewrite() - << ", event: " << ev->GetTypeName()); + YDB_LOG_CRIT("Unexpected event", + {"type", ev->GetTypeRewrite()}, + {"event", ev->GetTypeName()}); } } @@ -62,17 +58,22 @@ public: TVector<NKikimrKqp::TKqpNodeResources> resources; if (event->Status == TEvStateStorage::TEvBoardInfo::EStatus::Ok) { - LOG_I("WhiteBoard entries: " << event->InfoEntries.size()); + YDB_LOG_INFO("WhiteBoard", + {"entries", event->InfoEntries.size()}); resources.resize(event->InfoEntries.size()); int i = 0; for (auto& [_, entry] : event->InfoEntries) { Y_PROTOBUF_SUPPRESS_NODISCARD resources[i].ParseFromString(entry.Payload); - LOG_D("WhiteBoard [" << i << "]: " << resources[i].ShortDebugString()); + YDB_LOG_DEBUG("WhiteBoard resource entry", + {"index", i}, + {"resource", resources[i].ShortDebugString()}); i++; } } else { - LOG_E("WhiteBoard error: " << (int) event->Status << ", path: " << event->Path); + YDB_LOG_ERROR("WhiteBoard", + {"error", (int) event->Status}, + {"path", event->Path}); } Callback(std::move(resources)); diff --git a/ydb/core/kqp/rm_service/kqp_rm_service.cpp b/ydb/core/kqp/rm_service/kqp_rm_service.cpp index c833fd34bec..4bfb5aa001a 100644 --- a/ydb/core/kqp/rm_service/kqp_rm_service.cpp +++ b/ydb/core/kqp/rm_service/kqp_rm_service.cpp @@ -23,6 +23,8 @@ #include <library/cpp/containers/absl/flat_hash_map.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_RESOURCE_MANAGER + namespace NKikimr { namespace NKqp { namespace NRm { @@ -30,22 +32,6 @@ namespace NRm { using namespace NActors; using namespace NResourceBroker; -#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) - -#define LOG_AS_SAFE(log) {if (ActorSystem) { log; }} - -#define LOG_AS_C(stream) LOG_AS_SAFE(LOG_CRIT_S(*ActorSystem, NKikimrServices::KQP_RESOURCE_MANAGER, stream)) -#define LOG_AS_D(stream) LOG_AS_SAFE(LOG_DEBUG_S(*ActorSystem, NKikimrServices::KQP_RESOURCE_MANAGER, stream)) -#define LOG_AS_I(stream) LOG_AS_SAFE(LOG_INFO_S(*ActorSystem, NKikimrServices::KQP_RESOURCE_MANAGER, stream)) -#define LOG_AS_E(stream) LOG_AS_SAFE(LOG_ERROR_S(*ActorSystem, NKikimrServices::KQP_RESOURCE_MANAGER, stream)) -#define LOG_AS_W(stream) LOG_AS_SAFE(LOG_WARN_S(*ActorSystem, NKikimrServices::KQP_RESOURCE_MANAGER, stream)) -#define LOG_AS_N(stream) LOG_AS_SAFE(LOG_NOTICE_S(*ActorSystem, NKikimrServices::KQP_RESOURCE_MANAGER, stream)) - TTxState::TTxState(std::shared_ptr<IKqpResourceManager>& resourceManager, ui64 txId, TInstant now, const TString& poolId, const double memoryPoolPercent, const TString& database, bool collectBacktrace) : ResourceManager(resourceManager) @@ -353,7 +339,10 @@ public: TStringBuilder reason; reason << "TxId: " << txId << ", taskId: " << taskId << ". Not enough memory for query, requested: " << resources.Memory << ". " << tx.ToString(); - LOG_AS_N(reason); + if (ActorSystem) { + YDB_LOG_NOTICE_CTX(*ActorSystem, "", + {"reason", reason}); + } result.SetError(NKikimrKqp::TEvStartKqpTasksResponse::NOT_ENOUGH_MEMORY, reason); return result; } @@ -366,7 +355,12 @@ public: Y_ABORT_UNLESS(merged); } - LOG_AS_D("TxId: " << txId << ", taskId: " << taskId << ". Allocated " << resources.ToString()); + if (ActorSystem) { + YDB_LOG_DEBUG_CTX(*ActorSystem, "Allocated", + {"txId", txId}, + {"taskId", taskId}, + {"resources", resources}); + } FireResourcesPublishing(); return result; } @@ -406,11 +400,14 @@ public: } } - LOG_AS_D("TxId: " << tx.TxId << ", taskId: " << taskId - << ". Released resources, " - << "Memory: " << resources.Memory << ", " - << "Free Tier: " << resources.ExternalMemory << ", " - << "ExecutionUnits: " << resources.ExecutionUnits << "."); + if (ActorSystem) { + YDB_LOG_DEBUG_CTX(*ActorSystem, "Released resources, Free", + {"txId", tx.TxId}, + {"taskId", taskId}, + {"memory", resources.Memory}, + {"tier", resources.ExternalMemory}, + {"executionUnits", resources.ExecutionUnits}); + } FireResourcesPublishing(); } @@ -441,7 +438,9 @@ public: } void RequestClusterResourcesInfo(TOnResourcesSnapshotCallback&& callback) override { - LOG_AS_D("Schedule Snapshot request"); + if (ActorSystem) { + YDB_LOG_DEBUG_CTX(*ActorSystem, "Schedule Snapshot request"); + } std::shared_ptr<TVector<NKikimrKqp::TKqpNodeResources>> infos; with_lock (ResourceSnapshotState->Lock) { infos = ResourceSnapshotState->Snapshot; @@ -651,7 +650,9 @@ public: } void Bootstrap() { - LOG_D("Start KqpResourceManagerActor at " << SelfId() << " with ResourceBroker at " << ResourceBrokerId); + YDB_LOG_DEBUG("Start KqpResourceManagerActor at with ResourceBroker", + {"selfId", SelfId()}, + {"resourceBrokerId", ResourceBrokerId}); // Subscribe for tenant changes Send(MakeTenantPoolRootID(), new TEvents::TEvSubscribe); @@ -675,7 +676,8 @@ public: WhiteBoardService = NNodeWhiteboard::MakeNodeWhiteboardServiceId(SelfId().NodeId()); if (WarmupInProgress) { - LOG_I("Warmup in progress, resource publishing delayed for up to " << WarmupDeadline); + YDB_LOG_INFO("Warmup in progress, resource publishing delayed for up", + {"warmupDeadline", WarmupDeadline}); Schedule(WarmupDeadline, new TEvPrivate::TEvWarmupDeadline()); } @@ -694,21 +696,22 @@ public: void Handle(NNodeWhiteboard::TEvWhiteboard::TEvSystemStateResponse::TPtr& ev) { const auto& record = ev->Get()->Record; if (record.SystemStateInfoSize() != 1) { - LOG_D("Unexpected whiteboard info"); + YDB_LOG_DEBUG("Unexpected whiteboard info"); return; } const auto& info = record.GetSystemStateInfo(0); if (AppData()->UserPoolId >= info.PoolStatsSize()) { - LOG_D("Unexpected whiteboard info: pool size is smaller than user pool id" - << ", pool size: " << info.PoolStatsSize() - << ", user pool id: " << AppData()->UserPoolId); + YDB_LOG_DEBUG("Unexpected whiteboard info: pool size is smaller than user pool id pool user pool", + {"size", info.PoolStatsSize()}, + {"id", AppData()->UserPoolId}); return; } const auto& pool = info.GetPoolStats(AppData()->UserPoolId); - LOG_D("Received node white board pool stats: " << pool.usage()); + YDB_LOG_DEBUG("Received node white board pool", + {"stats", pool.usage()}); ProxyNodeResources.SetCpuUsage(pool.usage()); ProxyNodeResources.SetThreads(pool.threads()); } @@ -750,7 +753,9 @@ private: void HandleWork(TEvKqp::TEvKqpProxyPublishRequest::TPtr&) { SendWhiteboardRequest(); if (AppData()->TenantName.empty() || !SelfDataCenterId) { - LOG_I("Cannot start publishing usage for kqp_proxy, tenants: " << AppData()->TenantName << ", " << SelfDataCenterId.value_or("empty")); + YDB_LOG_INFO("Cannot start publishing usage for kqp_proxy", + {"tenants", AppData()->TenantName}, + {"selfDataCenterId", SelfDataCenterId.value_or("empty")}); return; } PublishResourceUsage("kqp_proxy"); @@ -758,7 +763,8 @@ private: void HandleWork(TEvResourceBroker::TEvConfigResponse::TPtr& ev) { if (!ev->Get()->QueueConfig) { - LOG_E(NLocalDb::KqpResourceManagerQueue << " not configured!"); + YDB_LOG_ERROR("Resource broker queue is not configured", + {"queueName", NLocalDb::KqpResourceManagerQueue}); return; } auto& queueConfig = *ev->Get()->QueueConfig; @@ -767,7 +773,8 @@ private: with_lock (ResourceManager->Lock) { ResourceManager->TotalMemoryResource->SetNewLimit(queueConfig.GetLimit().GetMemory(), (double)100, ResourceManager->SpillingPercent.load()); } - LOG_I("Total node memory for scan queries: " << queueConfig.GetLimit().GetMemory() << " bytes"); + YDB_LOG_INFO("Total node memory for scan bytes", + {"queries", queueConfig.GetLimit().GetMemory()}); } } @@ -800,7 +807,8 @@ private: if (tenant.empty()) { tenant = slot.GetAssignedTenant(); } else { - LOG_E("Multiple tenants are served by the node: " << ev->Get()->Record.ShortDebugString()); + YDB_LOG_ERROR("Multiple tenants are served by the", + {"node", ev->Get()->Record.ShortDebugString()}); } } } @@ -812,13 +820,15 @@ private: WbState.DomainNotFound = true; } - LOG_I("Received tenant pool status, serving tenant: " << tenant << ", board: " << WbState.BoardPath); + YDB_LOG_INFO("Received tenant pool status, serving", + {"tenant", tenant}, + {"board", WbState.BoardPath}); PublishResourceUsage("tenant updated"); } static void HandleWork(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) { - LOG_D("Subscribed for config changes"); + YDB_LOG_DEBUG("Subscribed for config changes"); } void HandleWork(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) { @@ -841,7 +851,8 @@ private: FORCE_VALUE(MinChannelBufferSize); #undef FORCE_VALUE - LOG_I("Updated table service config: " << config.DebugString()); + YDB_LOG_INFO("Updated table service", + {"config", config.DebugString()}); with_lock (ResourceManager->Lock) { i32 prev = ResourceManager->ExecutionUnitsLimit.load(); @@ -855,15 +866,16 @@ private: static void HandleWork(TEvents::TEvUndelivered::TPtr& ev) { switch (ev->Get()->SourceType) { case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest: - LOG_C("Failed to deliver subscription request to config dispatcher"); + YDB_LOG_CRIT("Failed to deliver subscription request to config dispatcher"); break; case NConsole::TEvConsole::EvConfigNotificationResponse: - LOG_E("Failed to deliver config notification response"); + YDB_LOG_ERROR("Failed to deliver config notification response"); break; default: - LOG_C("Undelivered event with unexpected source type: " << ev->Get()->SourceType); + YDB_LOG_CRIT("Undelivered event with unexpected source", + {"type", ev->Get()->SourceType}); break; } } @@ -944,7 +956,7 @@ private: void HandleWarmupComplete(TEvKqpWarmupComplete::TPtr&) { if (WarmupInProgress) { WarmupInProgress = false; - LOG_I("Warmup complete, starting resource publishing"); + YDB_LOG_INFO("Warmup complete, starting resource publishing"); PublishResourceUsage("warmup_complete"); } } @@ -952,7 +964,7 @@ private: void HandleWarmupDeadline() { if (WarmupInProgress) { WarmupInProgress = false; - LOG_W("Warmup deadline exceeded, forcing resource publishing"); + YDB_LOG_WARN("Warmup deadline exceeded, forcing resource publishing"); PublishResourceUsage("warmup_deadline"); } } @@ -968,7 +980,9 @@ private: PublishResourcesScheduledAt = *WbState.LastPublishTime + publishInterval; Schedule(*PublishResourcesScheduledAt - now, new TEvPrivate::TEvPublishResources); - LOG_D("Schedule publish at " << *PublishResourcesScheduledAt << ", after " << (*PublishResourcesScheduledAt - now)); + YDB_LOG_DEBUG("Scheduled resource usage publish", + {"publishAt", *PublishResourcesScheduledAt}, + {"delay", (*PublishResourcesScheduledAt - now)}); return; } @@ -988,7 +1002,7 @@ private: } } } else { - LOG_D("Don't set KqpProxySharedResources"); + YDB_LOG_DEBUG("Don't set KqpProxySharedResources"); } ActorIdToProto(MakeKqpResourceManagerServiceID(SelfId().NodeId()), payload.MutableResourceManagerActorId()); // legacy @@ -1015,10 +1029,10 @@ private: } } - LOG_I("Send to publish resource usage for " - << "reason: " << reason - << (WarmupInProgress ? " (warmup: zero resources)" : "") - << ", payload: " << payload.ShortDebugString()); + YDB_LOG_INFO("Sending resource usage to publish", + {"reason", reason}, + {"warmupInProgress", WarmupInProgress}, + {"payload", payload.ShortDebugString()}); WbState.LastPublishTime = now; if (ResourceManager->ResourceInfoExchanger) { Send(ResourceManager->ResourceInfoExchanger, diff --git a/ydb/core/kqp/rm_service/kqp_snapshot_manager.cpp b/ydb/core/kqp/rm_service/kqp_snapshot_manager.cpp index f13dc470607..78c77c0efb7 100644 --- a/ydb/core/kqp/rm_service/kqp_snapshot_manager.cpp +++ b/ydb/core/kqp/rm_service/kqp_snapshot_manager.cpp @@ -9,6 +9,8 @@ #include <ydb/library/actors/core/hfunc.h> #include <ydb/library/actors/core/log.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_RESOURCE_MANAGER + static IOutputStream& operator<<(IOutputStream& out, const NKikimr::NKqp::IKqpGateway::TKqpSnapshot snap) { out << "[step: " << snap.Step << ", txId: " << snap.TxId << "]"; return out; @@ -17,10 +19,6 @@ static IOutputStream& operator<<(IOutputStream& out, const NKikimr::NKqp::IKqpGa namespace NKikimr { namespace NKqp { -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_RESOURCE_MANAGER, stream) - namespace { class TSnapshotManagerActor: public TActorBootstrapped<TSnapshotManagerActor> { @@ -37,7 +35,8 @@ public: RequestTimeoutCookieHolder_.Reset(ISchedulerCookie::Make2Way()); CreateLongTimer(TlsActivationContext->AsActorContext(), RequestTimeout, ev, 0, RequestTimeoutCookieHolder_.Get()); - LOG_D("Start KqpSnapshotManager at " << SelfId()); + YDB_LOG_DEBUG("Start KqpSnapshotManager", + {"selfId", SelfId()}); Become(&TThis::StateAwaitRequest); } @@ -58,7 +57,8 @@ private: Orbit = std::move(ev->Get()->Orbit); Cookie = ev->Get()->Cookie; - LOG_D("KqpSnapshotManager: got snapshot request from " << ClientActorId); + YDB_LOG_DEBUG("KqpSnapshotManager: got snapshot request", + {"clientActorId", ClientActorId}); if (MvccSnapshot) { AFL_ENSURE(ev->Get()->Tables.empty()); @@ -120,7 +120,8 @@ private: if (msg->Status == Ydb::StatusIds::SUCCESS) { Snapshot = IKqpGateway::TKqpSnapshot(msg->Snapshot.Step, msg->Snapshot.TxId); - LOG_D("KqpSnapshotManager: snapshot: " << Snapshot << " acquired"); + YDB_LOG_DEBUG("KqpSnapshotManager: acquired", + {"snapshot", Snapshot}); Send(ClientActorId, new TEvKqpSnapshot::TEvCreateSnapshotResponse( Snapshot, std::move(ev->Get()->SnapshotHandle), NKikimrIssues::TStatusIds::SUCCESS, /* issues */ {}, std::move(Orbit)), @@ -129,8 +130,9 @@ private: PassAway(); } else { NYql::TIssues issues = msg->Issues; - LOG_E("KqpSnapshotManager: CreateSnapshot got unexpected status=" - << msg->Status << ", issues:" << issues.ToString()); + YDB_LOG_ERROR("KqpSnapshotManager: CreateSnapshot got unexpected", + {"status", msg->Status}, + {"issues", issues}); ReplyErrorAndDie(NKikimrIssues::TStatusIds::ERROR, std::move(issues)); } } @@ -146,7 +148,9 @@ private: if (status == EStatus::ExecComplete && msg->Record.GetStatusCode() == NKikimrIssues::TStatusIds::SUCCESS) { Snapshot = IKqpGateway::TKqpSnapshot(msg->Record.GetStep(), msg->Record.GetTxId()); - LOG_D("KqpSnapshotManager: snapshot " << Snapshot.Step << ":" << Snapshot.TxId << " created in cleanup state. Send discard"); + YDB_LOG_DEBUG("Snapshot manager created snapshot in cleanup state and will discard it", + {"snapshotStep", Snapshot.Step}, + {"snapshotTxId", Snapshot.TxId}); SendDiscard(); } @@ -173,7 +177,9 @@ private: if (status == EStatus::ExecComplete && msg->Record.GetStatusCode() == NKikimrIssues::TStatusIds::SUCCESS) { Snapshot = IKqpGateway::TKqpSnapshot(msg->Record.GetStep(), msg->Record.GetTxId()); - LOG_D("KqpSnapshotManager: snapshot " << Snapshot.Step << ":" << Snapshot.TxId << " created"); + YDB_LOG_DEBUG("Snapshot manager created snapshot", + {"snapshotStep", Snapshot.Step}, + {"snapshotTxId", Snapshot.TxId}); Send(ClientActorId, new TEvKqpSnapshot::TEvCreateSnapshotResponse( Snapshot, TSnapshotHandle(), NKikimrIssues::TStatusIds::SUCCESS, /* issues */ {}, std::move(Orbit)), @@ -185,7 +191,9 @@ private: NYql::TIssues issues; NYql::IssuesFromMessage(msg->Record.GetIssues(), issues); - LOG_E("KqpSnapshotManager: CreateSnapshot got unexpected status " << status << ": " << issues.ToString()); + YDB_LOG_ERROR("KqpSnapshotManager: CreateSnapshot got unexpected status", + {"status", status}, + {"issues", issues}); ReplyErrorAndDie(msg->Record.GetStatusCode(), std::move(issues)); } } @@ -213,7 +221,7 @@ private: refreshSnapshot->SetSnapshotStep(Snapshot.Step); refreshSnapshot->SetSnapshotTxId(Snapshot.TxId); - LOG_D("KqpSnapshotManager: refreshing snapshot"); + YDB_LOG_DEBUG("KqpSnapshotManager: refreshing snapshot"); Send(MakeTxProxyID(), req.Release()); ScheduleRefresh(); @@ -228,25 +236,27 @@ private: NYql::TIssues issues; NYql::IssuesFromMessage(msg->Record.GetIssues(), issues); - LOG_E("KqpSnapshotManager: RefreshSnapshot got unexpected status=" << status - << ", issues:" << issues.ToString()); + YDB_LOG_ERROR("KqpSnapshotManager: RefreshSnapshot got unexpected", + {"status", status}, + {"issues", issues}); ReplyErrorAndDie(msg->Record.GetStatusCode(), std::move(issues)); } } void HandleAwaitCreation(TEvKqpSnapshot::TEvDiscardSnapshot::TPtr&) { - LOG_D("KqpSnapshotManager: discarding snapshot in awaitCreation state; goto cleanup"); + YDB_LOG_DEBUG("KqpSnapshotManager: discarding snapshot in awaitCreation state; goto cleanup"); Become(&TThis::StateCleanup); } void HandleRefreshing(TEvKqpSnapshot::TEvDiscardSnapshot::TPtr&) { - LOG_W("KqpSnapshotManager: discarding snapshot; our snapshot: " << Snapshot << " shutting down"); + YDB_LOG_WARN("KqpSnapshotManager: discarding snapshot; our shutting down", + {"snapshot", Snapshot}); SendDiscard(); PassAway(); } void Handle(TEvents::TEvPoison::TPtr&) { - LOG_D("KqpSnapshotManager: shutting down on timeout"); + YDB_LOG_DEBUG("KqpSnapshotManager: shutting down on timeout"); ReplyErrorAndDie(NKikimrIssues::TStatusIds::TIMEOUT, {}); } @@ -271,8 +281,9 @@ private: } void HandleUnexpectedEvent(const TString& state, ui32 eventType) { - LOG_E("KqpSnapshotManager: unexpected event, state: " << state - << ", event type: " << eventType); + YDB_LOG_ERROR("KqpSnapshotManager: unexpected event, event", + {"state", state}, + {"type", eventType}); ReplyErrorAndDie(NKikimrIssues::TStatusIds::INTERNAL_ERROR, {}); } diff --git a/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp b/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp index e6baa1a6217..c138a5ca24f 100644 --- a/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp +++ b/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp @@ -29,12 +29,7 @@ #include <exception> #include <forward_list> -#define LOG_T(stream) LOG_TRACE_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_D(stream) LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_I(stream) LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_N(stream) LOG_NOTICE_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_W(stream) LOG_WARN_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_E(stream) LOG_ERROR_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER namespace NKikimr::NKqp { @@ -83,7 +78,9 @@ public: {} void Bootstrap() { - LOG_I("Bootstrap, StreamingDisposition: " << (Ctx->UserRequestContext->StreamingDisposition ? Ctx->UserRequestContext->StreamingDisposition->DebugString() : "null")); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Bootstrap", + {"logPrefix", LogPrefix()}, + {"streamingDisposition", (Ctx->UserRequestContext->StreamingDisposition ? Ctx->UserRequestContext->StreamingDisposition->DebugString() : "null")}); Become(&TThis::StateFuncCreating); } @@ -148,16 +145,20 @@ private: void HandleCreatingFinished() { if (FinishInfo.IsFinished()) { - LOG_N("Script execution metadata saved after failure, continue finishing"); + YDB_LOG_NOTICE_CTX(TActivationContext::AsActorContext(), "Script execution metadata saved after failure, continue finishing", + {"logPrefix", LogPrefix()}); Finish(); // Continue finishing return; } - LOG_I("Script execution metadata saved, creating new session"); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Script execution metadata saved, creating new session", + {"logPrefix", LogPrefix()}); Become(&TThis::StateFuncInitialize); ScriptLeaseWatcherActor.Id = RegisterWithSameMailbox(CreateScriptLeaseWatcherActor(Ctx)); - LOG_I("Started ScriptLeaseWatcherActor: " << ScriptLeaseWatcherActor.Id); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Started", + {"logPrefix", LogPrefix()}, + {"scriptLeaseWatcherActor", ScriptLeaseWatcherActor.Id}); auto ev = std::make_unique<TEvKqp::TEvCreateSessionRequest>(); ev->Record.SetTraceId(Ctx->UserRequestContext->TraceId); @@ -171,7 +172,9 @@ private: } void HandleCancellation(TEvKqp::TEvCancelScriptExecutionRequest::TPtr& ev) { - LOG_I("Got cancel request: " << ev->Sender); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Got cancel", + {"logPrefix", LogPrefix()}, + {"request", ev->Sender}); CancelRequests.emplace_front(std::move(ev)); @@ -181,7 +184,9 @@ private: } void HandleCheckAlive(TEvCheckAliveRequest::TPtr& ev) { - LOG_W("Lease was expired in database, checker actor: " << ev->Sender); + YDB_LOG_WARN_CTX(TActivationContext::AsActorContext(), "Lease was expired in database, checker", + {"logPrefix", LogPrefix()}, + {"actor", ev->Sender}); Send(ev->Sender, new TEvCheckAliveResponse()); } @@ -199,7 +204,10 @@ private: const auto& record = ev->Get()->Record; if (const auto status = record.GetYdbStatus(); status != Ydb::StatusIds::SUCCESS) { const auto resourceExhausted = record.GetResourceExhausted(); - LOG_E("Create new session failed: " << status << ", resource exhausted: " << resourceExhausted); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "Create new session resource", + {"logPrefix", LogPrefix()}, + {"failed", status}, + {"exhausted", resourceExhausted}); auto error = TStringBuilder() << "Create new session failed with " << status; @@ -218,13 +226,16 @@ private: QueryRequest->Record.MutableRequest()->SetSessionId(Ctx->UserRequestContext->SessionId); if (FinishInfo.IsFinished()) { - LOG_N("Session created after finish, continue finishing"); + YDB_LOG_NOTICE_CTX(TActivationContext::AsActorContext(), "Session created after finish, continue finishing", + {"logPrefix", LogPrefix()}); Finish(); return; } if (session.GetNodeId() != SelfId().NodeId()) { - LOG_E("New session started on unexpected node: " << session.GetNodeId()); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "New session started on unexpected", + {"logPrefix", LogPrefix()}, + {"node", session.GetNodeId()}); Finish(Ydb::StatusIds::INTERNAL_ERROR, TStringBuilder() << "Session created on wrong node " << session.GetNodeId() << ", expected local session on node " << SelfId().NodeId()); return; } @@ -233,7 +244,10 @@ private: const auto& physicalGraph = QueryRequest->GetQueryPhysicalGraph(); ScriptResultHandlerActor.Id = RegisterWithSameMailbox(CreateScriptResultHandlerActor(Ctx, physicalGraph ? std::optional(*physicalGraph) : std::nullopt, QueryServiceConfig)); - LOG_D("Started ScriptResultHandlerActor: " << ScriptResultHandlerActor.Id << ", starting query, has physical graph: " << (physicalGraph ? "YES" : "NO")); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Started starting query, has physical", + {"logPrefix", LogPrefix()}, + {"scriptResultHandlerActor", ScriptResultHandlerActor.Id}, + {"graph", (physicalGraph ? "YES" : "NO")}); Ctx->UserRequestContext->RunScriptActorId = ScriptResultHandlerActor.Id; ActorIdToProto(ScriptResultHandlerActor.Id, QueryRequest->Record.MutableRequestActorId()); @@ -245,10 +259,16 @@ private: if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS || !FinishInfo.IsFinished()) { const auto& issues = ev->Get()->Issues; - LOG_E("Got lease watcher finished: " << ev->Sender << " with status " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "Got lease watcher with status", + {"logPrefix", LogPrefix()}, + {"finished", ev->Sender}, + {"status", status}, + {"issues", issues.ToOneLineString()}); Finish(status, AddRootIssue("Script lease watcher error", issues)); } else { - LOG_I("Got lease watcher finished: " << ev->Sender); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Got lease watcher", + {"logPrefix", LogPrefix()}, + {"finished", ev->Sender}); Finish(); } } @@ -269,11 +289,17 @@ private: const auto& response = record.GetResponse(); NYql::TIssues issues; NYql::IssuesFromMessage(response.GetQueryIssues(), issues); - LOG_W("Ignored query response from " << ev->Sender << ", execution already finished, status: " << record.GetYdbStatus() << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN_CTX(TActivationContext::AsActorContext(), "Ignored query response from execution already finished", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", record.GetYdbStatus()}, + {"issues", issues.ToOneLineString()}); return; } - LOG_D("Forward query response from " << ev->Sender << " to result handler"); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Forward query response from to result handler", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); Forward(ev, ScriptResultHandlerActor.Id); } @@ -281,9 +307,15 @@ private: ScriptResultHandlerActor.Id = {}; if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { - LOG_E("Got result handler finished: " << ev->Sender << " with status " << status << ", issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "Got result handler with status", + {"logPrefix", LogPrefix()}, + {"finished", ev->Sender}, + {"status", status}, + {"issues", ev->Get()->Issues.ToOneLineString()}); } else { - LOG_I("Got result handler finished: " << ev->Sender); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Got result handler", + {"logPrefix", LogPrefix()}, + {"finished", ev->Sender}); } ExecutionInfo = std::move(ev->Get()->Info); @@ -309,9 +341,17 @@ private: const auto& issues = ev->Get()->Issues; const auto& info = ev->Get()->Info; if (status != Ydb::StatusIds::SUCCESS) { - LOG_E("Got finalize: " << ev->Sender << " with status " << status << ", issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "Got with status", + {"logPrefix", LogPrefix()}, + {"finalize", ev->Sender}, + {"status", status}, + {"issues", ev->Get()->Issues.ToOneLineString()}); } else { - LOG_I("Got finalize: " << ev->Sender << ", already finished: " << info.AlreadyStopped << ", execution entry exists: " << info.ExecutionEntryExists); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Got already execution entry", + {"logPrefix", LogPrefix()}, + {"finalize", ev->Sender}, + {"finished", info.AlreadyStopped}, + {"exists", info.ExecutionEntryExists}); } const auto alreadyStopped = info.AlreadyStopped || FinishInfo.IsSuccess(); @@ -334,37 +374,48 @@ private: void Finish(const Ydb::StatusIds::StatusCode status, NYql::TIssues issues = {}) { if (status != Ydb::StatusIds::SUCCESS) { - LOG_E("Finish with error " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "Finish with error", + {"logPrefix", LogPrefix()}, + {"status", status}, + {"issues", issues.ToOneLineString()}); } else if (!FinishInfo.IsFailed()) { - LOG_I("Finish successfully"); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Finish successfully", + {"logPrefix", LogPrefix()}); } FinishInfo.Update(status, std::move(issues)); if (ScriptResultHandlerActor.Id) { - LOG_D("Stop script result handler " << ScriptResultHandlerActor.Id); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Stopping script result handler", + {"logPrefix", LogPrefix()}, + {"scriptResultHandlerActorId", ScriptResultHandlerActor.Id}); ScriptResultHandlerActor.Stop(SelfId()); return; } if (SessionState.WaitCreation) { - LOG_D("Wait for session creation before exit"); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Wait for session creation before exit", + {"logPrefix", LogPrefix()}); return; } if (SessionState.SessionOpen) { - LOG_D("Close session"); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Close session", + {"logPrefix", LogPrefix()}); SessionState.Close(SelfId(), *Ctx); } if (ScriptLeaseWatcherActor.Id) { - LOG_D("Stop script lease watcher " << ScriptLeaseWatcherActor.Id); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Stopping script lease watcher", + {"logPrefix", LogPrefix()}, + {"scriptLeaseWatcherActorId", ScriptLeaseWatcherActor.Id}); ScriptLeaseWatcherActor.Stop(SelfId()); return; } if (!WaitFinalizationRequest) { - LOG_I("Start script execution finalization"); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Start script execution finalization", + {"logPrefix", LogPrefix()}); Become(&TThis::StateFuncFinalize); const auto cancelledByUser = !CancelRequests.empty(); @@ -394,7 +445,10 @@ private: Send(MakeKqpFinalizeScriptServiceId(SelfId().NodeId()), scriptFinalizeRequest.release()); WaitFinalizationRequest = true; } else { - LOG_N("Skip finish with error " << *FinishInfo.Status << ", issues: " << FinishInfo.Issues.ToOneLineString() << ", already waiting finalization"); + YDB_LOG_NOTICE_CTX(TActivationContext::AsActorContext(), "Skipping finish with error because finalization is already in progress", + {"logPrefix", LogPrefix()}, + {"finishStatus", *FinishInfo.Status}, + {"issues", FinishInfo.Issues.ToOneLineString()}); } } diff --git a/ydb/core/kqp/run_script_actor/kqp_script_lease_watcher_actor.cpp b/ydb/core/kqp/run_script_actor/kqp_script_lease_watcher_actor.cpp index 67f01958c02..477951cd138 100644 --- a/ydb/core/kqp/run_script_actor/kqp_script_lease_watcher_actor.cpp +++ b/ydb/core/kqp/run_script_actor/kqp_script_lease_watcher_actor.cpp @@ -22,19 +22,14 @@ #include <exception> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + namespace NKikimr::NKqp::NPrivate { namespace { using namespace NActors; -#define LOG_T(stream) LOG_TRACE_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_D(stream) LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_I(stream) LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_N(stream) LOG_NOTICE_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_W(stream) LOG_WARN_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_E(stream) LOG_ERROR_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); - class TScriptLeaseWatcherActor final : public TActorBootstrapped<TScriptLeaseWatcherActor>, IActorExceptionHandler { using TBase = TActorBootstrapped<TScriptLeaseWatcherActor>; @@ -50,7 +45,8 @@ public: } void Bootstrap() { - LOG_I("Bootstrap"); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Bootstrap", + {"logPrefix", LogPrefix()}); Become(&TThis::StateFunc); ScheduleLeaseUpdate(TInstant::Now() + Ctx->LeaseDuration); } @@ -83,9 +79,18 @@ private: const auto executionEntryExists = ev->Get()->ExecutionEntryExists; const auto currentDeadline = ev->Get()->CurrentDeadline; if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { - LOG_E("Lease update " << ev->Sender << " failed " << status << ", issues: " << issues.ToOneLineString() << ", execution entry exists: " << executionEntryExists); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "Lease update failed execution entry", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", issues.ToOneLineString()}, + {"exists", executionEntryExists}); } else { - LOG_D("Lease updated by " << ev->Sender << ", current deadline: " << currentDeadline << ", execution entry exists: " << executionEntryExists); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Lease updated by current execution entry", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"deadline", currentDeadline}, + {"exists", executionEntryExists}); } if (!executionEntryExists) { @@ -102,14 +107,18 @@ private: void ScheduleLeaseUpdate(const TInstant currentDeadline) { LeaseUpdateScheduleTime = TInstant::Now(); const auto leaseUpdateTime = std::max(currentDeadline - Ctx->LeaseDuration / LEASE_UPDATE_FREQUENCY, TInstant::Now() + TDuration::Seconds(1)); - LOG_D("Scheduling lease update on " << leaseUpdateTime); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Scheduling lease update", + {"logPrefix", LogPrefix()}, + {"leaseUpdateTime", leaseUpdateTime}); Schedule(leaseUpdateTime, new TEvents::TEvWakeup()); } void StartLeaseUpdate() { const auto& updaterId = Register(CreateScriptLeaseUpdateActor(SelfId(), Ctx->UserRequestContext->Database, Ctx->UserRequestContext->CurrentExecutionId, Ctx->LeaseDuration, Ctx->LeaseGeneration)); - LOG_D("Run lease updater " << updaterId); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Run lease updater", + {"logPrefix", LogPrefix()}, + {"updaterId", updaterId}); LeaseUpdateStartTime = TInstant::Now(); @@ -128,9 +137,13 @@ private: void Finish(const Ydb::StatusIds::StatusCode status, NYql::TIssues issues = {}) { if (status != Ydb::StatusIds::SUCCESS) { - LOG_E("Finish with error " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "Finish with error", + {"logPrefix", LogPrefix()}, + {"status", status}, + {"issues", issues.ToOneLineString()}); } else if (!FinishInfo.IsFailed()) { - LOG_I("Finish successfully"); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Finish successfully", + {"logPrefix", LogPrefix()}); } FinishInfo.Update(status, std::move(issues)); diff --git a/ydb/core/kqp/run_script_actor/kqp_script_result_handler.cpp b/ydb/core/kqp/run_script_actor/kqp_script_result_handler.cpp index 877774b5448..2eff5b25b68 100644 --- a/ydb/core/kqp/run_script_actor/kqp_script_result_handler.cpp +++ b/ydb/core/kqp/run_script_actor/kqp_script_result_handler.cpp @@ -31,19 +31,14 @@ #include <utility> #include <vector> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_EXECUTER + namespace NKikimr::NKqp::NPrivate { namespace { using namespace NActors; -#define LOG_T(stream) LOG_TRACE_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_D(stream) LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_I(stream) LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_N(stream) LOG_NOTICE_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_W(stream) LOG_WARN_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); -#define LOG_E(stream) LOG_ERROR_S(TActivationContext::AsActorContext(), NKikimrServices::KQP_EXECUTER, LogPrefix() << stream); - class TScriptResultHandlerActor final : public TActorBootstrapped<TScriptResultHandlerActor>, IActorExceptionHandler { using TBase = TActorBootstrapped<TScriptResultHandlerActor>; @@ -196,7 +191,8 @@ public: } void Bootstrap() { - LOG_I("Bootstrap"); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Bootstrap", + {"logPrefix", LogPrefix()}); Become(&TThis::StateFunc); ContinueExecute(); } @@ -242,13 +238,18 @@ private: auto& description = ev->Get()->Description; auto& sinks = description.Sinks; sinks = FilterExternalSinksWithEffects(sinks); - LOG_D("Got script external effect request from " << ev->Sender << ", sinks #" << sinks.size() << ", secrets #" << description.SecretNames.size()); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Received script external effect request", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"sinksCount", sinks.size()}, + {"secretNamesCount", description.SecretNames.size()}); if (!sinks.empty()) { SaveExternalEffectsState.Requests.emplace(ev->Sender, std::move(description)); ContinueExecute(); } else { - LOG_D("No external effects to save, reply immediately"); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "No external effects to save, reply immediately", + {"logPrefix", LogPrefix()}); Send(ev->Sender, new TEvSaveScriptExternalEffectResponse(Ydb::StatusIds::SUCCESS, {})); } } @@ -258,9 +259,15 @@ private: Y_VALIDATE(!SaveExternalEffectsState.Requests.empty(), "Unexpected event"); if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { - LOG_W("Failed to save external effects " << ev->Sender << ", fail: " << status << ", issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_WARN_CTX(TActivationContext::AsActorContext(), "Failed to save external effects", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"fail", status}, + {"issues", ev->Get()->Issues.ToOneLineString()}); } else { - LOG_D("External effects saved " << ev->Sender); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "External effects saved", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); } Forward(ev, SaveExternalEffectsState.Requests.front().first); @@ -269,7 +276,9 @@ private: } void Handle(TEvSaveScriptPhysicalGraphRequest::TPtr& ev) { - LOG_I("Got save script physical graph request from " << ev->Sender); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Got save script physical graph request", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); if (SavePhysicalGraphState.Sender) { Send(ev->Sender, new TEvSaveScriptPhysicalGraphResponse(Ydb::StatusIds::INTERNAL_ERROR, {NYql::TIssue(TStringBuilder() << "Can not save graph twice, previous sender was: " << SavePhysicalGraphState.Sender << ", got graph from: " << ev->Sender)})); @@ -297,9 +306,15 @@ private: const bool saveFailed = status != Ydb::StatusIds::SUCCESS; const auto& issues = ev->Get()->Issues; if (saveFailed) { - LOG_W("Failed to save physical graph " << ev->Sender << ", fail: " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN_CTX(TActivationContext::AsActorContext(), "Failed to save physical graph", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"fail", status}, + {"issues", issues.ToOneLineString()}); } else { - LOG_D("Physical graph saved " << ev->Sender); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Physical graph saved", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); } const auto sendResponse = SavePhysicalGraphState.GraphsToSave.front().first; @@ -317,7 +332,9 @@ private: } void Handle(NFq::TEvCheckpointCoordinator::TEvZeroCheckpointDone::TPtr& ev) { - LOG_I("Zero checkpoint saved by " << ev->Sender); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Zero checkpoint saved by", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); if (!PhysicalGraph) { Finish(Ydb::StatusIds::INTERNAL_ERROR, "Zero checkpoint saved before physical graph saved"); @@ -335,7 +352,11 @@ private: auto& record = ev->Get()->Record; const bool hasPlan = record.HasQueryPlan(); const bool hasAst = record.HasQueryAst(); - LOG_T("Got script progress from " << ev->Sender << ", has plan: " << hasPlan << ", has ast: " << hasAst); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "Got script progress from has has", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"plan", hasPlan}, + {"ast", hasAst}); if (hasPlan) { SaveProgressState.UpdateChanged(ExecutionInfo.QueryPlan, record.GetQueryPlan()); @@ -358,9 +379,17 @@ private: SaveProgressState.QueryStatsChanged = true; SaveProgressState.SuspendUntil = TInstant::Now() + TDuration::Seconds(1); Schedule(SaveProgressState.SuspendUntil, new TEvents::TEvWakeup()); - LOG_N("Script progress updated " << ev->Sender << ", fail: " << status << ", suspend until: " << SaveProgressState.SuspendUntil << ", issues: " << ev->Get()->Issues.ToOneLineString()); + YDB_LOG_NOTICE_CTX(TActivationContext::AsActorContext(), "Script progress updated suspend", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"fail", status}, + {"until", SaveProgressState.SuspendUntil}, + {"issues", ev->Get()->Issues.ToOneLineString()}); } else { - LOG_T("Script progress updated " << ev->Sender << ", ast saved: " << astSaved); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "Script progress updated ast", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"saved", astSaved}); SaveProgressState.AstSaved = SaveProgressState.AstSaved || astSaved; } @@ -374,12 +403,13 @@ private: const auto& resultSet = record.GetResultSet(); const auto rowsCount = resultSet.rows_size(); const auto finished = record.GetFinished(); - LOG_D("Compute stream data" - << ", seq no: " << record.GetSeqNo() - << ", query result index: " << resultSetIndex - << ", rows count: " << rowsCount - << ", finished: " << finished - << ", from: " << ev->Sender); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Compute stream data seq query result rows", + {"logPrefix", LogPrefix()}, + {"no", record.GetSeqNo()}, + {"index", resultSetIndex}, + {"count", rowsCount}, + {"finished", finished}, + {"from", ev->Sender}); auto& resultSetInfos = SaveResultsState.ResultSetInfos; if (resultSetIndex >= resultSetInfos.size()) { @@ -434,7 +464,9 @@ private: } } } else { - LOG_T("Skip truncated result part with #" << rowsCount << " rows"); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "Skip truncated result part with rows", + {"logPrefix", LogPrefix()}, + {"rowsCount", rowsCount}); } const auto channelId = record.GetChannelId(); @@ -454,12 +486,18 @@ private: if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { const auto& issues = ev->Get()->Issues; - LOG_E("Save result meta " << ev->Sender << " failed " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "Save result meta failed", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"status", status}, + {"issues", issues.ToOneLineString()}); Finish(status, AddRootIssue("Failed to save result set meta", issues)); return; } - LOG_D("Save result meta " << ev->Sender << " finished"); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Save result meta finished", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}); ContinueExecute(); } @@ -473,12 +511,20 @@ private: if (const auto status = ev->Get()->Status; status != Ydb::StatusIds::SUCCESS) { const auto& issues = ev->Get()->Issues; - LOG_E("Save result " << ev->Sender << " set #" << resultSetIndex << " failed " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "Save result set failed", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"resultSetIndex", resultSetIndex}, + {"status", status}, + {"issues", issues.ToOneLineString()}); Finish(status, AddRootIssue(TStringBuilder() << "Failed to save result set #" << resultSetIndex, issues)); return; } - LOG_D("Save result " << ev->Sender << " set #" << resultSetIndex << " finished"); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Save result set finished", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"resultSetIndex", resultSetIndex}); auto& resultSetInfo = infos[resultSetIndex]; auto& meta = resultSetInfo.Meta.MutableMeta(); @@ -490,7 +536,11 @@ private: if (const auto freeSpaceBytes = SaveResultsState.GetFreeSpaceBytes(); freeSpaceBytes > 0) { for (auto& [channelId, channel] : StreamChannels) { if (channel.ResumeIfStopped(SelfId(), freeSpaceBytes)) { - LOG_D("Resume execution, " << ", channel: " << channelId << ", seqNo: " << channel.LastSeqNo << ", freeSpace: " << freeSpaceBytes); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Resume execution", + {"logPrefix", LogPrefix()}, + {"channel", channelId}, + {"seqNo", channel.LastSeqNo}, + {"freeSpace", freeSpaceBytes}); } } } @@ -510,9 +560,16 @@ private: const auto status = record.GetYdbStatus(); if (status == Ydb::StatusIds::SUCCESS) { - LOG_I("Script query successfully finished from " << ev->Sender << ", issues: " << issues.ToOneLineString()); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Script query successfully finished", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"issues", issues.ToOneLineString()}); } else { - LOG_W("Script query failed from " << ev->Sender << " " << record.GetYdbStatus() << ", issues: " << issues.ToOneLineString()); + YDB_LOG_WARN_CTX(TActivationContext::AsActorContext(), "Script query failed", + {"logPrefix", LogPrefix()}, + {"sender", ev->Sender}, + {"ydbStatus", record.GetYdbStatus()}, + {"issues", issues.ToOneLineString()}); } if (status == Ydb::StatusIds::TIMEOUT) { @@ -541,7 +598,11 @@ private: if (const auto status = record.GetStatus(); status != Ydb::StatusIds::SUCCESS) { NYql::TIssues issues; NYql::IssuesFromMessage(record.GetIssues(), issues); - LOG_E("Failed to cancel query " << status << ", issues: " << issues.ToOneLineString() << ", response from: " << ev->Sender); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "Failed to cancel query response", + {"logPrefix", LogPrefix()}, + {"status", status}, + {"issues", issues.ToOneLineString()}, + {"from", ev->Sender}); // We can not finish query manually, consider it is already finished QueryIsRunning = false; @@ -550,7 +611,9 @@ private: } // Wait for normal query finish - LOG_I("Query cancelled, response from: " << ev->Sender); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Query cancelled, response", + {"logPrefix", LogPrefix()}, + {"from", ev->Sender}); } bool HasMetadataOperationInflight() const { @@ -572,17 +635,19 @@ private: if (!SaveResultsState.WaitSaveResult) { TryToDrainResults(); } else { - LOG_T("Wait for operations on table `result_sets` to finish"); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "Wait for operations on table `result_sets` to finish", + {"logPrefix", LogPrefix()}); } // Save info to script execution metadata table if (HasMetadataOperationInflight()) { - LOG_T("Wait for operations on table `script_executions` to finish" - << ", save progress: " << SaveProgressState.WaitSave - << ", save external effects: " << SaveExternalEffectsState.WaitSave - << ", save physical graph: " << SavePhysicalGraphState.WaitSave - << ", save results meta: " << SaveResultsState.WaitSaveMeta); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "Wait for operations on table `script_executions` to finish save save external save physical save results", + {"logPrefix", LogPrefix()}, + {"progress", SaveProgressState.WaitSave}, + {"effects", SaveExternalEffectsState.WaitSave}, + {"graph", SavePhysicalGraphState.WaitSave}, + {"meta", SaveResultsState.WaitSaveMeta}); return; } @@ -617,8 +682,11 @@ private: } const auto freeSpaceBytes = SaveResultsState.GetFreeSpaceBytes(); - const auto forceSaveResults = FinishInfo.IsSuccess() || freeSpaceBytes <= 0; - LOG_T("Try to drain results, free space: " << freeSpaceBytes << ", force save: " << forceSaveResults); + const auto forceSaveResults = FinishInfo.IsSuccess() || freeSpaceBytes <= 0; + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "Try to drain results, free force", + {"logPrefix", LogPrefix()}, + {"space", freeSpaceBytes}, + {"save", forceSaveResults}); // We save results when: // - Where is large enough result batch @@ -644,7 +712,10 @@ private: if (resultToSave) { auto& info = SaveResultsState.ResultSetInfos[*resultToSave]; const auto& saverId = Register(CreateSaveScriptExecutionResultActor(SelfId(), Ctx->UserRequestContext->Database, Ctx->UserRequestContext->CurrentExecutionId, *resultToSave, SaveResultsState.GetExpireAt(*Ctx), info.FirstRowId, info.AccumulatedSize, std::move(info.PendingResult))); - LOG_D("Save part for result set #" << *resultToSave << ", saver id: " << saverId); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Saving result set part", + {"logPrefix", LogPrefix()}, + {"resultSetId", *resultToSave}, + {"id", saverId}); SaveResultsState.WaitSaveResult = true; const auto bytes = info.GetBytesToSave(); @@ -660,7 +731,9 @@ private: Y_VALIDATE(!SaveExternalEffectsState.Requests.empty() && !SaveExternalEffectsState.WaitSave, "Unexpected call"); const auto& saverId = Register(CreateSaveScriptExternalEffectActor(SelfId(), Ctx->UserRequestContext->Database, Ctx->UserRequestContext->CurrentExecutionId, std::move(SaveExternalEffectsState.Requests.front().second), Ctx->LeaseGeneration)); - LOG_D("Save external effect, saver id: " << saverId); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Save external effect, saver", + {"logPrefix", LogPrefix()}, + {"id", saverId}); SaveExternalEffectsState.WaitSave = true; } @@ -668,7 +741,9 @@ private: Y_VALIDATE(!SavePhysicalGraphState.GraphsToSave.empty() && !SavePhysicalGraphState.WaitSave, "Unexpected call"); const auto& saverId = Register(CreateSaveScriptExecutionPhysicalGraphActor(SelfId(), Ctx->UserRequestContext->Database, Ctx->UserRequestContext->CurrentExecutionId, std::move(SavePhysicalGraphState.GraphsToSave.front().second), Ctx->LeaseGeneration, QueryServiceConfig)); - LOG_D("Save script physical graph, saver id: " << saverId); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Save script physical graph, saver", + {"logPrefix", LogPrefix()}, + {"id", saverId}); SavePhysicalGraphState.WaitSave = true; } @@ -681,7 +756,10 @@ private: }); const auto& saverId = Register(CreateSaveScriptExecutionResultMetaActor(SelfId(), Ctx->UserRequestContext->Database, Ctx->UserRequestContext->CurrentExecutionId, std::move(metas), Ctx->LeaseGeneration)); - LOG_D("Save result meta for result sets #" << resultsCount << ", saver id: " << saverId); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Save result meta for result sets saver", + {"logPrefix", LogPrefix()}, + {"resultsCount", resultsCount}, + {"id", saverId}); SaveResultsState.WaitSaveMeta = true; } @@ -697,7 +775,9 @@ private: Ctx->LeaseGeneration, QueryServiceConfig )); - LOG_T("Start TScriptProgressActor " << updaterId); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "Start TScriptProgressActor", + {"logPrefix", LogPrefix()}, + {"updaterId", updaterId}); SaveProgressState.WaitSave = true; } @@ -711,9 +791,13 @@ private: void Finish(const Ydb::StatusIds::StatusCode status, NYql::TIssues issues = {}) { if (status != Ydb::StatusIds::SUCCESS) { - LOG_E("Finish with error " << status << ", issues: " << issues.ToOneLineString()); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "Finish with error", + {"logPrefix", LogPrefix()}, + {"status", status}, + {"issues", issues.ToOneLineString()}); } else if (!FinishInfo.IsFailed()) { - LOG_I("Finish successfully"); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Finish successfully", + {"logPrefix", LogPrefix()}); } FinishInfo.Update(status, std::move(issues)); @@ -721,7 +805,9 @@ private: if (QueryIsRunning) { // We should abort query before finish FinishInfo.Update(Ydb::StatusIds::CANCELLED, {NYql::TIssue("Query was cancelled")}); - LOG_I("Wait for query finish, started cancel: " << QueryIsCancelling); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Wait for query finish, started", + {"logPrefix", LogPrefix()}, + {"cancel", QueryIsCancelling}); if (!QueryIsCancelling) { auto ev = MakeHolder<TEvKqp::TEvCancelQueryRequest>(); @@ -733,17 +819,21 @@ private: } if (FinishInfo.IsSuccess() && SaveResultsState.HasResultsToSave()) { - LOG_D("Wait for results to save"); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Wait for results to save", + {"logPrefix", LogPrefix()}); ContinueExecute(); return; } if (HasOperationInflight()) { - LOG_D("Wait for inflight queries to complete"); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "Wait for inflight queries to complete", + {"logPrefix", LogPrefix()}); return; } - LOG_I("Exit, send response to " << Owner); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "Exit, send response", + {"logPrefix", LogPrefix()}, + {"owner", Owner}); Send(Owner, new TEvRunScriptPrivate::TEvScriptResultHandlerFinished(*FinishInfo.Status, std::move(ExecutionInfo), std::move(FinishInfo.Issues))); PassAway(); } diff --git a/ydb/core/kqp/runtime/kqp_buffer_lock_actor.cpp b/ydb/core/kqp/runtime/kqp_buffer_lock_actor.cpp index 99cbee5a6c5..7fcce81aada 100644 --- a/ydb/core/kqp/runtime/kqp_buffer_lock_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_buffer_lock_actor.cpp @@ -13,6 +13,8 @@ #include <ydb/library/yql/dq/actors/protos/dq_stats.pb.h> #include <yql/essentials/public/issue/yql_issue_message.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE + namespace NKikimr { namespace NKqp { @@ -68,7 +70,8 @@ public: } void Bootstrap() { - CA_LOG_D("Start buffer lock actor"); + YDB_LOG_DEBUG("Starting buffer lock actor", + {"logPrefix", this->LogPrefix}); Settings.Counters->StreamLookupActorsCount->Inc(); Become(&TKqpBufferLockActor::StateFunc); @@ -78,7 +81,7 @@ public: void PassAway() final { Settings.Counters->StreamLookupActorsCount->Dec(); - + if (!LockSendTime.empty()) { TInstant now = AppData()->TimeProvider->Now(); TDuration maxInFlightTime = TDuration::Zero(); @@ -251,8 +254,11 @@ public: Settings.Counters->SentLocks->Inc(); auto& record = request->Record; - CA_LOG_D("Start locking of table: " << Settings.TablePath << ", requestId: " << record.GetRequestId() - << ", shardId: " << shardId); + YDB_LOG_DEBUG("Starting row lock request", + {"logPrefix", this->LogPrefix}, + {"table", Settings.TablePath}, + {"requestId", record.GetRequestId()}, + {"shardId", shardId}); Settings.TxManager->AddShard(shardId, false, Settings.TablePath); @@ -282,7 +288,7 @@ public: .ShardId = shardId, .Blocked = false, }).second); - + LockSendTime[requestId] = AppData()->TimeProvider->Now(); } @@ -291,7 +297,9 @@ public: auto requestIt = LockIdToState.find(record.GetRequestId()); if (requestIt == LockIdToState.end() || requestIt->second.Blocked) { - CA_LOG_D("Drop lock with requestId: " << record.GetRequestId() << ", because it's already completed or blocked"); + YDB_LOG_DEBUG("Dropping lock request because it is already completed or blocked", + {"logPrefix", this->LogPrefix}, + {"requestId", record.GetRequestId()}); return; } @@ -307,13 +315,15 @@ public: AFL_ENSURE(lockState.Worker); AFL_ENSURE(lockState.LocksInflight > 0); - CA_LOG_D("Recv TEvLockRowsResult (buffer lock) from ShardID=" << shardId - << ", Table = " << Settings.TablePath - << ", RequestId=" << record.GetRequestId() - << ", Status=" << NKikimrDataEvents::TEvLockRowsResult::EStatus_Name(record.GetStatus())); + YDB_LOG_DEBUG("Received TEvLockRowsResult for buffer lock", + {"logPrefix", this->LogPrefix}, + {"shardID", shardId}, + {"tablePath", Settings.TablePath}, + {"requestId", record.GetRequestId()}, + {"status", NKikimrDataEvents::TEvLockRowsResult::EStatus_Name(record.GetStatus())}); ui64 requestId = record.GetRequestId(); - + if (auto it = LockSendTime.find(requestId); it != LockSendTime.end()) { Settings.Counters->LockLatencyHistogram->Collect((AppData()->TimeProvider->Now() - it->second).MilliSeconds()); LockSendTime.erase(it); @@ -329,7 +339,9 @@ public: case NKikimrDataEvents::TEvLockRowsResult::STATUS_SUCCESS: break; case NKikimrDataEvents::TEvLockRowsResult::STATUS_LOCKS_BROKEN: { - CA_LOG_D("STATUS_LOCKS_BROKEN from shard: " << shardId); + YDB_LOG_DEBUG("Received STATUS_LOCKS_BROKEN from datashard", + {"logPrefix", this->LogPrefix}, + {"shard", shardId}); BrokenLocksCount += record.GetLocks().size(); Settings.TxManager->SetError(shardId); RuntimeError(NYql::NDqProto::StatusIds::ABORTED, @@ -338,7 +350,9 @@ public: return; } case NKikimrDataEvents::TEvLockRowsResult::STATUS_OVERLOADED: { - CA_LOG_D("STATUS_OVERLOADED from shard: " << shardId); + YDB_LOG_DEBUG("Received STATUS_OVERLOADED from datashard", + {"logPrefix", this->LogPrefix}, + {"shard", shardId}); if (!RetryLockRequest(record.GetRequestId(), false)) { return RuntimeError( NYql::NDqProto::StatusIds::OVERLOADED, @@ -349,7 +363,9 @@ public: return; } case NKikimrDataEvents::TEvLockRowsResult::STATUS_DEADLOCK: { - CA_LOG_D("STATUS_DEADLOCK from shard: " << shardId); + YDB_LOG_DEBUG("Received STATUS_DEADLOCK from datashard", + {"logPrefix", this->LogPrefix}, + {"shard", shardId}); return RuntimeError( NYql::NDqProto::StatusIds::ABORTED, NYql::TIssuesIds::KIKIMR_OPERATION_ABORTED, @@ -421,7 +437,9 @@ public: } void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - CA_LOG_D("TEvDeliveryProblem was received from tablet: " << ev->Get()->TabletId); + YDB_LOG_DEBUG("Received TEvDeliveryProblem from datashard", + {"logPrefix", this->LogPrefix}, + {"tablet", ev->Get()->TabletId}); ShardToState.at(ev->Get()->TabletId).HasPipe = false; TVector<ui64> toRetry; @@ -449,7 +467,9 @@ public: const ui64 failedRequestId = ev->Get()->RequestId; auto requestIt = LockIdToState.find(failedRequestId); if (requestIt == LockIdToState.end()) { - CA_LOG_D("received retry request for already finished/non-existing request, request_id: " << failedRequestId); + YDB_LOG_DEBUG("Received retry request for already finished/non-existing request", + {"logPrefix", this->LogPrefix}, + {"requestId", failedRequestId}); return; } @@ -461,8 +481,11 @@ public: bool RetryLockRequest(const ui64 failedRequestId, bool allowInstantRetry) { auto& failedRequest = LockIdToState.at(failedRequestId); auto& lockState = CookieToLockState.at(failedRequest.LockCookie); - CA_LOG_D("Retry locking of table: " << Settings.TablePath << ", failedRequestId: " << failedRequestId - << ", shardId: " << failedRequest.ShardId); + YDB_LOG_DEBUG("Retrying row lock request", + {"logPrefix", this->LogPrefix}, + {"table", Settings.TablePath}, + {"failedRequestId", failedRequestId}, + {"shardId", failedRequest.ShardId}); failedRequest.Blocked = true; if (failedRequest.RetryAttempts >= MaxShardRetries()) { diff --git a/ydb/core/kqp/runtime/kqp_buffer_lookup_actor.cpp b/ydb/core/kqp/runtime/kqp_buffer_lookup_actor.cpp index 69ae765f9e4..8474c135ab5 100644 --- a/ydb/core/kqp/runtime/kqp_buffer_lookup_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_buffer_lookup_actor.cpp @@ -12,6 +12,8 @@ #include <ydb/library/yql/dq/actors/protos/dq_stats.pb.h> #include <yql/essentials/public/issue/yql_issue_message.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE + namespace NKikimr { namespace NKqp { @@ -75,7 +77,8 @@ public: } void Bootstrap() { - CA_LOG_D("Start buffer lookup actor"); + YDB_LOG_DEBUG("Starting buffer lookup actor", + {"logPrefix", this->LogPrefix}); Settings.Counters->StreamLookupActorsCount->Inc(); Become(&TKqpBufferLookupActor::StateFunc); @@ -325,8 +328,11 @@ public: auto& worker = CookieToLookupState.at(cookie).Worker; - CA_LOG_D("Start reading of table: " << worker->GetTablePath() << ", readId: " << record.GetReadId() - << ", shardId: " << shardId); + YDB_LOG_DEBUG("Starting table read for buffer lookup", + {"logPrefix", this->LogPrefix}, + {"table", worker->GetTablePath()}, + {"readId", record.GetReadId()}, + {"shardId", shardId}); Settings.TxManager->AddShard(shardId, false, worker->GetTablePath()); Settings.TxManager->AddAction(shardId, IKqpTransactionManager::EAction::READ); @@ -358,12 +364,15 @@ public: record.SetMaxBytes(defaultSettings.GetMaxBytes()); record.SetResultFormat(NKikimrDataEvents::FORMAT_CELLVEC); - CA_LOG_D(TStringBuilder() << "Send EvRead (buffer lookup) to shardId=" << shardId - << ", readId = " << record.GetReadId() - << ", tablePath: " << worker->GetTablePath() - << ", snapshot=(txid=" << record.GetSnapshot().GetTxId() << ", step=" << record.GetSnapshot().GetStep() << ")" - << ", lockTxId=" << record.GetLockTxId() - << ", lockNodeId=" << record.GetLockNodeId()); + YDB_LOG_DEBUG("Sending EvRead request for buffer lookup", + {"logPrefix", this->LogPrefix}, + {"shardId", shardId}, + {"readId", record.GetReadId()}, + {"tablePath", worker->GetTablePath()}, + {"snapshotTxId", record.GetSnapshot().GetTxId()}, + {"step", record.GetSnapshot().GetStep()}, + {"lockTxId", record.GetLockTxId()}, + {"lockNodeId", record.GetLockNodeId()}); auto& shardState = ShardToState[shardId]; @@ -402,7 +411,9 @@ public: auto readIt = ReadIdToState.find(record.GetReadId()); if (readIt == ReadIdToState.end() || readIt->second.Blocked) { - CA_LOG_D("Drop read with readId: " << record.GetReadId() << ", because it's already completed or blocked"); + YDB_LOG_DEBUG("Dropping read because it is already completed or blocked", + {"logPrefix", this->LogPrefix}, + {"readId", record.GetReadId()}); return; } @@ -418,27 +429,28 @@ public: AFL_ENSURE(lookupState.Worker); AFL_ENSURE(lookupState.ReadsInflight > 0); - CA_LOG_D("Recv TEvReadResult (buffer lookup) from ShardID=" << shardId - << ", Table = " << lookupState.Worker->GetTablePath() - << ", ReadId=" << record.GetReadId() << " (current ReadId=" << ReadId << ")" - << ", SeqNo=" << record.GetSeqNo() - << ", Status=" << Ydb::StatusIds::StatusCode_Name(record.GetStatus().GetCode()) - << ", Finished=" << record.GetFinished() - << ", RowCount=" << record.GetRowCount() - << ", TxLocks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : record.GetTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }() - << ", BrokenTxLocks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : record.GetBrokenTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }()); + TStringBuilder txLocks; + for (const auto& lock : record.GetTxLocks()) { + txLocks << lock.ShortDebugString(); + } + + TStringBuilder borkenTxLocks; + for (const auto& lock : record.GetBrokenTxLocks()) { + borkenTxLocks << lock.ShortDebugString(); + } + + YDB_LOG_DEBUG("Received TEvReadResult for buffer lookup", + {"logPrefix", this->LogPrefix}, + {"shardID", shardId}, + {"tablePath", lookupState.Worker->GetTablePath()}, + {"readId", record.GetReadId()}, + {"currentReadId", ReadId}, + {"seqNo", record.GetSeqNo()}, + {"status", Ydb::StatusIds::StatusCode_Name(record.GetStatus().GetCode())}, + {"finished", record.GetFinished()}, + {"rowCount", record.GetRowCount()}, + {"txLocks", txLocks}, + {"brokenTxLocks", borkenTxLocks}); if (!record.GetBrokenTxLocks().empty()) { BrokenLocksCount += record.GetBrokenTxLocks().size(); @@ -482,8 +494,10 @@ public: getIssues()); } case Ydb::StatusIds::OVERLOADED: { - CA_LOG_D("OVERLOADED was received from tablet: " << shardId << "." - << getIssues().ToOneLineString()); + YDB_LOG_DEBUG("Received OVERLOADED status from datashard", + {"logPrefix", this->LogPrefix}, + {"tablet", shardId}, + {"issues", getIssues().ToOneLineString()}); const std::optional<TDuration> throttleDelay = record.HasThrottleDelayMs() ? std::make_optional(TDuration::MilliSeconds(record.GetThrottleDelayMs())) : std::nullopt; @@ -498,8 +512,10 @@ public: return; } case Ydb::StatusIds::INTERNAL_ERROR: { - CA_LOG_D("INTERNAL_ERROR was received from tablet: " << shardId << "." - << getIssues().ToOneLineString()); + YDB_LOG_DEBUG("Received INTERNAL_ERROR status from datashard", + {"logPrefix", this->LogPrefix}, + {"tablet", shardId}, + {"issues", getIssues().ToOneLineString()}); if (!RetryTableRead(record.GetReadId(), true)) { return RuntimeError( NYql::NDqProto::StatusIds::INTERNAL_ERROR, @@ -562,7 +578,9 @@ public: LookupActorSpan.GetTraceId()); shardState.HasPipe = true; - CA_LOG_D("TEvReadAck was sent to shard: " << shardId); + YDB_LOG_DEBUG("Sent TEvReadAck to datashard", + {"logPrefix", this->LogPrefix}, + {"shard", shardId}); } if (failOnUniqueCheck && record.GetRowCount() != 0) { @@ -583,7 +601,9 @@ public: } void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - CA_LOG_D("TEvDeliveryProblem was received from tablet: " << ev->Get()->TabletId); + YDB_LOG_DEBUG("Received TEvDeliveryProblem from datashard", + {"logPrefix", this->LogPrefix}, + {"tablet", ev->Get()->TabletId}); ShardToState.at(ev->Get()->TabletId).HasPipe = false; TVector<ui64> toRetry; @@ -612,7 +632,9 @@ public: const ui64 failedReadId = ev->Get()->ReadId; auto readIt = ReadIdToState.find(failedReadId); if (readIt == ReadIdToState.end()) { - CA_LOG_D("received retry request for already finished/non-existing read, read_id: " << failedReadId); + YDB_LOG_DEBUG("Received retry request for already finished/non-existing read", + {"logPrefix", this->LogPrefix}, + {"readId", failedReadId}); return; } @@ -627,8 +649,11 @@ public: bool RetryTableRead(const ui64 failedReadId, bool allowInstantRetry, std::optional<TDuration> throttleDelay = std::nullopt) { auto& failedRead = ReadIdToState.at(failedReadId); auto& lookupState = CookieToLookupState.at(failedRead.LookupCookie); - CA_LOG_D("Retry reading of table: " << lookupState.Worker->GetTablePath() << ", failedReadId: " << failedReadId - << ", shardId: " << failedRead.ShardId); + YDB_LOG_DEBUG("Retrying table read for buffer lookup", + {"logPrefix", this->LogPrefix}, + {"table", lookupState.Worker->GetTablePath()}, + {"failedReadId", failedReadId}, + {"shardId", failedRead.ShardId}); failedRead.Blocked = true; TDuration delay; diff --git a/ydb/core/kqp/runtime/kqp_full_text_source.cpp b/ydb/core/kqp/runtime/kqp_full_text_source.cpp index 2261606970c..dced5ba03e5 100644 --- a/ydb/core/kqp/runtime/kqp_full_text_source.cpp +++ b/ydb/core/kqp/runtime/kqp_full_text_source.cpp @@ -88,6 +88,8 @@ #include <cmath> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE + namespace NKikimr::NKqp { using namespace NYql; @@ -2125,7 +2127,10 @@ public: } } - CA_LOG_D("Sending ack for read #" << readId << " seqno = " << readInfo.LastSeqNo); + YDB_LOG_DEBUG("Sending read ack for sequence number", + {"logPrefix", this->LogPrefix}, + {"readId", readId}, + {"lastSeqNo", readInfo.LastSeqNo}); bool newPipe = PipesCreated.insert(shardId).second; TlsActivationContext->Send(new NActors::IEventHandle( @@ -2144,11 +2149,14 @@ public: auto readId = request->Record.GetReadId(); const bool needToCreatePipe = PipesCreated.insert(shardId).second; - CA_LOG_D(TStringBuilder() << "Send EvRead (full text source) to shardId=" << shardId - << ", readId = " << record.GetReadId() - << ", snapshot=(txid=" << record.GetSnapshot().GetTxId() << ", step=" << record.GetSnapshot().GetStep() << ")" - << ", lockTxId=" << record.GetLockTxId() - << ", lockNodeId=" << record.GetLockNodeId()); + YDB_LOG_DEBUG("Sending EvRead request from full text source", + {"logPrefix", this->LogPrefix}, + {"shardId", shardId}, + {"readId", record.GetReadId()}, + {"snapshotTxId", record.GetSnapshot().GetTxId()}, + {"step", record.GetSnapshot().GetStep()}, + {"lockTxId", record.GetLockTxId()}, + {"lockNodeId", record.GetLockNodeId()}); TlsActivationContext->Send(new NActors::IEventHandle( NKikimr::MakePipePerNodeCacheID(false), @@ -2843,8 +2851,12 @@ private: } } if (IsNgram && bestTokenLimit < Words.size()) { - CA_LOG_I("Selecting " << bestTokenLimit << " balanced ngrams out of " << Words.size() - << " (imbalance: " << Words[byFreq[0]]->Frequency << " vs " << Words[byFreq[bestTokenLimit]]->Frequency << ")"); + YDB_LOG_INFO("Selecting balanced ngrams from token frequency list", + {"logPrefix", this->LogPrefix}, + {"bestTokenLimit", bestTokenLimit}, + {"wordCount", Words.size()}, + {"maxFrequency", Words[byFreq[0]]->Frequency}, + {"cutoffFrequency", Words[byFreq[bestTokenLimit]]->Frequency}); TVector<TWordStatePtr> newWords; for (size_t i = 0; i < bestTokenLimit; i++) { newWords.emplace_back(std::move(Words[byFreq[i]])); @@ -2852,8 +2864,12 @@ private: } std::swap(Words, newWords); } else if (MainTableReader->GetWithRelevance() && bestTokenLimit < Words.size() && defaultOperator == EDefaultOperator::And) { - CA_LOG_I("Selecting " << bestTokenLimit << " balanced tokens out of " << Words.size() - << " (imbalance: " << Words[byFreq[0]]->Frequency << " vs " << Words[byFreq[bestTokenLimit]]->Frequency << ")"); + YDB_LOG_INFO("Selecting balanced tokens from token frequency list", + {"logPrefix", this->LogPrefix}, + {"bestTokenLimit", bestTokenLimit}, + {"wordCount", Words.size()}, + {"maxFrequency", Words[byFreq[0]]->Frequency}, + {"cutoffFrequency", Words[byFreq[bestTokenLimit]]->Frequency}); needL2Layer = true; TVector<TWordStatePtr> newWords; @@ -3301,7 +3317,9 @@ public: // Handle broken pipe to a datashard tablet. // Resets pipe tracking and schedules a retry for all reads on that shard. void HandleError(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - CA_LOG_E("TEvDeliveryProblem was received from tablet: " << ev->Get()->TabletId); + YDB_LOG_ERROR("Received TEvDeliveryProblem from datashard", + {"logPrefix", this->LogPrefix}, + {"tablet", ev->Get()->TabletId}); ui64 shardId = ev->Get()->TabletId; ReadsState.UntrackPipe(shardId); @@ -3316,12 +3334,16 @@ public: // - If relevance mode: read stats + dict tables, then proceed to StartWordReads. // - If plain mode: go directly to StartWordReads. void HandleResolve(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) { - CA_LOG_D("TEvResolveKeySetResult was received for table."); + YDB_LOG_DEBUG("Received TEvResolveKeySetResult", + {"logPrefix", this->LogPrefix}); ResolveInProgress = false; if (ev->Get()->Request->ErrorCount > 0) { for(const auto& entry : ev->Get()->Request->ResultSet) { - CA_LOG_E("Table " << entry.KeyDescription->TableId << " error status: " << entry.Status); + YDB_LOG_ERROR("Table resolve error", + {"logPrefix", this->LogPrefix}, + {"tableId", entry.KeyDescription->TableId}, + {"status", entry.Status}); } TString errorMsg = TStringBuilder() << "Failed to get partitioning for table. "; @@ -3659,7 +3681,9 @@ public: L1MergedDocuments.insert(L1MergedDocuments.end(), l1matched.begin(), l1matched.end()); std::vector<TDocInfoPtr> matches = L2MergeAlgo->FindMatches(); - CA_LOG_D("L2Merge done: " << L2MergeAlgo->Done()); + YDB_LOG_DEBUG("Running L2 merge step", + {"logPrefix", this->LogPrefix}, + {"done", L2MergeAlgo->Done()}); MergeL2MatchFrequencies(matches); FetchDocumentDetails(matches); } @@ -3840,12 +3864,14 @@ public: NYql::IssuesFromMessage(record.GetStatus().GetIssues(), shardIssues); const TString tablePath = GetReadTablePath(static_cast<EReadKind>(readInfo.ReadKind)); - CA_LOG_W("Read result error, ReadId=" << readId - << ", ShardId=" << shardId - << ", ReadKind=" << ReadKindName(static_cast<EReadKind>(readInfo.ReadKind)) - << ", Table=" << tablePath - << ", Status=" << Ydb::StatusIds::StatusCode_Name(statusCode) - << ", Issues=[" << shardIssues.ToOneLineString() << "]"); + YDB_LOG_WARN("Read result returned an error", + {"logPrefix", this->LogPrefix}, + {"readId", readId}, + {"shardId", shardId}, + {"readKind", ReadKindName(static_cast<EReadKind>(readInfo.ReadKind))}, + {"table", tablePath}, + {"status", Ydb::StatusIds::StatusCode_Name(statusCode)}, + {"issues", shardIssues.ToOneLineString()}); switch (statusCode) { case Ydb::StatusIds::OVERLOADED: { @@ -3889,30 +3915,29 @@ public: auto& readInfo = *it; - CA_LOG_D("Recv TEvReadResult (full text source)" - << ", Cookie=" << readInfo.Cookie - << ", ReadKind=" << (ui32)readInfo.ReadKind - << ", ShardId=" << readInfo.ShardId - << ", ReadId=" << record.GetReadId() - << ", SeqNo=" << record.GetSeqNo() - << ", Status=" << Ydb::StatusIds::StatusCode_Name(record.GetStatus().GetCode()) - << ", Finished=" << record.GetFinished() - << ", RowCount=" << record.GetRowCount() - << ", ResultFormat=" << NKikimrDataEvents::EDataFormat_Name(record.GetResultFormat()) - << ", TxLocks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : record.GetTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }() - << ", BrokenTxLocks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : record.GetBrokenTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }()); + TStringBuilder txLocks; + for (const auto& lock : record.GetTxLocks()) { + txLocks << lock.ShortDebugString(); + } + + TStringBuilder borkenTxlocks; + for (const auto& lock : record.GetBrokenTxLocks()) { + borkenTxlocks << lock.ShortDebugString(); + } + + YDB_LOG_DEBUG("Received TEvReadResult from full text source", + {"logPrefix", this->LogPrefix}, + {"cookie", readInfo.Cookie}, + {"readKind", (ui32)readInfo.ReadKind}, + {"shardId", readInfo.ShardId}, + {"readId", record.GetReadId()}, + {"seqNo", record.GetSeqNo()}, + {"status", Ydb::StatusIds::StatusCode_Name(record.GetStatus().GetCode())}, + {"finished", record.GetFinished()}, + {"rowCount", record.GetRowCount()}, + {"resultFormat", NKikimrDataEvents::EDataFormat_Name(record.GetResultFormat())}, + {"txLocks", txLocks}, + {"brokenTxLocks", borkenTxlocks}); if (record.GetStatus().GetCode() != Ydb::StatusIds::SUCCESS) { HandleReadResultError(readId, readInfo, record); diff --git a/ydb/core/kqp/runtime/kqp_read_actor.cpp b/ydb/core/kqp/runtime/kqp_read_actor.cpp index 27b6d3e18f7..574b379bac6 100644 --- a/ydb/core/kqp/runtime/kqp_read_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_read_actor.cpp @@ -24,6 +24,8 @@ #include <util/generic/intrlist.h> #include <util/string/vector.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE + namespace { bool IsDebugLogEnabled(const NActors::TActorSystem* actorSystem, NActors::NLog::EComponent component) { @@ -448,7 +450,9 @@ public: } } - CA_LOG_D("Shards State: " << state.ToString(KeyColumnTypes)); + YDB_LOG_DEBUG("Started table scan with initial shard state", + {"logPrefix", this->LogPrefix}, + {"state", state.ToString(KeyColumnTypes)}); if (!Settings->HasShardIdHint()) { state.IsFake = true; @@ -462,11 +466,18 @@ public: bool StartShards() { const ui32 maxAllowedInFlight = Settings->GetSorted() || Settings->GetIsBatch() ? 1 : MaxInFlight; - CA_LOG_D("effective maxinflight " << maxAllowedInFlight << " sorted " << Settings->GetSorted()); + YDB_LOG_DEBUG("Computed effective max in-flight shard count", + {"logPrefix", this->LogPrefix}, + {"maxAllowedInFlight", maxAllowedInFlight}, + {"sorted", Settings->GetSorted()}); bool isFirst = true; while (!PendingShards.Empty() && RunningReads() + 1 <= maxAllowedInFlight) { if (isFirst) { - CA_LOG_D("BEFORE: " << PendingShards.Size() << "." << RunningReads()); + YDB_LOG_DEBUG("Starting next batch of shard reads", + {"logPrefix", this->LogPrefix}, + {"event", "startShardsBefore"}, + {"pendingShardsBefore", PendingShards.Size()}, + {"runningReads", RunningReads()}); isFirst = false; } if (Settings->GetReverse()) { @@ -480,11 +491,17 @@ public: } } if (!isFirst) { - CA_LOG_D("AFTER: " << PendingShards.Size() << "." << RunningReads()); + YDB_LOG_DEBUG("Finished starting batch of shard reads", + {"logPrefix", this->LogPrefix}, + {"event", "startShardsAfter"}, + {"pendingShardsAfter", PendingShards.Size()}, + {"runningReads", RunningReads()}); } - CA_LOG_D("Scheduled table scans, in flight: " << RunningReads() << " shards. " - << "pending shards to read: " << PendingShards.Size() << ", "); + YDB_LOG_DEBUG("Scheduled table scans across shards", + {"logPrefix", this->LogPrefix}, + {"runningReads", RunningReads()}, + {"pendingShards", PendingShards.Size()}); return RunningReads() > 0 || !PendingShards.Empty(); } @@ -513,9 +530,11 @@ public: auto keyDesc = MakeHolder<TKeyDesc>(TableId, range, TKeyDesc::ERowOperation::Read, KeyColumnTypes, columns); - CA_LOG_D("Sending TEvResolveKeySet update for table '" << Settings->GetTable().GetTablePath() << "'" - << ", range: " << DebugPrintRange(KeyColumnTypes, range, *AppData()->TypeRegistry) - << ", attempt #" << state->ResolveAttempt); + YDB_LOG_DEBUG("Sending TEvResolveKeySet to scheme cache", + {"logPrefix", this->LogPrefix}, + {"tablePath", Settings->GetTable().GetTablePath()}, + {"range", DebugPrintRange(KeyColumnTypes, range, *AppData()->TypeRegistry)}, + {"resolveAttempt", state->ResolveAttempt}); auto request = MakeHolder<NSchemeCache::TSchemeCacheRequest>(); request->DatabaseName = Settings->GetDatabase(); @@ -532,7 +551,9 @@ public: } void HandleResolve(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) { - CA_LOG_D("Received TEvResolveKeySetResult update for table '" << Settings->GetTable().GetTablePath() << "'"); + YDB_LOG_DEBUG("Received TEvResolveKeySetResult from scheme cache", + {"logPrefix", this->LogPrefix}, + {"tablePath", Settings->GetTable().GetTablePath()}); auto* request = ev->Get()->Request.Get(); THolder<TShardState> state; @@ -544,7 +565,10 @@ public: } if (request->ErrorCount > 0 || !state) { - CA_LOG_E("Resolve request failed for table '" << Settings->GetTable().GetTablePath() << "', ErrorCount# " << request->ErrorCount); + YDB_LOG_ERROR("Shard resolve request failed", + {"logPrefix", this->LogPrefix}, + {"tablePath", Settings->GetTable().GetTablePath()}, + {"errorCount", request->ErrorCount}); auto statusCode = NDqProto::StatusIds::UNAVAILABLE; TString error; @@ -583,7 +607,9 @@ public: if (keyDesc->GetPartitions().empty()) { TString error = TStringBuilder() << "No partitions to read from '" << Settings->GetTable().GetTablePath() << "'"; - CA_LOG_E(error); + YDB_LOG_ERROR("No partitions found for resolved table", + {"logPrefix", this->LogPrefix}, + {"error", error}); return RuntimeError(error, NDqProto::StatusIds::SCHEME_ERROR); } else if (keyDesc->GetPartitions().size() == 1) { auto& partition = keyDesc->GetPartitions()[0]; @@ -637,26 +663,34 @@ public: keyDesc->GetPartitions()[idx].Range->IsInclusive }; - CA_LOG_D("Processing resolved ShardId# " << partition.ShardId - << ", partition range: " << DebugPrintRange(KeyColumnTypes, partitionRange, tr) - << ", i: " << rangeIndex << ", state ranges: " << ranges.size() - << ", points: " << points.size()); + YDB_LOG_DEBUG("Processing resolved partition for shard split", + {"logPrefix", this->LogPrefix}, + {"shardId", partition.ShardId}, + {"range", DebugPrintRange(KeyColumnTypes, partitionRange, tr)}, + {"rangeIndex", rangeIndex}, + {"rangesCount", ranges.size()}, + {"pointsCount", points.size()}); auto newShard = MakeHolder<TShardState>(partition.ShardId); if (state->HasRanges()) { for (ui64 j = rangeIndex; j < ranges.size(); ++j) { auto comparison = CompareRanges(partitionRange, ranges[j].ToTableRange(), KeyColumnTypes); - CA_LOG_D("Compare range #" << j << " " << DebugPrintRange(KeyColumnTypes, ranges[j].ToTableRange(), tr) - << " with partition range " << DebugPrintRange(KeyColumnTypes, partitionRange, tr) - << " : " << comparison); + YDB_LOG_DEBUG("Comparing read range with partition range", + {"logPrefix", this->LogPrefix}, + {"rangeIndex", j}, + {"range", DebugPrintRange(KeyColumnTypes, ranges[j].ToTableRange(), tr)}, + {"partitionRange", DebugPrintRange(KeyColumnTypes, partitionRange, tr)}, + {"comparison", comparison}); if (comparison > 0) { continue; } else if (comparison == 0) { auto intersection = Intersect(KeyColumnTypes, partitionRange, ranges[j].ToTableRange()); - CA_LOG_D("Add range to new shardId: " << partition.ShardId - << ", range: " << DebugPrintRange(KeyColumnTypes, intersection, tr)); + YDB_LOG_DEBUG("Adding intersected range to new shard", + {"logPrefix", this->LogPrefix}, + {"shardId", partition.ShardId}, + {"range", DebugPrintRange(KeyColumnTypes, intersection, tr)}); newShard->AddRange(TSerializedTableRange(intersection)); } else { @@ -678,7 +712,9 @@ public: if (intersection == 0) { newShard->AddPoint(std::move(points[pointIndex])); - CA_LOG_D("Add point to new shardId: " << partition.ShardId); + YDB_LOG_DEBUG("Adding point to new shard", + {"logPrefix", this->LogPrefix}, + {"shardId", partition.ShardId}); } else { YQL_ENSURE(intersection > 0, "Missed intersection of point and partition ranges."); break; @@ -727,7 +763,10 @@ public: sb << st.ToString(KeyColumnTypes) << "; "; } } - CA_LOG_D(sb); + YDB_LOG_DEBUG("Shard queue state after resolve", + {"logPrefix", this->LogPrefix}, + {"event", "shardQueueAfterResolve"}, + {"shardStates", sb}); } StartShards(); } @@ -782,7 +821,10 @@ public: return DoRetryRead(id); } - CA_LOG_D("schedule retry #" << id << " after " << delay); + YDB_LOG_DEBUG("Scheduled read retry after delay", + {"logPrefix", this->LogPrefix}, + {"readId", id}, + {"delay", delay}); TlsActivationContext->Schedule(delay, new IEventHandle(SelfId(), SelfId(), new TEvRetryShard(id, Reads[id].LastSeqNo))); } @@ -792,7 +834,9 @@ public: } auto state = Reads[id].Shard; - CA_LOG_D("Retrying read #" << id); + YDB_LOG_DEBUG("Retrying read", + {"logPrefix", this->LogPrefix}, + {"readId", id}); ResetRead(id); @@ -903,14 +947,18 @@ public: record.SetPoolId(Settings->GetPoolId()); } - CA_LOG_D(TStringBuilder() << "Send EvRead to shardId: " << state->TabletId << ", tablePath: " << Settings->GetTable().GetTablePath() - << ", ranges: " << DebugPrintRanges(KeyColumnTypes, ev->Ranges, *AppData()->TypeRegistry) - << ", limit: " << limit - << ", readId = " << id - << ", reverse = " << record.GetReverse() - << ", snapshot = (txid=" << Settings->GetSnapshot().GetTxId() << ",step=" << Settings->GetSnapshot().GetStep() << ")" - << ", lockTxId = " << Settings->GetLockTxId() - << ", lockNodeId = " << Settings->GetLockNodeId()); + YDB_LOG_DEBUG("Sending TEvRead to data shard", + {"logPrefix", this->LogPrefix}, + {"shardId", state->TabletId}, + {"tablePath", Settings->GetTable().GetTablePath()}, + {"ranges", DebugPrintRanges(KeyColumnTypes, ev->Ranges, *AppData()->TypeRegistry)}, + {"limit", limit}, + {"readId", id}, + {"reverse", record.GetReverse()}, + {"snapshotTxId", Settings->GetSnapshot().GetTxId()}, + {"snapshotStep", Settings->GetSnapshot().GetStep()}, + {"lockTxId", Settings->GetLockTxId()}, + {"lockNodeId", Settings->GetLockNodeId()}); Counters->CreatedIterators->Inc(); ReadIdByTabletId[state->TabletId].push_back(id); @@ -950,9 +998,10 @@ public: } void ReportNullValue(const THolder<TEventHandle<TEvDataShard::TEvReadResult>>& result, size_t columnIndex) { - CA_LOG_D(TStringBuilder() << "validation failed, " - << " seqno = " << result->Get()->Record.GetSeqNo() - << " finished = " << result->Get()->Record.GetFinished()); + YDB_LOG_DEBUG("Read validation failed: NULL value in NOT NULL column", + {"logPrefix", this->LogPrefix}, + {"seqNo", result->Get()->Record.GetSeqNo()}, + {"finished", result->Get()->Record.GetFinished()}); NYql::TIssue issue; issue.SetCode(NYql::TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION, NYql::TSeverityIds::S_FATAL); issue.SetMessage(TStringBuilder() @@ -971,25 +1020,24 @@ public: return; } - CA_LOG_D("Recv TEvReadResult from ShardID=" << Reads[id].Shard->TabletId - << ", ReadId=" << id - << ", Status=" << Ydb::StatusIds::StatusCode_Name(record.GetStatus().GetCode()) - << ", Finished=" << record.GetFinished() - << ", RowCount=" << record.GetRowCount() - << ", TxLocks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : record.GetTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }() - << ", BrokenTxLocks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : record.GetBrokenTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }()); + TStringBuilder txLocks; + for (const auto& lock : record.GetTxLocks()) { + txLocks << lock.ShortDebugString(); + } + TStringBuilder brokenTxlocks; + for (const auto& lock : record.GetBrokenTxLocks()) { + brokenTxlocks << lock.ShortDebugString(); + } + + YDB_LOG_DEBUG("Received TEvReadResult from data shard", + {"logPrefix", this->LogPrefix}, + {"shardId", Reads[id].Shard->TabletId}, + {"readId", id}, + {"status", Ydb::StatusIds::StatusCode_Name(record.GetStatus().GetCode())}, + {"finished", record.GetFinished()}, + {"rowCount", record.GetRowCount()}, + {"txLocks", txLocks}, + {"brokenTxLocks", brokenTxlocks}); if (!record.HasNodeId()) { Counters->ReadActorAbsentNodeId->Inc(); @@ -997,14 +1045,20 @@ public: auto* state = Reads[id].Shard; if (!state->NodeId) { state->NodeId = record.GetNodeId(); - CA_LOG_D("Node mismatch for tablet " << state->TabletId << " " << *state->NodeId << " != SelfId: " << SelfId().NodeId()); + YDB_LOG_DEBUG("Detected node mismatch for tablet read", + {"logPrefix", this->LogPrefix}, + {"tabletId", state->TabletId}, + {"nodeId", *state->NodeId}, + {"selfNodeId", SelfId().NodeId()}); if (state->IsFirst) { Counters->ReadActorRemoteFirstFetch->Inc(); } Counters->ReadActorRemoteFetch->Inc(); } } else { - CA_LOG_T("Node match for tablet " << Reads[id].Shard->TabletId); + YDB_LOG_TRACE("Node matches local node for tablet read", + {"logPrefix", this->LogPrefix}, + {"tabletId", Reads[id].Shard->TabletId}); } Counters->DataShardIteratorMessages->Inc(); @@ -1013,7 +1067,10 @@ public: } for (auto& issue : record.GetStatus().GetIssues()) { - CA_LOG_D("read id #" << id << " got issue " << issue.Getmessage()); + YDB_LOG_DEBUG("Read result contains issue", + {"logPrefix", this->LogPrefix}, + {"readId", id}, + {"issueMessage", issue.Getmessage()}); Reads[id].Shard->Issues.push_back(issue); } @@ -1097,7 +1154,9 @@ public: YQL_ENSURE(Locks.empty()); } - CA_LOG_D("Taken " << Locks.size() << " locks"); + YDB_LOG_DEBUG("Collected transaction locks from read result", + {"logPrefix", this->LogPrefix}, + {"locksCount", Locks.size()}); Reads[id].SerializedContinuationToken = record.GetContinuationToken(); ui64 seqNo = record.GetSeqNo(); @@ -1110,10 +1169,16 @@ public: ReceivedRowCount += msg.GetRowsCount(); - CA_LOG_D(TStringBuilder() << "new data for read #" << id - << " seqno = " << seqNo - << " finished = " << record.GetFinished()); - CA_LOG_T(TStringBuilder() << "read #" << id << " pushed " << DebugPrintCells(&msg) << " continuation token " << DebugPrintContinuationToken(record.GetContinuationToken())); + YDB_LOG_DEBUG("Queued new read result batch", + {"logPrefix", this->LogPrefix}, + {"readId", id}, + {"seqNo", seqNo}, + {"finished", record.GetFinished()}); + YDB_LOG_TRACE("Read result pushed with continuation token", + {"logPrefix", this->LogPrefix}, + {"readId", id}, + {"cells", DebugPrintCells(&msg)}, + {"continuationToken", DebugPrintContinuationToken(record.GetContinuationToken())}); Results.push({Reads[id].Shard->TabletId, THolder<TEventHandle<TEvDataShard::TEvReadResult>>(ev.Release()), id, seqNo}); NotifyCA(); @@ -1125,7 +1190,10 @@ public: HasEstablishedPipe.erase(msg.TabletId); TVector<ui32> reads; reads = ReadIdByTabletId[msg.TabletId]; - CA_LOG_W("Got EvDeliveryProblem, TabletId: " << msg.TabletId << ", NotDelivered: " << msg.NotDelivered); + YDB_LOG_WARN("Received TEvDeliveryProblem from pipe cache", + {"logPrefix", this->LogPrefix}, + {"tabletId", msg.TabletId}, + {"notDelivered", msg.NotDelivered}); for (auto read : reads) { if (Reads[read]) { Counters->IteratorDeliveryProblems->Inc(); @@ -1266,11 +1334,12 @@ public: auto& [shardId, result, batch, processedRows, packed, readId, seqNo] = handle; NMiniKQL::TBytesStatistics stats; batch->reserve(batch->size()); - CA_LOG_D(TStringBuilder() << "enter pack cells method " - << " shardId: " << shardId - << " processedRows: " << processedRows - << " packed rows: " << packed - << " freeSpace: " << freeSpace); + YDB_LOG_DEBUG("Entering PackCells for read result", + {"logPrefix", this->LogPrefix}, + {"shardId", shardId}, + {"processedRows", processedRows}, + {"packedRows", packed}, + {"freeSpace", freeSpace}); for (size_t rowIndex = packed; rowIndex < result->Get()->GetRowsCount(); ++rowIndex) { const auto& row = result->Get()->GetCells(rowIndex); @@ -1313,11 +1382,12 @@ public: } } - CA_LOG_D(TStringBuilder() << "exit pack cells method " - << " shardId: " << shardId - << " processedRows: " << processedRows - << " packed rows: " << packed - << " freeSpace: " << freeSpace); + YDB_LOG_DEBUG("Exiting PackCells for read result", + {"logPrefix", this->LogPrefix}, + {"shardId", shardId}, + {"processedRows", processedRows}, + {"packedRows", packed}, + {"freeSpace", freeSpace}); return stats; } @@ -1339,8 +1409,10 @@ public: YQL_ENSURE(!resultBatch.IsWide(), "Wide stream is not supported"); - CA_LOG_D(TStringBuilder() << " enter getasyncinputdata results size " << Results.size() - << ", freeSpace " << freeSpace); + YDB_LOG_DEBUG("Entering GetAsyncInputData", + {"logPrefix", this->LogPrefix}, + {"resultsCount", Results.size()}, + {"freeSpace", freeSpace}); ui64 bytes = 0; while (!Results.empty()) { @@ -1375,11 +1447,15 @@ public: bytes += rowSize.AllocatedBytes; if (ProcessedRowCount == Settings->GetItemsLimit()) { finished = true; - CA_LOG_D(TStringBuilder() << " returned async data because limit reached"); + YDB_LOG_DEBUG("Returned async data because limit reached", + {"logPrefix", this->LogPrefix}); return bytes; } } - CA_LOG_D(TStringBuilder() << "returned " << resultBatch.RowCount() << " rows; processed " << ProcessedRowCount << " rows"); + YDB_LOG_DEBUG("Returned rows from result batch", + {"logPrefix", this->LogPrefix}, + {"resultBatchRowCount", resultBatch.RowCount()}, + {"processedRowCount", ProcessedRowCount}); size_t rowCount = result.ReadResult.Get()->Get()->GetRowsCount(); if (rowCount == result.ProcessedRows) { @@ -1399,7 +1475,11 @@ public: request->Record.SetMaxRows(*limit); } Counters->SentIteratorAcks->Inc(); - CA_LOG_D("sending ack for read #" << id << " limit " << limit << " seqno = " << record.GetSeqNo()); + YDB_LOG_DEBUG("Sending TEvReadAck to data shard", + {"logPrefix", this->LogPrefix}, + {"readId", id}, + {"limit", limit}, + {"seqNo", record.GetSeqNo()}); bool newPipe = HasEstablishedPipe.insert(Reads[id].Shard->TabletId).second; Send(PipeCacheId, new TEvPipeCache::TEvForward(request.Release(), Reads[id].Shard->TabletId, TEvPipeCache::TEvForwardOptions{ .AutoConnect = newPipe, @@ -1419,7 +1499,9 @@ public: } Results.pop(); - CA_LOG_D("dropping batch for read #" << id); + YDB_LOG_DEBUG("Dropped processed read result batch", + {"logPrefix", this->LogPrefix}, + {"readId", id}); if (LimitReached()) { finished = true; @@ -1436,15 +1518,16 @@ public: finished = true; } - CA_LOG_D(TStringBuilder() << "returned async data" - << " processed rows " << ProcessedRowCount - << " left freeSpace " << freeSpace - << " received rows " << ReceivedRowCount - << " running reads " << RunningReads() - << " pending shards " << PendingShards.Size() - << " finished = " << finished - << " has limit " << (Settings->GetItemsLimit() != 0) - << " limit reached " << LimitReached()); + YDB_LOG_DEBUG("Returning async input data to compute actor", + {"logPrefix", this->LogPrefix}, + {"processedRowCount", ProcessedRowCount}, + {"freeSpace", freeSpace}, + {"receivedRowCount", ReceivedRowCount}, + {"runningReads", RunningReads()}, + {"pendingShardsCount", PendingShards.Size()}, + {"finished", finished}, + {"hasItemsLimit", (Settings->GetItemsLimit() != 0)}, + {"limitReached", LimitReached()}); return bytes; } diff --git a/ydb/core/kqp/runtime/kqp_sequencer_actor.cpp b/ydb/core/kqp/runtime/kqp_sequencer_actor.cpp index a140eda96ef..4ba601e991f 100644 --- a/ydb/core/kqp/runtime/kqp_sequencer_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_sequencer_actor.cpp @@ -22,6 +22,8 @@ #include <list> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE + namespace NKikimr { namespace NKqp { @@ -143,7 +145,8 @@ public: void Bootstrap() { Counters->SequencerActorsCount->Inc(); - CA_LOG_D("Start stream lookup actor"); + YDB_LOG_DEBUG("Start stream lookup actor", + {"logPrefix", this->LogPrefix}); Become(&TKqpSequencerActor::StateFunc); } @@ -189,9 +192,12 @@ private: if (PendingRows.size() > 0 && WaitingReplies == 0) { Send(ComputeActorId, new TEvNewAsyncInputDataArrived(InputIndex)); - } + } - CA_LOG_D("Returned " << totalDataSize << " bytes, finished: " << finished); + YDB_LOG_DEBUG("Returned bytes", + {"logPrefix", this->LogPrefix}, + {"totalDataSize", totalDataSize}, + {"finished", finished}); return totalDataSize; } @@ -291,7 +297,7 @@ private: Counters->SequencerErrors->Inc(); TStringBuilder result; result << "Failed to get next val for sequence: " << ColumnSequenceInfo[ev->Cookie].DefaultFromSequence - << ", status: " << ev->Get()->Status; + << ", status: " << ev->Get()->Status; RuntimeError(result, NYql::NDq::YdbStatusToDqStatus(ev->Get()->Status), ev->Get()->Issues); return; } diff --git a/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp b/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp index 2259b4b93a3..7de344e907d 100644 --- a/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp @@ -19,6 +19,8 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h> #include <ydb/library/wilson_ids/wilson.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE + namespace NKikimr { namespace NKqp { @@ -121,7 +123,8 @@ public: } void Bootstrap() { - CA_LOG_D("Start stream lookup actor"); + YDB_LOG_DEBUG("Stream lookup actor started", + {"logPrefix", this->LogPrefix}); Counters->StreamLookupActorsCount->Inc(); ResolveTableShards(); @@ -530,8 +533,11 @@ private: finished = inputRowsFinished && allReadsFinished && allRowsProcessed; - CA_LOG_D("Returned " << replyResultStats.ResultBytesCount << " bytes, " << replyResultStats.ResultRowsCount - << " rows, finished: " << finished); + YDB_LOG_DEBUG("Returned lookup results", + {"logPrefix", this->LogPrefix}, + {"resultBytesCount", replyResultStats.ResultBytesCount}, + {"resultRowsCount", replyResultStats.ResultRowsCount}, + {"finished", finished}); return replyResultStats.ResultBytesCount; } @@ -578,7 +584,9 @@ private: } void Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr& ev) { - CA_LOG_D("TEvResolveKeySetResult was received for table: " << StreamLookupWorker->GetTablePath()); + YDB_LOG_DEBUG("Received TEvResolveKeySetResult", + {"logPrefix", this->LogPrefix}, + {"table", StreamLookupWorker->GetTablePath()}); if (!ResolveShardsInProgress) { return; } @@ -622,34 +630,36 @@ private: auto readIt = Reads.find(record.GetReadId()); if (readIt == Reads.end() || readIt->second.State != EReadState::Running) { - CA_LOG_D("Drop read with readId: " << record.GetReadId() << ", because it's already completed or blocked"); + YDB_LOG_DEBUG("Dropped read: already completed or blocked", + {"logPrefix", this->LogPrefix}, + {"readId", record.GetReadId()}); return; } auto& read = readIt->second; ui64 shardId = read.ShardId; - CA_LOG_D("Recv TEvReadResult (stream lookup) from ShardID=" << read.ShardId - << ", Table = " << StreamLookupWorker->GetTablePath() - << ", ReadId=" << record.GetReadId() << " (current OperationId=" << OperationId << ")" - << ", SeqNo=" << record.GetSeqNo() - << ", Status=" << Ydb::StatusIds::StatusCode_Name(record.GetStatus().GetCode()) - << ", Finished=" << record.GetFinished() - << ", RowCount=" << record.GetRowCount() - << ", Locks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : record.GetTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }() - << ", BrokenTxLocks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : record.GetBrokenTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }()); + TStringBuilder txLocks; + for (const auto& lock : record.GetTxLocks()) { + txLocks << lock.ShortDebugString(); + } + TStringBuilder brokenTxLocks; + for (const auto& lock : record.GetBrokenTxLocks()) { + brokenTxLocks << lock.ShortDebugString(); + } + + YDB_LOG_DEBUG("Received TEvReadResult from table shard", + {"logPrefix", this->LogPrefix}, + {"shardId", read.ShardId}, + {"tablePath", StreamLookupWorker->GetTablePath()}, + {"readId", record.GetReadId()}, + {"operationId", OperationId}, + {"seqNo", record.GetSeqNo()}, + {"status", Ydb::StatusIds::StatusCode_Name(record.GetStatus().GetCode())}, + {"finished", record.GetFinished()}, + {"rowCount", record.GetRowCount()}, + {"locks", txLocks}, + {"brokenTxLocks", brokenTxLocks}); for (auto& lock : record.GetBrokenTxLocks()) { BrokenLocks.push_back(lock); @@ -699,8 +709,10 @@ private: case Ydb::StatusIds::NOT_FOUND: { StreamLookupWorker->ResetRowsProcessing(read.Id); - CA_LOG_D("NOT_FOUND was received from tablet: " << read.ShardId << ". " - << getIssues().ToOneLineString()); + YDB_LOG_DEBUG("Received NOT_FOUND status", + {"logPrefix", this->LogPrefix}, + {"tablet", read.ShardId}, + {"issues", getIssues().ToOneLineString()}); Reads.eraseRead(read); return ResolveTableShards(); } @@ -713,8 +725,10 @@ private: TStringBuilder() << "Table '" << StreamLookupWorker->GetTablePath() << "' retry limit exceeded.", NYql::NDqProto::StatusIds::OVERLOADED); } - CA_LOG_D("OVERLOADED was received from tablet: " << read.ShardId << "." - << getIssues().ToOneLineString()); + YDB_LOG_DEBUG("Received OVERLOADED status", + {"logPrefix", this->LogPrefix}, + {"tablet", read.ShardId}, + {"issues", getIssues().ToOneLineString()}); read.SetBlocked(); return RetryTableRead(read, /*allowInstantRetry = */false, throttleDelay); } @@ -724,8 +738,10 @@ private: TStringBuilder() << "Table '" << StreamLookupWorker->GetTablePath() << "' retry limit exceeded.", NYql::NDqProto::StatusIds::INTERNAL_ERROR); } - CA_LOG_D("INTERNAL_ERROR was received from tablet: " << read.ShardId << "." - << getIssues().ToOneLineString()); + YDB_LOG_DEBUG("Received INTERNAL_ERROR status", + {"logPrefix", this->LogPrefix}, + {"tablet", read.ShardId}, + {"issues", getIssues().ToOneLineString()}); read.SetBlocked(); return RetryTableRead(read); } @@ -768,7 +784,9 @@ private: Reads.SetPipeCreated(read.ShardId); - CA_LOG_D("TEvReadAck was sent to shard: " << read.ShardId); + YDB_LOG_DEBUG("TEvReadAck sent", + {"logPrefix", this->LogPrefix}, + {"shard", read.ShardId}); if (auto delay = ShardTimeout()) { TlsActivationContext->Schedule( @@ -789,7 +807,9 @@ private: } void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - CA_LOG_D("TEvDeliveryProblem was received from tablet: " << ev->Get()->TabletId); + YDB_LOG_DEBUG("Received TEvDeliveryProblem", + {"logPrefix", this->LogPrefix}, + {"tablet", ev->Get()->TabletId}); const auto& tabletId = ev->Get()->TabletId; @@ -820,12 +840,16 @@ private: } void Handle(TEvPrivate::TEvSchemeCacheRequestTimeout::TPtr&) { - CA_LOG_D("TEvSchemeCacheRequestTimeout was received, shards for table " << StreamLookupWorker->GetTablePath() - << " was resolved: " << !!Partitioning); + YDB_LOG_DEBUG("Scheme cache request timed out while resolving table shards", + {"logPrefix", this->LogPrefix}, + {"tablePath", StreamLookupWorker->GetTablePath()}, + {"resolved", !!Partitioning}); if (!Partitioning) { LookupActorStateSpan.EndError("timeout exceeded"); - CA_LOG_D("Retry attempt to resolve shards for table: " << StreamLookupWorker->GetTablePath()); + YDB_LOG_DEBUG("Retry attempt to resolve shards", + {"logPrefix", this->LogPrefix}, + {"table", StreamLookupWorker->GetTablePath()}); ResolveShardsInProgress = false; ResolveTableShards(); } @@ -834,7 +858,9 @@ private: void Handle(TEvPrivate::TEvRetryRead::TPtr& ev) { auto readIt = Reads.find(ev->Get()->ReadId); if (readIt == Reads.end()) { - CA_LOG_D("received retry request for already finished/non-existing read, read_id: " << ev->Get()->ReadId); + YDB_LOG_DEBUG("Received retry request for already finished/non-existing read", + {"logPrefix", this->LogPrefix}, + {"readId", ev->Get()->ReadId}); return; } @@ -857,7 +883,9 @@ private: void Handle(TEvPrivate::TEvRetryLock::TPtr& ev) { auto lockIt = Reads.findLock(ev->Get()->LockId); if (lockIt == Reads.endLocks()) { - CA_LOG_D("received retry request for already finished/non-existing lock, lock_id: " << ev->Get()->LockId); + YDB_LOG_DEBUG("Received retry request for already finished/non-existing lock", + {"logPrefix", this->LogPrefix}, + {"lockId", ev->Get()->LockId}); return; } @@ -965,7 +993,9 @@ private: } void SendLockRequest(ui64 shardId, THolder<NEvents::TDataEvents::TEvLockRows> request) { - CA_LOG_D("Send lock request to shard: " << shardId); + YDB_LOG_DEBUG("Send lock request", + {"logPrefix", this->LogPrefix}, + {"shard", shardId}); Counters->SentLocks->Inc(); ui64 requestId = request->Record.GetRequestId(); @@ -1000,8 +1030,10 @@ private: void Handle(NEvents::TDataEvents::TEvLockRowsResult::TPtr& ev) { const auto& record = ev->Get()->Record; - CA_LOG_D("Received lock result, requestId: " << record.GetRequestId() - << ", status: " << record.GetStatus()); + YDB_LOG_DEBUG("Received lock result", + {"logPrefix", this->LogPrefix}, + {"requestId", record.GetRequestId()}, + {"status", record.GetStatus()}); ui64 requestId = record.GetRequestId(); @@ -1020,14 +1052,18 @@ private: case NKikimrDataEvents::TEvLockRowsResult::STATUS_SUCCESS: break; case NKikimrDataEvents::TEvLockRowsResult::STATUS_LOCKS_BROKEN: { - CA_LOG_D("STATUS_LOCKS_BROKEN from shard: " << record.GetTabletId()); + YDB_LOG_DEBUG("Lock request returned STATUS_LOCKS_BROKEN", + {"logPrefix", this->LogPrefix}, + {"shard", record.GetTabletId()}); return RuntimeError( TStringBuilder() << "Table: `" << StreamLookupWorker->GetTablePath() << "`. Locks Invalidated.", NYql::NDqProto::StatusIds::ABORTED, getIssues()); } case NKikimrDataEvents::TEvLockRowsResult::STATUS_OVERLOADED: { - CA_LOG_D("STATUS_OVERLOADED from shard: " << record.GetTabletId()); + YDB_LOG_DEBUG("Lock request returned STATUS_OVERLOADED", + {"logPrefix", this->LogPrefix}, + {"shard", record.GetTabletId()}); auto lockIt = Reads.findLock(record.GetRequestId()); if (lockIt != Reads.endLocks()) { return RetryLock(lockIt->second, false); @@ -1036,7 +1072,9 @@ private: return; } case NKikimrDataEvents::TEvLockRowsResult::STATUS_DEADLOCK: { - CA_LOG_D("STATUS_DEADLOCK from shard: " << record.GetTabletId()); + YDB_LOG_DEBUG("Lock request returned STATUS_DEADLOCK", + {"logPrefix", this->LogPrefix}, + {"shard", record.GetTabletId()}); return RuntimeError( TStringBuilder() << "Table: `" << StreamLookupWorker->GetTablePath() << "`. " << "Deadlock detected", NYql::NDqProto::StatusIds::ABORTED, @@ -1128,8 +1166,11 @@ private: Counters->CreatedIterators->Inc(); auto& record = request->Record; - CA_LOG_D("Start reading of table: " << StreamLookupWorker->GetTablePath() << ", readId: " << record.GetReadId() - << ", shardId: " << shardId); + YDB_LOG_DEBUG("Started table read", + {"logPrefix", this->LogPrefix}, + {"table", StreamLookupWorker->GetTablePath()}, + {"readId", record.GetReadId()}, + {"shardId", shardId}); TReadState read(record.GetReadId(), shardId); @@ -1172,12 +1213,15 @@ private: Counters->StreamLookupIteratorTotalQuotaBytesExceeded->Inc(); } - CA_LOG_D(TStringBuilder() << "Send EvRead (stream lookup) to shardId=" << shardId - << ", readId = " << record.GetReadId() - << ", tablePath: " << StreamLookupWorker->GetTablePath() - << ", snapshot=(txid=" << record.GetSnapshot().GetTxId() << ", step=" << record.GetSnapshot().GetStep() << ")" - << ", lockTxId=" << record.GetLockTxId() - << ", lockNodeId=" << record.GetLockNodeId()); + YDB_LOG_DEBUG("Sent EvRead to shard", + {"logPrefix", this->LogPrefix}, + {"shardId", shardId}, + {"readId", record.GetReadId()}, + {"tablePath", StreamLookupWorker->GetTablePath()}, + {"snapshotTxId", record.GetSnapshot().GetTxId()}, + {"step", record.GetSnapshot().GetStep()}, + {"lockTxId", record.GetLockTxId()}, + {"lockNodeId", record.GetLockNodeId()}); const bool needToCreatePipe = Reads.NeedToCreatePipe(read.ShardId); @@ -1214,8 +1258,11 @@ private: } void RetryTableRead(TReadState& failedRead, bool allowInstantRetry = true, std::optional<TDuration> throttleDelay = std::nullopt) { - CA_LOG_D("Retry reading of table: " << StreamLookupWorker->GetTablePath() << ", readId: " << failedRead.Id - << ", shardId: " << failedRead.ShardId); + YDB_LOG_DEBUG("Retrying table read", + {"logPrefix", this->LogPrefix}, + {"table", StreamLookupWorker->GetTablePath()}, + {"readId", failedRead.Id}, + {"shardId", failedRead.ShardId}); TDuration delay; if (!throttleDelay) { @@ -1243,7 +1290,10 @@ private: Reads.eraseRead(failedRead); ScheduleNextReads(); } else { - CA_LOG_D("Schedule retry attempt for readId: " << failedRead.Id << " after " << delay); + YDB_LOG_DEBUG("Scheduled read retry", + {"logPrefix", this->LogPrefix}, + {"readId", failedRead.Id}, + {"delay", delay}); TlsActivationContext->Schedule( delay, new IEventHandle(SelfId(), SelfId(), new TEvPrivate::TEvRetryRead(failedRead.Id, failedRead.LastSeqNo, /*instantStart = */ true)) ); @@ -1251,7 +1301,10 @@ private: } void RetryLock(TLockState& failedLock, bool allowInstantRetry = true) { - CA_LOG_D("Retry locking for shard: " << failedLock.ShardId << ", lockId: " << failedLock.Id); + YDB_LOG_DEBUG("Retry locking", + {"logPrefix", this->LogPrefix}, + {"shard", failedLock.ShardId}, + {"lockId", failedLock.Id}); if (CheckTotalRetriesExceeded()) { return RuntimeError(TStringBuilder() << "Table '" << StreamLookupWorker->GetTablePath() << "' lock retry limit exceeded", @@ -1273,7 +1326,10 @@ private: Reads.eraseLock(failedLock); DrainPendingLocks(); } else { - CA_LOG_D("Schedule retry for lockId: " << failedLock.Id << " after " << delay); + YDB_LOG_DEBUG("Scheduled lock retry", + {"logPrefix", this->LogPrefix}, + {"lockId", failedLock.Id}, + {"delay", delay}); TlsActivationContext->Schedule( delay, new IEventHandle(SelfId(), SelfId(), new TEvPrivate::TEvRetryLock(failedLock.Id, /*instantStart = */ true)) ); @@ -1290,7 +1346,9 @@ private: NYql::NDqProto::StatusIds::UNAVAILABLE); } - CA_LOG_D("Resolve shards for table: " << StreamLookupWorker->GetTablePath()); + YDB_LOG_DEBUG("Resolve shards", + {"logPrefix", this->LogPrefix}, + {"table", StreamLookupWorker->GetTablePath()}); ResolveShardsInProgress = true; Partitioning.reset(); diff --git a/ydb/core/kqp/runtime/kqp_sys_view_source.cpp b/ydb/core/kqp/runtime/kqp_sys_view_source.cpp index 7c3f63053bf..de95adac35a 100644 --- a/ydb/core/kqp/runtime/kqp_sys_view_source.cpp +++ b/ydb/core/kqp/runtime/kqp_sys_view_source.cpp @@ -22,6 +22,8 @@ #include <yql/essentials/minikql/computation/mkql_computation_node_holders.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE + namespace NKikimr::NKqp { using namespace NActors; @@ -31,11 +33,6 @@ using namespace NKikimr::NMiniKQL; namespace { -#define LOG_PREFIX "TKqpSysViewSource " -#define LOG_D(msg) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, LOG_PREFIX << SelfId() << " " << msg) -#define LOG_W(msg) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, LOG_PREFIX << SelfId() << " " << msg) -#define LOG_E(msg) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, LOG_PREFIX << SelfId() << " " << msg) - class TKqpSysViewSource : public TActorBootstrapped<TKqpSysViewSource>, public IDqComputeActorAsyncInput { public: TKqpSysViewSource( @@ -57,7 +54,8 @@ public: } void Bootstrap() { - LOG_D("Bootstrap"); + YDB_LOG_DEBUG("TKqpSysViewSource Bootstrap", + {"selfId", SelfId()}); const auto& table = Settings->GetTable(); TTableId tableId(table.GetOwnerId(), table.GetTableId(), table.GetSysView()); @@ -113,7 +111,9 @@ public: if (!scanActor) { auto issue = TStringBuilder() << "Failed to create system view scan, table id: " << tableId; - LOG_E(issue); + YDB_LOG_ERROR("TKqpSysViewSource", + {"selfId", SelfId()}, + {"issue", issue}); Send(ComputeActorId, new TEvAsyncInputError(InputIndex, TIssues({TIssue(issue)}), NDqProto::StatusIds::INTERNAL_ERROR)); @@ -121,7 +121,9 @@ public: } ScanActorId = Register(scanActor.Release()); - LOG_D("Registered scan actor: " << ScanActorId); + YDB_LOG_DEBUG("TKqpSysViewSource Registered scan", + {"selfId", SelfId()}, + {"actor", ScanActorId}); // Send initial ack to start data flow Send(ScanActorId, new TEvKqpCompute::TEvScanDataAck(BufferSize)); @@ -139,13 +141,16 @@ public: hFunc(TEvKqpCompute::TEvScanData, Handle); hFunc(TEvKqpCompute::TEvScanError, Handle); default: - LOG_W("Unexpected event: " << ev->GetTypeRewrite()); + YDB_LOG_WARN("TKqpSysViewSource Unexpected", + {"selfId", SelfId()}, + {"event", ev->GetTypeRewrite()}); } } private: void Handle(TEvKqpCompute::TEvScanInitActor::TPtr& ev) { - LOG_D("Got scan init actor event"); + YDB_LOG_DEBUG("TKqpSysViewSource Got scan init actor event", + {"selfId", SelfId()}); // Respond with ack so the scan actor knows we're ready Send(ev->Sender, new TEvKqpCompute::TEvScanDataAck(BufferSize)); } @@ -153,9 +158,11 @@ private: void Handle(TEvKqpCompute::TEvScanData::TPtr& ev) { auto& msg = *ev->Get(); - LOG_D("Got scan data, rows: " << msg.Rows.size() - << ", finished: " << msg.Finished - << ", from: " << ev->Sender); + YDB_LOG_DEBUG("TKqpSysViewSource Got scan data", + {"selfId", SelfId()}, + {"rows", msg.Rows.size()}, + {"finished", msg.Finished}, + {"from", ev->Sender}); if (!msg.Rows.empty()) { auto guard = Guard(*Alloc); @@ -165,7 +172,8 @@ private: } if (msg.ArrowBatch && msg.ArrowBatch->num_rows() > 0) { - LOG_W("Arrow batches not supported in sys view source, ignoring"); + YDB_LOG_WARN("TKqpSysViewSource Arrow batches not supported in sys view source, ignoring", + {"selfId", SelfId()}); } if (msg.Finished) { @@ -184,12 +192,16 @@ private: if (status == Ydb::StatusIds::SUCCESS) { ScanWarnings.AddIssues(issues); - LOG_W("Got partial compile cache scan warning: " << issues.ToOneLineString()); + YDB_LOG_WARN("TKqpSysViewSource Got partial compile cache scan", + {"selfId", SelfId()}, + {"warning", issues.ToOneLineString()}); return; } - LOG_E("Got scan error, status: " << Ydb::StatusIds::StatusCode_Name(status) - << ", issues: " << issues.ToOneLineString()); + YDB_LOG_ERROR("TKqpSysViewSource Got scan error", + {"selfId", SelfId()}, + {"status", Ydb::StatusIds::StatusCode_Name(status)}, + {"issues", issues.ToOneLineString()}); Send(ComputeActorId, new TEvAsyncInputError(InputIndex, issues, NYql::NDq::YdbStatusToDqStatus(status, NYql::NDq::EStatusCompatibilityLevel::WithUnauthorized))); @@ -200,7 +212,9 @@ private: return; } ScanWarningsSent = true; - LOG_W("Forwarding compile cache scan warnings to compute: " << ScanWarnings.ToOneLineString()); + YDB_LOG_WARN("TKqpSysViewSource Forwarding compile cache scan warnings", + {"selfId", SelfId()}, + {"compute", ScanWarnings.ToOneLineString()}); Send(ComputeActorId, new NYql::NDq::IDqComputeActorAsyncInput::TEvAsyncInputError( InputIndex, ScanWarnings, NYql::NDqProto::StatusIds::UNSPECIFIED)); } diff --git a/ydb/core/kqp/runtime/kqp_vector_actor.cpp b/ydb/core/kqp/runtime/kqp_vector_actor.cpp index 5f0734e518b..8b016ca71df 100644 --- a/ydb/core/kqp/runtime/kqp_vector_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_vector_actor.cpp @@ -13,6 +13,8 @@ #include <util/string/vector.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE + namespace NKikimr { namespace NKqp { @@ -74,7 +76,8 @@ public: void Bootstrap() { //Counters->VectorResolveActorsCount->Inc(); - CA_LOG_D("Start vector resolve actor"); + YDB_LOG_DEBUG("Start vector resolve actor", + {"logPrefix", this->LogPrefix}); Become(&TKqpVectorResolveActor::StateFunc); } @@ -138,7 +141,10 @@ private: } } - CA_LOG_D("Returned " << totalDataSize << " bytes, finished: " << finished); + YDB_LOG_DEBUG("Returned bytes", + {"logPrefix", this->LogPrefix}, + {"totalDataSize", totalDataSize}, + {"finished", finished}); return totalDataSize; } diff --git a/ydb/core/kqp/runtime/kqp_vector_index_levels_cache.cpp b/ydb/core/kqp/runtime/kqp_vector_index_levels_cache.cpp index 6fd13f807a8..51551d702db 100644 --- a/ydb/core/kqp/runtime/kqp_vector_index_levels_cache.cpp +++ b/ydb/core/kqp/runtime/kqp_vector_index_levels_cache.cpp @@ -22,16 +22,14 @@ #include <util/generic/algorithm.h> #include <util/generic/hash.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE + namespace NKikimr::NKqp { namespace { constexpr ui64 LevelCacheTxId = std::numeric_limits<ui64>::max() - 1; -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "VectorIndexLevelsCacheMaintainer: " << stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_COMPUTE, "VectorIndexLevelsCacheMaintainer: " << stream) - - class TVectorIndexLevelsCacheMaintainer : public TActorBootstrapped<TVectorIndexLevelsCacheMaintainer> { @@ -83,13 +81,14 @@ public: void Handle(TEvents::TEvUndelivered::TPtr& ev) { switch (ev->Get()->SourceType) { case NConsole::TEvConfigsDispatcher::EvSetConfigSubscriptionRequest: - LOG_E("Failed to deliver subscription request to config dispatcher"); + YDB_LOG_ERROR("VectorIndexLevelsCacheMaintainer: Failed to deliver subscription request to config dispatcher"); break; case NConsole::TEvConsole::EvConfigNotificationResponse: - LOG_E("Failed to deliver config notification response"); + YDB_LOG_ERROR("VectorIndexLevelsCacheMaintainer: Failed to deliver config notification response"); break; default: - LOG_E("Undelivered event with unexpected source type: " << ev->Get()->SourceType); + YDB_LOG_ERROR("VectorIndexLevelsCacheMaintainer: Undelivered event with unexpected source", + {"type", ev->Get()->SourceType}); break; } } @@ -116,8 +115,9 @@ public: auto res = ResourceManager->AllocateResources(*Tx, 1, NRm::TKqpResourcesRequest{.Memory=increaseBatchSize}); if (res) { Cache_->SetMaxBytes(maxCurrentSizeBytes + increaseBatchSize); - LOG_N("Altered max bytes to " << HumanReadableSize(maxCurrentSizeBytes + increaseBatchSize, ESizeFormat::SF_BYTES) - << ", prev size " << HumanReadableSize(maxCurrentSizeBytes, ESizeFormat::SF_BYTES)); + YDB_LOG_NOTICE("Vector index levels cache max size increased", + {"newMaxSize", HumanReadableSize(maxCurrentSizeBytes + increaseBatchSize, ESizeFormat::SF_BYTES)}, + {"previousMaxSize", HumanReadableSize(maxCurrentSizeBytes, ESizeFormat::SF_BYTES)}); } } else if (maxAllowedSizeBytes < static_cast<ui64>(maxCurrentSizeBytes)) { @@ -126,8 +126,9 @@ public: ResourceManager->FreeResources(*Tx, 1, NRm::TKqpResourcesRequest{.Memory=change}); i64 newSize = maxCurrentSizeBytes - static_cast<i64>(change); Cache_->SetMaxBytes(newSize); - LOG_N("Altered max bytes to " << HumanReadableSize(newSize, ESizeFormat::SF_BYTES) - << ", prev size " << HumanReadableSize(maxCurrentSizeBytes, ESizeFormat::SF_BYTES)); + YDB_LOG_NOTICE("Vector index levels cache max size decreased", + {"newMaxSize", HumanReadableSize(newSize, ESizeFormat::SF_BYTES)}, + {"previousMaxSize", HumanReadableSize(maxCurrentSizeBytes, ESizeFormat::SF_BYTES)}); } Schedule(TDuration::Seconds(1), new TEvPrivate::TEvIncreaseCacheSize); @@ -141,9 +142,6 @@ private: NKikimrConfig::TTableServiceConfig::TResourceManager RmConfig; }; -#undef LOG_E -#undef LOG_N - } // anonymous namespace IActor* CreateVectorIndexLevelsCacheMaintainer( diff --git a/ydb/core/kqp/runtime/kqp_write_actor.cpp b/ydb/core/kqp/runtime/kqp_write_actor.cpp index ae71723000d..7e2aea2c457 100644 --- a/ydb/core/kqp/runtime/kqp_write_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_write_actor.cpp @@ -35,6 +35,8 @@ #include <ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h> #include <yql/essentials/public/issue/yql_issue_message.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE + namespace { TDuration CalculateNextAttemptDelay(const NKikimr::NKqp::TWriteActorSettings& settings, ui64 attempt) { @@ -583,7 +585,9 @@ public: // even when batches from multiple queries are later combined into a single EvWrite. ShardedWriteController->SetTokenQuerySpanId(token, CurrentQuerySpanId); - CA_LOG_D("Open: token=" << token); + YDB_LOG_DEBUG("Open", + {"logPrefix", this->LogPrefix}, + {"token", token}); } void Write( @@ -591,14 +595,18 @@ public: IDataBatchPtr data) { YQL_ENSURE(!Closed); YQL_ENSURE(ShardedWriteController); - CA_LOG_D("Write: token=" << token); + YDB_LOG_DEBUG("Write", + {"logPrefix", this->LogPrefix}, + {"token", token}); ShardedWriteController->Write(token, std::move(data)); } void Close(TWriteToken token) { YQL_ENSURE(!Closed); YQL_ENSURE(ShardedWriteController); - CA_LOG_D("Close: token=" << token); + YDB_LOG_DEBUG("Close", + {"logPrefix", this->LogPrefix}, + {"token", token}); ShardedWriteController->Close(token); } @@ -687,7 +695,9 @@ public: } void PlanResolve() { - CA_LOG_D("Plan resolve with delay " << CalculateNextAttemptDelay(MessageSettings, ResolveAttempts)); + YDB_LOG_DEBUG("Scheduling table resolve retry with delay.", + {"logPrefix", this->LogPrefix}, + {"nextAttemptDelay", CalculateNextAttemptDelay(MessageSettings, ResolveAttempts)}); TlsActivationContext->Schedule( CalculateNextAttemptDelay(MessageSettings, ResolveAttempts), new IEventHandle(SelfId(), SelfId(), new TEvPrivate::TEvResolveRequestPlanned{}, 0, 0)); @@ -703,8 +713,10 @@ public: Partitioning.reset(); if (ResolveAttempts++ >= MessageSettings.MaxResolveAttempts) { - CA_LOG_E(TStringBuilder() - << "Too many table resolve attempts for table `" << TablePath << "` (" << TableId << ")."); + YDB_LOG_ERROR("Too many table resolve attempts exceeded.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"tableId", TableId}); RuntimeError( NYql::NDqProto::StatusIds::SCHEME_ERROR, NYql::TIssuesIds::KIKIMR_SCHEME_ERROR, @@ -713,7 +725,9 @@ public: return; } - CA_LOG_D("Resolve TableId=" << TableId); + YDB_LOG_DEBUG("Resolve", + {"logPrefix", this->LogPrefix}, + {"tableId", TableId}); TAutoPtr<NSchemeCache::TSchemeCacheNavigate> request(new NSchemeCache::TSchemeCacheNavigate()); request->DatabaseName = Database; NSchemeCache::TSchemeCacheNavigate::TEntry entry; @@ -732,17 +746,21 @@ public: YQL_ENSURE(resultSet.size() == 1); if (ev->Get()->Request->ErrorCount > 0) { - CA_LOG_E(TStringBuilder() << "Failed to get table: " - << TableId << "'. Entry: " << resultSet[0].ToString()); + YDB_LOG_ERROR("Failed to resolve table from scheme cache.", + {"logPrefix", this->LogPrefix}, + {"table", TableId}, + {"entry", resultSet[0]}); PlanResolve(); return; } SchemeEntry = resultSet[0]; - CA_LOG_D("Resolved TableId=" << TableId << " (" - << TablePath << " " - << TableId.SchemaVersion << ")"); + YDB_LOG_DEBUG("Table resolved from scheme cache.", + {"logPrefix", this->LogPrefix}, + {"tableId", TableId}, + {"tablePath", TablePath}, + {"schemaVersion", TableId.SchemaVersion}); if (TableId.SchemaVersion != SchemeEntry->TableId.SchemaVersion) { RuntimeError( @@ -759,7 +777,9 @@ public: void ResolveShards() { YQL_ENSURE(!KeyColumnTypes.empty()); - CA_LOG_D("Resolve shards for TableId=" << TableId); + YDB_LOG_DEBUG("Resolve shards", + {"logPrefix", this->LogPrefix}, + {"tableId", TableId}); AFL_ENSURE(InconsistentTx); // Only for CTAS @@ -785,8 +805,9 @@ public: auto* request = ev->Get()->Request.Get(); if (request->ErrorCount > 0) { - CA_LOG_E(TStringBuilder() << "Failed to get table: " - << TableId << "'"); + YDB_LOG_ERROR("Failed to resolve table shards from scheme cache.", + {"logPrefix", this->LogPrefix}, + {"table", TableId}); PlanResolve(); return; } @@ -794,7 +815,10 @@ public: YQL_ENSURE(request->ResultSet.size() == 1); Partitioning = std::move(request->ResultSet[0].KeyDescription->Partitioning); - CA_LOG_D("Resolved shards for TableId=" << TableId << ". PartitionsCount=" << Partitioning->Size() << "."); + YDB_LOG_DEBUG("Table shards resolved from scheme cache.", + {"logPrefix", this->LogPrefix}, + {"tableId", TableId}, + {"partitionsCount", Partitioning->Size()}); Prepare(); } @@ -802,7 +826,9 @@ public: void OnOverloadReady(const ui64 shardId, const ui64 seqNo) { const auto metadata = ShardedWriteController->GetMessageMetadata(shardId); if (metadata && seqNo + 1 == metadata->NextOverloadSeqNo) { - CA_LOG_D("Retry Overloaded ShardID=" << shardId); + YDB_LOG_DEBUG("Retry Overloaded", + {"logPrefix", this->LogPrefix}, + {"shardID", shardId}); ResetShardRetries(shardId, metadata->Cookie); SendDataToShard(shardId); } @@ -832,17 +858,18 @@ public: return issues; }; - CA_LOG_D("Recv EvWriteResult from ShardID=" << ev->Get()->Record.GetOrigin() - << ", Status=" << NKikimrDataEvents::TEvWriteResult::EStatus_Name(ev->Get()->GetStatus()) - << ", TxId=" << ev->Get()->Record.GetTxId() - << ", Locks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : ev->Get()->Record.GetTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }() - << ", Cookie=" << ev->Cookie); + TStringBuilder txLocks; + for (const auto& lock : ev->Get()->Record.GetTxLocks()) { + txLocks << lock.ShortDebugString(); + } + + YDB_LOG_DEBUG("Recv EvWriteResult", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"status", NKikimrDataEvents::TEvWriteResult::EStatus_Name(ev->Get()->GetStatus())}, + {"txId", ev->Get()->Record.GetTxId()}, + {"locks", txLocks}, + {"cookie", ev->Cookie}); TxManager->AddParticipantNode(ev->Sender.NodeId()); @@ -850,11 +877,12 @@ public: || ev->Get()->GetStatus() == NKikimrDataEvents::TEvWriteResult::STATUS_OVERLOADED; if (ev->Get()->Record.HasOverloadSubscribed() && handleOverload) { - CA_LOG_I("Got OverloadSubscribed for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_INFO("Received EvWriteResult with overload subscription.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); const auto metadata = ShardedWriteController->GetMessageMetadata(ev->Get()->Record.GetOrigin()); if (metadata && ev->Get()->Record.GetOverloadSubscribed() + 1 == metadata->NextOverloadSeqNo) { @@ -866,11 +894,12 @@ public: switch (ev->Get()->GetStatus()) { case NKikimrDataEvents::TEvWriteResult::STATUS_UNSPECIFIED: { - CA_LOG_E("Got UNSPECIFIED for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received EvWriteResult with unspecified status.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); UpdateStats(ev->Get()->Record.GetTxStats()); TxManager->SetError(ev->Get()->Record.GetOrigin()); RuntimeError( @@ -890,11 +919,12 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_ABORTED: { - CA_LOG_E("Got ABORTED for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received EvWriteResult with aborted status.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); UpdateStats(ev->Get()->Record.GetTxStats()); TxManager->SetError(ev->Get()->Record.GetOrigin()); RuntimeError( @@ -905,11 +935,12 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_WRONG_SHARD_STATE: - CA_LOG_E("Got WRONG SHARD STATE for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received EvWriteResult with wrong shard state.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); if (InconsistentTx) { ResetShardRetries(ev->Get()->Record.GetOrigin(), ev->Cookie); @@ -926,11 +957,12 @@ public: } return; case NKikimrDataEvents::TEvWriteResult::STATUS_INTERNAL_ERROR: { - CA_LOG_E("Got INTERNAL ERROR for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received EvWriteResult with internal error.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); UpdateStats(ev->Get()->Record.GetTxStats()); TxManager->SetError(ev->Get()->Record.GetOrigin()); RuntimeError( @@ -941,11 +973,12 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_DATABASE_DISK_SPACE_QUOTA_EXCEEDED: { - CA_LOG_E("Got DATABASE_DISK_SPACE_QUOTA_EXCEEDED for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received EvWriteResult with database disk space quota exceeded status.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); UpdateStats(ev->Get()->Record.GetTxStats()); TxManager->SetError(ev->Get()->Record.GetOrigin()); RuntimeError( @@ -957,12 +990,12 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_DISK_GROUP_OUT_OF_SPACE: { - CA_LOG_W("Got DISK_GROUP_OUT_OF_SPACE for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << " Ignored this error." - << getIssues().ToOneLineString()); + YDB_LOG_WARN("Received EvWriteResult with disk group out of space status.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); // TODO: support waiting if (!InconsistentTx) { UpdateStats(ev->Get()->Record.GetTxStats()); @@ -977,12 +1010,12 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_OVERLOADED: { - CA_LOG_W("Got OVERLOADED for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << " Ignored this error." - << getIssues().ToOneLineString()); + YDB_LOG_WARN("Received EvWriteResult with overloaded status.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); // TODO: support waiting if (!InconsistentTx) { UpdateStats(ev->Get()->Record.GetTxStats()); @@ -998,11 +1031,12 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_CANCELLED: { - CA_LOG_E("Got CANCELLED for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received EvWriteResult with cancelled status.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); UpdateStats(ev->Get()->Record.GetTxStats()); TxManager->SetError(ev->Get()->Record.GetOrigin()); RuntimeError( @@ -1013,11 +1047,12 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST: { - CA_LOG_E("Got BAD REQUEST for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received EvWriteResult with bad request status.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); UpdateStats(ev->Get()->Record.GetTxStats()); TxManager->SetError(ev->Get()->Record.GetOrigin()); RuntimeError( @@ -1029,11 +1064,12 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_SCHEME_CHANGED: { - CA_LOG_E("Got SCHEME CHANGED for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received EvWriteResult with scheme changed status.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); if (InconsistentTx) { ResetShardRetries(ev->Get()->Record.GetOrigin(), ev->Cookie); RetryResolve(); @@ -1050,11 +1086,12 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_LOCKS_BROKEN: { - CA_LOG_E("Got LOCKS BROKEN for table `" - << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received EvWriteResult with locks broken status.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); const ui64 brokenShardId = ev->Get()->Record.GetOrigin(); @@ -1070,10 +1107,12 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_CONSTRAINT_VIOLATION: { - CA_LOG_E("Got CONSTRAINT VIOLATION for table `" << TablePath << "`." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received EvWriteResult with constraint violation.", + {"logPrefix", this->LogPrefix}, + {"tablePath", TablePath}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); UpdateStats(ev->Get()->Record.GetTxStats()); TxManager->SetError(ev->Get()->Record.GetOrigin()); RuntimeError( @@ -1116,17 +1155,19 @@ public: } void ProcessWriteCompletedShard(NKikimr::NEvents::TDataEvents::TEvWriteResult::TPtr& ev) { - CA_LOG_D("Got completed result TxId=" << ev->Get()->Record.GetTxId() - << ", TabletId=" << ev->Get()->Record.GetOrigin() - << ", Cookie=" << ev->Cookie - << ", Mode=" << static_cast<int>(Mode) - << ", Locks=" << [&]() { - TStringBuilder builder; - for (const auto& lock : ev->Get()->Record.GetTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }()); + + TStringBuilder txLocks; + for (const auto& lock : ev->Get()->Record.GetTxLocks()) { + txLocks << lock.ShortDebugString(); + } + + YDB_LOG_DEBUG("Got completed result", + {"logPrefix", this->LogPrefix}, + {"txId", ev->Get()->Record.GetTxId()}, + {"tabletId", ev->Get()->Record.GetOrigin()}, + {"cookie", ev->Cookie}, + {"mode", static_cast<int>(Mode)}, + {"locks", txLocks}); // Only collect locks in WRITE mode (COLLECTING state required by AddLock) if (Mode == EMode::WRITE) { @@ -1168,7 +1209,9 @@ public: } void SetPrepare(ui64 txId) { - CA_LOG_D("SetPrepare; txId=" << txId); + YDB_LOG_DEBUG("Switching to prepare mode.", + {"logPrefix", this->LogPrefix}, + {"txId", txId}); YQL_ENSURE(Mode == EMode::WRITE); Mode = EMode::PREPARE; TxId = txId; @@ -1176,13 +1219,16 @@ public: } void SetDistributedCommit() { - CA_LOG_D("SetDistributedCommit; txId=" << *TxId); + YDB_LOG_DEBUG("Switching to distributed commit mode.", + {"logPrefix", this->LogPrefix}, + {"txId", *TxId}); YQL_ENSURE(Mode == EMode::PREPARE); Mode = EMode::COMMIT; } void SetImmediateCommit() { - CA_LOG_D("SetImmediateCommit"); + YDB_LOG_DEBUG("Switching to immediate commit mode.", + {"logPrefix", this->LogPrefix}); YQL_ENSURE(Mode == EMode::WRITE); Mode = EMode::IMMEDIATE_COMMIT; @@ -1235,10 +1281,11 @@ public: YQL_ENSURE(metadata); YQL_ENSURE(metadata->SendAttempts == 0 || InconsistentTx); if (metadata->SendAttempts >= MessageSettings.MaxWriteAttempts) { - CA_LOG_W("ShardId=" << shardId - << " for table '" << TablePath - << "': retry limit exceeded." - << " Sink=" << this->SelfId() << "."); + YDB_LOG_WARN("Write retry limit exceeded for table.", + {"logPrefix", this->LogPrefix}, + {"shardId", shardId}, + {"tablePath", TablePath}, + {"sink", this->SelfId()}); RetryResolve(); return false; } @@ -1326,19 +1373,26 @@ public: NDataIntegrity::LogIntegrityTrails("EvWriteTx", evWrite->Record.GetTxId(), shardId, TlsActivationContext->AsActorContext(), "WriteActor"); - CA_LOG_D("Send EvWrite to ShardID=" << shardId << ", isPrepare=" << isPrepare << ", isImmediateCommit=" << isImmediateCommit << ", TxId=" << evWrite->Record.GetTxId() - << ", LockTxId=" << evWrite->Record.GetLockTxId() << ", LockNodeId=" << evWrite->Record.GetLockNodeId() - << ", Locks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : evWrite->Record.GetLocks().GetLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }() - << ", Size=" << serializationResult.TotalDataSize << ", Cookie=" << metadata->Cookie - << ", OperationsCount=" << evWrite->Record.OperationsSize() << ", IsFinal=" << metadata->IsFinal - << ", Attempts=" << metadata->SendAttempts << ", Mode=" << static_cast<int>(Mode) - << ", BufferMemory=" << GetMemory()); + TStringBuilder locks; + for (const auto& lock : evWrite->Record.GetLocks().GetLocks()) { + locks << lock.ShortDebugString(); + }; + YDB_LOG_DEBUG("Send EvWrite", + {"logPrefix", this->LogPrefix}, + {"shardID", shardId}, + {"isPrepare", isPrepare}, + {"isImmediateCommit", isImmediateCommit}, + {"txId", evWrite->Record.GetTxId()}, + {"lockTxId", evWrite->Record.GetLockTxId()}, + {"lockNodeId", evWrite->Record.GetLockNodeId()}, + {"locks", locks}, + {"size", serializationResult.TotalDataSize}, + {"cookie", metadata->Cookie}, + {"operationsCount", evWrite->Record.OperationsSize()}, + {"isFinal", metadata->IsFinal}, + {"attempts", metadata->SendAttempts}, + {"mode", static_cast<int>(Mode)}, + {"bufferMemory", GetMemory()}); AFL_ENSURE(Mode == EMode::WRITE || metadata->IsFinal); @@ -1369,14 +1423,19 @@ public: void RetryShard(const ui64 shardId, const std::optional<ui64> ifCookieEqual) { const auto metadata = ShardedWriteController->GetMessageMetadata(shardId); if (!metadata || (ifCookieEqual && metadata->Cookie != ifCookieEqual)) { - CA_LOG_I("Retry failed: not found ShardID=" << shardId << " with Cookie=" << ifCookieEqual.value_or(0)); + YDB_LOG_INFO("Shard retry skipped because metadata was not found for the given cookie.", + {"logPrefix", this->LogPrefix}, + {"shardID", shardId}, + {"cookie", ifCookieEqual.value_or(0)}); return; } - CA_LOG_D("Retry ShardID=" << shardId - << ", Cookie=" << ifCookieEqual.value_or(0) - << ", Attempt=" << metadata->SendAttempts - << ", Next Delay=" << CalculateNextAttemptDelay(MessageSettings, metadata->SendAttempts)); + YDB_LOG_DEBUG("Retry Next", + {"logPrefix", this->LogPrefix}, + {"shardID", shardId}, + {"cookie", ifCookieEqual.value_or(0)}, + {"attempt", metadata->SendAttempts}, + {"delay", CalculateNextAttemptDelay(MessageSettings, metadata->SendAttempts)}); SendDataToShard(shardId); } @@ -1385,16 +1444,22 @@ public: } void Handle(TEvPrivate::TEvShardRequestTimeout::TPtr& ev) { - CA_LOG_I("Timeout shardID=" << ev->Get()->ShardId); + YDB_LOG_INFO("Timeout", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->ShardId}); YQL_ENSURE(InconsistentTx); RetryShard(ev->Get()->ShardId, ev->Cookie); } void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - CA_LOG_W("TEvDeliveryProblem was received from tablet: " << ev->Get()->TabletId); + YDB_LOG_WARN("TEvDeliveryProblem was received", + {"logPrefix", this->LogPrefix}, + {"tablet", ev->Get()->TabletId}); if (!LinkedPipeCache) { - CA_LOG_W("Ignoring TEvDeliveryProblem from tablet " << ev->Get()->TabletId << " after pipe unlink"); + YDB_LOG_WARN("Ignoring TEvDeliveryProblem from tablet after pipe unlink.", + {"logPrefix", this->LogPrefix}, + {"tabletId", ev->Get()->TabletId}); return; } @@ -1410,8 +1475,10 @@ public: || state == IKqpTransactionManager::EXECUTING) && TxManager->ShouldReattach(ev->Get()->TabletId, TlsActivationContext->Now())) { // Disconnected while waiting for other shards to prepare - CA_LOG_N("Shard " << ev->Get()->TabletId << " delivery problem (reattaching in " - << reattachState.ReattachInfo.Delay << ")"); + YDB_LOG_NOTICE("Shard delivery problem detected; scheduling reattach to shard.", + {"logPrefix", this->LogPrefix}, + {"tabletId", ev->Get()->TabletId}, + {"reattachDelay", reattachState.ReattachInfo.Delay}); Schedule(reattachState.ReattachInfo.Delay, new TEvPrivate::TEvReattachToShard(ev->Get()->TabletId)); } else if (state == IKqpTransactionManager::EXECUTING && (!ev->Get()->NotDelivered || reattachState.Cookie != 0)) { @@ -1448,7 +1515,9 @@ public: return; } if (*result) { - CA_LOG_D("Reattached to shard " << ev->Get()->Record.GetTabletId()); + YDB_LOG_DEBUG("Reattached to shard.", + {"logPrefix", this->LogPrefix}, + {"tabletId", ev->Get()->Record.GetTabletId()}); return; } @@ -1472,7 +1541,9 @@ public: } void Handle(TEvDataShard::TEvProposeTransactionRestart::TPtr& ev) { - CA_LOG_D("Got transaction restart event from tabletId: " << ev->Get()->Record.GetTabletId()); + YDB_LOG_DEBUG("Got transaction restart event", + {"logPrefix", this->LogPrefix}, + {"tabletId", ev->Get()->Record.GetTabletId()}); if (!HandleTransactionRestart(TxManager, ev)) { RuntimeError( NYql::NDqProto::StatusIds::UNAVAILABLE, @@ -1487,7 +1558,9 @@ public: const ui64 tabletId = ev->Get()->TabletId; auto& state = TxManager->GetReattachState(tabletId); - CA_LOG_D("Reattach to shard " << tabletId); + YDB_LOG_DEBUG("Reattach to shard", + {"logPrefix", this->LogPrefix}, + {"tabletId", tabletId}); YQL_ENSURE(TxId); Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvForward( @@ -3010,7 +3083,8 @@ private: } if (Closed && WriteTableActor->IsFinished()) { - CA_LOG_D("Write actor finished"); + YDB_LOG_DEBUG("Write actor finished", + {"logPrefix", this->LogPrefix}); Callbacks->OnAsyncOutputFinished(GetOutputIndex()); } } catch (const TMemoryLimitExceededException&) { @@ -3060,7 +3134,8 @@ private: } void ResumeExecution() { - CA_LOG_D("Resuming execution."); + YDB_LOG_DEBUG("Resuming execution", + {"logPrefix", this->LogPrefix}); WaitingForTableActor = false; Callbacks->ResumeExecution(); } @@ -3377,7 +3452,9 @@ public: hFunc(TEvPipeCache::TEvDeliveryProblem, HandleRollback); default: - CA_LOG_W("StateRollback: unknown message " << ev->GetTypeRewrite()); + YDB_LOG_WARN("StateRollback: received unknown message.", + {"logPrefix", this->LogPrefix}, + {"eventType", ev->GetTypeRewrite()}); } } catch (const TMemoryLimitExceededException&) { ReplyMemoryLimitError(); @@ -3393,7 +3470,9 @@ public: hFunc(TEvKqpBuffer::TEvRollback, Handle); default: - CA_LOG_W("StateRollback: unknown message " << ev->GetTypeRewrite()); + YDB_LOG_WARN("StateError: received unknown message.", + {"logPrefix", this->LogPrefix}, + {"eventType", ev->GetTypeRewrite()}); } } catch (...) { ReplyCurrentExceptionError(); @@ -3499,7 +3578,12 @@ public: UserCtx); ptr->SetParentTraceId(BufferWriteActorStateSpan.GetTraceId()); TActorId id = RegisterWithSameMailbox(ptr); - CA_LOG_D("Create new TableWriteActor for table `" << tablePath << "` (" << tableId << "). lockId=" << LockTxId << ". ActorId=" << id); + YDB_LOG_DEBUG("Created new TableWriteActor.", + {"logPrefix", this->LogPrefix}, + {"tablePath", tablePath}, + {"tableId", tableId}, + {"lockId", LockTxId}, + {"actorId", id}); actors.emplace(tableId.PathId, TWriteInfo::TActorInfo{ .WriteActor = ptr, @@ -3544,7 +3628,12 @@ public: }); TActorId id = RegisterWithSameMailbox(actor); - CA_LOG_D("Create new KqpBufferTableLookup for table `" << tablePath << "` (" << tableId << "). lockId=" << LockTxId << ". ActorId=" << id); + YDB_LOG_DEBUG("Created new KqpBufferTableLookup actor.", + {"logPrefix", this->LogPrefix}, + {"tablePath", tablePath}, + {"tableId", tableId}, + {"lockId", LockTxId}, + {"actorId", id}); AFL_ENSURE(actors.emplace(tableId.PathId, TLookupInfo::TActorInfo{ .LookupActor = ptr, @@ -3587,7 +3676,12 @@ public: }); TActorId id = RegisterWithSameMailbox(actor); - CA_LOG_D("Create new KqpBufferTableLock for table `" << tablePath << "` (" << tableId << "). lockId=" << LockTxId << ". ActorId=" << id); + YDB_LOG_DEBUG("Created new KqpBufferTableLock actor.", + {"logPrefix", this->LogPrefix}, + {"tablePath", tablePath}, + {"tableId", tableId}, + {"lockId", LockTxId}, + {"actorId", id}); AFL_ENSURE(actors.emplace(tableId.PathId, TLockInfo::TActorInfo{ .LockActor = ptr, @@ -3598,8 +3692,9 @@ public: bool CheckSchemaVersion(auto* actor, const TTableId& tableId, const TString& tablePath) { if (actor->GetTableId().SchemaVersion != tableId.SchemaVersion) { - CA_LOG_E("Scheme changed for table `" - << tablePath << "`."); + YDB_LOG_ERROR("Table schema version mismatch detected.", + {"logPrefix", this->LogPrefix}, + {"tablePath", tablePath}); ReplyError( NYql::NDqProto::StatusIds::SCHEME_ERROR, NYql::TIssuesIds::KIKIMR_SCHEME_MISMATCH, @@ -4319,7 +4414,8 @@ public: } if (NeedToFlush()) { - CA_LOG_D("Flush data"); + YDB_LOG_DEBUG("Flush data", + {"logPrefix", this->LogPrefix}); bool flushFailed = false; ForEachWriteActor([&](TKqpTableWriteActor* actor, const TActorId) { @@ -4351,7 +4447,8 @@ public: actor->FlushBuffers(); }); - CA_LOG_D("Start flush"); + YDB_LOG_DEBUG("Start flush", + {"logPrefix", this->LogPrefix}); CheckQueuesEmpty(); return Process(); } @@ -4396,7 +4493,8 @@ public: UpdateTracingState("Commit", std::move(traceId)); OperationStartTime = TInstant::Now(); - CA_LOG_D("Start prepare for distributed commit"); + YDB_LOG_DEBUG("Start prepare for distributed commit", + {"logPrefix", this->LogPrefix}); AFL_ENSURE(CurrentStateFunc() == &TThis::StateWaitTasks || CurrentStateFunc() == &TThis::StateFlush); Become(&TThis::StatePrepare); @@ -4423,7 +4521,8 @@ public: UpdateTracingState("Commit", std::move(traceId)); OperationStartTime = TInstant::Now(); - CA_LOG_D("Start immediate commit"); + YDB_LOG_DEBUG("Start immediate commit", + {"logPrefix", this->LogPrefix}); YQL_ENSURE(CurrentStateFunc() == &TThis::StateWaitTasks); Become(&TThis::StateCommit); PendingCommitShards = CountParticipatingShards(); @@ -4446,7 +4545,9 @@ public: Counters->BufferActorDistributedCommits->Inc(); OperationStartTime = TInstant::Now(); - CA_LOG_D("Start distributed commit with TxId=" << *TxId); + YDB_LOG_DEBUG("Start distributed commit with", + {"logPrefix", this->LogPrefix}, + {"txId", *TxId}); YQL_ENSURE(CurrentStateFunc() == &TThis::StatePrepare); Become(&TThis::StateCommit); PendingCommitShards = CountParticipatingShards(); @@ -4465,7 +4566,8 @@ public: Counters->BufferActorRollbacks->Inc(); UpdateTracingState("RollBack", std::move(traceId)); - CA_LOG_D("Start rollback"); + YDB_LOG_DEBUG("Start rollback", + {"logPrefix", this->LogPrefix}); const auto& shardsToRollback = TxManager->StartRollback(); if (shardsToRollback.empty() && waitForResult) { @@ -4477,7 +4579,9 @@ public: SendToExternalShard(shardId, true, /* isRollback */ true); } } catch (...) { - CA_LOG_E("Failed to rollback transaction. Error: " << CurrentExceptionMessage() << "."); + YDB_LOG_ERROR("Failed to rollback transaction", + {"logPrefix", this->LogPrefix}, + {"error", CurrentExceptionMessage()}); } } @@ -4566,18 +4670,26 @@ public: const auto rollbackCookie = isRollback ? RollbackMessageCookie : 0; SendTime[shardId] = TInstant::Now(); - CA_LOG_D("Send EvWrite (external) to ShardID=" << shardId << ", isPrepare=" << !isImmediate << ", isRollback=" << isRollback << ", TxId=" << evWrite->Record.GetTxId() - << ", LockTxId=" << evWrite->Record.GetLockTxId() << ", LockNodeId=" << evWrite->Record.GetLockNodeId() - << ", Locks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : evWrite->Record.GetLocks().GetLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }() - << ", Size=" << 0 << ", Cookie=" << rollbackCookie - << ", OperationsCount=" << 0 << ", IsFinal=" << 1 - << ", Attempts=" << 0); + + TStringBuilder locks; + for (const auto& lock : evWrite->Record.GetLocks().GetLocks()) { + locks << lock.ShortDebugString(); + } + + YDB_LOG_DEBUG("Send EvWrite (external)", + {"logPrefix", this->LogPrefix}, + {"shardID", shardId}, + {"isPrepare", !isImmediate}, + {"isRollback", isRollback}, + {"txId", evWrite->Record.GetTxId()}, + {"lockTxId", evWrite->Record.GetLockTxId()}, + {"lockNodeId", evWrite->Record.GetLockNodeId()}, + {"locks", locks}, + {"size", 0}, + {"cookie", rollbackCookie}, + {"operationsCount", 0}, + {"isFinal", 1}, + {"attempts", 0}); Send( NKikimr::MakePipePerNodeCacheID(false), @@ -4650,8 +4762,11 @@ public: SendTime[tabletId] = TInstant::Now(); - CA_LOG_D("Executing KQP transaction on topic tablet: " << tabletId - << ", writeId: " << writeId << ", isImmediateCommit: " << isImmediateCommit); + YDB_LOG_DEBUG("Executing KQP transaction on topic", + {"logPrefix", this->LogPrefix}, + {"tablet", tabletId}, + {"writeId", writeId}, + {"isImmediateCommit", isImmediateCommit}); Send( MakePipePerNodeCacheID(false), @@ -4692,9 +4807,11 @@ public: NDataIntegrity::LogIntegrityTrails("PlannedTx", *TxId, {}, TlsActivationContext->AsActorContext(), "BufferActor"); - CA_LOG_D("Execute planned transaction, coordinator: " << *Coordinator - << ", volitale: " << ((transaction.GetFlags() & TEvTxProxy::TEvProposeTransaction::FlagVolatile) != 0) - << ", shards: " << affectedSet.size()); + YDB_LOG_DEBUG("Execute planned transaction", + {"logPrefix", this->LogPrefix}, + {"coordinator", *Coordinator}, + {"volitale", ((transaction.GetFlags() & TEvTxProxy::TEvProposeTransaction::FlagVolatile) != 0)}, + {"shards", affectedSet.size()}); Send( MakePipePerNodeCacheID(false), new TEvPipeCache::TEvForward(ev.Release(), *Coordinator, /* subscribe */ true), @@ -4776,7 +4893,9 @@ public: void Handle(TEvTxProxy::TEvProposeTransactionStatus::TPtr &ev) { TEvTxProxy::TEvProposeTransactionStatus* res = ev->Get(); - CA_LOG_D("Got transaction status, status: " << res->GetStatus()); + YDB_LOG_DEBUG("Got transaction status", + {"logPrefix", this->LogPrefix}, + {"status", res->GetStatus()}); switch (res->GetStatus()) { case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusAccepted: @@ -4828,9 +4947,10 @@ public: auto& event = ev->Get()->Record; const ui64 tabletId = event.GetOrigin(); - CA_LOG_D("Got ProposeTransactionResult" << - ", PQ tablet: " << tabletId << - ", status: " << NKikimrPQ::TEvProposeTransactionResult_EStatus_Name(event.GetStatus())); + YDB_LOG_DEBUG("Got ProposeTransactionResult PQ", + {"logPrefix", this->LogPrefix}, + {"tablet", tabletId}, + {"status", NKikimrPQ::TEvProposeTransactionResult_EStatus_Name(event.GetStatus())}); switch (event.GetStatus()) { case NKikimrPQ::TEvProposeTransactionResult::PREPARED: @@ -4845,9 +4965,10 @@ public: auto& event = ev->Get()->Record; const ui64 tabletId = event.GetOrigin(); - CA_LOG_D("Got ProposeTransactionResult" << - ", PQ tablet: " << tabletId << - ", status: " << NKikimrPQ::TEvProposeTransactionResult_EStatus_Name(event.GetStatus())); + YDB_LOG_DEBUG("Got ProposeTransactionResult PQ", + {"logPrefix", this->LogPrefix}, + {"tablet", tabletId}, + {"status", NKikimrPQ::TEvProposeTransactionResult_EStatus_Name(event.GetStatus())}); switch (event.GetStatus()) { case NKikimrPQ::TEvProposeTransactionResult::COMPLETE: @@ -4873,9 +4994,10 @@ public: case NKikimrPQ::TEvProposeTransactionResult::COMPLETE: AFL_ENSURE(false); case NKikimrPQ::TEvProposeTransactionResult::ABORTED: - CA_LOG_E("Got ABORTED ProposeTransactionResult for PQ." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "."); + YDB_LOG_ERROR("Received ABORTED ProposeTransactionResult from pers queue.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}); ReplyError( NYql::NDqProto::StatusIds::ABORTED, NYql::TIssuesIds::KIKIMR_OPERATION_ABORTED, @@ -4883,9 +5005,10 @@ public: {}); return; case NKikimrPQ::TEvProposeTransactionResult::BAD_REQUEST: - CA_LOG_E("Got BAD REQUEST ProposeTransactionResult for PQ." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "."); + YDB_LOG_ERROR("Received BAD REQUEST ProposeTransactionResult from pers queue.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}); ReplyError( NYql::NDqProto::StatusIds::BAD_REQUEST, NYql::TIssuesIds::KIKIMR_BAD_REQUEST, @@ -4893,9 +5016,10 @@ public: {}); return; case NKikimrPQ::TEvProposeTransactionResult::OVERLOADED: - CA_LOG_E("Got OVERLOADED ProposeTransactionResult for PQ." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "."); + YDB_LOG_ERROR("Received OVERLOADED ProposeTransactionResult from pers queue.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}); ReplyError( NYql::NDqProto::StatusIds::OVERLOADED, NYql::TIssuesIds::KIKIMR_OVERLOADED, @@ -4903,9 +5027,10 @@ public: {}); return; case NKikimrPQ::TEvProposeTransactionResult::CANCELLED: - CA_LOG_E("Got CANCELLED ProposeTransactionResult for PQ." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "."); + YDB_LOG_ERROR("Received CANCELLED ProposeTransactionResult from pers queue.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}); ReplyError( NYql::NDqProto::StatusIds::CANCELLED, NYql::TIssuesIds::KIKIMR_OPERATION_CANCELLED, @@ -4913,9 +5038,10 @@ public: {}); return; default: - CA_LOG_E("Got undefined ProposeTransactionResult for PQ." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "."); + YDB_LOG_ERROR("Received undefined ProposeTransactionResult from pers queue.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}); ReplyError( NYql::NDqProto::StatusIds::INTERNAL_ERROR, NYql::TIssuesIds::KIKIMR_INTERNAL_ERROR, @@ -4931,7 +5057,9 @@ public: return; } if (*result) { - CA_LOG_D("Reattached to shard " << ev->Get()->Record.GetTabletId()); + YDB_LOG_DEBUG("Reattached to shard.", + {"logPrefix", this->LogPrefix}, + {"tabletId", ev->Get()->Record.GetTabletId()}); return; } @@ -4949,7 +5077,9 @@ public: return; } if (*result) { - CA_LOG_D("Reattached to shard " << ev->Get()->Record.GetTabletId()); + YDB_LOG_DEBUG("Reattached to shard.", + {"logPrefix", this->LogPrefix}, + {"tabletId", ev->Get()->Record.GetTabletId()}); return; } @@ -4962,7 +5092,9 @@ public: } void Handle(TEvDataShard::TEvProposeTransactionRestart::TPtr& ev) { - CA_LOG_D("Got transaction restart event from tabletId: " << ev->Get()->Record.GetTabletId()); + YDB_LOG_DEBUG("Got transaction restart event", + {"logPrefix", this->LogPrefix}, + {"tabletId", ev->Get()->Record.GetTabletId()}); if (!HandleTransactionRestart(TxManager, ev)) { ReplyError( NYql::NDqProto::StatusIds::UNAVAILABLE, @@ -4977,7 +5109,9 @@ public: const ui64 tabletId = ev->Get()->TabletId; auto& state = TxManager->GetReattachState(tabletId); - CA_LOG_D("Reattach to shard " << tabletId); + YDB_LOG_DEBUG("Reattach to shard", + {"logPrefix", this->LogPrefix}, + {"tabletId", tabletId}); YQL_ENSURE(TxId); Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvForward( @@ -4986,7 +5120,9 @@ public: } void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - CA_LOG_W("TEvDeliveryProblem was received from tablet: " << ev->Get()->TabletId); + YDB_LOG_WARN("TEvDeliveryProblem was received", + {"logPrefix", this->LogPrefix}, + {"tablet", ev->Get()->TabletId}); ReplyError( NYql::NDqProto::StatusIds::UNAVAILABLE, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE, @@ -4997,13 +5133,17 @@ public: } void HandlePrepare(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - CA_LOG_W("TEvDeliveryProblem was received from tablet: " << ev->Get()->TabletId); + YDB_LOG_WARN("TEvDeliveryProblem was received", + {"logPrefix", this->LogPrefix}, + {"tablet", ev->Get()->TabletId}); const auto state = TxManager->GetState(ev->Get()->TabletId); if (state == IKqpTransactionManager::PREPARED && TxManager->ShouldReattach(ev->Get()->TabletId, TlsActivationContext->Now())) { const auto& reattachState = TxManager->GetReattachState(ev->Get()->TabletId); - CA_LOG_N("Shard " << ev->Get()->TabletId << " delivery problem (reattaching in " - << reattachState.ReattachInfo.Delay << ")"); + YDB_LOG_NOTICE("Shard delivery problem detected; scheduling reattach to shard.", + {"logPrefix", this->LogPrefix}, + {"tabletId", ev->Get()->TabletId}, + {"reattachDelay", reattachState.ReattachInfo.Delay}); Schedule(reattachState.ReattachInfo.Delay, new TEvPrivate::TEvReattachToShard(ev->Get()->TabletId)); return; @@ -5020,7 +5160,9 @@ public: } void HandleCommit(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - CA_LOG_W("TEvDeliveryProblem was received from tablet: " << ev->Get()->TabletId); + YDB_LOG_WARN("TEvDeliveryProblem was received", + {"logPrefix", this->LogPrefix}, + {"tablet", ev->Get()->TabletId}); if (Coordinator == ev->Get()->TabletId) { if (ev->Get()->NotDelivered) { @@ -5052,8 +5194,10 @@ public: const auto state = TxManager->GetState(ev->Get()->TabletId); if (state == IKqpTransactionManager::EXECUTING && TxManager->ShouldReattach(ev->Get()->TabletId, TlsActivationContext->Now())) { const auto& reattachState = TxManager->GetReattachState(ev->Get()->TabletId); - CA_LOG_N("Shard " << ev->Get()->TabletId << " delivery problem (reattaching in " - << reattachState.ReattachInfo.Delay << ")"); + YDB_LOG_NOTICE("Shard delivery problem detected; scheduling reattach to shard.", + {"logPrefix", this->LogPrefix}, + {"tabletId", ev->Get()->TabletId}, + {"reattachDelay", reattachState.ReattachInfo.Delay}); Schedule(reattachState.ReattachInfo.Delay, new TEvPrivate::TEvReattachToShard(ev->Get()->TabletId)); return; @@ -5070,7 +5214,9 @@ public: } void HandleRollback(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { - CA_LOG_W("TEvDeliveryProblem was received from tablet: " << ev->Get()->TabletId); + YDB_LOG_WARN("TEvDeliveryProblem was received", + {"logPrefix", this->LogPrefix}, + {"tablet", ev->Get()->TabletId}); if (Coordinator == ev->Get()->TabletId) { return; } @@ -5146,17 +5292,18 @@ public: } void HandlePrepare(NKikimr::NEvents::TDataEvents::TEvWriteResult::TPtr& ev) { - CA_LOG_D("Recv EvWriteResult (external) from ShardID=" << ev->Get()->Record.GetOrigin() - << ", Status=" << NKikimrDataEvents::TEvWriteResult::EStatus_Name(ev->Get()->GetStatus()) - << ", TxId=" << ev->Get()->Record.GetTxId() - << ", Locks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : ev->Get()->Record.GetTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }() - << ", Cookie=" << ev->Cookie); + TStringBuilder txLocks; + for (const auto& lock : ev->Get()->Record.GetTxLocks()) { + txLocks << lock.ShortDebugString(); + } + + YDB_LOG_DEBUG("Recv EvWriteResult (external)", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"status", NKikimrDataEvents::TEvWriteResult::EStatus_Name(ev->Get()->GetStatus())}, + {"txId", ev->Get()->Record.GetTxId()}, + {"locks", txLocks}, + {"cookie", ev->Cookie}); TxManager->AddParticipantNode(ev->Sender.NodeId()); @@ -5168,7 +5315,9 @@ public: case NKikimrDataEvents::TEvWriteResult::STATUS_DISK_GROUP_OUT_OF_SPACE: case NKikimrDataEvents::TEvWriteResult::STATUS_OVERLOADED: { if (ev->Get()->Record.HasOverloadSubscribed()) { - CA_LOG_D("Shard " << ev->Get()->Record.GetOrigin() << " is overloaded. Waiting."); + YDB_LOG_DEBUG("Shard is overloaded; waiting for overload ready notification.", + {"logPrefix", this->LogPrefix}, + {"shardId", ev->Get()->Record.GetOrigin()}); return; } } @@ -5179,17 +5328,17 @@ public: } void HandleCommit(NKikimr::NEvents::TDataEvents::TEvWriteResult::TPtr& ev) { - CA_LOG_D("Recv EvWriteResult (external) from ShardID=" << ev->Get()->Record.GetOrigin() - << ", Status=" << NKikimrDataEvents::TEvWriteResult::EStatus_Name(ev->Get()->GetStatus()) - << ", TxId=" << ev->Get()->Record.GetTxId() - << ", Locks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : ev->Get()->Record.GetTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }() - << ", Cookie=" << ev->Cookie); + TStringBuilder locks; + for (const auto& lock : ev->Get()->Record.GetTxLocks()) { + locks << lock.ShortDebugString(); + } + YDB_LOG_DEBUG("Recv EvWriteResult (external)", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"status", NKikimrDataEvents::TEvWriteResult::EStatus_Name(ev->Get()->GetStatus())}, + {"txId", ev->Get()->Record.GetTxId()}, + {"locks", locks}, + {"cookie", ev->Cookie}); TxManager->AddParticipantNode(ev->Sender.NodeId()); @@ -5201,7 +5350,9 @@ public: case NKikimrDataEvents::TEvWriteResult::STATUS_DISK_GROUP_OUT_OF_SPACE: case NKikimrDataEvents::TEvWriteResult::STATUS_OVERLOADED: { if (ev->Get()->Record.HasOverloadSubscribed()) { - CA_LOG_D("Shard " << ev->Get()->Record.GetOrigin() << " is overloaded. Waiting."); + YDB_LOG_DEBUG("Shard is overloaded; waiting for overload ready notification.", + {"logPrefix", this->LogPrefix}, + {"shardId", ev->Get()->Record.GetOrigin()}); return; } } @@ -5212,17 +5363,18 @@ public: } void HandleRollback(NKikimr::NEvents::TDataEvents::TEvWriteResult::TPtr& ev) { - CA_LOG_D("Recv EvWriteResult (external) from ShardID=" << ev->Get()->Record.GetOrigin() - << ", Status=" << NKikimrDataEvents::TEvWriteResult::EStatus_Name(ev->Get()->GetStatus()) - << ", TxId=" << ev->Get()->Record.GetTxId() - << ", Locks= " << [&]() { - TStringBuilder builder; - for (const auto& lock : ev->Get()->Record.GetTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }() - << ", Cookie=" << ev->Cookie); + TStringBuilder locks; + for (const auto& lock : ev->Get()->Record.GetTxLocks()) { + locks << lock.ShortDebugString(); + } + + YDB_LOG_DEBUG("Recv EvWriteResult (external)", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"status", NKikimrDataEvents::TEvWriteResult::EStatus_Name(ev->Get()->GetStatus())}, + {"txId", ev->Get()->Record.GetTxId()}, + {"locks", locks}, + {"cookie", ev->Cookie}); TxManager->AddParticipantNode(ev->Sender.NodeId()); @@ -5234,7 +5386,9 @@ public: void OnOverloadReady(const ui64 shardId, const ui64 seqNo) { if (seqNo == ExternalShardIdToOverloadSeqNo.at(shardId)) { - CA_LOG_D("Retry Overloaded ShardID=" << shardId); + YDB_LOG_DEBUG("Retry Overloaded", + {"logPrefix", this->LogPrefix}, + {"shardID", shardId}); SendToExternalShard(shardId, IsImmediateCommit, false); } } @@ -5264,10 +5418,11 @@ public: switch (ev->Get()->GetStatus()) { case NKikimrDataEvents::TEvWriteResult::STATUS_UNSPECIFIED: { - CA_LOG_E("Got UNSPECIFIED for tables." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received external EvWriteResult with unspecified status.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); TxManager->SetError(ev->Get()->Record.GetOrigin()); ReplyError( NYql::NDqProto::StatusIds::UNSPECIFIED, @@ -5281,10 +5436,11 @@ public: case NKikimrDataEvents::TEvWriteResult::STATUS_COMPLETED: AFL_ENSURE(false); case NKikimrDataEvents::TEvWriteResult::STATUS_ABORTED: { - CA_LOG_E("Got ABORTED for tables." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received external EvWriteResult with aborted status.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); TxManager->SetError(ev->Get()->Record.GetOrigin()); ReplyError( NYql::NDqProto::StatusIds::ABORTED, @@ -5294,10 +5450,11 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_WRONG_SHARD_STATE: { - CA_LOG_E("Got WRONG SHARD STATE for tables." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received external EvWriteResult with wrong shard state.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); TxManager->SetError(ev->Get()->Record.GetOrigin()); ReplyError( NYql::NDqProto::StatusIds::UNAVAILABLE, @@ -5307,10 +5464,11 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_INTERNAL_ERROR: { - CA_LOG_E("Got INTERNAL ERROR for tables." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received external EvWriteResult with internal error.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); TxManager->SetError(ev->Get()->Record.GetOrigin()); ReplyError( NYql::NDqProto::StatusIds::INTERNAL_ERROR, @@ -5320,10 +5478,11 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_DATABASE_DISK_SPACE_QUOTA_EXCEEDED: { - CA_LOG_E("Got DATABASE_DISK_SPACE_QUOTA_EXCEEDED for tables." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received external EvWriteResult with database disk space quota exceeded status.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); TxManager->SetError(ev->Get()->Record.GetOrigin()); ReplyError( NYql::NDqProto::StatusIds::UNAVAILABLE, @@ -5333,11 +5492,11 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_DISK_GROUP_OUT_OF_SPACE: { - CA_LOG_W("Got DISK_GROUP_OUT_OF_SPACE for tables." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << " Ignored this error." - << getIssues().ToOneLineString()); + YDB_LOG_WARN("Received external EvWriteResult with disk group out of space status.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); TxManager->SetError(ev->Get()->Record.GetOrigin()); ReplyError( NYql::NDqProto::StatusIds::UNAVAILABLE, @@ -5348,11 +5507,11 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_OVERLOADED: { - CA_LOG_W("Got OVERLOADED for tables." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << " Ignored this error." - << getIssues().ToOneLineString()); + YDB_LOG_WARN("Received external EvWriteResult with overloaded status.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); TxManager->SetError(ev->Get()->Record.GetOrigin()); ReplyError( NYql::NDqProto::StatusIds::OVERLOADED, @@ -5364,10 +5523,11 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_CANCELLED: { - CA_LOG_E("Got CANCELLED for tables." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received external EvWriteResult with cancelled status.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); TxManager->SetError(ev->Get()->Record.GetOrigin()); ReplyError( NYql::NDqProto::StatusIds::CANCELLED, @@ -5377,10 +5537,11 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST: { - CA_LOG_E("Got BAD REQUEST for tables." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received external EvWriteResult with bad request status.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); TxManager->SetError(ev->Get()->Record.GetOrigin()); ReplyError( NYql::NDqProto::StatusIds::BAD_REQUEST, @@ -5390,10 +5551,11 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_SCHEME_CHANGED: { - CA_LOG_E("Got SCHEME CHANGED for table." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received external EvWriteResult with scheme changed status.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); TxManager->SetError(ev->Get()->Record.GetOrigin()); ReplyError( NYql::NDqProto::StatusIds::SCHEME_ERROR, @@ -5403,10 +5565,11 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_LOCKS_BROKEN: { - CA_LOG_E("Got LOCKS BROKEN for table." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received external EvWriteResult with locks broken status.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); if (CurrentStateFunc() == &TThis::StateCommit && PendingCommitShards > 0) { --PendingCommitShards; } @@ -5422,10 +5585,11 @@ public: return; } case NKikimrDataEvents::TEvWriteResult::STATUS_CONSTRAINT_VIOLATION: { - CA_LOG_E("Got CONSTRAINT VIOLATION for table." - << " ShardID=" << ev->Get()->Record.GetOrigin() << "," - << " Sink=" << this->SelfId() << "." - << getIssues().ToOneLineString()); + YDB_LOG_ERROR("Received external EvWriteResult with constraint violation.", + {"logPrefix", this->LogPrefix}, + {"shardID", ev->Get()->Record.GetOrigin()}, + {"sink", this->SelfId()}, + {"issues", getIssues().ToOneLineString()}); TxManager->SetError(ev->Get()->Record.GetOrigin()); ReplyError( NYql::NDqProto::StatusIds::PRECONDITION_FAILED, @@ -5450,9 +5614,11 @@ public: void ProcessPreparedTopic(TEvPersQueue::TEvProposeTransactionResult::TPtr& ev) { OnMessageReceived(ev->Get()->Record.GetOrigin()); - CA_LOG_D("Got propose prepared result TxId=" << ev->Get()->Record.GetTxId() - << ", TabletId=" << ev->Get()->Record.GetOrigin() - << ", Cookie=" << ev->Cookie); + YDB_LOG_DEBUG("Got propose prepared result", + {"logPrefix", this->LogPrefix}, + {"txId", ev->Get()->Record.GetTxId()}, + {"tabletId", ev->Get()->Record.GetOrigin()}, + {"cookie", ev->Cookie}); const auto& record = ev->Get()->Record; IKqpTransactionManager::TPrepareResult preparedInfo; @@ -5474,18 +5640,21 @@ public: NKikimrPQ::TEvProposeTransactionResult& event = ev->Get()->Record; OnMessageReceived(event.GetOrigin()); - CA_LOG_D("Got propose completed result" << - ", topic tablet: " << event.GetOrigin() << - ", status: " << NKikimrPQ::TEvProposeTransactionResult_EStatus_Name(event.GetStatus())); + YDB_LOG_DEBUG("Got propose completed result topic", + {"logPrefix", this->LogPrefix}, + {"tablet", event.GetOrigin()}, + {"status", NKikimrPQ::TEvProposeTransactionResult_EStatus_Name(event.GetStatus())}); OnCommitted(event.GetOrigin(), 0); } void ProcessWritePreparedShard(NKikimr::NEvents::TDataEvents::TEvWriteResult::TPtr& ev) { OnMessageReceived(ev->Get()->Record.GetOrigin()); - CA_LOG_D("Got prepared result TxId=" << ev->Get()->Record.GetTxId() - << ", TabletId=" << ev->Get()->Record.GetOrigin() - << ", Cookie=" << ev->Cookie); + YDB_LOG_DEBUG("Got prepared result", + {"logPrefix", this->LogPrefix}, + {"txId", ev->Get()->Record.GetTxId()}, + {"tabletId", ev->Get()->Record.GetOrigin()}, + {"cookie", ev->Cookie}); CollectTliStats(ev->Get()->Record); @@ -5507,16 +5676,17 @@ public: void ProcessWriteCompletedShard(NKikimr::NEvents::TDataEvents::TEvWriteResult::TPtr& ev) { OnMessageReceived(ev->Get()->Record.GetOrigin()); - CA_LOG_D("Got completed result TxId=" << ev->Get()->Record.GetTxId() - << ", TabletId=" << ev->Get()->Record.GetOrigin() - << ", Cookie=" << ev->Cookie - << ", Locks=" << [&]() { - TStringBuilder builder; - for (const auto& lock : ev->Get()->Record.GetTxLocks()) { - builder << lock.ShortDebugString(); - } - return builder; - }()); + TStringBuilder locks; + for (const auto& lock : ev->Get()->Record.GetTxLocks()) { + locks << lock.ShortDebugString(); + } + + YDB_LOG_DEBUG("Got completed result", + {"logPrefix", this->LogPrefix}, + {"txId", ev->Get()->Record.GetTxId()}, + {"tabletId", ev->Get()->Record.GetOrigin()}, + {"cookie", ev->Cookie}, + {"locks", locks}); CollectTliStats(ev->Get()->Record); @@ -5530,9 +5700,11 @@ public: void OnPrepared(IKqpTransactionManager::TPrepareResult&& preparedInfo, ui64) override { if (HandleDeferredLocksBrokenOnPrepare()) return; if (!preparedInfo.Coordinator || (TxManager->GetCoordinator() && preparedInfo.Coordinator != TxManager->GetCoordinator())) { - CA_LOG_E("Handle TEvWriteResult: unable to select coordinator. Tx canceled, actorId: " << SelfId() - << ", previously selected coordinator: " << TxManager->GetCoordinator() - << ", coordinator selected at propose result: " << preparedInfo.Coordinator); + YDB_LOG_ERROR("Handle TEvWriteResult: unable to select coordinator. Tx canceled, previously selected coordinator selected at propose", + {"logPrefix", this->LogPrefix}, + {"actorId", SelfId()}, + {"coordinator", TxManager->GetCoordinator()}, + {"result", preparedInfo.Coordinator}); TxProxyMon->TxResultAborted->Inc(); ReplyError(NYql::NDqProto::StatusIds::CANCELLED, @@ -5556,7 +5728,9 @@ public: } if (TxManager->ConsumeCommitResult(shardId)) { if (FlushDeferredLocksBrokenIfPending()) return; - CA_LOG_D("Committed TxId=" << TxId.value_or(0)); + YDB_LOG_DEBUG("Committed", + {"logPrefix", this->LogPrefix}, + {"txId", TxId.value_or(0)}); OnOperationFinished(Counters->BufferActorCommitLatencyHistogram); Send<ESendingType::Tail>(ExecuterActorId, new TEvKqpBuffer::TEvResult{ BuildStats(), @@ -5583,7 +5757,9 @@ public: } void OnRollbackFinished() { - CA_LOG_D("RolledBack TxId=" << TxId.value_or(0)); + YDB_LOG_DEBUG("RolledBack", + {"logPrefix", this->LogPrefix}, + {"txId", TxId.value_or(0)}); OnOperationFinished(Counters->BufferActorRollbackLatencyHistogram); Send<ESendingType::Tail>(ExecuterActorId, new TEvKqpBuffer::TEvResult{ BuildStats() @@ -5823,7 +5999,11 @@ public: } void ReplyErrorImpl(NYql::NDqProto::StatusIds::StatusCode statusCode, NYql::TIssues&& issues) { - CA_LOG_E("statusCode=" << NYql::NDqProto::StatusIds_StatusCode_Name(statusCode) << ". Issue=" << issues.ToString() << ". sessionActorId=" << SessionActorId << "."); + YDB_LOG_ERROR("Buffer write actor is replying with error to session.", + {"logPrefix", this->LogPrefix}, + {"statusCode", NYql::NDqProto::StatusIds_StatusCode_Name(statusCode)}, + {"issue", issues}, + {"sessionActorId", SessionActorId}); TxManager->SetError(); CancelProposal(); @@ -5883,7 +6063,9 @@ public: } } } catch (...) { - CA_LOG_E("Failed to cancel transaction proposals. Error: " << CurrentExceptionMessage() << "."); + YDB_LOG_ERROR("Failed to cancel transaction proposals.", + {"logPrefix", this->LogPrefix}, + {"error", CurrentExceptionMessage()}); } } @@ -6148,7 +6330,9 @@ private: } void Handle(TEvBufferWriteResult::TPtr& result) { - CA_LOG_D("TKqpForwardWriteActor receive EvBufferWriteResult from " << BufferActorId); + YDB_LOG_DEBUG("TKqpForwardWriteActor receive EvBufferWriteResult", + {"logPrefix", this->LogPrefix}, + {"bufferActorId", BufferActorId}); AFL_ENSURE(!PendingResult); WriteToken = result->Get()->Token; @@ -6220,10 +6404,13 @@ private: if (TransformOutput) { TransformOutput->Finish(); } - CA_LOG_D("Finished"); + YDB_LOG_DEBUG("Finished", + {"logPrefix", this->LogPrefix}); Callbacks->OnAsyncOutputFinished(GetOutputIndex()); } else { - CA_LOG_D("Resume with freeSpace=" << GetFreeSpace()); + YDB_LOG_DEBUG("Resume with", + {"logPrefix", this->LogPrefix}, + {"freeSpace", GetFreeSpace()}); Callbacks->ResumeExecution(); } } @@ -6356,7 +6543,11 @@ private: ev->SendTime = TInstant::Now(); - CA_LOG_D("Send data=" << DataSize << ", closed=" << Closed << ", bufferActorId=" << BufferActorId); + YDB_LOG_DEBUG("Send", + {"logPrefix", this->LogPrefix}, + {"data", DataSize}, + {"closed", Closed}, + {"bufferActorId", BufferActorId}); AFL_ENSURE(Send(BufferActorId, ev.release())); } @@ -6388,7 +6579,10 @@ private: Batcher->AddData(data); DataSize = Batcher->GetMemory(); - CA_LOG_D("Add data: " << DataSize << " / " << DataSize); + YDB_LOG_DEBUG("Added data to forward write actor batch.", + {"logPrefix", this->LogPrefix}, + {"data", DataSize}, + {"dataSize", DataSize}); if (Closed || GetFreeSpace() <= 0) { WriteToBuffer(); } @@ -6397,7 +6591,9 @@ private: void RuntimeError(const TString& message, NYql::NDqProto::StatusIds::StatusCode statusCode, const NYql::TIssues& subIssues = {}) { ForwardWriteActorSpan.EndError(message); - CA_LOG_E("RuntimeError: " << message); + YDB_LOG_ERROR("Forward write actor reported runtime error.", + {"logPrefix", this->LogPrefix}, + {"runtimeError", message}); NYql::TIssue issue(message); for (const auto& i : subIssues) { issue.AddSubIssue(MakeIntrusive<NYql::TIssue>(i)); diff --git a/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service.cpp b/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service.cpp index 8288d5f2b5b..b7198a94fe8 100644 --- a/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service.cpp +++ b/ydb/core/kqp/runtime/scheduler/kqp_compute_scheduler_service.cpp @@ -1,6 +1,6 @@ #include "kqp_compute_scheduler_service.h" -#include "log.h" +#include <ydb/library/actors/core/log.h> #include "tree/dynamic.h" #include <ydb/core/base/appdata_fwd.h> @@ -16,6 +16,8 @@ #include <ydb/library/actors/core/events.h> #include <ydb/library/actors/core/subsystems/stats.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE_SCHEDULER + using namespace NKikimr; using namespace NKikimr::NKqp; using namespace NKikimr::NKqp::NScheduler; @@ -40,9 +42,9 @@ public: ); if (Scheduler->IsEnabled()) { - LOG_I("Enabled on start"); + YDB_LOG_INFO("Enabled on start"); } else { - LOG_I("Disabled on start"); + YDB_LOG_INFO("Disabled on start"); } Scheduler->SetTotalCpuLimit(CalculateTotalCpuLimit()); // TODO: take total cpu limit from outside @@ -67,12 +69,13 @@ public: hFunc(NActors::TEvents::TEvWakeup, Handle); default: - LOG_E("Unexpected event: " << ev->GetTypeRewrite()); + YDB_LOG_ERROR("Unexpected", + {"event", ev->GetTypeRewrite()}); } } void Handle(NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionResponse::TPtr&) { - LOG_D("Subscribed to config changes"); + YDB_LOG_DEBUG("Subscribed to config changes"); } void Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) { @@ -80,9 +83,9 @@ public: Scheduler->ToggleEnabled(event.GetConfig().GetFeatureFlags().GetEnableResourcePoolsScheduler()); if (Scheduler->IsEnabled()) { - LOG_I("Become enabled"); + YDB_LOG_INFO("Become enabled"); } else { - LOG_I("Become disabled"); + YDB_LOG_INFO("Become disabled"); } auto responseEvent = std::make_unique<NKikimr::NConsole::TEvConsole::TEvConfigNotificationResponse>(event); @@ -95,7 +98,9 @@ public: }; Scheduler->AddOrUpdateDatabase(ev->Get()->DatabaseId, attrs); - LOG_D("Add database: " << ev->Get()->DatabaseId << " (" << attrs.ToString() << ")"); + YDB_LOG_DEBUG("Add", + {"database", ev->Get()->DatabaseId}, + {"attrs", attrs}); } void Handle(TEvRemoveDatabase::TPtr&) { @@ -122,7 +127,10 @@ public: Y_ASSERT(!poolId.empty()); - LOG_D("Add pool: " << databaseId << "/" << poolId << " (" << attrs.ToString() << ")"); + YDB_LOG_DEBUG("Add", + {"pool", databaseId}, + {"poolId", poolId}, + {"attrs", attrs}); if (PoolSubscribtions.insert({std::make_pair(databaseId, poolId), {.IsFirstRemoval=false, .ExternalWeight=resourceWeight}}).second) { PoolExternalWeightSum += resourceWeight; @@ -168,7 +176,10 @@ public: Scheduler->AddOrUpdatePool(databaseId, poolId, attrs); - LOG_D("Update pool: " << databaseId << "/" << poolId << " (" << attrs.ToString() << ")"); + YDB_LOG_DEBUG("Update", + {"pool", databaseId}, + {"poolId", poolId}, + {"attrs", attrs}); } else if (poolIt != PoolSubscribtions.end()) { if (!poolIt->second.IsFirstRemoval) { // The first removal - try to re-subscribe in case it's just the pool removal from cache. @@ -181,7 +192,9 @@ public: // TODO: Scheduler->UpdatePool(…); } } else { - LOG_E("Trying to remove unknown pool: " << databaseId << "/" << poolId); + YDB_LOG_ERROR("Trying to remove unknown", + {"pool", databaseId}, + {"poolId", poolId}); // TODO: the removing message for unknown pool - should we check? } } @@ -198,7 +211,10 @@ public: if (Scheduler->IsEnabled()) { auto query = Scheduler->AddOrUpdateQuery(databaseId, poolId.empty() ? NKikimr::NResourcePool::DEFAULT_POOL_ID : poolId, queryId, attrs); response->Query = query; - LOG_D("Add query: " << databaseId << "/" << poolId << ", TxId: " << queryId); + YDB_LOG_DEBUG("Add", + {"query", databaseId}, + {"poolId", poolId}, + {"txId", queryId}); } Send(ev->Sender, response.Release(), 0, queryId); } @@ -206,9 +222,11 @@ public: void Handle(TEvRemoveQuery::TPtr& ev) { const auto& queryId = ev->Get()->QueryId; if (!Scheduler->RemoveQuery(queryId)) { - LOG_E("Trying to remove unknown query: " << queryId); + YDB_LOG_ERROR("Trying to remove unknown", + {"query", queryId}); } else { - LOG_D("Remove query: TxId: " << queryId); + YDB_LOG_DEBUG("Remove query", + {"txId", queryId}); } } diff --git a/ydb/core/kqp/runtime/scheduler/kqp_schedulable_read.cpp b/ydb/core/kqp/runtime/scheduler/kqp_schedulable_read.cpp index ef60cd9e001..4545719fc5d 100644 --- a/ydb/core/kqp/runtime/scheduler/kqp_schedulable_read.cpp +++ b/ydb/core/kqp/runtime/scheduler/kqp_schedulable_read.cpp @@ -1,11 +1,13 @@ #include "kqp_schedulable_read.h" -#include "log.h" +#include <ydb/library/actors/core/log.h> #include "tree/dynamic.h" #include <yql/essentials/utils/yql_panic.h> #include <yt/yt/core/utilex/random.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_COMPUTE_SCHEDULER + namespace NKikimr::NKqp::NScheduler { TSchedulableRead::TSchedulableRead(const NHdrf::NDynamic::TQueryPtr& query) @@ -25,7 +27,9 @@ TSchedulableRead::TSchedulableRead(const NHdrf::NDynamic::TQueryPtr& query) AvailableQuotaMs = MaxQuotaMs; LastRefill = TMonotonic::Now(); - LOG_T("TSchedulableRead [" << uintptr_t(this) << "] MaxQuotaMs: " << MaxQuotaMs); + YDB_LOG_TRACE("Created schedulable read with quota limit", + {"schedulableReadPtr", uintptr_t(this)}, + {"maxQuotaMs", MaxQuotaMs}); YQL_ENSURE(MaxQuotaMs <= 1000); } @@ -34,7 +38,9 @@ bool TSchedulableRead::TryConsumeQuota(TDuration expectedQuota) { // TODO: support update of the pool's read quota on AddOrUpdatePool(). auto expectedQuotaMs = std::min(expectedQuota.MilliSeconds(), MaxQuotaMs); - LOG_T("TSchedulableRead [" << uintptr_t(this) << "] ExpectedQuotaMs: " << expectedQuotaMs); + YDB_LOG_TRACE("Trying to consume read quota", + {"schedulableReadPtr", uintptr_t(this)}, + {"expectedQuotaMs", expectedQuotaMs}); // Refill quota if (const auto now = TMonotonic::Now(); Y_LIKELY(now >= LastRefill)) { @@ -43,7 +49,9 @@ bool TSchedulableRead::TryConsumeQuota(TDuration expectedQuota) { LastRefill = now; } - LOG_T("TSchedulableRead [" << uintptr_t(this) << "] AvailableQuotaMs: " << AvailableQuotaMs); + YDB_LOG_TRACE("Refilled read quota", + {"schedulableReadPtr", uintptr_t(this)}, + {"availableQuotaMs", AvailableQuotaMs}); if (AvailableQuotaMs <= 0 || !TryIncreaseUsage()) { return false; @@ -53,7 +61,9 @@ bool TSchedulableRead::TryConsumeQuota(TDuration expectedQuota) { AvailableQuotaMs -= expectedQuotaMs; ReservedQuotaMs = expectedQuotaMs; - LOG_T("TSchedulableRead [" << uintptr_t(this) << "] ReservedQuotaMs: " << ReservedQuotaMs); + YDB_LOG_TRACE("Reserved read quota", + {"schedulableReadPtr", uintptr_t(this)}, + {"reservedQuotaMs", ReservedQuotaMs}); return true; } @@ -67,8 +77,12 @@ void TSchedulableRead::ReturnQuota(NHPTimer::STime elapsedCycles) { AvailableQuotaMs = std::min<i64>(MaxQuotaMs, AvailableQuotaMs + ReservedQuotaMs - ms); ReservedQuotaMs = 0; - LOG_T("TSchedulableRead [" << uintptr_t(this) << "] ReturnedQuotaMs: " << ms); - LOG_T("TSchedulableRead [" << uintptr_t(this) << "] AvailableQuotaMs: " << AvailableQuotaMs); + YDB_LOG_TRACE("Returned unused read quota", + {"schedulableReadPtr", uintptr_t(this)}, + {"returnedQuotaMs", ms}); + YDB_LOG_TRACE("Updated available read quota after return", + {"schedulableReadPtr", uintptr_t(this)}, + {"availableQuotaMs", AvailableQuotaMs}); DecreaseUsage(TDuration::MilliSeconds(ms), READ_DEFAULT); } diff --git a/ydb/core/kqp/session_actor/kqp_log_query.cpp b/ydb/core/kqp/session_actor/kqp_log_query.cpp index baf636f6338..76acf4931ec 100644 --- a/ydb/core/kqp/session_actor/kqp_log_query.cpp +++ b/ydb/core/kqp/session_actor/kqp_log_query.cpp @@ -10,6 +10,8 @@ #include <library/cpp/json/writer/json.h> #include <yql/essentials/public/issue/yql_issue_message.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_REQUEST + namespace NKikimr::NKqp { namespace { @@ -26,8 +28,6 @@ constexpr size_t ISSUES_CHUNK_SOLO = SQL_TEXT_MAX_SIZE + ISSUES_CHUNK_WITH_DATA; constexpr size_t ISSUES_TEXT_MAX_TOTAL = 64_KB; static_assert(SQL_TEXT_MAX_SIZE + ISSUES_CHUNK_WITH_DATA + LOG_LINE_OVERHEAD + PART1_JSON_OVERHEAD <= RSYSLOG_MAX_MESSAGE_SIZE); -#define _KQP_REQ_LOG_AT(prio, stream) \ - LOG_LOG_S(*TlsActivationContext, (prio), NKikimrServices::KQP_REQUEST, "[REQ_JSON] " << stream) // BUILTIN_ACL_METADATA traffic dominates KQP_REQUEST volume — silence SUCCESS, keep failures. bool IsMetadataServiceQuery(const TKqpQueryState& state) { @@ -208,7 +208,8 @@ void WriteJsonChunks(NActors::NLog::EPriority prio, json.EndObject(); json.EndObject(); - _KQP_REQ_LOG_AT(prio, ss.Str()); + YDB_LOG((prio), "[REQ_JSON]", + {"requestJson", ss.Str()}); } } diff --git a/ydb/core/kqp/session_actor/kqp_query_state.cpp b/ydb/core/kqp/session_actor/kqp_query_state.cpp index dd2eddac15a..f1ce84ca9b1 100644 --- a/ydb/core/kqp/session_actor/kqp_query_state.cpp +++ b/ydb/core/kqp/session_actor/kqp_query_state.cpp @@ -3,19 +3,12 @@ #include <ydb/core/kqp/compile_service/kqp_compile_service.h> #include <ydb/library/persqueue/topic_parser/topic_parser.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_SESSION + namespace NKikimr::NKqp { using namespace NSchemeCache; -#define LOG_C(msg) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, msg) -#define LOG_E(msg) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, msg) -#define LOG_W(msg) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, msg) -#define LOG_N(msg) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, msg) -#define LOG_I(msg) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, msg) -#define LOG_D(msg) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, msg) -#define LOG_T(msg) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, msg) - - TKqpQueryState::TQueryTxId::TQueryTxId(const TQueryTxId& other) { YQL_ENSURE(!Id); Id = other.Id; @@ -49,8 +42,8 @@ bool TKqpQueryState::EnsureTableVersions(const TEvTxProxySchemeCache::TEvNavigat case TSchemeCacheNavigate::EStatus::Ok: { auto expectedVersion = TableVersions.FindPtr(TTableId(entry.TableId.PathId)); if (!expectedVersion) { - LOG_W("Unexpected tableId in scheme cache navigate reply" - << ", tableId: " << entry.TableId); + YDB_LOG_WARN("Unexpected tableId in scheme cache navigate reply", + {"tableId", entry.TableId}); continue; } @@ -60,10 +53,10 @@ bool TKqpQueryState::EnsureTableVersions(const TEvTxProxySchemeCache::TEvNavigat } if (entry.TableId.SchemaVersion && entry.TableId.SchemaVersion != *expectedVersion) { - LOG_I("Scheme version mismatch" - << ", pathId: " << entry.TableId.PathId - << ", expected version: " << *expectedVersion - << ", actual version: " << entry.TableId.SchemaVersion); + YDB_LOG_INFO("Scheme version mismatch", + {"pathId", entry.TableId.PathId}, + {"expectedVersion", *expectedVersion}, + {"actualVersion", entry.TableId.SchemaVersion}); return false; } @@ -73,9 +66,9 @@ bool TKqpQueryState::EnsureTableVersions(const TEvTxProxySchemeCache::TEvNavigat case TSchemeCacheNavigate::EStatus::PathErrorUnknown: case TSchemeCacheNavigate::EStatus::PathNotTable: case TSchemeCacheNavigate::EStatus::TableCreationNotComplete: - LOG_I("Scheme error" - << ", pathId: " << entry.TableId.PathId - << ", status: " << entry.Status); + YDB_LOG_INFO("Scheme error", + {"pathId", entry.TableId.PathId}, + {"status", entry.Status}); return false; case TSchemeCacheNavigate::EStatus::LookupError: @@ -87,9 +80,9 @@ bool TKqpQueryState::EnsureTableVersions(const TEvTxProxySchemeCache::TEvNavigat default: // Unexpected reply, do not invalidate the query as it may block the query execution. // Hard validation will be performed later during the query execution. - LOG_E("Unexpected reply from scheme cache" - << ", pathId: " << entry.TableId.PathId - << ", status: " << entry.Status); + YDB_LOG_ERROR("Unexpected reply from scheme cache", + {"pathId", entry.TableId.PathId}, + {"status", entry.Status}); break; } } diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index 7f981c0289d..5c5a2d70657 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -62,6 +62,8 @@ #include <ydb/library/actors/wilson/wilson_span.h> #include <ydb/library/actors/wilson/wilson_trace.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_SESSION + LWTRACE_USING(KQP_PROVIDER); namespace NKikimr { @@ -72,14 +74,6 @@ using namespace NSchemeCache; namespace { -#define STLOG_C(MESSAGE, ...) STLOG(PRI_CRIT, NKikimrServices::KQP_SESSION, KQPSA, LogPrefix() << MESSAGE, __VA_ARGS__) -#define STLOG_E(MESSAGE, ...) STLOG(PRI_ERROR, NKikimrServices::KQP_SESSION, KQPSA, LogPrefix() << MESSAGE, __VA_ARGS__) -#define STLOG_W(MESSAGE, ...) STLOG(PRI_WARN, NKikimrServices::KQP_SESSION, KQPSA, LogPrefix() << MESSAGE, __VA_ARGS__) -#define STLOG_N(MESSAGE, ...) STLOG(PRI_NOTICE, NKikimrServices::KQP_SESSION, KQPSA, LogPrefix() << MESSAGE, __VA_ARGS__) -#define STLOG_I(MESSAGE, ...) STLOG(PRI_INFO, NKikimrServices::KQP_SESSION, KQPSA, LogPrefix() << MESSAGE, __VA_ARGS__) -#define STLOG_D(MESSAGE, ...) STLOG(PRI_DEBUG, NKikimrServices::KQP_SESSION, KQPSA, LogPrefix() << MESSAGE, __VA_ARGS__) -#define STLOG_T(MESSAGE, ...) STLOG(PRI_TRACE, NKikimrServices::KQP_SESSION, KQPSA, LogPrefix() << MESSAGE, __VA_ARGS__) - void FillColumnsMeta(const NKqpProto::TKqpPhyQuery& phyQuery, NKikimrKqp::TQueryResponse& resp) { for (size_t i = 0; i < phyQuery.ResultBindingsSize(); ++i) { const auto& binding = phyQuery.GetResultBindings(i); @@ -285,15 +279,19 @@ public: TempTablesState.Database = Settings.Database; TempTablesState.TempDirName = TAppData::RandomProvider->GenUuid4().AsUuidString(); - STLOG_D("Create session actor", - (session_id, SessionId), - (temp_dir_name, TempTablesState.TempDirName), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Create session actor", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"sessionId", SessionId}, + {"tempDirName", TempTablesState.TempDirName}, + {"traceId", TraceId()}); } void Bootstrap() { - STLOG_D("Session actor bootstrapped", - (trace_id, TraceId())); + YDB_LOG_DEBUG("Session actor bootstrapped", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); Counters->ReportSessionActorCreated(Settings.DbCounters); CreationTime = TInstant::Now(); FillGUCSettings(); @@ -407,10 +405,12 @@ public: auto txId = TTxId::FromString(txControl.tx_id()); auto txCtx = Transactions.Find(txId); - STLOG_D("QueryRequest", - (tx_control, txControl.DebugString()), - (tx_ctx, (uintptr_t)txCtx.Get()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("QueryRequest", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"txControl", txControl.DebugString()}, + {"txCtx", (uintptr_t)txCtx.Get()}, + {"traceId", TraceId()}); if (!txCtx) { ReplyTransactionNotFound(txControl.tx_id()); return; @@ -451,9 +451,11 @@ public: } void ClientLost() { - STLOG_D("Got ClientLost event, send AbortExecution to executer", - (executer_id, ExecuterId), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Got ClientLost event, send AbortExecution to executer", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"executerId", ExecuterId}, + {"traceId", TraceId()}); if (ExecuterId) { auto abortEv = TEvKqp::TEvAbortExecution::Aborted("Client lost"); // any status code can be here @@ -486,18 +488,22 @@ public: TString errMsg = issues.ToString(); auto status = ev->Get()->GetYdbStatus(); - STLOG_N("Got invalid query request, reply with error", - (status, status), - (error_msg, errMsg), - (trace_id, TraceId())); + YDB_LOG_NOTICE("Got invalid query request, reply with error", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"status", status}, + {"errorMsg", errMsg}, + {"traceId", TraceId()}); ReplyProcessError(ev, status, errMsg); return; } if (ShutdownState && ShutdownState->SoftTimeoutReached()) { // we reached the soft timeout, so at this point we don't allow to accept new queries for session. - STLOG_N("System shutdown requested: soft timeout reached, no queries can be accepted", - (trace_id, TraceId())); + YDB_LOG_NOTICE("System shutdown requested: soft timeout reached, no queries can be accepted", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); ReplyProcessError(ev, Ydb::StatusIds::BAD_SESSION, "Session is under shutdown"); CleanupAndPassAway(); return; @@ -517,18 +523,20 @@ public: action, QueryState->GetQuery()); - STLOG_D("Received request", - (proxy_request_id, proxyRequestId), - (prepared, QueryState->HasPreparedQuery()), - (has_tx_control, QueryState->HasTxControl()), - (action, action), - (type, QueryState->GetType()), - (text, QueryState->GetQuery()), - (rpc_actor, QueryState->RequestActorId), - (database, QueryState->GetDatabase()), - (database_id, QueryState->UserRequestContext->DatabaseId), - (pool_id, QueryState->UserRequestContext->PoolId), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Received request", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"proxyRequestId", proxyRequestId}, + {"prepared", QueryState->HasPreparedQuery()}, + {"hasTxControl", QueryState->HasTxControl()}, + {"action", action}, + {"type", QueryState->GetType()}, + {"text", QueryState->GetQuery()}, + {"rpcActor", QueryState->RequestActorId}, + {"database", QueryState->GetDatabase()}, + {"databaseId", QueryState->UserRequestContext->DatabaseId}, + {"poolId", QueryState->UserRequestContext->PoolId}, + {"traceId", TraceId()}); switch (action) { case NKikimrKqp::QUERY_ACTION_EXPLAIN: @@ -600,10 +608,13 @@ public: using TError = std::optional<std::pair<Ydb::StatusIds::StatusCode, TString>>; auto error = std::visit(TOverloaded { [this, &sent](const NWorkloadManager::IQueryClassifier::TResolvedPoolId& s) -> TError { - STLOG_D("PreCompile Classify resolved", - (pool_id, s.PoolId), - (skip_admission, s.SkipAdmission), - (trace_id, TraceId())); + YDB_LOG_DEBUG("PreCompile Classify resolved", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"poolId", s.PoolId}, + {"skipAdmission", s.SkipAdmission}, + {"traceId", TraceId()}); + QueryState->UserRequestContext->PoolId = s.PoolId; if (s.SkipAdmission) { QueryState->UserRequestContext->PoolConfig = s.PoolConfig; @@ -614,19 +625,25 @@ public: return std::nullopt; }, [this](const NWorkloadManager::IQueryClassifier::TReject& r) -> TError { - STLOG_N("PreCompile Classify rejected", - (trace_id, TraceId())); + YDB_LOG_NOTICE("PreCompile Classify rejected", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); return std::make_pair(r.Code, r.Message); }, [this](const NWorkloadManager::IQueryClassifier::TBypass&) -> TError { - STLOG_D("PreCompile Classify bypass, compiling", - (trace_id, TraceId())); + YDB_LOG_DEBUG("PreCompile Classify bypass, compiling", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); QueryState->UserRequestContext->PoolId = NResourcePool::DEFAULT_POOL_ID; return std::nullopt; }, [this](const NWorkloadManager::IQueryClassifier::TPendingCompilation&) -> TError { - STLOG_D("PreCompile Classify pending, compiling", - (trace_id, TraceId())); + YDB_LOG_DEBUG("PreCompile Classify pending, compiling", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); return std::nullopt; }, }, result); @@ -641,8 +658,10 @@ public: void Handle(TEvents::TEvUndelivered::TPtr& ev) { if (ev->Get()->SourceType == NWorkloadManager::TWorkloadManagerEvents::EvPlaceRequestIntoPool) { - STLOG_W("Failed to deliver request to workload service, bypassing WLM", - (trace_id, TraceId())); + YDB_LOG_WARN("Failed to deliver request to workload service, bypassing WLM", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); ContinueAfterWmAdmission(); return; } @@ -653,8 +672,10 @@ public: // client-lost before our cross-node subscription arrived. Treat it as // client lost and tear down now, instead of hanging until the query // deadline. See SubscribeRemoteCancel (FlagTrackDelivery). - STLOG_D("Grpc cancel subscription undelivered, treat as client lost", - (trace_id, TraceId())); + YDB_LOG_DEBUG("Grpc cancel subscription undelivered, treat as client lost", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); ClientLost(); return; } @@ -665,8 +686,10 @@ public: QueryState->ContinueTime = TInstant::Now(); if (ev->Get()->Status == Ydb::StatusIds::UNSUPPORTED) { - STLOG_T("Failed to place request in resource pool, feature flag is disabled", - (trace_id, TraceId())); + YDB_LOG_TRACE("Failed to place request in resource pool, feature flag is disabled", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); QueryState->UserRequestContext->PoolId.clear(); ContinueAfterWmAdmission(); return; @@ -681,14 +704,19 @@ public: } if (ev->Get()->IsDiskFull()) { - STLOG_W("Database disks are without free space", - (pool_id, poolId),(trace_id, TraceId())); + YDB_LOG_WARN("Database disks are without free space", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"poolId", poolId}, + {"traceId", TraceId()}); FillQueryIssues(ev->Get()->Issues); } - STLOG_D("Continue request", - (pool_id, poolId), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Continue request", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"poolId", poolId}, + {"traceId", TraceId()}); QueryState->PoolHandlerActor = ev->Sender; QueryState->UserRequestContext->PoolId = poolId; @@ -707,10 +735,16 @@ public: auto state = classifier->GetState(); if (state == NWorkloadManager::IQueryClassifier::EState::PreCompileDone) { - STLOG_D("Pre-compile admission completed, compiling", (trace_id, TraceId())); + YDB_LOG_DEBUG("Pre-compile admission completed, compiling", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); CompileQuery(); } else if (state == NWorkloadManager::IQueryClassifier::EState::PostCompileDone) { - STLOG_D("Post-compile admission completed, executing", (trace_id, TraceId())); + YDB_LOG_DEBUG("Post-compile admission completed, executing", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); OnSuccessCompileRequest(); } else { Y_VALIDATE(false, TStringBuilder() @@ -836,8 +870,10 @@ public: // for extra sanity we make extra hop to the compile service, which might handle the issue better auto ev = QueryState->BuildCompileRequest(CompilationCookie, GUCSettings, txCtx); - STLOG_D("Sending CompileQuery request", - (trace_id, TraceId())); + YDB_LOG_DEBUG("Sending CompileQuery request", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); Send(MakeKqpCompileServiceID(SelfId().NodeId()), ev.release(), 0, QueryState->QueryId, QueryState->KqpSessionSpan.GetTraceId()); @@ -847,8 +883,10 @@ public: YQL_ENSURE(QueryState); auto txCtx = GetTxContextForCompilation(); auto ev = QueryState->BuildCompileSplittedRequest(CompilationCookie, GUCSettings, txCtx); - STLOG_D("Sending CompileSplittedQuery request", - (trace_id, TraceId())); + YDB_LOG_DEBUG("Sending CompileSplittedQuery request", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); Send(MakeKqpCompileServiceID(SelfId().NodeId()), ev.release(), 0, QueryState->QueryId, QueryState->KqpSessionSpan.GetTraceId()); @@ -965,8 +1003,10 @@ public: // for extra sanity we make extra hop to the compile service, which might handle the issue better auto request = QueryState->BuildCompileRequest(CompilationCookie, GUCSettings, txCtx); - STLOG_D("Sending CompileQuery request (statement)", - (trace_id, TraceId())); + YDB_LOG_DEBUG("Sending CompileQuery request (statement)", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); Send(MakeKqpCompileServiceID(SelfId().NodeId()), request.release(), 0, QueryState->QueryId, QueryState->KqpSessionSpan.GetTraceId()); @@ -1013,10 +1053,12 @@ public: using TError = std::optional<std::pair<Ydb::StatusIds::StatusCode, TString>>; auto error = std::visit(TOverloaded { [this, &sent](const NWorkloadManager::IQueryClassifier::TResolvedPoolId& r) -> TError { - STLOG_D("PostCompile Classify resolved", - (pool_id, r.PoolId), - (skip_admission, r.SkipAdmission), - (trace_id, TraceId())); + YDB_LOG_DEBUG("PostCompile Classify resolved", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"poolId", r.PoolId}, + {"skipAdmission", r.SkipAdmission}, + {"traceId", TraceId()}); QueryState->UserRequestContext->PoolId = r.PoolId; if (r.SkipAdmission) { QueryState->UserRequestContext->PoolConfig = r.PoolConfig; @@ -1027,13 +1069,17 @@ public: return std::nullopt; }, [this](const NWorkloadManager::IQueryClassifier::TBypass&) -> TError { - STLOG_D("PostCompile Classify bypass", - (trace_id, TraceId())); + YDB_LOG_DEBUG("PostCompile Classify bypass", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); return std::nullopt; }, [this](const NWorkloadManager::IQueryClassifier::TReject& r) -> TError { - STLOG_N("PostCompile Classify rejected", - (trace_id, TraceId())); + YDB_LOG_NOTICE("PostCompile Classify rejected", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); return std::make_pair(r.Code, r.Message); } }, result); @@ -1181,8 +1227,10 @@ public: } void AcquirePersistentSnapshot() { - STLOG_D("Acquire persistent snapshot", - (trace_id, TraceId())); + YDB_LOG_DEBUG("Acquire persistent snapshot", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); AcquireSnapshotSpan = NWilson::TSpan(TWilsonKqp::SessionAcquireSnapshot, QueryState->KqpSessionSpan.GetTraceId(), "SessionActor.AcquirePersistentSnapshot"); auto timeout = QueryState->QueryDeadlines.TimeoutAt - TAppData::TimeProvider->Now(); @@ -1205,8 +1253,10 @@ public: void AcquireMvccSnapshot() { AcquireSnapshotSpan = NWilson::TSpan(TWilsonKqp::SessionAcquireSnapshot, QueryState->KqpSessionSpan.GetTraceId(), "SessionActor.AcquireMvccSnapshot"); - STLOG_D("Acquire mvcc snapshot", - (trace_id, TraceId())); + YDB_LOG_DEBUG("Acquire mvcc snapshot", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); auto timeout = QueryState->QueryDeadlines.TimeoutAt - TAppData::TimeProvider->Now(); auto* snapMgr = CreateKqpSnapshotManager(Settings.Database, timeout); @@ -1243,11 +1293,13 @@ public: QueryState->Orbit = std::move(response->Orbit); } - STLOG_T("Read snapshot result", - (status, StatusForSnapshotError(response->Status)), - (step, response->Snapshot.Step), - (tx_id, response->Snapshot.TxId), - (trace_id, TraceId())); + YDB_LOG_TRACE("Read snapshot result", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"status", StatusForSnapshotError(response->Status)}, + {"step", response->Snapshot.Step}, + {"txId", response->Snapshot.TxId}, + {"traceId", TraceId()}); if (response->Status != NKikimrIssues::TStatusIds::SUCCESS) { auto& issues = response->Issues; AcquireSnapshotSpan.EndError(issues.ToString()); @@ -1279,10 +1331,12 @@ public: alloc->Alloc->SetLimit(mkqlInitialLimit); alloc->Alloc->Ref().SetIncreaseMemoryLimitCallback([this, &alloc, mkqlMaxLimit](ui64 currentLimit, ui64 required) { if (required < mkqlMaxLimit) { - STLOG_D("Increase memory limit", - (current_limit, currentLimit), - (required, required), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Increase memory limit", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"currentLimit", currentLimit}, + {"required", required}, + {"traceId", TraceId()}); alloc->Alloc->SetLimit(required); } }); @@ -1910,9 +1964,11 @@ public: for (const auto& effect : txCtx.DeferredEffects) { request.Transactions.emplace_back(effect.PhysicalTx, effect.Params, effect.QuerySpanId); - STLOG_D("TExecPhysicalRequest, add DeferredEffect to Transaction", - (transactions_size, request.Transactions.size()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("TExecPhysicalRequest, add DeferredEffect to Transaction", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"transactionsSize", request.Transactions.size()}, + {"traceId", TraceId()}); } SendToPartitionedExecuter(QueryState->TxCtx.Get(), std::move(request)); @@ -1928,9 +1984,11 @@ public: for (const auto& effect : txCtx.DeferredEffects) { request.Transactions.emplace_back(effect.PhysicalTx, effect.Params, effect.QuerySpanId); - STLOG_D("TExecPhysicalRequest, add DeferredEffect to Transaction", - (transactions_size, request.Transactions.size()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("TExecPhysicalRequest, add DeferredEffect to Transaction", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"transactionsSize", request.Transactions.size()}, + {"traceId", TraceId()}); } request.AcquireLocksTxId = txCtx.LockHandle.GetLockId(); @@ -1992,12 +2050,14 @@ public: auto request = PrepareRequest(tx, literal, QueryState.get()); - STLOG_D("ExecutePhyTx", - (literal, literal), - (commit, commit), - (deferred_effects_size, txCtx.DeferredEffects.Size()), - (tx, (uintptr_t)tx.get()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("ExecutePhyTx", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"literal", literal}, + {"commit", commit}, + {"deferredEffectsSize", txCtx.DeferredEffects.Size()}, + {"tx", (uintptr_t)tx.get()}, + {"traceId", TraceId()}); if (!CheckTopicOperations()) { return true; @@ -2042,9 +2102,11 @@ public: for (const auto& effect : txCtx.DeferredEffects) { request.Transactions.emplace_back(effect.PhysicalTx, effect.Params, effect.QuerySpanId); - STLOG_D("TExecPhysicalRequest, add DeferredEffect to Transaction", - (transactions_size, request.Transactions.size()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("TExecPhysicalRequest, add DeferredEffect to Transaction", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"transactionsSize", request.Transactions.size()}, + {"traceId", TraceId()}); } if (!txCtx.DeferredEffects.Empty()) { @@ -2057,17 +2119,23 @@ public: if (hasLocks || txCtx.TopicOperations.HasOperations()) { if (!txCtx.GetSnapshot().IsValid() || txCtx.TxHasEffects() || txCtx.TopicOperations.HasOperations()) { - STLOG_D("TExecPhysicalRequest, tx has commit locks", - (trace_id, TraceId())); + YDB_LOG_DEBUG("TExecPhysicalRequest, tx has commit locks", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); request.LocksOp = ELocksOp::Commit; } else { - STLOG_D("TExecPhysicalRequest, tx has rollback locks", - (trace_id, TraceId())); + YDB_LOG_DEBUG("TExecPhysicalRequest, tx has rollback locks", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); request.LocksOp = ELocksOp::Rollback; } } else if (txCtx.TxHasEffects()) { - STLOG_D("TExecPhysicalRequest, need commit locks", - (trace_id, TraceId())); + YDB_LOG_DEBUG("TExecPhysicalRequest, need commit locks", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); request.LocksOp = ELocksOp::Commit; } @@ -2193,9 +2261,11 @@ public: request.ResourceManager_ = ResourceManager_; request.SaveQueryPhysicalGraph = allowSaveState && QueryState->SaveQueryPhysicalGraph; request.QueryPhysicalGraph = allowSaveState ? QueryState->QueryPhysicalGraph : nullptr; - STLOG_D("Sending to Executer", - (span_id_size, request.TraceId.GetSpanIdSize()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Sending to Executer", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"spanIdSize", request.TraceId.GetSpanIdSize()}, + {"traceId", TraceId()}); txCtx->TxManager->SetSkipTopicsConflictCheck(AppData()->FeatureFlags.GetEnableSkipConflictCheckForTopicsInTransaction()); @@ -2218,10 +2288,12 @@ public: alloc->SetLimit(writeBufferInitialMemoryLimit); alloc->Ref().SetIncreaseMemoryLimitCallback([this, alloc=alloc.get(), writeBufferMemoryLimit](ui64 currentLimit, ui64 required) { if (required < writeBufferMemoryLimit) { - STLOG_D("Increase memory limit", - (current_limit, currentLimit), - (required, required), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Increase memory limit", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"currentLimit", currentLimit}, + {"required", required}, + {"traceId", TraceId()}); alloc->SetLimit(required); } }); @@ -2275,10 +2347,12 @@ public: ); auto exId = RegisterWithSameMailbox(executerActor); - STLOG_D("Created new KQP executer", - (executer_id, exId), - (is_rollback, isRollback), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Created new KQP executer", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"executerId", exId}, + {"isRollback", isRollback}, + {"traceId", TraceId()}); auto ev = std::make_unique<TEvTxUserProxy::TEvProposeKqpTransaction>(exId); Send(MakeTxProxyID(), ev.release()); if (!isRollback) { @@ -2349,9 +2423,11 @@ public: auto executerActor = CreateKqpPartitionedExecuter(std::move(settings), ChannelService); ExecuterId = RegisterWithSameMailbox(executerActor); - STLOG_D("Created new KQP partitioned executer", - (executer_id, ExecuterId), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Created new KQP partitioned executer", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"executerId", ExecuterId}, + {"traceId", TraceId()}); } @@ -2404,8 +2480,11 @@ public: } } - STLOG_D("Forwarded TEvExecuterProgress to " << QueryState->RequestActorId, - (trace_id, TraceId())); + YDB_LOG_DEBUG("Forwarded TEvExecuterProgress", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"requestActorId", QueryState->RequestActorId}, + {"traceId", TraceId()}); Send(QueryState->RequestActorId, ev->Release().Release(), 0, QueryState->ProxyRequestId); } } @@ -2463,10 +2542,11 @@ public: void FillQueryIssues(const NYql::TIssues& issues) { if (!QueryState) { - STLOG_W("Try to put issues into empty QueryState", - (issues, issues.ToOneLineString()), - (trace_id, TraceId()) - ); + YDB_LOG_WARN("Try to put issues into empty QueryState", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"issues", issues.ToOneLineString()}, + {"traceId", TraceId()}); return; } @@ -2641,11 +2721,13 @@ public: auto* response = ev->Record.MutableResponse(); - STLOG_D("TEvTxResponse", - (current_tx, QueryState->CurrentTx), - (transactions_size, QueryState->PreparedQuery ? QueryState->PreparedQuery->GetPhysicalQuery().TransactionsSize() : 0), - (status, response->GetStatus()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("TEvTxResponse", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"currentTx", QueryState->CurrentTx}, + {"transactionsSize", QueryState->PreparedQuery ? QueryState->PreparedQuery->GetPhysicalQuery().TransactionsSize() : 0}, + {"status", response->GetStatus()}, + {"traceId", TraceId()}); ExecuterId = TActorId{}; @@ -2667,11 +2749,13 @@ public: if (response->GetStatus() != Ydb::StatusIds::SUCCESS) { const auto executionType = ev->ExecutionType; - STLOG_D("TEvTxResponse has non-success status", - (current_tx, QueryState->CurrentTx), - (execution_type, executionType), - (status, response->GetStatus()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("TEvTxResponse has non-success status", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"currentTx", QueryState->CurrentTx}, + {"executionType", executionType}, + {"status", response->GetStatus()}, + {"traceId", TraceId()}); auto status = response->GetStatus(); TIssues issues; @@ -2750,8 +2834,11 @@ public: void HandleExecute(TEvKqpExecuter::TEvStreamData::TPtr& ev) { YQL_ENSURE(QueryState && QueryState->RequestActorId); - STLOG_D("Forwarded TEvStreamData to " << QueryState->RequestActorId, - (trace_id, TraceId())); + YDB_LOG_DEBUG("Forwarded TEvStreamData", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"requestActorId", QueryState->RequestActorId}, + {"traceId", TraceId()}); QueryState->QueryData->AddBuiltResultIndex(ev->Get()->Record.GetQueryResultIndex()); @@ -2765,10 +2852,12 @@ public: void HandleExecute(TEvKqp::TEvAbortExecution::TPtr& ev) { auto& msg = ev->Get()->Record; - STLOG_I("Got TEvAbortExecution, send it to Executer", - (status_code, NYql::NDqProto::StatusIds_StatusCode_Name(msg.GetStatusCode())), - (executer_id, ExecuterId), - (trace_id, TraceId())); + YDB_LOG_INFO("Got TEvAbortExecution, send it to Executer", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"statusCode", NYql::NDqProto::StatusIds_StatusCode_Name(msg.GetStatusCode())}, + {"executerId", ExecuterId}, + {"traceId", TraceId()}); auto issues = ev->Get()->GetIssues(); TStringBuilder reason = TStringBuilder() << "Cancelling after " << (AppData()->MonotonicTimeProvider->Now() - QueryState->StartedAt).MilliSeconds() << "ms"; @@ -2796,12 +2885,18 @@ public: << ", status: " << NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode) << " send to: " << ExecuterId << " from: " << ev->Sender; if (!QueryState || !QueryState->TxCtx || QueryState->TxCtx->BufferActorId != ev->Sender) { - STLOG_E(logMsg << ": Ignored error.", - (trace_id, TraceId())); + YDB_LOG_ERROR("Ignored error", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"logMsg", logMsg}, + {"traceId", TraceId()}); return; } else { - STLOG_W(logMsg, - (trace_id, TraceId())); + YDB_LOG_WARN("Buffer actor reported error", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"logMsg", logMsg}, + {"traceId", TraceId()}); } if (ExecuterId) { @@ -2826,8 +2921,11 @@ public: << ", status: " << NYql::NDqProto::StatusIds_StatusCode_Name(msg.StatusCode) << " send to: " << ExecuterId << " from: " << ev->Sender; if (CleanupCtx->TransactionsToBeAborted.empty()) { - STLOG_E(logMsg << ": Ignored error. TransactionsToBeAborted is empty.", - (trace_id, TraceId())); + YDB_LOG_ERROR("Ignored error. TransactionsToBeAborted is empty", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"logMsg", logMsg}, + {"traceId", TraceId()}); } AFL_ENSURE(ExecuterId); // ExecuterId can't be empty during cleanup if TransactionsToBeAborted is not empty. @@ -2835,12 +2933,18 @@ public: const auto& txCtx = CleanupCtx->TransactionsToBeAborted.front(); AFL_ENSURE(txCtx); if (txCtx->BufferActorId != ev->Sender) { - STLOG_E(logMsg << ": Ignored error. Current BufferActorId is not sender.", - (trace_id, TraceId())); + YDB_LOG_ERROR("Ignored error. Current BufferActorId is not sender", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"logMsg", logMsg}, + {"traceId", TraceId()}); return; } else { - STLOG_W(logMsg, - (trace_id, TraceId())); + YDB_LOG_WARN("Buffer actor reported error, forwarding to executer", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"logMsg", logMsg}, + {"traceId", TraceId()}); } Send(ExecuterId, new TEvKqpBuffer::TEvError{msg.StatusCode, std::move(msg.Issues), std::move(msg.Stats)}, IEventHandle::FlagTrackDelivery); @@ -2944,8 +3048,10 @@ public: template<class TEvRecord> void AddTrailingInfo(TEvRecord& record) { if (ShutdownState) { - STLOG_D("Session is closing, set trailing metadata to request session shutdown", - (trace_id, TraceId())); + YDB_LOG_DEBUG("Session is closing, set trailing metadata to request session shutdown", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); record.SetWorkerIsClosing(true); } } @@ -2956,13 +3062,15 @@ public: if (QueryState->TxCtx) { auto txInfo = QueryState->TxCtx->GetInfo(); - STLOG_I("TxInfo", - (status, txInfo.Status), - (kind, txInfo.Kind), - (total_duration, txInfo.TotalDuration.SecondsFloat()*1e3), - (server_duration, txInfo.ServerDuration.SecondsFloat()*1e3), - (queries_count, txInfo.QueriesCount), - (trace_id, TraceId())); + YDB_LOG_INFO("TxInfo", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"status", txInfo.Status}, + {"kind", txInfo.Kind}, + {"totalDuration", txInfo.TotalDuration.SecondsFloat()*1e3}, + {"serverDuration", txInfo.ServerDuration.SecondsFloat()*1e3}, + {"queriesCount", txInfo.QueriesCount}, + {"traceId", TraceId()}); Counters->ReportTransaction(Settings.DbCounters, txInfo); } } @@ -3140,9 +3248,11 @@ public: } resEv->Record.SetYdbStatus(Ydb::StatusIds::SUCCESS); - STLOG_D("Create QueryResponse for action with SUCCESS status", - (action, QueryState->GetAction()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Create QueryResponse for action with SUCCESS status", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"action", QueryState->GetAction()}, + {"traceId", TraceId()}); QueryResponse = std::move(resEv); @@ -3176,11 +3286,13 @@ public: } } - STLOG_W("ReplyQueryCompileError, remove tx", - (status, QueryState->CompileResult->Status), - (issues, Join(", ", QueryResponse->Record.GetResponse().GetQueryIssues())), - (tx_id, txId.GetHumanStr()), - (trace_id, TraceId())); + YDB_LOG_WARN("ReplyQueryCompileError, remove tx", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"status", QueryState->CompileResult->Status}, + {"issues", Join(", ", QueryResponse->Record.GetResponse().GetQueryIssues())}, + {"txId", txId.GetHumanStr()}, + {"traceId", TraceId()}); if (auto ctx = Transactions.ReleaseTransaction(txId)) { ctx->Invalidate(); @@ -3220,9 +3332,12 @@ public: const TString& message) { ui64 proxyRequestId = request->Cookie; - STLOG_W("Reply query error, msg: " << message, - (proxy_request_id, proxyRequestId), - (trace_id, TraceId())); + YDB_LOG_WARN("Reply query error", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"msg", message}, + {"proxyRequestId", proxyRequestId}, + {"traceId", TraceId()}); auto response = std::make_unique<TEvKqp::TEvQueryResponse>(); response->Record.SetYdbStatus(ydbStatus); auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, message); @@ -3343,15 +3458,19 @@ public: KQP_REQ_LOG(TLogQuery::Completed(*QueryState, record, responseByteSize)); Send<ESendingType::Tail>(QueryState->Sender, QueryResponse.release(), 0, QueryState->ProxyRequestId); - STLOG_D("Sent query response back to proxy", - (proxy_request_id, QueryState->ProxyRequestId), - (proxy_id, QueryState->Sender.ToString()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Sent query response back to proxy", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"proxyRequestId", QueryState->ProxyRequestId}, + {"proxyId", QueryState->Sender}, + {"traceId", TraceId()}); if (IsFatalError(status)) { - STLOG_N("SessionActor destroyed", - (status, status), - (trace_id, TraceId())); + YDB_LOG_NOTICE("SessionActor destroyed", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"status", status}, + {"traceId", TraceId()}); Counters->ReportSessionActorClosedError(Settings.DbCounters); } } @@ -3393,8 +3512,10 @@ public: } void HandleReady(TEvKqp::TEvCloseSessionRequest::TPtr&) { - STLOG_I("Session closed due to explicit close event", - (trace_id, TraceId())); + YDB_LOG_INFO("Session closed due to explicit close event", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); Counters->ReportSessionActorClosedRequest(Settings.DbCounters); CleanupAndPassAway(); } @@ -3418,8 +3539,10 @@ public: void Handle(TEvKqp::TEvInitiateSessionShutdown::TPtr& ev) { if (!ShutdownState) { - STLOG_N("Started session shutdown", - (trace_id, TraceId())); + YDB_LOG_NOTICE("Started session shutdown", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); ShutdownState = TSessionShutdownState(ev->Get()->SoftTimeoutMs, ev->Get()->HardTimeoutMs); ScheduleNextShutdownTick(); } @@ -3433,13 +3556,17 @@ public: YQL_ENSURE(ShutdownState); ShutdownState->MoveToNextState(); if (ShutdownState->HardTimeoutReached()) { - STLOG_N("Reached hard shutdown timeout", - (trace_id, TraceId())); + YDB_LOG_NOTICE("Reached hard shutdown timeout", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); Send(SelfId(), new TEvKqp::TEvCloseSessionRequest()); } else { ScheduleNextShutdownTick(); - STLOG_I("Schedule next shutdown tick", - (trace_id, TraceId())); + YDB_LOG_INFO("Schedule next shutdown tick", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); } } @@ -3538,13 +3665,15 @@ public: } } - STLOG_I("Cleanup start", - (is_final, isFinal), - (has_cleanup_ctx, bool{CleanupCtx}), - (transactions_to_be_aborted_size, CleanupCtx ? CleanupCtx->TransactionsToBeAborted.size() : 0), - (worker_id, workerId ? *workerId : TActorId()), - (workload_service_cleanup, CleanupCtx ? CleanupCtx->IsWaitingForWorkloadServiceCleanup : false), - (trace_id, TraceId())); + YDB_LOG_INFO("Cleanup start", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"isFinal", isFinal}, + {"hasCleanupCtx", bool{CleanupCtx}}, + {"transactionsToBeAbortedSize", CleanupCtx ? CleanupCtx->TransactionsToBeAborted.size() : 0}, + {"workerId", workerId ? *workerId : TActorId()}, + {"workloadServiceCleanup", CleanupCtx ? CleanupCtx->IsWaitingForWorkloadServiceCleanup : false}, + {"traceId", TraceId()}); if (CleanupCtx) { Become(&TKqpSessionActor::CleanupState); } else { @@ -3578,9 +3707,11 @@ public: if (response.GetStatus() != Ydb::StatusIds::SUCCESS) { TIssues issues; IssuesFromMessage(response.GetIssues(), issues); - STLOG_E("Failed to cleanup", - (issues, issues.ToString()), - (trace_id, TraceId())); + YDB_LOG_ERROR("Failed to cleanup", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"issues", issues}, + {"traceId", TraceId()}); for (const auto& txCtx : CleanupCtx->TransactionsToBeAborted) { AFL_ENSURE(txCtx); @@ -3607,10 +3738,12 @@ public: CleanupCtx->IsWaitingForWorkloadServiceCleanup = false; if (ev->Get()->Status != Ydb::StatusIds::SUCCESS && ev->Get()->Status != Ydb::StatusIds::NOT_FOUND) { - STLOG_E("Failed to cleanup workload service", - (status, ev->Get()->Status), - (issues, ev->Get()->Issues.ToOneLineString()), - (trace_id, TraceId())); + YDB_LOG_ERROR("Failed to cleanup workload service", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"status", ev->Get()->Status}, + {"issues", ev->Get()->Issues.ToOneLineString()}, + {"traceId", TraceId()}); } if (CleanupCtx->CleanupFinished()) { @@ -3619,9 +3752,11 @@ public: } void EndCleanup(bool isFinal) { - STLOG_D("EndCleanup", - (is_final, isFinal), - (trace_id, TraceId())); + YDB_LOG_DEBUG("EndCleanup", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"isFinal", isFinal}, + {"traceId", TraceId()}); if (QueryResponse) Reply(); @@ -3633,9 +3768,11 @@ public: auto userToken = QueryState ? QueryState->UserToken : TIntrusiveConstPtr<NACLib::TUserToken>(); Become(&TKqpSessionActor::FinalCleanupState); - STLOG_D("Cleanup temp tables", - (temp_tables_size, TempTablesState.TempTables.size()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Cleanup temp tables", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"tempTablesSize", TempTablesState.TempTables.size()}, + {"traceId", TraceId()}); auto tempTablesManager = CreateKqpTempTablesManager( std::move(TempTablesState), std::move(userToken), SelfId(), Settings.Database); @@ -3669,10 +3806,13 @@ public: { // DEBUG, not WARN: the [REQ_JSON] completed envelope already carries // status/issues/trace_id at WARN on failure (see kqp_log_query.cpp). - STLOG_D("Create QueryResponse for error on request, msg: " << message, - (status, ydbStatus), - (issues, issues ? Join(", ", *issues) : TString()), - (trace_id, TraceId())); + YDB_LOG_DEBUG("Create QueryResponse for error on request", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"msg", message}, + {"status", ydbStatus}, + {"issues", issues ? Join(", ", *issues) : TString()}, + {"traceId", TraceId()}); QueryResponse = std::make_unique<TEvKqp::TEvQueryResponse>(); QueryResponse->Record.SetYdbStatus(ydbStatus); @@ -3731,8 +3871,10 @@ public: closeEv->Record.MutableResponse()->SetClosed(true); Send(Owner, closeEv.release()); - STLOG_D("Session actor destroyed", - (trace_id, TraceId())); + YDB_LOG_DEBUG("Session actor destroyed", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"traceId", TraceId()}); PassAway(); } @@ -3910,8 +4052,11 @@ private: } void InternalError(const TString& message) { - STLOG_E("Internal error" << message, - (trace_id, TraceId())); + YDB_LOG_ERROR("Internal error", + {"marker", "KQPSA"}, + {"logPrefix", LogPrefix()}, + {"message", message}, + {"traceId", TraceId()}); if (QueryState) { ReplyQueryError(Ydb::StatusIds::INTERNAL_ERROR, message); } else { diff --git a/ydb/core/kqp/session_actor/kqp_temp_tables_manager.cpp b/ydb/core/kqp/session_actor/kqp_temp_tables_manager.cpp index 772c8d01226..ed3aa5b3020 100644 --- a/ydb/core/kqp/session_actor/kqp_temp_tables_manager.cpp +++ b/ydb/core/kqp/session_actor/kqp_temp_tables_manager.cpp @@ -12,14 +12,9 @@ #include <ydb/library/actors/core/log.h> #include <ydb/library/yql/utils/actor_log/log.h> -namespace NKikimr::NKqp { +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_SESSION -#define LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, stream) -#define LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, stream) -#define LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, stream) -#define LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, stream) -#define LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, stream) -#define LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, stream) +namespace NKikimr::NKqp { using namespace NThreading; @@ -120,7 +115,8 @@ private: void HandleNavigate(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { const NSchemeCache::TSchemeCacheNavigate* navigate = ev->Get()->Request.Get(); if (navigate->ErrorCount != 0) { - LOG_E(TStringBuilder() << "Navigate errors: " << navigate->ErrorCount); + YDB_LOG_ERROR("Navigate", + {"errors", navigate->ErrorCount}); } for (const auto& entry : navigate->ResultSet) { @@ -136,7 +132,8 @@ private: } } } else { - LOG_E(TStringBuilder() << "Navigate error. Entry: " << entry.ToString()); + YDB_LOG_ERROR("Navigate error", + {"entry", entry}); } } diff --git a/ydb/core/kqp/session_actor/kqp_worker_actor.cpp b/ydb/core/kqp/session_actor/kqp_worker_actor.cpp index 51d7a132007..eaef080c4a0 100644 --- a/ydb/core/kqp/session_actor/kqp_worker_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_worker_actor.cpp @@ -24,6 +24,8 @@ #include <util/string/escape.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_WORKER + namespace NKikimr { namespace NKqp { @@ -35,14 +37,6 @@ using namespace NRuCalc; namespace { -#define LOG_C(msg) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::KQP_WORKER, LogPrefix() << msg) -#define LOG_E(msg) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::KQP_WORKER, LogPrefix() << msg) -#define LOG_W(msg) LOG_WARN_S(*TlsActivationContext, NKikimrServices::KQP_WORKER, LogPrefix() << msg) -#define LOG_N(msg) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::KQP_WORKER, LogPrefix() << msg) -#define LOG_I(msg) LOG_INFO_S(*TlsActivationContext, NKikimrServices::KQP_WORKER, LogPrefix() << msg) -#define LOG_D(msg) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_WORKER, LogPrefix() << msg) -#define LOG_T(msg) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::KQP_WORKER, LogPrefix() << msg) - using TQueryResult = IKqpHost::TQueryResult; enum EReplyFlags : ui32 { @@ -141,7 +135,8 @@ public: } void Bootstrap(const TActorContext&) { - LOG_D("Worker bootstrapped"); + YDB_LOG_DEBUG("Worker bootstrapped", + {"logPrefix", LogPrefix()}); Counters->ReportWorkerCreated(Settings.DbCounters); Become(&TKqpWorkerActor::ReadyState); } @@ -158,7 +153,8 @@ public: void HandleReady(TEvKqp::TEvCloseSessionRequest::TPtr &ev, const TActorContext &ctx) { ui64 proxyRequestId = ev->Cookie; if (CheckRequest(ev->Get()->Record.GetRequest().GetSessionId(), ev->Sender, proxyRequestId, ctx)) { - LOG_I("Session closed due to explicit close event"); + YDB_LOG_INFO("Session closed due to explicit close event", + {"logPrefix", LogPrefix()}); Counters->ReportWorkerClosedRequest(Settings.DbCounters); FinalCleanup(ctx); } @@ -175,8 +171,10 @@ public: return; } - LOG_D("Received request, proxyRequestId: " << proxyRequestId - << " rpcCtx: " << (void*)(ev->Get()->GetRequestCtx().get())); + YDB_LOG_DEBUG("Received request", + {"logPrefix", LogPrefix()}, + {"proxyRequestId", proxyRequestId}, + {"rpcCtx", (void*)(ev->Get()->GetRequestCtx().get())}); Y_ABORT_UNLESS(!QueryState); @@ -302,7 +300,8 @@ public: } void HandlePerformQuery(TEvKqp::TEvCloseSessionRequest::TPtr &ev, const TActorContext &ctx) { - LOG_D("Got TEvCloseSessionRequest during PerformQuery state"); + YDB_LOG_DEBUG("Got TEvCloseSessionRequest during PerformQuery state", + {"logPrefix", LogPrefix()}); Y_UNUSED(ev); Y_UNUSED(ctx); QueryState->KeepSession = false; @@ -366,7 +365,9 @@ public: Y_ABORT_UNLESS(CleanupState); auto result = CleanupState->AsyncResult->GetResult(); if (!result.Success()) { - LOG_E("Failed to cleanup: " << result.Issues().ToString()); + YDB_LOG_ERROR("Failed", + {"logPrefix", LogPrefix()}, + {"cleanup", result.Issues()}); } EndCleanup(ctx); @@ -439,10 +440,11 @@ private: return true; } - LOG_N("Legacy YQL request" - << ", action: " << (ui32)queryRequest->GetAction() - << ", type: " << (ui32)queryRequest->GetType() - << ", query: \"" << queryRequest->GetQuery().substr(0, 1000) << "\""); + YDB_LOG_NOTICE("Received legacy YQL request", + {"logPrefix", LogPrefix()}, + {"action", (ui32)queryRequest->GetAction()}, + {"type", (ui32)queryRequest->GetType()}, + {"queryPreview", queryRequest->GetQuery().substr(0, 1000)}); return false; } @@ -749,32 +751,38 @@ private: } ctx.Send<ESendingType::Tail>(QueryState->Sender, responseEv.Release(), 0, QueryState->ProxyRequestId); - LOG_D("Sent query response back to proxy, proxyRequestId: " << QueryState->ProxyRequestId - << ", proxyId: " << QueryState->Sender.ToString()); + YDB_LOG_DEBUG("Sent query response back to proxy", + {"logPrefix", LogPrefix()}, + {"proxyRequestId", QueryState->ProxyRequestId}, + {"proxyId", QueryState->Sender}); QueryState.Reset(); if (Settings.LongSession) { if (status == Ydb::StatusIds::INTERNAL_ERROR) { - LOG_D("Worker destroyed due to internal error"); + YDB_LOG_DEBUG("Worker destroyed due to internal error", + {"logPrefix", LogPrefix()}); Counters->ReportWorkerClosedError(Settings.DbCounters); return false; } if (status == Ydb::StatusIds::BAD_SESSION) { - LOG_D("Worker destroyed due to session error"); + YDB_LOG_DEBUG("Worker destroyed due to session error", + {"logPrefix", LogPrefix()}); Counters->ReportWorkerClosedError(Settings.DbCounters); return false; } } else { if (status != Ydb::StatusIds::SUCCESS) { - LOG_D("Worker destroyed due to query error"); + YDB_LOG_DEBUG("Worker destroyed due to query error", + {"logPrefix", LogPrefix()}); Counters->ReportWorkerClosedError(Settings.DbCounters); return false; } } if (!keepSession) { - LOG_D("Worker destroyed due to negative keep session flag"); + YDB_LOG_DEBUG("Worker destroyed due to negative keep session flag", + {"logPrefix", LogPrefix()}); Counters->ReportWorkerClosedRequest(Settings.DbCounters); return false; } @@ -854,7 +862,8 @@ private: template<class TEvRecord> void AddTrailingInfo(TEvRecord& record) { if (ShutdownState) { - LOG_D("Session is closing, set trailing metadata to request session shutdown"); + YDB_LOG_DEBUG("Session is closing, set trailing metadata to request session shutdown", + {"logPrefix", LogPrefix()}); record.SetWorkerIsClosing(true); } } @@ -862,7 +871,9 @@ private: bool ReplyProcessError(const TActorId& sender, ui64 proxyRequestId, Ydb::StatusIds::StatusCode ydbStatus, const TString& message) { - LOG_W(message); + YDB_LOG_WARN("Replying with process error", + {"logPrefix", LogPrefix()}, + {"message", message}); auto response = std::make_unique<TEvKqp::TEvQueryResponse>(); response->Record.SetYdbStatus(ydbStatus); auto issue = MakeIssue(NKikimrIssues::TIssuesIds::DEFAULT_ERROR, message); @@ -1046,7 +1057,9 @@ private: } void InternalError(const TString& message) { - LOG_E("Internal error, message: " << message); + YDB_LOG_ERROR("Internal error", + {"logPrefix", LogPrefix()}, + {"message", message}); if (QueryState) { ReplyProcessError(QueryState->Sender, QueryState->ProxyRequestId, Ydb::StatusIds::INTERNAL_ERROR, message); } diff --git a/ydb/core/kqp/session_actor/kqp_worker_common.cpp b/ydb/core/kqp/session_actor/kqp_worker_common.cpp index 8bdf60b0c4c..17ea5639ba7 100644 --- a/ydb/core/kqp/session_actor/kqp_worker_common.cpp +++ b/ydb/core/kqp/session_actor/kqp_worker_common.cpp @@ -2,6 +2,8 @@ #include <ydb/library/security/util.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_SLOW_LOG + namespace NKikimr::NKqp { using namespace NYql; @@ -125,13 +127,14 @@ void SlowLogQuery(const TActorContext &ctx, const TKikimrConfiguration* config, resultsSize += result.ByteSize(); } - LOG_LOG_S(ctx, priority, NKikimrServices::KQP_SLOW_LOG, requestInfo - << "Slow query, duration: " << duration.ToString() - << ", status: " << status - << ", user: " << username - << ", results: " << resultsSize << 'b' - << ", text: \"" << EscapeC(protectedQueryText) << '"' - << ", parameters: " << paramsSize); + YDB_LOG_CTX(ctx, priority, "Slow query, b", + {"requestInfo", requestInfo}, + {"duration", duration}, + {"status", status}, + {"user", username}, + {"results", resultsSize}, + {"text", EscapeC(protectedQueryText)}, + {"parameters", paramsSize}); } } diff --git a/ydb/core/kqp/topics/kqp_topics.cpp b/ydb/core/kqp/topics/kqp_topics.cpp index f291f2cad2e..61488229804 100644 --- a/ydb/core/kqp/topics/kqp_topics.cpp +++ b/ydb/core/kqp/topics/kqp_topics.cpp @@ -9,7 +9,7 @@ #include <util/generic/set.h> -#define LOG_D(msg) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::KQP_SESSION, msg) +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_SESSION namespace NKikimr::NKqp::NTopic { @@ -726,10 +726,10 @@ bool TTopicOperations::ProcessSchemeCacheNavigate(const NSchemeCache::TSchemeCac TTopicPartition key{path, partition.GetPartitionId()}; if (auto p = Operations_.find(key); p != Operations_.end()) { - LOG_D(TStringBuilder() << "(topic, partition, tablet): " - << "'" << key.Topic_ << "'" - << ", " << partition.GetPartitionId() - << ", " << partition.GetTabletId()); + YDB_LOG_DEBUG("Resolved topic partition tablet mapping", + {"topic", key.Topic_}, + {"partitionId", partition.GetPartitionId()}, + {"tabletId", partition.GetTabletId()}); p->second.SetTabletId(partition.GetTabletId()); } diff --git a/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp b/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp index 1c272f4dbc6..384dbbbd9ce 100644 --- a/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp +++ b/ydb/core/kqp/ut/channels/dq_channel_service_ut.cpp @@ -15,6 +15,8 @@ #include <ydb/library/yql/dq/actors/dq.h> #include <util/random/random.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::KQP_CHANNELS + using namespace NKikimr::NKqp; using namespace NYql::NDq; @@ -102,7 +104,10 @@ public: virtual void HandleStart(TEvTestPrivate::TEvStart::TPtr& ev) { RunnerId = ev->Sender; PeerId = ev->Get()->PeerId; - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, LogPrefix << "TEST START SelfId=" << this->SelfId() << ", ChannelId=" << ChannelId << ", PeerId=" << PeerId); + YDB_LOG_DEBUG("TEST START", + {"selfId", this->SelfId()}, + {"channelId", ChannelId}, + {"peerId", PeerId}); if (Settings.StartDelayMs) { this->Schedule(TDuration::MilliSeconds(RandomNumber<ui64>(Settings.StartDelayMs) + 1), new NActors::TEvents::TEvWakeup()); } else { @@ -111,9 +116,10 @@ public: } virtual void HandleAbort(NYql::NDq::TEvDq::TEvAbortExecution::TPtr& ev) { - LOG_DEBUG_S(*NActors::TlsActivationContext, NKikimrServices::KQP_CHANNELS, - LogPrefix << "TEST ABORT SelfId=" << this->SelfId() << ", ChannelId=" << ChannelId << ", " << ev->Get()->GetIssues().ToOneLineString() - ); + YDB_LOG_DEBUG("Test worker received abort execution", + {"selfId", this->SelfId()}, + {"channelId", ChannelId}, + {"issues", ev->Get()->GetIssues().ToOneLineString()}); this->Send(RunnerId, new TEvTestPrivate::TEvFinished(TEvTestPrivate::ERole::Producer, true)); this->PassAway(); } diff --git a/ydb/core/kqp/ut/federated_query/datastreams/common.cpp b/ydb/core/kqp/ut/federated_query/datastreams/common.cpp index f429c895ad6..5c114b5d29f 100644 --- a/ydb/core/kqp/ut/federated_query/datastreams/common.cpp +++ b/ydb/core/kqp/ut/federated_query/datastreams/common.cpp @@ -1,5 +1,7 @@ #include "common.h" +#include <ydb/services/workload_manager/ut/common/workload_service_ut_common.h> + #include <ydb/core/base/counters.h> #include <ydb/core/cms/console/console.h> #include <ydb/core/kqp/common/kqp_script_executions.h> @@ -1029,4 +1031,8 @@ void TTabletKiller::KillTablet() const { Schedule(KillerInterval, new TEvents::TEvWakeup()); } +void TStreamingTestFixture::WaitForClassifierPropagation() { + NWorkloadManager::WaitForClassifierPropagation(GetRuntime()); +} + } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/federated_query/datastreams/common.h b/ydb/core/kqp/ut/federated_query/datastreams/common.h index e641d48eced..6167d1c2461 100644 --- a/ydb/core/kqp/ut/federated_query/datastreams/common.h +++ b/ydb/core/kqp/ut/federated_query/datastreams/common.h @@ -143,6 +143,8 @@ public: void ExecSchemeQuery(const std::string& query, NYdb::EStatus expectedStatus = NYdb::EStatus::SUCCESS); + void WaitForClassifierPropagation(); + // Query client SDK std::vector<NYdb::TResultSet> ExecQuery(const std::string& query, NYdb::EStatus expectedStatus = NYdb::EStatus::SUCCESS, const std::string& expectedError = "", std::function<void(const std::string&)> astValidator = nullptr); diff --git a/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp b/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp index 9d0e4f288b0..b2a4bc18d1e 100644 --- a/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/datastreams/kqp_has_path_ut.cpp @@ -25,41 +25,6 @@ TString RejectClassifierDdl(TStringBuf classifierName, TStringBuf hasPath) { )"; } -void WaitClassifierVisible(TStreamingTestFixture& fixture, - const std::string& classifierName, - TDuration timeout = TDuration::Seconds(20)) -{ - const auto probe = fmt::format(R"( - SELECT COUNT(*) FROM `.metadata/workload_manager/classifiers/resource_pool_classifiers` - WHERE name = "{name}"; - )", "name"_a = classifierName); - - auto session = fixture.GetQueryClient()->GetSession().GetValueSync().GetSession(); - const auto execSettings = NYdb::NQuery::TExecuteQuerySettings() - .ResourcePool(std::string(NResourcePool::DEFAULT_POOL_ID)); - const auto deadline = TInstant::Now() + timeout; - - while (TInstant::Now() < deadline) { - auto result = session.ExecuteQuery( - probe, NYdb::NQuery::TTxControl::NoTx(), execSettings).ExtractValueSync(); - - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, - result.GetIssues().ToOneLineString()); - - NYdb::TResultSetParser parser(result.GetResultSet(0)); - UNIT_ASSERT(parser.TryNextRow()); - - if (parser.ColumnParser(0).GetUint64() == 1) { - Sleep(TDuration::MilliSeconds(500)); // pad for workload service snapshot refresh - return; - } - - Sleep(TDuration::MilliSeconds(200)); - } - - UNIT_ASSERT_C(false, "Classifier '" << classifierName << "' not visible within " << timeout); -} - } // anonymous namespace @@ -83,7 +48,7 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { ExecSchemeQuery(RejectClassifierDdl( "hp_direct_topic", "/Root/test_topic")); - WaitClassifierVisible(*this, "hp_direct_topic"); + WaitForClassifierPropagation(); ExecQuery(R"( SELECT * FROM `/Root/test_topic` WITH ( @@ -105,7 +70,7 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { ExecSchemeQuery(RejectClassifierDdl( "hp_cdc", "/Root/t_cdc/cf")); - WaitClassifierVisible(*this, "hp_cdc"); + WaitForClassifierPropagation(); ExecSchemeQuery(R"( CREATE TABLE t_cdc ( @@ -141,7 +106,7 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { ExecSchemeQuery(RejectClassifierDdl( "hp_eds_local", "/Root/eds")); - WaitClassifierVisible(*this, "hp_eds_local"); + WaitForClassifierPropagation(); ExecSchemeQuery(fmt::format(R"( CREATE EXTERNAL DATA SOURCE eds WITH ( @@ -177,7 +142,7 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { ExecSchemeQuery(RejectClassifierDdl( "hp_remote_name", "/remote_by_name_topic")); - WaitClassifierVisible(*this, "hp_remote_name"); + WaitForClassifierPropagation(); ExecSchemeQuery(fmt::format(R"( CREATE EXTERNAL DATA SOURCE eds WITH ( @@ -207,7 +172,7 @@ Y_UNIT_TEST_SUITE(HasPathDatastreams) { ExecSchemeQuery(RejectClassifierDdl( "hp_et", "/Root/et_s3")); - WaitClassifierVisible(*this, "hp_et"); + WaitForClassifierPropagation(); ExecSchemeQuery(R"( CREATE EXTERNAL DATA SOURCE eds_s3 WITH ( diff --git a/ydb/core/kqp/ut/federated_query/datastreams/ya.make b/ydb/core/kqp/ut/federated_query/datastreams/ya.make index d38503fba29..7f4bb747ed8 100644 --- a/ydb/core/kqp/ut/federated_query/datastreams/ya.make +++ b/ydb/core/kqp/ut/federated_query/datastreams/ya.make @@ -36,6 +36,7 @@ PEERDIR( ydb/library/testlib/solomon_helpers ydb/library/yql/providers/generic/connector/libcpp ydb/library/yql/providers/generic/connector/libcpp/ut_helpers + ydb/services/workload_manager/ut/common yql/essentials/sql/pg yql/essentials/parser/pg_wrapper yql/essentials/udfs/common/yson2 diff --git a/ydb/core/kqp/ut/indexes/fulltext/kqp_fulltext_build_ut.cpp b/ydb/core/kqp/ut/indexes/fulltext/kqp_fulltext_build_ut.cpp index 30e2b880d81..2ee2c36a50c 100644 --- a/ydb/core/kqp/ut/indexes/fulltext/kqp_fulltext_build_ut.cpp +++ b/ydb/core/kqp/ut/indexes/fulltext/kqp_fulltext_build_ut.cpp @@ -2662,7 +2662,6 @@ Y_UNIT_TEST(Utf8) { Y_UNIT_TEST(AddFullTextFlatIndexWithTruncateWithSelect) { NKikimrConfig::TFeatureFlags featureFlags; featureFlags.SetEnableFulltextIndex(true); - featureFlags.SetEnableTruncateTable(true); auto kikimr = Kikimr(std::move(featureFlags)); @@ -2717,7 +2716,6 @@ Y_UNIT_TEST(AddFullTextFlatIndexWithTruncateWithSelect) { Y_UNIT_TEST(AddFullTextRelevanceIndexWithTruncate) { NKikimrConfig::TFeatureFlags featureFlags; featureFlags.SetEnableFulltextIndex(true); - featureFlags.SetEnableTruncateTable(true); auto kikimr = Kikimr(std::move(featureFlags)); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); diff --git a/ydb/core/kqp/ut/indexes/json/kqp_indexes_json_ut.cpp b/ydb/core/kqp/ut/indexes/json/kqp_indexes_json_ut.cpp index 62afa59b357..bcba0b8c364 100644 --- a/ydb/core/kqp/ut/indexes/json/kqp_indexes_json_ut.cpp +++ b/ydb/core/kqp/ut/indexes/json/kqp_indexes_json_ut.cpp @@ -2086,7 +2086,6 @@ Y_UNIT_TEST_SUITE(KqpJsonIndexes) { Y_UNIT_TEST(TruncateTable) { NKikimrConfig::TFeatureFlags featureFlags; featureFlags.SetEnableJsonIndex(true); - featureFlags.SetEnableTruncateTable(true); auto kikimr = TKikimrRunner(TKikimrSettings().SetFeatureFlags(featureFlags)); auto db = kikimr.GetQueryClient(); diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp index f0e1b7bab97..168c44b27ca 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp @@ -7714,7 +7714,6 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda Y_UNIT_TEST(TruncateTableWithAsyncIndexFails) { NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableTruncateTable(true); TKikimrRunner kikimr(TKikimrSettings().SetFeatureFlags(featureFlags)); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); diff --git a/ydb/core/kqp/ut/indexes/prefixed_vector/kqp_indexes_prefixed_vector_ut.cpp b/ydb/core/kqp/ut/indexes/prefixed_vector/kqp_indexes_prefixed_vector_ut.cpp index bbfe1ede947..dbfb6ca5cb6 100644 --- a/ydb/core/kqp/ut/indexes/prefixed_vector/kqp_indexes_prefixed_vector_ut.cpp +++ b/ydb/core/kqp/ut/indexes/prefixed_vector/kqp_indexes_prefixed_vector_ut.cpp @@ -1179,7 +1179,6 @@ Y_UNIT_TEST_SUITE(KqpPrefixedVectorIndexes) { Y_UNIT_TEST_QUAD(PrefixedVectorIndexTruncateTable, Covered, Overlap) { NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableTruncateTable(true); auto serverSettings = TKikimrSettings().SetFeatureFlags(featureFlags); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); diff --git a/ydb/core/kqp/ut/indexes/vector/kqp_indexes_vector_ut.cpp b/ydb/core/kqp/ut/indexes/vector/kqp_indexes_vector_ut.cpp index aa1a6ddf6b0..52ffc78d625 100644 --- a/ydb/core/kqp/ut/indexes/vector/kqp_indexes_vector_ut.cpp +++ b/ydb/core/kqp/ut/indexes/vector/kqp_indexes_vector_ut.cpp @@ -1588,7 +1588,6 @@ Y_UNIT_TEST_SUITE(KqpVectorIndexes) { Y_UNIT_TEST_QUAD(VectorIndexTruncateTable, Covered, Overlap) { NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableTruncateTable(true); auto serverSettings = TKikimrSettings().SetFeatureFlags(featureFlags); TKikimrRunner kikimr(serverSettings); kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE); diff --git a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp index 2e0606d0220..3586ba7ce53 100644 --- a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp +++ b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp @@ -15,6 +15,8 @@ #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/operation/operation.h> #include <ydb/public/sdk/cpp/src/client/ss_tasks/task.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_COLUMNSHARD + namespace NKikimr::NKqp { Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { @@ -45,18 +47,26 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { } virtual void DoProposeSuccess(const TString& sessionId) const override { CSTransferStatus->SetProposed(true); - AFL_NOTICE(NKikimrServices::TX_COLUMNSHARD)("event", "sharing_proposed")("session_id", sessionId); + YDB_LOG_NOTICE("Column shard sharing proposed", + {"event", "sharing_proposed"}, + {"sessionId", sessionId}); } virtual void DoConfirmSuccess(const TString& sessionId) const override { CSTransferStatus->SetConfirmed(true); - AFL_NOTICE(NKikimrServices::TX_COLUMNSHARD)("event", "sharing_confirmed")("session_id", sessionId); + YDB_LOG_NOTICE("Column shard sharing confirmed", + {"event", "sharing_confirmed"}, + {"sessionId", sessionId}); } virtual void DoFinished(const TString& sessionId) const override { CSTransferStatus->SetFinished(true); - AFL_NOTICE(NKikimrServices::TX_COLUMNSHARD)("event", "sharing_finished")("session_id", sessionId); + YDB_LOG_NOTICE("Column shard sharing finished", + {"event", "sharing_finished"}, + {"sessionId", sessionId}); } virtual void DoStatus(const NOlap::NDataSharing::TStatusContainer& status) const override { - AFL_NOTICE(NKikimrServices::TX_COLUMNSHARD)("event", "status")("info", status.SerializeToProto().DebugString()); + YDB_LOG_NOTICE("Column shard sharing status updated", + {"event", "status"}, + {"info", status.SerializeToProto().DebugString()}); } virtual TConclusionStatus DoDeserializeFromProto(const NKikimrColumnShardDataSharingProto::TInitiator::TController& /*proto*/) override { return TConclusionStatus::Success(); diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index ecd82fa6c69..a228e67cecf 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -5194,7 +5194,6 @@ Y_UNIT_TEST_SUITE(KqpOlap) { Y_UNIT_TEST(TruncateColumnTableFails) { NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableTruncateTable(true); auto settings = TKikimrSettings().SetWithSampleTables(false).SetFeatureFlags(featureFlags); TKikimrRunner kikimr(settings); auto client = kikimr.GetQueryClient(); diff --git a/ydb/core/kqp/ut/olap/sys_view_ut.cpp b/ydb/core/kqp/ut/olap/sys_view_ut.cpp index b41904b7f29..7638b38d1ad 100644 --- a/ydb/core/kqp/ut/olap/sys_view_ut.cpp +++ b/ydb/core/kqp/ut/olap/sys_view_ut.cpp @@ -9,6 +9,8 @@ #include <ydb/core/tx/columnshard/hooks/testing/controller.h> #include <ydb/core/tx/columnshard/test_helper/controllers.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_COLUMNSHARD + namespace NKikimr::NKqp { Y_UNIT_TEST_SUITE(KqpOlapSysView) { @@ -501,7 +503,8 @@ Y_UNIT_TEST_SUITE(KqpOlapSysView) { // AFL_VERIFY(i.GetArraySafe()[0]["entity_id"].GetInteger() == 4); // AFL_VERIFY(i.GetArraySafe()[0]["data"].GetIntegerRobust() >= 799992); // AFL_VERIFY(i.GetArraySafe()[0]["data"].GetIntegerRobust() <= 799999); -// AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("json", i); +// YDB_LOG_INFO("", +// {"json", i}); // } } } @@ -534,7 +537,8 @@ Y_UNIT_TEST_SUITE(KqpOlapSysView) { // AFL_VERIFY(i.GetArraySafe()[0]["entity_id"].GetInteger() == 5)("json", i); // AFL_VERIFY(i.GetArraySafe()[0]["data"].GetIntegerRobust() >= 799992); // AFL_VERIFY(i.GetArraySafe()[0]["data"].GetIntegerRobust() <= 799999); -// AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("json", i); +// YDB_LOG_INFO("", +// {"json", i}); // } } } diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 80b73b4b5a0..6d6f6fcd192 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -73,7 +73,6 @@ void CreateSecret(const TString& secretName, const TString& secretValue, TSessio void TestTruncateTable(const TString& tablePath, bool useQueryClient = false, bool createSecondaryIndex = false) { NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableTruncateTable(true); TKikimrRunner kikimr(featureFlags); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -15092,7 +15091,6 @@ END DO)", Y_UNIT_TEST(TruncateTableEraseRowPermission) { NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableTruncateTable(true); TKikimrRunner kikimr(featureFlags); auto rootSession = kikimr.GetTableClient().CreateSession().GetValueSync().GetSession(); @@ -15156,7 +15154,6 @@ END DO)", Y_UNIT_TEST(TruncateTableDoesNotResetSerialSequence) { NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableTruncateTable(true); TKikimrRunner kikimr(featureFlags); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -15220,7 +15217,6 @@ END DO)", Y_UNIT_TEST(TruncateTableWithTtl) { NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableTruncateTable(true); TKikimrRunner kikimr(featureFlags); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -15272,7 +15268,6 @@ END DO)", Y_UNIT_TEST(TruncateNonExistentTable) { NKikimrConfig::TFeatureFlags featureFlags; - featureFlags.SetEnableTruncateTable(true); TKikimrRunner kikimr(featureFlags); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); diff --git a/ydb/core/kqp/ut/tx/kqp_read_committed_pg_ut.cpp b/ydb/core/kqp/ut/tx/kqp_read_committed_pg_ut.cpp index aa895182968..5164ac329e5 100644 --- a/ydb/core/kqp/ut/tx/kqp_read_committed_pg_ut.cpp +++ b/ydb/core/kqp/ut/tx/kqp_read_committed_pg_ut.cpp @@ -1365,6 +1365,145 @@ Y_UNIT_TEST_SUITE(KqpReadCommittedPg) { tester.Execute(); } + // ========================================================================= + // INSERT then UPDATE within the same RC transaction should see the inserted row + // ========================================================================= + class TInsertThenUpdateSameTx : public TTableDataModificationTester { + protected: + void DoExecute() override { + auto client = Kikimr->GetQueryClient(); + auto session = Kikimr->RunCall([&] { return client.GetSession().GetValueSync().GetSession(); }); + + auto result1 = Kikimr->RunCall([&] { + return session.ExecuteQuery(Q_(R"( + INSERT INTO `/Root/Test` (Group, Name, Amount, Comment) VALUES (998u, "NewRow", 500u, "Initial"); + )"), TTxControl::BeginTx(TTxSettings::ReadCommittedRW())).ExtractValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(result1.GetStatus(), EStatus::SUCCESS, result1.GetIssues().ToString()); + auto tx = result1.GetTransaction(); + UNIT_ASSERT(tx && tx->IsActive()); + + auto result2 = Kikimr->RunCall([&] { + return session.ExecuteQuery(Q_(R"( + UPDATE `/Root/Test` SET Comment = "Updated" WHERE Group = 998u AND Name = "NewRow"; + )"), TTxControl::Tx(*tx)).ExtractValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), EStatus::SUCCESS, result2.GetIssues().ToString()); + tx = result2.GetTransaction(); + UNIT_ASSERT(tx && tx->IsActive()); + + auto result3 = Kikimr->RunCall([&] { + return session.ExecuteQuery(Q_(R"( + SELECT Comment FROM `/Root/Test` WHERE Group = 998u AND Name = "NewRow"; + )"), TTxControl::Tx(*tx).CommitTx()).ExtractValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(result3.GetStatus(), EStatus::SUCCESS, result3.GetIssues().ToString()); + CompareYson(R"([[["Updated"]]])", FormatResultSetYson(result3.GetResultSet(0))); + } + }; + + Y_UNIT_TEST(TInsertThenUpdateSameTx) { + TInsertThenUpdateSameTx tester; + tester.SetIsOlap(false); + tester.SetUseRealThreads(false); + tester.Execute(); + } + + // ========================================================================= + // INSERT then DELETE within the same RC transaction should delete the inserted row + // ========================================================================= + class TInsertThenDeleteSameTx : public TTableDataModificationTester { + protected: + void DoExecute() override { + auto client = Kikimr->GetQueryClient(); + auto session = Kikimr->RunCall([&] { return client.GetSession().GetValueSync().GetSession(); }); + + auto result1 = Kikimr->RunCall([&] { + return session.ExecuteQuery(Q_(R"( + INSERT INTO `/Root/Test` (Group, Name, Amount, Comment) VALUES (998u, "NewRow", 500u, "Initial"); + )"), TTxControl::BeginTx(TTxSettings::ReadCommittedRW())).ExtractValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(result1.GetStatus(), EStatus::SUCCESS, result1.GetIssues().ToString()); + auto tx = result1.GetTransaction(); + UNIT_ASSERT(tx && tx->IsActive()); + + auto result2 = Kikimr->RunCall([&] { + return session.ExecuteQuery(Q_(R"( + DELETE FROM `/Root/Test` WHERE Group = 998u AND Name = "NewRow"; + )"), TTxControl::Tx(*tx)).ExtractValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), EStatus::SUCCESS, result2.GetIssues().ToString()); + tx = result2.GetTransaction(); + UNIT_ASSERT(tx && tx->IsActive()); + + auto result3 = Kikimr->RunCall([&] { + return session.ExecuteQuery(Q_(R"( + SELECT Comment FROM `/Root/Test` WHERE Group = 998u AND Name = "NewRow"; + )"), TTxControl::Tx(*tx).CommitTx()).ExtractValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(result3.GetStatus(), EStatus::SUCCESS, result3.GetIssues().ToString()); + CompareYson(R"([])", FormatResultSetYson(result3.GetResultSet(0))); + } + }; + + Y_UNIT_TEST(TInsertThenDeleteSameTx) { + TInsertThenDeleteSameTx tester; + tester.SetIsOlap(false); + tester.SetUseRealThreads(false); + tester.Execute(); + } + + // ========================================================================= + // INSERT, DELETE, INSERT within the same RC transaction: final row should survive + // ========================================================================= + class TInsertDeleteInsertSameTx : public TTableDataModificationTester { + protected: + void DoExecute() override { + auto client = Kikimr->GetQueryClient(); + auto session = Kikimr->RunCall([&] { return client.GetSession().GetValueSync().GetSession(); }); + + auto result1 = Kikimr->RunCall([&] { + return session.ExecuteQuery(Q_(R"( + INSERT INTO `/Root/Test` (Group, Name, Amount, Comment) VALUES (998u, "NewRow", 500u, "First"); + )"), TTxControl::BeginTx(TTxSettings::ReadCommittedRW())).ExtractValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(result1.GetStatus(), EStatus::SUCCESS, result1.GetIssues().ToString()); + auto tx = result1.GetTransaction(); + UNIT_ASSERT(tx && tx->IsActive()); + + auto result2 = Kikimr->RunCall([&] { + return session.ExecuteQuery(Q_(R"( + DELETE FROM `/Root/Test` WHERE Group = 998u AND Name = "NewRow"; + )"), TTxControl::Tx(*tx)).ExtractValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(result2.GetStatus(), EStatus::SUCCESS, result2.GetIssues().ToString()); + tx = result2.GetTransaction(); + UNIT_ASSERT(tx && tx->IsActive()); + + auto result3 = Kikimr->RunCall([&] { + return session.ExecuteQuery(Q_(R"( + INSERT INTO `/Root/Test` (Group, Name, Amount, Comment) VALUES (998u, "NewRow", 500u, "Second"); + )"), TTxControl::Tx(*tx).CommitTx()).ExtractValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(result3.GetStatus(), EStatus::SUCCESS, result3.GetIssues().ToString()); + + auto verify = Kikimr->RunCall([&] { + return session.ExecuteQuery(Q_(R"( + SELECT Comment FROM `/Root/Test` WHERE Group = 998u AND Name = "NewRow"; + )"), TTxControl::BeginTx(TTxSettings::SnapshotRW()).CommitTx()).ExtractValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(verify.GetStatus(), EStatus::SUCCESS, verify.GetIssues().ToString()); + CompareYson(R"([[["Second"]]])", FormatResultSetYson(verify.GetResultSet(0))); + } + }; + + Y_UNIT_TEST(TInsertDeleteInsertSameTx) { + TInsertDeleteInsertSameTx tester; + tester.SetIsOlap(false); + tester.SetUseRealThreads(false); + tester.Execute(); + } + class TLockOnlyShardDistributedCommit : public TTableDataModificationTester { protected: void DoExecute() override { diff --git a/ydb/core/mind/node_broker_ut.cpp b/ydb/core/mind/node_broker_ut.cpp index 120bfc19d40..6d8c3cfcc3c 100644 --- a/ydb/core/mind/node_broker_ut.cpp +++ b/ydb/core/mind/node_broker_ut.cpp @@ -80,7 +80,8 @@ THashMap<ui32, TIntrusivePtr<TNodeWardenConfig>> NodeWardenConfigs; void SetupServices(TTestActorRuntime &runtime, ui32 maxDynNodes, bool enableNodeBrokerDeltaProtocol, - bool enableNodeBrokerLongLease = false) + bool enableNodeBrokerLongLease = false, + bool forceDistconfDisable = false) { const ui32 domainsNum = 1; const ui32 disksInDomain = 1; @@ -196,6 +197,12 @@ void SetupServices(TTestActorRuntime &runtime, app.FeatureFlags.SetEnableStableNodeNames(true); app.FeatureFlags.SetEnableNodeBrokerDeltaProtocol(enableNodeBrokerDeltaProtocol); app.FeatureFlags.SetEnableNodeBrokerLongLease(enableNodeBrokerLongLease); + // The node warden's TDistributedConfigKeeper on every runtime node also + // subscribes to static node changes. Tests that push a fabricated static + // node table (SetNameserverStaticNodes) would make the keeper re-process a + // list that excludes its own static node id and abort. Such tests disable + // distconf, which is incidental to the node broker / nameservice under test. + app.FeatureFlags.SetForceDistconfDisable(forceDistconfDisable); runtime.Initialize(app.Unwrap()); @@ -335,7 +342,8 @@ void Setup(TTestActorRuntime& runtime, ui32 maxDynNodes = 3, const TVector<TString>& databases = {}, bool enableNodeBrokerDeltaProtocol = false, - bool enableNodeBrokerLongLease = false) + bool enableNodeBrokerLongLease = false, + bool forceDistconfDisable = false) { using namespace NMalloc; TMallocInfo mallocInfo = MallocInfo(); @@ -351,7 +359,7 @@ void Setup(TTestActorRuntime& runtime, runtime.SetScheduledEventFilter(scheduledFilter); SetupLogging(runtime); - SetupServices(runtime, maxDynNodes, enableNodeBrokerDeltaProtocol, enableNodeBrokerLongLease); + SetupServices(runtime, maxDynNodes, enableNodeBrokerDeltaProtocol, enableNodeBrokerLongLease, forceDistconfDisable); TActorId sender = runtime.AllocateEdgeActor(); ui32 txId = 100; @@ -5543,7 +5551,11 @@ Y_UNIT_TEST_SUITE(TDynamicNameserverTest) { Y_UNIT_TEST(OnlyAliveDynamicNodesStaticNodeChangeSubscribers) { TTestBasicRuntime runtime(8, false); - Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true); + // Disable distconf: SetNameserverStaticNodes below pushes a fabricated + // static node table that excludes the runtime's own static nodes, which + // would make the node warden's TDistributedConfigKeeper abort. + Setup(runtime, 4, {}, false, /* enableNodeBrokerLongLease */ true, + /* forceDistconfDisable */ true); TActorId sender = runtime.AllocateEdgeActor(); SetLeaseDuration(runtime, sender, TDuration::Minutes(5)); diff --git a/ydb/core/mon/mon.cpp b/ydb/core/mon/mon.cpp index d69f16cd2dd..0cf8d214102 100644 --- a/ydb/core/mon/mon.cpp +++ b/ydb/core/mon/mon.cpp @@ -55,7 +55,7 @@ struct TIssueInfo { } }; -bool HasJsonContent(NHttp::THttpIncomingRequest* request) { +bool HasJsonContent(const NHttp::THttpIncomingRequest* request) { if (request->Method == "POST") { const TStringBuf header = request->ContentType.Before(';'); return header.empty() || AsciiEqualsIgnoreCase(header, "application/json"); // by default we will try to parse json, no error will be generated if parsing fails @@ -63,7 +63,7 @@ bool HasJsonContent(NHttp::THttpIncomingRequest* request) { return false; } -TString GetDatabase(NHttp::THttpIncomingRequest* request) { +TString GetDatabase(const NHttp::THttpIncomingRequest* request) { NHttp::TUrlParameters urlParams(request->URL); TString database = urlParams["database"]; if (database) { @@ -78,6 +78,32 @@ TString GetDatabase(NHttp::THttpIncomingRequest* request) { return {}; } +void LogAuthorizedHttpRequest( + const TAppData* appData, + const NGRpcService::TEvRequestAuthAndCheckResult* result, + const NHttp::THttpIncomingRequest& request) +{ + const TString address = request.Address ? request.Address->ToString() : ""; + const TString user = (result && result->UserToken) ? result->UserToken->GetUserSID() : "anonymous"; + const NACLib::TUserToken* userToken = (result && result->UserToken) ? result->UserToken.Get() : nullptr; + const TString accessLevel = ToString(GetHighestAccessLevel(appData, userToken)); + const TString database = result ? result->Database : GetDatabase(&request); + YDB_LOG_NOTICE( + "Send request" + << " [" << address << "]" + << " " << user + << " " << request.Method + << " " << request.URL + << " highest_access_level=" << accessLevel + << " database=" << database, + {"address", address}, + {"user", user}, + {"method", request.Method}, + {"url", request.URL}, + {"highest_access_level", accessLevel}, + {"database", database}); +} + const Ydb::Issue::IssueMessage* FindDeepestIssue(const google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& issues) { std::queue<TIssueInfo> issuesQueue; ui32 minimalSeverity = std::numeric_limits<ui32>::max(); @@ -594,12 +620,7 @@ public: void SendRequest(const NKikimr::NGRpcService::TEvRequestAuthAndCheckResult* result = nullptr) { NHttp::THttpIncomingRequestPtr request = Event->Get()->Request; if (ActorMonPage->Authorizer) { - TString user = (result && result->UserToken) ? result->UserToken->GetUserSID() : "anonymous"; - YDB_LOG_NOTICE("Send request", - {"address", (request->Address ? request->Address->ToString() : "")}, - {"user", user}, - {"method", request->Method}, - {"url", request->URL}); + LogAuthorizedHttpRequest(AppData(), result, *request); } TString serializedToken = result && result->UserToken ? result->UserToken->GetSerializedToken() : TString(); Send(ActorMonPage->TargetActorId, new NMon::TEvHttpInfo( @@ -1248,12 +1269,7 @@ public: void SendRequest(const NKikimr::NGRpcService::TEvRequestAuthAndCheckResult* result = nullptr) { if (Authorizer) { - TString user = (result && result->UserToken) ? result->UserToken->GetUserSID() : "anonymous"; - YDB_LOG_NOTICE("", - {"address", (Request->Address ? Request->Address->ToString() : "")}, - {"user", user}, - {"method", Request->Method}, - {"url", Request->URL}); + LogAuthorizedHttpRequest(AppData(), result, *Request); } Send(new IEventHandle(Fields.Handler, SelfId(), Event->ReleaseBase().Release(), IEventHandle::FlagTrackDelivery, Event->Cookie)); } diff --git a/ydb/core/nbs/cloud/blockstore/config/protos/ddisk_config.proto b/ydb/core/nbs/cloud/blockstore/config/protos/ddisk_config.proto index 2649ae72ed8..3cfc542d223 100644 --- a/ydb/core/nbs/cloud/blockstore/config/protos/ddisk_config.proto +++ b/ydb/core/nbs/cloud/blockstore/config/protos/ddisk_config.proto @@ -61,4 +61,11 @@ message TPBufferConfig // Deallocate a chunk proactively when it has been freed for this many seconds. // Default 30 seconds. optional uint32 DeallocateThresholdSeconds = 15; + + // TEvListPersistentBuffer must not observe a partially-applied write/erase for its tablet: the + // listing is deferred (queued and retried) while any disk operation is in flight for the + // requesting tablet. These parameters bound how long/how often we wait before giving up and + // replying with an OVERLOADED error to avoid returning a potentially-stale view. + optional uint32 ListPersistentBufferMaxRetries = 16; + optional uint32 ListPersistentBufferRetryPeriodMilliseconds = 17; } diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.cpp index 2e0b2cc1c14..94e520c8dd3 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.cpp @@ -295,7 +295,9 @@ void TInflightInfo::RemoveHosts(THostMask removed) EraseConfirmed = EraseConfirmed.Exclude(removed); // Check if flush became complete after removing hosts. - if (State == EState::PBufferFlushing && FlushDesired == FlushConfirmed) { + const bool flushDone = + !FlushConfirmed.Empty() && FlushDesired == FlushConfirmed; + if (State == EState::PBufferFlushing && flushDone) { SetState(EState::PBufferFlushed); } diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info_ut.cpp index cf46e88e604..f0fba70ab75 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info_ut.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info_ut.cpp @@ -670,6 +670,35 @@ Y_UNIT_TEST_SUITE(TInflightInfoTests) inflightInfo.UnlockPBuffer(); UNIT_ASSERT_VALUES_EQUAL(true, readyQueue.ReadyToErase.contains(123)); } + + Y_UNIT_TEST(ShouldRemoveHostsNotCompleteFlushWithEmptyConfirmed) + { + TTestReadyQueue readyQueue; + TInflightInfo inflightInfo(&readyQueue, 123, 4096); + inflightInfo.OnWritten(MakePrimaryHosts(), MakePrimaryHosts()); + + // Request flush for host 0 only; do NOT confirm it. + // FlushDesired = {0}, FlushConfirmed = {}. + UNIT_ASSERT_VALUES_EQUAL( + THostIndex{0}, + inflightInfo.RequestFlush(THostIndex{0}, THostMask())); + UNIT_ASSERT_VALUES_EQUAL( + TInflightInfo::EState::PBufferFlushing, + inflightInfo.GetState()); + + // Remove host 0 — this empties FlushDesired while FlushConfirmed is + // still empty. The flush must NOT be considered complete. + THostMask removed; + removed.Set(THostIndex{0}); + inflightInfo.RemoveHosts(removed); + + // State must stay PBufferFlushing and erase must NOT be registered. + UNIT_ASSERT_VALUES_EQUAL( + TInflightInfo::EState::PBufferFlushing, + inflightInfo.GetState()); + UNIT_ASSERT_VALUES_EQUAL(false, readyQueue.ReadyToErase.contains(123)); + UNIT_ASSERT_VALUES_EQUAL(true, readyQueue.ReadyToFlush.contains(123)); + } } //////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/core/protos/blob_depot_config.proto b/ydb/core/protos/blob_depot_config.proto index 4b974b42803..7d4d71ed863 100644 --- a/ydb/core/protos/blob_depot_config.proto +++ b/ydb/core/protos/blob_depot_config.proto @@ -58,6 +58,10 @@ message TS3BackendSettings { optional string BalancerHost = 4; optional uint32 BalancerRefreshSecMin = 5 [default = 10]; optional uint32 BalancerRefreshSecMax = 6 [default = 15]; + // ProxyPort when balancer returns a hostname without an explicit port. + optional uint32 BalancerProxyPort = 7 [default = 4080]; + // ProxyScheme for connections via the balancer-provided host. + optional NKikimrSchemeOp.TS3Settings.EScheme BalancerProxyScheme = 8 [default = HTTP]; } message TBlobDepotConfig { diff --git a/ydb/core/protos/data_events.proto b/ydb/core/protos/data_events.proto index 55176e68bc0..cc4a7a7a71b 100644 --- a/ydb/core/protos/data_events.proto +++ b/ydb/core/protos/data_events.proto @@ -212,6 +212,7 @@ message TEvLockRows { // When true, keys that are absent or deleted in the latest version are skipped // and not locked. If the skipped key was modified/deleted above the snapshot, // its index will NOT appear in the ModifiedKeys result field. + // If this transaction already locked a key, it will NOT be skipped. optional bool SkipAbsent = 13; // Destination global table id, must include SchemaVersion to detect schema changes diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 9c0a55787ee..127175149f6 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -251,7 +251,7 @@ message TFeatureFlags { optional bool EnableFsBackups = 222 [default = false]; optional bool EnableCmsLocksPriority = 223 [default = false]; optional bool EnableResourcePoolsScheduler = 224 [default = true]; // works only with EnableResourcePools = true - optional bool EnableTruncateTable = 225 [default = true]; + reserved 225; // EnableTruncateTable optional bool EnableTopicsSqlIoOperations = 226 [default = true]; optional bool EnableSysViewPermissionsExport = 227 [default = false]; optional bool EnableCmsSmartAvailabilityMode = 228 [default = false]; @@ -353,7 +353,7 @@ message TFeatureFlags { // RequireRestart: the in-memory counter is rebuilt from all FileStores at SchemeShard start, // the counter can be re-enabled, which requires restarting the SchemeShard tablet. optional bool DisableFileStoreSSDSystemSpaceAccounting = 302 [default = false, (RequireRestart) = true]; - optional bool EnableHasPredicatesInResourcePoolClassifiers = 303 [default = false]; - optional bool EnableRejectActionInResourcePoolClassifiers = 304 [default = false]; + optional bool EnableHasPredicatesInResourcePoolClassifiers = 303 [default = true]; + optional bool EnableRejectActionInResourcePoolClassifiers = 304 [default = true]; optional bool EnableMoveWithColumnTableReplace = 305 [default = false]; } diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index 2cc96998dd7..1d2b27d7356 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -5554,10 +5554,19 @@ void TExecutor::MoveData(TEvTablet::TEvMoveData::TPtr& ev) { } } +void TExecutor::StartMoveDataVacuumFromOwner() { + MoveDataVacuumInProgress = true; + StartVacuum(TNoTag()); +} + void TExecutor::VacuumComplete(TVacuumGeneration generation, const TActorContext& ctx) { if (generation) { Owner->VacuumComplete(generation, ctx); } + if (MoveDataVacuumInProgress) { + Owner->MoveDataCompleted(ctx); + } + MoveDataVacuumInProgress = false; for (const auto& actor : MoveDataSubscribers) { ctx.Send(actor, new TEvTablet::TEvMoveDataResponse(TabletId())); } diff --git a/ydb/core/tablet_flat/flat_executor.h b/ydb/core/tablet_flat/flat_executor.h index 862a57fe50a..4e3bc18a2f1 100644 --- a/ydb/core/tablet_flat/flat_executor.h +++ b/ydb/core/tablet_flat/flat_executor.h @@ -516,6 +516,7 @@ class TExecutor TControlWrapper MaxTxInFly; THashSet<TActorId> MoveDataSubscribers; + bool MoveDataVacuumInProgress = false; ui64 Stamp() const noexcept; void Registered(TActorSystem*, const TActorId&) override; @@ -713,10 +714,11 @@ public: bool CompactTables() override; THolder<TDirectPartWriter> BeginWritePart(ui32 tableId) override; void ReleaseWritePart(ui32 step) override; - void MoveData(TEvTablet::TEvMoveData::TPtr &ev) override; void StartVacuum(TVacuumTag tag) override; void VacuumComplete(TVacuumGeneration generation, const TActorContext& ctx) override; + void MoveData(TEvTablet::TEvMoveData::TPtr &ev) override; + void StartMoveDataVacuumFromOwner() override; void Handle(NMemory::TEvMemTableRegistered::TPtr &ev); void Handle(NMemory::TEvMemTableCompact::TPtr &ev); diff --git a/ydb/core/tablet_flat/flat_executor_recovery.cpp b/ydb/core/tablet_flat/flat_executor_recovery.cpp index 130ca86e6bb..394cbb0052a 100644 --- a/ydb/core/tablet_flat/flat_executor_recovery.cpp +++ b/ydb/core/tablet_flat/flat_executor_recovery.cpp @@ -333,6 +333,7 @@ public: void DisableReadMissingReferences() override { Y_TABLET_ERROR("Not supported"); } ui64 MissingReferencesSize() const override { Y_TABLET_ERROR("Not supported"); } void MoveData(TEvTablet::TEvMoveData::TPtr&) override { Y_TABLET_ERROR("Not supported"); } + void StartMoveDataVacuumFromOwner() override { Y_TABLET_ERROR("Not supported"); } private: void ScheduleProcessPending() { diff --git a/ydb/core/tablet_flat/tablet_flat_executor.cpp b/ydb/core/tablet_flat/tablet_flat_executor.cpp index d88e03bc4dd..f922c49e4e5 100644 --- a/ydb/core/tablet_flat/tablet_flat_executor.cpp +++ b/ydb/core/tablet_flat/tablet_flat_executor.cpp @@ -27,6 +27,10 @@ namespace NFlatExecutorSetup { Y_UNUSED(ctx); } + void ITablet::MoveDataCompleted(const TActorContext& ctx) { + Y_UNUSED(ctx); + } + void ITablet::CompletedLoansChanged(const TActorContext &ctx) { Y_UNUSED(ctx); } diff --git a/ydb/core/tablet_flat/tablet_flat_executor.h b/ydb/core/tablet_flat/tablet_flat_executor.h index 6ce1cc794fd..b0fbcd83d47 100644 --- a/ydb/core/tablet_flat/tablet_flat_executor.h +++ b/ydb/core/tablet_flat/tablet_flat_executor.h @@ -507,6 +507,7 @@ namespace NFlatExecutorSetup { virtual void CompletedLoansChanged(const TActorContext &ctx); // would be no-op in default implementation virtual void CompactionComplete(ui32 tableId, const TActorContext &ctx); // would be no-op in default implementation virtual void VacuumComplete(ui64 vacuumGeneration, const TActorContext& ctx); + virtual void MoveDataCompleted(const TActorContext& ctx); virtual void BackupSnapshotComplete(const TActorContext &ctx); // would be no-op in default implementation virtual void ScanComplete(NTable::EStatus status, TAutoPtr<IDestructable> prod, ui64 cookie, const TActorContext &ctx); @@ -676,7 +677,8 @@ namespace NFlatExecutorSetup { virtual void StartVacuum(TVacuumTag tag) = 0; virtual void VacuumComplete(TVacuumGeneration generation, const TActorContext& ctx) = 0; - virtual void MoveData(TEvTablet::TEvMoveData::TPtr&) = 0; + virtual void MoveData(TEvTablet::TEvMoveData::TPtr&) = 0; // TEvMoveData is handled by basic executor + virtual void StartMoveDataVacuumFromOwner() = 0; // TEvMoveData is handled by tablet ui32 Generation() const { return Generation0; } ui32 Step() const { return Step0; } diff --git a/ydb/core/tx/columnshard/engines/changes/actualization/construction/context.cpp b/ydb/core/tx/columnshard/engines/changes/actualization/construction/context.cpp index ef8ed214895..2adee5da495 100644 --- a/ydb/core/tx/columnshard/engines/changes/actualization/construction/context.cpp +++ b/ydb/core/tx/columnshard/engines/changes/actualization/construction/context.cpp @@ -34,6 +34,9 @@ TTieringProcessContext::EAddPortionResult TTieringProcessContext::AddPortion( }; auto it = Tasks.find(features.GetRWAddress()); if (it == Tasks.end()) { + if (!Controller->IsNewTaskAvailable(features.GetRWAddress(), 0)) { + return EAddPortionResult::TASK_LIMIT_EXCEEDED; + } std::vector<TTaskConstructor> tasks = { buildNewTask() }; it = Tasks.emplace(features.GetRWAddress(), std::move(tasks)).first; } diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/iterator.h b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/iterator.h index 542c2da144d..d96522e7b7c 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/iterator.h +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/iterator.h @@ -24,18 +24,19 @@ private: TRemapColumns& Remapper; const ui32 SourceIdx; + std::shared_ptr<TSubColumnsArray> MaterializeSubColumnsArray(const std::shared_ptr<IChunkedArray>& array) const { + if (array->GetType() == IChunkedArray::EType::SubColumnsArray) { + return std::static_pointer_cast<TSubColumnsArray>(array); + } + return std::static_pointer_cast<TSubColumnsArray>( + Loader->GetAccessorConstructor()->Construct(array, Loader->BuildAccessorContext(array->GetRecordsCount())).DetachResult()); + } + void InitArraysImpl(const ui32 position) { if (OriginalArray) { CurrentChunk = OriginalArray->GetArray(CurrentChunk, position, OriginalArray); CurrentChunkStartPosition = CurrentChunk->GetAddress().GetGlobalStartPosition(); - if (CurrentChunk->GetArray()->GetType() == IChunkedArray::EType::SubColumnsArray) { - CurrentSubColumnsArray = std::static_pointer_cast<TSubColumnsArray>(CurrentChunk->GetArray()); - } else { - CurrentSubColumnsArray = std::static_pointer_cast<TSubColumnsArray>( - Loader->GetAccessorConstructor() - ->Construct(CurrentChunk->GetArray(), Loader->BuildAccessorContext(CurrentChunk->GetArray()->GetRecordsCount())) - .DetachResult()); - } + CurrentSubColumnsArray = MaterializeSubColumnsArray(CurrentChunk->GetArray()); } } @@ -65,6 +66,21 @@ public: InitArray(0); } + std::vector<std::shared_ptr<TSubColumnsArray>> MaterializePerChunkArrays() const { + std::vector<std::shared_ptr<TSubColumnsArray>> result; + if (!OriginalArray) { + return result; + } + std::optional<IChunkedArray::TFullChunkedArrayAddress> chunk; + ui32 position = 0; + while (position < OriginalArray->GetRecordsCount()) { + chunk = OriginalArray->GetArray(chunk, position, OriginalArray); + result.emplace_back(MaterializeSubColumnsArray(chunk->GetArray())); + position = chunk->GetAddress().GetGlobalStartPosition() + chunk->GetArray()->GetRecordsCount(); + } + return result; + } + template <class TStartRecordActor, class TKVActor, class TFinishRecordActor> void ReadRecord(const ui32 recordIndex, const TStartRecordActor& startRecordActor, const TKVActor& kvActor, const TFinishRecordActor& finishRecordActor) { diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp index 372845d292a..9a874e59015 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/logic.cpp @@ -15,13 +15,15 @@ void TSubColumnsMerger::DoStart(const std::vector<std::shared_ptr<NArrow::NAcces for (auto&& i : input) { OrderedIterators.emplace_back(NSubColumns::TChunksIterator(i, Context.GetLoader(), RemapKeyIndex, OrderedIterators.size())); } - std::vector<const TDictStats*> stats; + // Deduce value types over every chunk of every source, not just the first one: a portion's chunks + // may disagree on a key's native scalar type, and sampling one chunk would mislabel the merged column. + std::vector<TDictStats> stats; ui32 statRecordsCount = 0; for (auto&& i : OrderedIterators) { - if (i.GetCurrentSubColumnsArray()) { - stats.emplace_back(&i.GetCurrentSubColumnsArray()->GetColumnsData().GetStats()); - stats.emplace_back(&i.GetCurrentSubColumnsArray()->GetOthersData().GetStats()); - statRecordsCount += i.GetCurrentSubColumnsArray()->GetRecordsCount(); + for (auto&& sub : i.MaterializePerChunkArrays()) { + stats.emplace_back(sub->GetColumnsData().GetStats()); + stats.emplace_back(sub->GetOthersData().GetStats()); + statRecordsCount += sub->GetRecordsCount(); } } AFL_VERIFY(stats.size()); diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/ut/ut_merger.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/ut/ut_merger.cpp new file mode 100644 index 00000000000..09adf5e7f4b --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/ut/ut_merger.cpp @@ -0,0 +1,147 @@ +#include <ydb/core/formats/arrow/accessor/composite/accessor.h> +#include <ydb/core/formats/arrow/accessor/plain/accessor.h> +#include <ydb/core/formats/arrow/accessor/sub_columns/accessor.h> +#include <ydb/core/formats/arrow/accessor/sub_columns/constructor.h> +#include <ydb/core/formats/arrow/accessor/sub_columns/settings.h> +#include <ydb/core/tx/columnshard/counters/indexation.h> +#include <ydb/core/tx/columnshard/engines/changes/compaction/abstract/merger.h> +#include <ydb/core/tx/columnshard/engines/changes/compaction/common/context.h> +#include <ydb/core/tx/columnshard/engines/scheme/index_info.h> +#include <ydb/core/tx/columnshard/engines/scheme/versions/snapshot_scheme.h> +#include <ydb/core/tx/columnshard/test_helper/helper.h> + +#include <contrib/libs/apache/arrow/cpp/src/arrow/array/array_binary.h> +#include <contrib/libs/apache/arrow/cpp/src/arrow/array/builder_primitive.h> +#include <contrib/libs/apache/arrow/cpp/src/arrow/record_batch.h> +#include <library/cpp/testing/unittest/registar.h> +#include <yql/essentials/types/binary_json/read.h> +#include <yql/essentials/types/binary_json/write.h> + +// Reproduces the compaction crash where the sub-columns merger deduces a native scalar value type from +// only the first chunk of a multi-chunk source. When a key's scalar type differs between chunks of the +// same source, the merged column is stamped with the first chunk's type (e.g. String) yet the builder +// stores re-encoded BinaryJson bytes for the diverging chunk. +Y_UNIT_TEST_SUITE(SubColumnsCompaction) { + using namespace NKikimr; + using namespace NKikimr::NOlap; + using namespace NKikimr::NOlap::NCompaction; + using namespace NKikimr::NArrow::NAccessor; + using namespace NKikimr::NArrow::NAccessor::NSubColumns; + + TSettings MakeSettings() { + TSettings s(20, 1024, 50u * 1024 * 1024, 0.0, TDataAdapterContainer::GetDefault(), 0.0); + s.SetEnableNativeColumns(true); + return s; + } + + std::shared_ptr<TSubColumnsArray> BuildChunk(const std::vector<TString>& jsons, const TSettings& settings) { + TTrivialArray::TPlainBuilder<arrow::BinaryType> b; + for (ui32 i = 0; i < jsons.size(); ++i) { + auto v = NBinaryJson::SerializeToBinaryJson(jsons[i]); + auto* bj = std::get_if<NBinaryJson::TBinaryJson>(&v); + UNIT_ASSERT_C(bj, "cannot serialize " << jsons[i]); + b.AddRecord(i, std::string_view(bj->data(), bj->size())); + } + auto arr = b.Finish(jsons.size()); + return TSubColumnsArray::Make(arr, settings, arr->GetDataType()).DetachResult(); + } + + ISnapshotSchema::TPtr MakeSchema(const TSettings& settings) { + auto storages = TTestStoragesManager::GetInstance(); + auto cache = std::make_shared<TSchemaObjectsCache>(); + + NKikimrSchemeOp::TColumnTableSchema proto; + *proto.MutableColumns()->Add() = NArrow::NTest::TTestColumn("pk", NScheme::TTypeInfo(NScheme::NTypeIds::Uint64)).CreateColumn(1); + + auto dataCol = NArrow::NTest::TTestColumn("data", NScheme::TTypeInfo(NScheme::NTypeIds::JsonDocument)).CreateColumn(2); + auto* dac = dataCol.MutableDataAccessorConstructor(); + dac->SetClassName(TConstructor::GetClassNameStatic()); + *dac->MutableSubColumns()->MutableSettings() = settings.SerializeToProto(); + *proto.MutableColumns()->Add() = dataCol; + + proto.AddKeyColumnNames("pk"); + proto.SetVersion(1); + proto.MutableOptions()->MutableCompactionPlannerConstructor()->SetClassName("l-buckets"); + *proto.MutableOptions()->MutableCompactionPlannerConstructor()->MutableLBuckets() = + NKikimrSchemeOp::TCompactionPlannerConstructorContainer::TLOptimizer(); + + auto info = TIndexInfo::BuildFromProto(1, proto, storages, cache); + UNIT_ASSERT(info); + return std::make_shared<TSnapshotSchema>(cache->UpsertIndexInfo(std::move(*info)), TSnapshot(1, 1)); + } + + // Render each stored BinaryJson document as text. For the corrupt portion this will trigger rendering + // of incorrectly labeled binary data. + TString RenderDocs(const std::shared_ptr<IChunkedArray>& arr) { + auto ca = arr->GetChunkedArray(); + TStringBuilder out; + for (int c = 0; c < ca->num_chunks(); ++c) { + const auto& bin = static_cast<const arrow::BinaryArray&>(*ca->chunk(c)); + for (i64 i = 0; i < bin.length(); ++i) { + if (bin.IsNull(i)) { + out << "null;"; + continue; + } + const auto view = bin.GetView(i); + out << NBinaryJson::SerializeToJson(TStringBuf(view.data(), view.size())) << ";"; + } + } + return out; + } + + Y_UNIT_TEST(MultiChunkDivergentScalarType) { + const auto settings = MakeSettings(); + + const std::vector<TString> docs0 = { R"({"a":"xxxx"})", R"({"a":"yyyy"})", R"({"a":"zzzz"})", R"({"a":"wwww"})" }; + const std::vector<TString> docs1 = { R"({"a":1})", R"({"a":2})", R"({"a":3})", R"({"a":4})" }; + + auto chunk0 = BuildChunk(docs0, settings); + auto chunk1 = BuildChunk(docs1, settings); + UNIT_ASSERT_VALUES_EQUAL(chunk0->GetColumnsData().GetStats().GetColumnsCount(), 1); + UNIT_ASSERT_VALUES_EQUAL_C((ui32)chunk0->GetColumnsData().GetStats().GetValueType(0), (ui32)EValueType::String, + "chunk0 key 'a' must be a native String column"); + UNIT_ASSERT_VALUES_EQUAL_C((ui32)chunk1->GetColumnsData().GetStats().GetValueType(0), (ui32)EValueType::Double, + "chunk1 key 'a' must be a native Double column"); + + NArrow::NAccessor::TCompositeChunkedArray::TBuilder cb(chunk0->GetDataType()); + cb.AddChunk(chunk0); + cb.AddChunk(chunk1); + std::vector<std::shared_ptr<IChunkedArray>> inputs = { cb.Finish() }; + + const ui32 columnId = 2; + auto schema = MakeSchema(settings); + TColumnMergeContext mergeCtx(columnId, schema, 8u * 1024 * 1024, std::nullopt); + + THolder<IColumnMerger> merger = IColumnMerger::TFactory::MakeHolder(TConstructor::GetClassNameStatic(), mergeCtx); + UNIT_ASSERT(merger); + + TMergingContext mergingCtx({}); + merger->Start(inputs, mergingCtx); + + const ui32 total = docs0.size() + docs1.size(); + arrow::UInt16Builder idxB; + arrow::UInt32Builder recB; + for (ui32 i = 0; i < total; ++i) { + UNIT_ASSERT(idxB.Append(0).ok()); + UNIT_ASSERT(recB.Append(i).ok()); + } + auto pkSchema = arrow::schema({ arrow::field(IColumnMerger::PortionIdFieldName, arrow::uint16()), + arrow::field(IColumnMerger::PortionRecordIndexFieldName, arrow::uint32()) }); + auto pkBatch = arrow::RecordBatch::Make(pkSchema, total, { idxB.Finish().ValueOrDie(), recB.Finish().ValueOrDie() }); + + TMergingChunkContext chunkCtxOwner(pkBatch); + NColumnShard::TIndexationCounters counters("Compaction"); + TChunkMergeContext chunkCtx(counters, chunkCtxOwner.Slice(0, total)); + + auto result = merger->Execute(chunkCtx, mergingCtx); + UNIT_ASSERT_VALUES_EQUAL(result.GetChunks().size(), 1); + + auto loader = schema->GetIndexInfo().GetColumnLoaderVerified(columnId); + auto merged = std::static_pointer_cast<TSubColumnsArray>(loader->ApplyVerified(result.GetChunks()[0]->GetData(), total)); + + // The merged portion must round-trip to the input documents. + const TString expected = + RenderDocs(BuildChunk({ docs0[0], docs0[1], docs0[2], docs0[3], docs1[0], docs1[1], docs1[2], docs1[3] }, settings)); + UNIT_ASSERT_VALUES_EQUAL(RenderDocs(merged), expected); + } +} diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/ut/ya.make b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/ut/ya.make new file mode 100644 index 00000000000..5dfbbe131ea --- /dev/null +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/ut/ya.make @@ -0,0 +1,22 @@ +UNITTEST_FOR(ydb/core/tx/columnshard/engines/changes/compaction/sub_columns) + +SIZE(SMALL) + +PEERDIR( + ydb/core/tx/columnshard/engines + ydb/core/tx/columnshard/test_helper + ydb/core/tx/columnshard/counters + ydb/core/formats/arrow/accessor/composite + ydb/core/formats/arrow/accessor/plain + ydb/core/formats/arrow/accessor/sub_columns + yql/essentials/sql/pg_dummy + yql/essentials/udfs/common/json2 +) + +YQL_LAST_ABI_VERSION() + +SRCS( + ut_merger.cpp +) + +END() diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/ya.make b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/ya.make index 6c45cdb4270..333a31e9394 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/ya.make +++ b/ydb/core/tx/columnshard/engines/changes/compaction/sub_columns/ya.make @@ -15,3 +15,7 @@ PEERDIR( ) END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/tx/columnshard/engines/storage/actualizer/common/address.h b/ydb/core/tx/columnshard/engines/storage/actualizer/common/address.h index 4fb1937686a..08a82eee225 100644 --- a/ydb/core/tx/columnshard/engines/storage/actualizer/common/address.h +++ b/ydb/core/tx/columnshard/engines/storage/actualizer/common/address.h @@ -1,5 +1,6 @@ #pragma once #include <util/generic/string.h> +#include <util/stream/output.h> #include <set> @@ -22,6 +23,10 @@ public: TString DebugString() const; + friend IOutputStream& operator<<(IOutputStream& out, const TRWAddress& address) { + return out << address.DebugString(); + } + TRWAddress(std::set<TString>&& readStorages, std::set<TString>&& writeStorages); bool operator==(const TRWAddress& item) const { diff --git a/ydb/core/tx/columnshard/engines/storage/actualizer/scheme/scheme.cpp b/ydb/core/tx/columnshard/engines/storage/actualizer/scheme/scheme.cpp index 69f5da4a905..fb8b526d1a4 100644 --- a/ydb/core/tx/columnshard/engines/storage/actualizer/scheme/scheme.cpp +++ b/ydb/core/tx/columnshard/engines/storage/actualizer/scheme/scheme.cpp @@ -90,6 +90,7 @@ void TSchemeActualizer::DoExtractTasks( auto info = BuildActualizationInfo(*portion); if (!info) { // its possible through chains with equivalent schemas collapsed portionsToRemove.emplace(portion->GetPortionId()); + continue; } auto portionScheme = portion->GetSchema(VersionedIndex); TPortionEvictionFeatures features( @@ -103,6 +104,7 @@ void TSchemeActualizer::DoExtractTasks( {"event", "cannot_add_portion"}, {"reason", "limit_exceeded"}, {"context", tasksContext.DebugString()}); + TSchemeGlobalCounters::OnSkipNotReadyWrite(); limitExceeded = true; break; case TTieringProcessContext::EAddPortionResult::PORTION_LOCKED: diff --git a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp index 9ad1d0d1322..9a586da9bf7 100644 --- a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp +++ b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp @@ -202,6 +202,7 @@ void TTieringActualizer::DoExtractTasks( Counters.SkipEvictionForTooEarly->Add(1); continue; } + // This is a best-effort check, address might differ from the queue key when the task is built if (!tasksContext.IsRWAddressAvailable(address)) { Counters.SkipEvictionForLimit->Add(1); continue; @@ -226,6 +227,7 @@ void TTieringActualizer::DoExtractTasks( switch (tasksContext.AddPortion(portion, std::move(features), info->GetLateness())) { case TTieringProcessContext::EAddPortionResult::TASK_LIMIT_EXCEEDED: + Counters.SkipEvictionForLimit->Add(1); limitEnriched = true; break; case TTieringProcessContext::EAddPortionResult::PORTION_LOCKED: diff --git a/ydb/core/tx/columnshard/engines/ut/ut_actualizer.cpp b/ydb/core/tx/columnshard/engines/ut/ut_actualizer.cpp new file mode 100644 index 00000000000..45ac6fd5a41 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/ut/ut_actualizer.cpp @@ -0,0 +1,221 @@ +#include <ydb/core/tx/columnshard/data_locks/manager/manager.h> +#include <ydb/core/tx/columnshard/engines/changes/actualization/construction/context.h> +#include <ydb/core/tx/columnshard/engines/scheme/objects_cache.h> +#include <ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h> +#include <ydb/core/tx/columnshard/engines/storage/actualizer/scheme/scheme.h> +#include <ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.h> +#include <ydb/core/tx/columnshard/test_helper/portion_test_helper.h> + +#include <library/cpp/testing/unittest/registar.h> + +namespace NKikimr::NOlap::NActualizer { + +namespace { + +const TInternalPathId TestPathId = TInternalPathId::FromRawValue(1); +constexpr ui64 TestPortionId = 1; +constexpr ui32 PkColumnId = 1; +constexpr ui64 MemoryLimit = 512 * 1024 * 1024; + +const TString Tier1 = NColumnShard::NTiers::TExternalStorageId("/Root/tier1").GetConfigPath(); +const TString Tier2 = NColumnShard::NTiers::TExternalStorageId("/Root/tier2").GetConfigPath(); +const TDuration Tier1EvictAfter = TDuration::Hours(1); +const TDuration Tier2EvictAfter = TDuration::Hours(2); + +TIndexInfo MakeTestIndexInfo(const std::shared_ptr<TSchemaObjectsCache>& cache, const ui64 version, const bool schemeNeedActualization) { + NKikimrSchemeOp::TColumnTableSchema proto; + *proto.MutableColumns()->Add() = NArrow::NTest::TTestColumn("pk", NScheme::TTypeInfo(NScheme::NTypeIds::Uint64)).CreateColumn(PkColumnId); + proto.AddKeyColumnNames("pk"); + proto.SetVersion(version); + proto.MutableOptions()->SetSchemeNeedActualization(schemeNeedActualization); + proto.MutableOptions()->MutableCompactionPlannerConstructor()->SetClassName("l-buckets"); + *proto.MutableOptions()->MutableCompactionPlannerConstructor()->MutableLBuckets() = + NKikimrSchemeOp::TCompactionPlannerConstructorContainer::TLOptimizer(); + + auto result = TIndexInfo::BuildFromProto(version, proto, TTestStoragesManager::GetInstance(), cache); + UNIT_ASSERT(result); + return std::move(*result); +} + +TTiering MakeTestTiering() { + TTiering result; + UNIT_ASSERT(result.Add(std::make_shared<TTierInfo>(NColumnShard::NTiers::TExternalStorageId(Tier1), Tier1EvictAfter, "pk", 1))); + UNIT_ASSERT(result.Add(std::make_shared<TTierInfo>(NColumnShard::NTiers::TExternalStorageId(Tier2), Tier2EvictAfter, "pk", 1))); + return result; +} + +class TTestEnv { +private: + const TInstant TestStart = TInstant::Now(); + + const std::shared_ptr<TSchemaObjectsCache> Cache = std::make_shared<TSchemaObjectsCache>(); + const std::shared_ptr<IStoragesManager> Storages = TTestStoragesManager::GetInstance(); + const std::shared_ptr<NDataLocks::TManager> DataLocksManager = std::make_shared<NDataLocks::TManager>(); + TVersionedIndex VersionedIndex; + THashMap<ui64, std::shared_ptr<TPortionInfo>> Portions; + std::optional<TTieringActualizer> Actualizer; + + YDB_READONLY_DEF(std::shared_ptr<TController>, Controller); + +public: + TTestEnv() + : Controller(std::make_shared<TController>()) + { + VersionedIndex.AddIndex(TSnapshot(1, 1), Cache->UpsertIndexInfo(MakeTestIndexInfo(Cache, 1, false))); + Actualizer.emplace(TestPathId, VersionedIndex, Storages); + } + + TRWAddress MakeAddress(const TString& targetTier) const { + const auto& indexInfo = VersionedIndex.GetLastSchema()->GetIndexInfo(); + return TRWAddress(indexInfo.GetUsedStorageIds(IStoragesManager::DefaultStorageId), indexInfo.GetUsedStorageIds(targetTier)); + } + + // Eviction of a portion living on the default storage to targetTier, i.e. features addressed as MakeAddress(targetTier). + TPortionEvictionFeatures MakeEvictionFeatures(const TString& targetTier) const { + const auto schema = VersionedIndex.GetLastSchema(); + TPortionEvictionFeatures result(schema, schema, IStoragesManager::DefaultStorageId); + result.SetTargetTierName(targetTier); + return result; + } + + // A portion on the default storage whose eviction column (the pk) holds T - dataAge. + std::shared_ptr<TPortionInfo> MakePortion(const TDuration dataAge) const { + const TInstant maxPk = TestStart - dataAge; + auto result = + NTest::MakeTestCompactedPortion(TestPathId, TestPortionId, maxPk.Seconds(), maxPk.Seconds(), 10, TSnapshot(1, 1), std::nullopt); + result->AddRuntimeFeature(TPortionInfo::ERuntimeFeature::Optimized); + return result; + } + + // Builds the context the way StartTtl does: a fresh one per round, sharing the tablet-wide controller. + TTieringProcessContext MakeProcessContext() const { + TSaverContext saverContext(Storages); + return TTieringProcessContext( + MemoryLimit, saverContext, DataLocksManager, VersionedIndex, NColumnShard::TEngineLogsCounters(), Controller); + } + + // Queues a portion at T - insertedAgo, with its pk holding T - insertedAgo - ageOnInsert. The queue key is + // decided at that instant and is never recomputed afterwards. + void QueuePortion(const TDuration insertedAgo, const TDuration ageOnInsert) { + Portions.emplace(TestPortionId, MakePortion(insertedAgo + ageOnInsert)); + Actualizer->Refresh(MakeTestTiering(), TAddExternalContext(TestStart - insertedAgo, Portions)); + } + + THashMap<TRWAddress, std::vector<TTaskConstructor>> ExtractTasks() { + auto context = MakeProcessContext(); + TInternalTasksContext internalContext; + Actualizer->ExtractTasks(context, TExternalTasksContext(Portions), internalContext); + return context.GetTasks(); + } +}; + +// The scheme actualizer rewrites portions onto the latest critical schema. Its version is registered as v2, so a +// portion written under v1 is a rewrite candidate until it resolves to v2. +class TSchemeTestEnv { +private: + const std::shared_ptr<TSchemaObjectsCache> Cache = std::make_shared<TSchemaObjectsCache>(); + const std::shared_ptr<IStoragesManager> Storages = TTestStoragesManager::GetInstance(); + const std::shared_ptr<NDataLocks::TManager> DataLocksManager = std::make_shared<NDataLocks::TManager>(); + const std::shared_ptr<TController> Controller = std::make_shared<TController>(); + TVersionedIndex VersionedIndex; + THashMap<ui64, std::shared_ptr<TPortionInfo>> Portions; + std::optional<TSchemeActualizer> Actualizer; + +public: + TSchemeTestEnv() { + VersionedIndex.AddIndex(TSnapshot(1, 1), Cache->UpsertIndexInfo(MakeTestIndexInfo(Cache, 1, false))); + VersionedIndex.AddIndex(TSnapshot(2, 1), Cache->UpsertIndexInfo(MakeTestIndexInfo(Cache, 2, true))); + Actualizer.emplace(TestPathId, VersionedIndex); + } + + // Queues a portion written under schema v1 for rewrite to the critical schema v2. + void QueuePortion() { + Portions.emplace(TestPortionId, NTest::MakeTestCompactedPortion(TestPathId, TestPortionId, 10, 19, 10, TSnapshot(1, 1), std::nullopt)); + Actualizer->Refresh(TAddExternalContext(TInstant::Now(), Portions)); + } + + // Drops schema v1, so a portion written under it now resolves to v2 and no longer needs a rewrite. + void CollapseSchemaOntoTarget() { + VersionedIndex.EraseVersion(1); + } + + THashMap<TRWAddress, std::vector<TTaskConstructor>> ExtractTasks() { + TSaverContext saverContext(Storages); + TTieringProcessContext context( + MemoryLimit, saverContext, DataLocksManager, VersionedIndex, NColumnShard::TEngineLogsCounters(), Controller); + TInternalTasksContext internalContext; + Actualizer->ExtractTasks(context, TExternalTasksContext(Portions), internalContext); + return context.GetTasks(); + } +}; + +} // namespace + +Y_UNIT_TEST_SUITE(TTieringProcessContextTests) { + Y_UNIT_TEST(FirstTaskIsBuiltForFreeAddress) { + TTestEnv env; + auto context = env.MakeProcessContext(); + UNIT_ASSERT_EQUAL(context.AddPortion(env.MakePortion(TDuration::Zero()), env.MakeEvictionFeatures(Tier2), TDuration::Zero()), + TTieringProcessContext::EAddPortionResult::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL(context.GetTasks().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(context.GetTasks().begin()->first, env.MakeAddress(Tier2)); + } + + Y_UNIT_TEST(FirstTaskIsNotBuiltForAddressAtLimit) { + TTestEnv env; + // tier2 already holds the single task its address is allowed to have in progress + env.GetController()->StartActualization(env.MakeAddress(Tier2)); + + auto context = env.MakeProcessContext(); + UNIT_ASSERT_EQUAL(context.AddPortion(env.MakePortion(TDuration::Zero()), env.MakeEvictionFeatures(Tier2), TDuration::Zero()), + TTieringProcessContext::EAddPortionResult::TASK_LIMIT_EXCEEDED); + UNIT_ASSERT(context.GetTasks().empty()); + } +} + +// A portion's target tier depends on how old its data is, so the address it is queued under and the address its task +// ends up under are computed at different instants and need not match. That is why the caller's per-queue check +// cannot stand in for the check above. +Y_UNIT_TEST_SUITE(TTieringActualizerTests) { + Y_UNIT_TEST(TargetTierIsRecomputedOnTaskBuild) { + TTestEnv env; + // at T-2h the data (T-2h30m) is 30 minutes old: due for no tier at all, tier1 being merely its next hop in + // another 30 minutes, so tier1 is the only target the queueing could possibly settle on + env.QueuePortion(TDuration::Hours(2), TDuration::Minutes(30)); + + // at T the same data is 2h30m old, past tier2's 2h boundary. The task is addressed to tier2, so its address + // was decided while it was being built and not while its portion was being queued. + const auto tasks = env.ExtractTasks(); + UNIT_ASSERT_VALUES_EQUAL(tasks.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(tasks.begin()->first, env.MakeAddress(Tier2)); + UNIT_ASSERT_VALUES_EQUAL(tasks.begin()->second.size(), 1); + } + + Y_UNIT_TEST(NoTasksOverLimitForRecomputedAddress) { + TTestEnv env; + // queued while headed for tier1, addressed to tier2 once built (see TargetTierIsRecomputedOnTaskBuild) + env.QueuePortion(TDuration::Hours(2), TDuration::Minutes(30)); + + // An eviction to tier2 started by an earlier round, before T. An address counts as busy from + // StartActualization (TTTLColumnEngineChanges::DoStart) until FinishActualization (DoOnFinish), which runs + // only when that task completes; nothing completes it here, so at T tier2 still holds its one allowed task. + env.GetController()->StartActualization(env.MakeAddress(Tier2)); + + // at T the queue key (tier1) is free, so the portion reaches the task building, but its task would belong to + // the busy tier2 address and must not be built + UNIT_ASSERT(env.ExtractTasks().empty()); + } +} + +Y_UNIT_TEST_SUITE(TSchemeActualizerTests) { + Y_UNIT_TEST(CollapsedSchemaPortionIsDropped) { + TSchemeTestEnv env; + env.QueuePortion(); + // once v1 collapses onto v2, the queued portion already carries the target schema: it must be dropped, not + // turned into a task from an empty actualization info + env.CollapseSchemaOntoTarget(); + UNIT_ASSERT(env.ExtractTasks().empty()); + } +} + +} // namespace NKikimr::NOlap::NActualizer diff --git a/ydb/core/tx/columnshard/engines/ut/ya.make b/ydb/core/tx/columnshard/engines/ut/ya.make index 789f82494e6..7eef5669291 100644 --- a/ydb/core/tx/columnshard/engines/ut/ya.make +++ b/ydb/core/tx/columnshard/engines/ut/ya.make @@ -40,6 +40,7 @@ ENDIF() YQL_LAST_ABI_VERSION() SRCS( + ut_actualizer.cpp ut_program.cpp ut_snapshot_holders.cpp ut_scan_snapshot_guard.cpp diff --git a/ydb/core/tx/datashard/alter_table_unit.cpp b/ydb/core/tx/datashard/alter_table_unit.cpp index a960ff3dd7d..c10159aa91c 100644 --- a/ydb/core/tx/datashard/alter_table_unit.cpp +++ b/ydb/core/tx/datashard/alter_table_unit.cpp @@ -5,6 +5,8 @@ #include <ydb/library/aclib/user_context.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -151,9 +153,9 @@ EExecutionStatus TAlterTableUnit::Execute(TOperation::TPtr op, const auto version = alterTableTx.GetTableSchemaVersion(); Y_ENSURE(version); - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, - "Trying to ALTER TABLE at " << DataShard.TabletID() - << " version " << version); + YDB_LOG_INFO_CTX(ctx, "TAlterTableUnit::Execute: trying to alter table", + {"tabletId", DataShard.TabletID()}, + {"version", version}); TPathId tableId(DataShard.GetPathOwnerId(), alterTableTx.GetId_Deprecated()); if (alterTableTx.HasPathId()) { diff --git a/ydb/core/tx/datashard/backup_restore_common.h b/ydb/core/tx/datashard/backup_restore_common.h index 31a75280466..34bf14382a8 100644 --- a/ydb/core/tx/datashard/backup_restore_common.h +++ b/ydb/core/tx/datashard/backup_restore_common.h @@ -33,7 +33,8 @@ protected: TActiveTransaction* tx = dynamic_cast<TActiveTransaction*>(op.Get()); Y_ENSURE(tx, "cannot cast operation of kind " << op->GetKind()); - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, error); + YDB_LOG_NOTICE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TBackupRestoreUnitBase::Abort: aborting operation", + {"errorMessage", error}); BuildResult(op)->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, error); ResetWaiting(op); @@ -94,8 +95,9 @@ public: } if (!IsWaiting(op)) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Starting a " << GetKind() << " operation" - << " at " << DataShard.TabletID()); + YDB_LOG_DEBUG_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "Starting a operation", + {"kind", GetKind()}, + {"tabletId", DataShard.TabletID()}); if (!Run(op, txc, ctx)) { return EExecutionStatus::Executed; @@ -106,8 +108,9 @@ public: } if (HasResult(op)) { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, "" << GetKind() << " complete" - << " at " << DataShard.TabletID()); + YDB_LOG_INFO_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "Complete", + {"kind", GetKind()}, + {"tabletId", DataShard.TabletID()}); ResetWaiting(op); if (ProcessResult(op, ctx)) { diff --git a/ydb/core/tx/datashard/build_and_wait_dependencies_unit.cpp b/ydb/core/tx/datashard/build_and_wait_dependencies_unit.cpp index 411f8d809f5..d7cb6390b75 100644 --- a/ydb/core/tx/datashard/build_and_wait_dependencies_unit.cpp +++ b/ydb/core/tx/datashard/build_and_wait_dependencies_unit.cpp @@ -4,6 +4,8 @@ #include <ydb/core/tx/locks/time_counters.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -102,9 +104,9 @@ EExecutionStatus TBuildAndWaitDependenciesUnit::Execute(TOperation::TPtr op, if (DataShard.TryCaptureTxCache(mem)) { tx->SetTxCacheUsage(mem); } else { - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, - "TBuildAndWaitDependenciesUnit at " << DataShard.TabletID() - << " released data for tx " << tx->GetTxId()); + YDB_LOG_INFO_CTX(ctx, "TBuildAndWaitDependenciesUnit::Execute: released data for tx", + {"tabletId", DataShard.TabletID()}, + {"txId", tx->GetTxId()}); DataShard.IncCounter(COUNTER_INACTIVE_TX_DATA_RELEASES); tx->ReleaseTxData(txc, ctx); diff --git a/ydb/core/tx/datashard/build_data_tx_out_rs_unit.cpp b/ydb/core/tx/datashard/build_data_tx_out_rs_unit.cpp index eeb78cf9c05..29810f0c79d 100644 --- a/ydb/core/tx/datashard/build_data_tx_out_rs_unit.cpp +++ b/ydb/core/tx/datashard/build_data_tx_out_rs_unit.cpp @@ -4,6 +4,8 @@ #include "setup_sys_locks.h" #include "datashard_locks_db.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -88,11 +90,11 @@ EExecutionStatus TBuildDataTxOutRSUnit::Execute(TOperation::TPtr op, engine->AfterOutgoingReadsetsExtracted(); } catch (const TMemoryLimitExceededException &) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "Operation " << *op << " at " << DataShard.TabletID() - << " exceeded memory limit " << txc.GetMemoryLimit() - << " and requests " << txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR - << " more for the next try"); + YDB_LOG_TRACE_CTX(ctx, "TBuildDataTxOutRSUnit::Execute: exceeded memory limit and requests more for the next try", + {"operation", *op}, + {"tabletId", DataShard.TabletID()}, + {"memoryLimit", txc.GetMemoryLimit()}, + {"memoryLimitWithFactor", txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR}); txc.NotEnoughMemory(); DataShard.IncCounter(DataShard.NotEnoughMemoryCounter(txc.GetNotEnoughMemoryCount())); @@ -104,9 +106,9 @@ EExecutionStatus TBuildDataTxOutRSUnit::Execute(TOperation::TPtr op, return EExecutionStatus::Restart; } catch (const TNotReadyTabletException&) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Tablet " << DataShard.TabletID() << " is not ready for " << *op - << " execution"); + YDB_LOG_DEBUG_CTX(ctx, "TBuildDataTxOutRSUnit::Execute: tablet is not ready for execution", + {"tabletId", DataShard.TabletID()}, + {"operation", *op}); DataShard.IncCounter(COUNTER_TX_TABLET_NOT_READY); diff --git a/ydb/core/tx/datashard/build_write_out_rs_unit.cpp b/ydb/core/tx/datashard/build_write_out_rs_unit.cpp index 3c2abccde09..173784fd7d7 100644 --- a/ydb/core/tx/datashard/build_write_out_rs_unit.cpp +++ b/ydb/core/tx/datashard/build_write_out_rs_unit.cpp @@ -5,17 +5,13 @@ #include "setup_sys_locks.h" #include "datashard_locks_db.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { using namespace NMiniKQL; -#define LOG_T(stream) LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, stream) -#define LOG_D(stream) LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, stream) -#define LOG_E(stream) LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, stream) -#define LOG_C(stream) LOG_CRIT_S(ctx, NKikimrServices::TX_DATASHARD, stream) -#define LOG_W(stream) LOG_WARN_S(ctx, NKikimrServices::TX_DATASHARD, stream) - class TBuildWriteOutRSUnit : public TExecutionUnit { public: TBuildWriteOutRSUnit(TDataShard& dataShard, TPipeline& pipeline); @@ -74,10 +70,13 @@ EExecutionStatus TBuildWriteOutRSUnit::Execute(TOperation::TPtr op, TTransaction const auto& kqpLocks = writeTx->GetKqpLocks() ? writeTx->GetKqpLocks().value() : NKikimrDataEvents::TKqpLocks{}; KqpFillOutReadSets(op->OutReadSets(), kqpLocks, true, DataShard.SysLocksTable(), tabletId); } catch (const TNotReadyTabletException&) { - LOG_C("Unexpected TNotReadyTabletException exception at build out rs"); + YDB_LOG_CRIT_CTX(ctx, "TBuildWriteOutRSUnit::Execute: unexpected TNotReadyTabletException while building out readset"); return OnTabletNotReady(*writeOp, txc, ctx); } catch (const yexception& e) { - LOG_C("Exception while preparing out-readsets for KQP transaction " << *op << " at " << DataShard.TabletID() << ": " << e.what()); + YDB_LOG_CRIT_CTX(ctx, "Exception while preparing out-readsets for KQP transaction", + {"operation", *op}, + {"tabletId", DataShard.TabletID()}, + {"exceptionMessage", e.what()}); if (op->IsImmediate()) { writeOp->ReleaseTxData(txc); writeOp->SetError(NKikimrDataEvents::TEvWriteResult::STATUS_INTERNAL_ERROR, TStringBuilder() << "Tx was terminated: " << e.what()); @@ -94,7 +93,9 @@ void TBuildWriteOutRSUnit::Complete(TOperation::TPtr, const TActorContext&) {} EExecutionStatus TBuildWriteOutRSUnit::OnTabletNotReady(TWriteOperation& writeOp, TTransactionContext& txc, const TActorContext& ctx) { - LOG_T("Tablet " << DataShard.TabletID() << " is not ready for " << writeOp << " execution"); + YDB_LOG_TRACE_CTX(ctx, "Tablet is not ready for execution", + {"tabletId", DataShard.TabletID()}, + {"writeOp", writeOp}); DataShard.IncCounter(COUNTER_TX_TABLET_NOT_READY); diff --git a/ydb/core/tx/datashard/check_commit_writes_tx_unit.cpp b/ydb/core/tx/datashard/check_commit_writes_tx_unit.cpp index 9178a4a3fea..c42f2c2b98c 100644 --- a/ydb/core/tx/datashard/check_commit_writes_tx_unit.cpp +++ b/ydb/core/tx/datashard/check_commit_writes_tx_unit.cpp @@ -3,6 +3,8 @@ #include "datashard_pipeline.h" #include "execution_unit_ctors.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -78,7 +80,8 @@ public: << "Can't propose tx " << op->GetTxId() << " at blocked shard " << DataShard.TabletID(); - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_NOTICE_CTX(ctx, "TCheckCommitWritesTxUnit::Execute: cannot propose tx at blocked shard", + {"errorMessage", err}); return buildUnsuccessfulResult( err, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR, @@ -87,10 +90,10 @@ public: BuildResult(op)->SetPrepared(op->GetMinStep(), op->GetMaxStep(), op->GetReceivedAt()); - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Prepared " << op->GetKind() - << " transaction txId " << op->GetTxId() - << " at shard " << DataShard.TabletID()); + YDB_LOG_DEBUG_CTX(ctx, "TCheckCommitWritesTxUnit::Execute: prepared transaction", + {"opKind", op->GetKind()}, + {"txId", op->GetTxId()}, + {"tabletId", DataShard.TabletID()}); return EExecutionStatus::Executed; } diff --git a/ydb/core/tx/datashard/check_data_tx_unit.cpp b/ydb/core/tx/datashard/check_data_tx_unit.cpp index c52bdbd1981..e2b3ddca5b8 100644 --- a/ydb/core/tx/datashard/check_data_tx_unit.cpp +++ b/ydb/core/tx/datashard/check_data_tx_unit.cpp @@ -5,6 +5,8 @@ #include <ydb/core/tablet/tablet_exception.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -62,9 +64,10 @@ EExecutionStatus TCheckDataTxUnit::Execute(TOperation::TPtr op, DataShard.IncCounter(COUNTER_MINIKQL_PROGRAM_SIZE, dataTx->ProgramSize()); } else { Y_ENSURE(dataTx->RequirePrepare()); - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Require prepare Tx " << op->GetTxId() << " at " << DataShard.TabletID() - << ": " << dataTx->GetErrors()); + YDB_LOG_DEBUG_CTX(ctx, "Require prepare Tx", + {"txId", op->GetTxId()}, + {"tabletId", DataShard.TabletID()}, + {"errors", dataTx->GetErrors()}); } // Check if we are out of space and tx wants to update user @@ -96,7 +99,8 @@ EExecutionStatus TCheckDataTxUnit::Execute(TOperation::TPtr op, ->AddError(NKikimrTxDataShard::TError::BAD_ARGUMENT, err); op->Abort(EExecutionUnitKind::FinishPropose); - LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_ERROR_CTX(ctx, "TCheckDataTxUnit::Execute: cannot read from snapshot on follower", + {"errorMessage", err}); return EExecutionStatus::Executed; } else if (snapshot < DataShard.GetSnapshotManager().GetLowWatermark()) { @@ -108,7 +112,8 @@ EExecutionStatus TCheckDataTxUnit::Execute(TOperation::TPtr op, ->AddError(NKikimrTxDataShard::TError::SNAPSHOT_NOT_EXIST, err); op->Abort(EExecutionUnitKind::FinishPropose); - LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_ERROR_CTX(ctx, "TCheckDataTxUnit::Execute: reads from stale snapshot", + {"errorMessage", err}); return EExecutionStatus::Executed; } @@ -130,7 +135,8 @@ EExecutionStatus TCheckDataTxUnit::Execute(TOperation::TPtr op, ->AddError(NKikimrTxDataShard::TError::READ_SIZE_EXECEEDED, err); op->Abort(EExecutionUnitKind::FinishPropose); - LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_ERROR_CTX(ctx, "TCheckDataTxUnit::Execute: transaction read size exceeds limit", + {"errorMessage", err}); return EExecutionStatus::Executed; } @@ -146,7 +152,8 @@ EExecutionStatus TCheckDataTxUnit::Execute(TOperation::TPtr op, ->AddError(NKikimrTxDataShard::TError::READ_SIZE_EXECEEDED, err); op->Abort(EExecutionUnitKind::FinishPropose); - LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_ERROR_CTX(ctx, "TCheckDataTxUnit::Execute: transaction total keys size exceeds limit", + {"errorMessage", err}); return EExecutionStatus::Executed; } @@ -167,7 +174,8 @@ EExecutionStatus TCheckDataTxUnit::Execute(TOperation::TPtr op, ->AddError(NKikimrTxDataShard::TError::BAD_ARGUMENT, err); op->Abort(EExecutionUnitKind::FinishPropose); - LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_ERROR_CTX(ctx, "TCheckDataTxUnit::Execute: write key size exceeds limit", + {"errorMessage", err}); return EExecutionStatus::Executed; } @@ -183,7 +191,8 @@ EExecutionStatus TCheckDataTxUnit::Execute(TOperation::TPtr op, BuildResult(op, NKikimrTxDataShard::TEvProposeTransactionResult::EXEC_ERROR)->AddError(NKikimrTxDataShard::TError::BAD_ARGUMENT, err); op->Abort(EExecutionUnitKind::FinishPropose); - LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_ERROR_CTX(ctx, "TCheckDataTxUnit::Execute: write column value exceeds limit", + {"errorMessage", err}); return EExecutionStatus::Executed; } @@ -289,7 +298,8 @@ EExecutionStatus TCheckDataTxUnit::Execute(TOperation::TPtr op, BuildResult(op)->AddError(NKikimrTxDataShard::TError::SHARD_IS_BLOCKED, err); op->Abort(EExecutionUnitKind::FinishPropose); - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_NOTICE_CTX(ctx, "TCheckDataTxUnit::Execute: cannot propose tx at blocked shard", + {"errorMessage", err}); return EExecutionStatus::Executed; } @@ -308,9 +318,10 @@ EExecutionStatus TCheckDataTxUnit::Execute(TOperation::TPtr op, } } - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Prepared " << op->GetKind() << " transaction txId " << op->GetTxId() - << " at tablet " << DataShard.TabletID()); + YDB_LOG_DEBUG_CTX(ctx, "TCheckDataTxUnit::Execute: prepared transaction", + {"opKind", op->GetKind()}, + {"txId", op->GetTxId()}, + {"tabletId", DataShard.TabletID()}); } return EExecutionStatus::Executed; diff --git a/ydb/core/tx/datashard/check_distributed_erase_tx_unit.cpp b/ydb/core/tx/datashard/check_distributed_erase_tx_unit.cpp index 5f75b8e22cc..cb0b8bd04fa 100644 --- a/ydb/core/tx/datashard/check_distributed_erase_tx_unit.cpp +++ b/ydb/core/tx/datashard/check_distributed_erase_tx_unit.cpp @@ -7,6 +7,8 @@ #include <util/generic/bitmap.h> #include <util/string/builder.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -99,7 +101,8 @@ public: << " tx " << op->GetTxId() << " at blocked shard " << DataShard.TabletID(); - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_NOTICE_CTX(ctx, "TCheckDistributedEraseTxUnit::Execute: cannot propose tx at blocked shard", + {"errorMessage", err}); return buildUnsuccessfulResult( err, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR, @@ -109,10 +112,10 @@ public: BuildResult(op)->SetPrepared(op->GetMinStep(), op->GetMaxStep(), op->GetReceivedAt()); - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Prepared" - << " " << op->GetKind() - << " transaction txId " << op->GetTxId() - << " at tablet " << DataShard.TabletID()); + YDB_LOG_DEBUG_CTX(ctx, "TCheckDistributedEraseTxUnit::Execute: prepared transaction", + {"opKind", op->GetKind()}, + {"txId", op->GetTxId()}, + {"tabletId", DataShard.TabletID()}); return EExecutionStatus::Executed; } diff --git a/ydb/core/tx/datashard/check_scheme_tx_unit.cpp b/ydb/core/tx/datashard/check_scheme_tx_unit.cpp index 01d33c4ddf2..d8217793fd8 100644 --- a/ydb/core/tx/datashard/check_scheme_tx_unit.cpp +++ b/ydb/core/tx/datashard/check_scheme_tx_unit.cpp @@ -5,6 +5,8 @@ #include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/core/tablet/tablet_exception.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -102,7 +104,8 @@ EExecutionStatus TCheckSchemeTxUnit::Execute(TOperation::TPtr op, error); op->Abort(EExecutionUnitKind::FinishPropose); - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, error); + YDB_LOG_NOTICE_CTX(ctx, "TCheckSchemeTxUnit::Execute: wrong shard state for scheme transaction", + {"errorMessage", error}); return EExecutionStatus::Executed; } @@ -120,7 +123,8 @@ EExecutionStatus TCheckSchemeTxUnit::Execute(TOperation::TPtr op, << " lastSeqNo " << lastSeqNo.Generation << ":" << lastSeqNo.Round; - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, error); + YDB_LOG_INFO_CTX(ctx, "TCheckSchemeTxUnit::Execute: ignoring outdated schema tx proposal", + {"errorMessage", error}); BuildResult(op)->SetProcessError(NKikimrTxDataShard::TError::SCHEME_CHANGED, error); op->Abort(EExecutionUnitKind::FinishPropose); @@ -128,10 +132,12 @@ EExecutionStatus TCheckSchemeTxUnit::Execute(TOperation::TPtr op, return EExecutionStatus::Executed; } - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Propose scheme transaction at tablet " << DataShard.TabletID() - << " txId " << op->GetTxId() << " ssId " << activeTx->GetSchemeShardId() - << " seqNo " << seqNo.Generation << ":" << seqNo.Round); + YDB_LOG_DEBUG_CTX(ctx, "TCheckSchemeTxUnit::Execute: propose scheme transaction", + {"tabletId", DataShard.TabletID()}, + {"txId", op->GetTxId()}, + {"schemeShardId", activeTx->GetSchemeShardId()}, + {"generation", seqNo.Generation}, + {"round", seqNo.Round}); // Preserve new seqno to correctly filter out tx duplicates. DataShard.UpdateLastSchemeOpSeqNo(seqNo, txc); @@ -217,9 +223,9 @@ EExecutionStatus TCheckSchemeTxUnit::Execute(TOperation::TPtr op, op->SetMinStep(Pipeline.AllowedSchemaStep()); op->SetMaxStep(Max<ui64>()); - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Prepared scheme transaction txId " << op->GetTxId() << " at tablet " - << DataShard.TabletID()); + YDB_LOG_DEBUG_CTX(ctx, "TCheckSchemeTxUnit::Execute: prepared scheme transaction", + {"txId", op->GetTxId()}, + {"tabletId", DataShard.TabletID()}); BuildResult(op)->SetPrepared(op->GetMinStep(), op->GetMaxStep(), op->GetReceivedAt()); @@ -229,11 +235,12 @@ EExecutionStatus TCheckSchemeTxUnit::Execute(TOperation::TPtr op, bool TCheckSchemeTxUnit::CheckSchemaVersion(TActiveTransaction *activeTx, ui64 proposedSchemaVersion, ui64 currentSchemaVersion, ui64 expectedSchemaVersion) { - LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Check scheme tx, proposed scheme version# " << proposedSchemaVersion << - " current version# " << currentSchemaVersion << - " expected version# " << expectedSchemaVersion << - " at tablet# " << DataShard.TabletID() << " txId# " << activeTx->GetTxId()); + YDB_LOG_INFO_CTX(TActivationContext::AsActorContext(), "TCheckSchemeTxUnit::CheckSchemaVersion: check scheme tx schema versions", + {"schemeVersion", proposedSchemaVersion}, + {"currentVersion", currentSchemaVersion}, + {"expectedVersion", expectedSchemaVersion}, + {"tablet", DataShard.TabletID()}, + {"txId", activeTx->GetTxId()}); // Allow scheme tx if proposed or current schema version is zero. This simplify migration a lot. if (proposedSchemaVersion && currentSchemaVersion && expectedSchemaVersion != proposedSchemaVersion) { @@ -243,7 +250,8 @@ bool TCheckSchemeTxUnit::CheckSchemaVersion(TActiveTransaction *activeTx, " expected version# " << expectedSchemaVersion << " at tablet# " << DataShard.TabletID() << " txId# " << activeTx->GetTxId(); - LOG_CRIT_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, err); + YDB_LOG_CRIT_CTX(TActivationContext::AsActorContext(), "TCheckSchemeTxUnit::CheckSchemaVersion: wrong schema version", + {"errorMessage", err}); return false; } @@ -255,11 +263,11 @@ bool TCheckSchemeTxUnit::HasDuplicate(TActiveTransaction *activeTx, const TStrin return false; } - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Ignoring " << kind << " duplicate" - << " at tablet " << DataShard.TabletID() - << " txId " << activeTx->GetTxId() - << " currentTxId " << Pipeline.CurrentSchemaTxId()); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "TCheckSchemeTxUnit::HasDuplicate: ignoring duplicate scheme tx", + {"kind", kind}, + {"tabletId", DataShard.TabletID()}, + {"activeTxId", activeTx->GetTxId()}, + {"currentSchemaTxId", Pipeline.CurrentSchemaTxId()}); BuildResult(activeTx)->SetSchemeTxDuplicate(Pipeline.CurrentSchemaTxId() == activeTx->GetTxId()); return true; @@ -274,11 +282,12 @@ bool TCheckSchemeTxUnit::HasConflicts(TActiveTransaction *activeTx, const TStrin continue; } - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Ignoring " << kind << " during ongoing " << conflicting - << " at tablet " << DataShard.TabletID() - << " txId " << activeTx->GetTxId() - << " currentTxId " << Pipeline.CurrentSchemaTxId()); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "TCheckSchemeTxUnit::HasConflicts: ignoring during ongoing scheme tx", + {"kind", kind}, + {"conflicting", conflicting}, + {"tabletId", DataShard.TabletID()}, + {"activeTxId", activeTx->GetTxId()}, + {"currentSchemaTxId", Pipeline.CurrentSchemaTxId()}); BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR); return true; @@ -293,11 +302,11 @@ bool TCheckSchemeTxUnit::HasPathId(TActiveTransaction *activeTx, const T &op, co return true; } - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - kind << " description has no PathId" - << " at tablet " << DataShard.TabletID() - << " txId " << activeTx->GetTxId() - << " currentTxId " << Pipeline.CurrentSchemaTxId()); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "TCheckSchemeTxUnit::HasPathId: description has no PathId", + {"kind", kind}, + {"tabletId", DataShard.TabletID()}, + {"activeTxId", activeTx->GetTxId()}, + {"currentSchemaTxId", Pipeline.CurrentSchemaTxId()}); BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR); return false; @@ -398,10 +407,10 @@ bool TCheckSchemeTxUnit::CheckSchemeTx(TActiveTransaction *activeTx) res = CheckTruncate(activeTx); break; default: - LOG_ERROR_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Unknown scheme tx type detected at tablet " - << DataShard.TabletID() << " txId " << activeTx->GetTxId() - << " txBody " << tx.ShortDebugString()); + YDB_LOG_ERROR_CTX(TActivationContext::AsActorContext(), "TCheckSchemeTxUnit::CheckSchemeTx: unknown scheme tx type detected", + {"tabletId", DataShard.TabletID()}, + {"activeTxId", activeTx->GetTxId()}, + {"tx", tx.ShortDebugString()}); BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR); } @@ -468,19 +477,19 @@ bool TCheckSchemeTxUnit::CheckAlter(TActiveTransaction *activeTx) if (alter.HasPartitionConfig() && alter.GetPartitionConfig().HasFreezeState()) { if (alter.ColumnsSize() || alter.DropColumnsSize()) { - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Ignoring alter, combine freeze with other actions is forbiden, tablet " << DataShard.TabletID() - << " txId " << activeTx->GetTxId() << " currentTxId " - << Pipeline.CurrentSchemaTxId()); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "TCheckSchemeTxUnit::CheckAlter: ignoring alter, combine freeze with other actions is forbidden", + {"tabletId", DataShard.TabletID()}, + {"activeTxId", activeTx->GetTxId()}, + {"currentSchemaTxId", Pipeline.CurrentSchemaTxId()}); BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST); return false; } if (DataShard.IsFollower()) { - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Ignoring alter, attempt to freeze follower, tablet " << DataShard.TabletID() - << " txId " << activeTx->GetTxId() << " currentTxId " - << Pipeline.CurrentSchemaTxId()); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "TCheckSchemeTxUnit::CheckAlter: ignoring alter, attempt to freeze follower", + {"tabletId", DataShard.TabletID()}, + {"activeTxId", activeTx->GetTxId()}, + {"currentSchemaTxId", Pipeline.CurrentSchemaTxId()}); BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::BAD_REQUEST); return false; } @@ -505,7 +514,8 @@ bool TCheckSchemeTxUnit::CheckAlter(TActiveTransaction *activeTx) << DatashardStateName(curState) << " tablet "<< DataShard.TabletID() << " txId " << activeTx->GetTxId() << " currentTxId " << Pipeline.CurrentSchemaTxId(); - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, errText); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "TCheckSchemeTxUnit::CheckAlter: ignoring alter due to wrong shard state", + {"errorMessage", errText}); BuildResult(activeTx)->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, errText); return false; } @@ -680,12 +690,11 @@ bool TCheckSchemeTxUnit::CheckFinalizeBuildIndex(TActiveTransaction *activeTx) { const auto snapshotKey = TSnapshotKey(pathId, finalize.GetSnapshotStep(), finalize.GetSnapshotTxId()); if (DataShard.GetSnapshotManager().FindAvailable(snapshotKey) == nullptr) { - LOG_DEBUG_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "FinalizeBuildIndex description has unexisting snapshot key." - << " Tablet " << DataShard.TabletID() - << " txId " << activeTx->GetTxId() - << " pathId " << pathId - << " snapshotKey " << snapshotKey); + YDB_LOG_DEBUG_CTX(TActivationContext::AsActorContext(), "TCheckSchemeTxUnit::CheckFinalizeBuildIndex: description has nonexistent snapshot key", + {"tabletId", DataShard.TabletID()}, + {"activeTxId", activeTx->GetTxId()}, + {"pathId", pathId}, + {"snapshotKey", snapshotKey}); BuildResult(activeTx, NKikimrTxDataShard::TEvProposeTransactionResult::ERROR); return false; } diff --git a/ydb/core/tx/datashard/check_snapshot_tx_unit.cpp b/ydb/core/tx/datashard/check_snapshot_tx_unit.cpp index 825bee3f37f..4c0db0612c7 100644 --- a/ydb/core/tx/datashard/check_snapshot_tx_unit.cpp +++ b/ydb/core/tx/datashard/check_snapshot_tx_unit.cpp @@ -2,6 +2,8 @@ #include "datashard_pipeline.h" #include "execution_unit_ctors.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -141,16 +143,18 @@ public: BuildResult(op)->AddError(NKikimrTxDataShard::TError::SHARD_IS_BLOCKED, err); op->Abort(EExecutionUnitKind::FinishPropose); - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_NOTICE_CTX(ctx, "TCheckSnapshotTxUnit::Execute: cannot propose tx at blocked shard", + {"errorMessage", err}); return EExecutionStatus::Executed; } BuildResult(op)->SetPrepared(op->GetMinStep(), op->GetMaxStep(), op->GetReceivedAt()); - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Prepared " << op->GetKind() << " transaction txId " << op->GetTxId() - << " at tablet " << DataShard.TabletID()); + YDB_LOG_DEBUG_CTX(ctx, "TCheckSnapshotTxUnit::Execute: prepared transaction", + {"opKind", op->GetKind()}, + {"txId", op->GetTxId()}, + {"tabletId", DataShard.TabletID()}); } return EExecutionStatus::Executed; diff --git a/ydb/core/tx/datashard/check_write_unit.cpp b/ydb/core/tx/datashard/check_write_unit.cpp index 41ded8f0ff9..fde7e7ec59c 100644 --- a/ydb/core/tx/datashard/check_write_unit.cpp +++ b/ydb/core/tx/datashard/check_write_unit.cpp @@ -5,6 +5,8 @@ #include "ydb/core/tx/datashard/datashard_write_operation.h" #include <ydb/core/tablet/tablet_exception.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -118,7 +120,8 @@ EExecutionStatus TCheckWriteUnit::Execute(TOperation::TPtr op, writeOp->SetError(NKikimrDataEvents::TEvWriteResult::STATUS_OVERLOADED, err); op->Abort(EExecutionUnitKind::FinishProposeWrite); - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_NOTICE_CTX(ctx, "TCheckWriteUnit::Execute: cannot propose tx at blocked shard", + {"errorMessage", err}); return EExecutionStatus::Executed; } @@ -132,7 +135,9 @@ EExecutionStatus TCheckWriteUnit::Execute(TOperation::TPtr op, DataShard.GetProcessingParams() ? DataShard.GetProcessingParams()->GetCoordinators() : google::protobuf::RepeatedField<ui64>{} } )); - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Prepared write transaction " << *op << " at tablet " << DataShard.TabletID()); + YDB_LOG_DEBUG_CTX(ctx, "TCheckWriteUnit::Execute: prepared write transaction", + {"operation", *op}, + {"tabletId", DataShard.TabletID()}); } return EExecutionStatus::Executed; diff --git a/ydb/core/tx/datashard/complete_data_tx_unit.cpp b/ydb/core/tx/datashard/complete_data_tx_unit.cpp index 178b4901e78..0b38103e501 100644 --- a/ydb/core/tx/datashard/complete_data_tx_unit.cpp +++ b/ydb/core/tx/datashard/complete_data_tx_unit.cpp @@ -7,6 +7,8 @@ #include <ydb/core/engine/minikql/minikql_engine_host.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + LWTRACE_USING(DATASHARD_PROVIDER) namespace NKikimr { @@ -85,8 +87,8 @@ void TCompleteOperationUnit::CompleteOperation(TOperation::TPtr op, if (DataShard.GetDataTxProfileLogThresholdMs() && duration.MilliSeconds() >= DataShard.GetDataTxProfileLogThresholdMs()) { - LOG_WARN_S(ctx, NKikimrServices::TX_DATASHARD, - op->ExecutionProfileLogString(DataShard.TabletID())); + YDB_LOG_WARN_CTX(ctx, "TCompleteOperationUnit::CompleteOperation: slow transaction execution profile", + {"executionProfile", op->ExecutionProfileLogString(DataShard.TabletID())}); } if (DataShard.GetDataTxProfileBufferThresholdMs() diff --git a/ydb/core/tx/datashard/complete_write_unit.cpp b/ydb/core/tx/datashard/complete_write_unit.cpp index 946b30d1d93..4ad4417810f 100644 --- a/ydb/core/tx/datashard/complete_write_unit.cpp +++ b/ydb/core/tx/datashard/complete_write_unit.cpp @@ -7,6 +7,8 @@ #include <ydb/core/engine/minikql/minikql_engine_host.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + LWTRACE_USING(DATASHARD_PROVIDER) namespace NKikimr { @@ -68,8 +70,8 @@ void TCompleteWriteUnit::CompleteWrite(TOperation::TPtr op, const TActorContext& if (DataShard.GetDataTxProfileLogThresholdMs() && duration.MilliSeconds() >= DataShard.GetDataTxProfileLogThresholdMs()) { - LOG_WARN_S(ctx, NKikimrServices::TX_DATASHARD, - op->ExecutionProfileLogString(DataShard.TabletID())); + YDB_LOG_WARN_CTX(ctx, "TCompleteWriteUnit::CompleteWrite: slow transaction execution profile", + {"executionProfile", op->ExecutionProfileLogString(DataShard.TabletID())}); } if (DataShard.GetDataTxProfileBufferThresholdMs() diff --git a/ydb/core/tx/datashard/create_table_unit.cpp b/ydb/core/tx/datashard/create_table_unit.cpp index 1d90d25f66a..f69566a555e 100644 --- a/ydb/core/tx/datashard/create_table_unit.cpp +++ b/ydb/core/tx/datashard/create_table_unit.cpp @@ -2,6 +2,8 @@ #include "datashard_pipeline.h" #include "execution_unit_ctors.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -63,10 +65,10 @@ EExecutionStatus TCreateTableUnit::Execute(TOperation::TPtr op, const ui64 schemaVersion = createTableTx.HasTableSchemaVersion() ? createTableTx.GetTableSchemaVersion() : 0u; - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, - "Trying to CREATE TABLE at " << DataShard.TabletID() - << " tableId# " << tableId - << " schema version# " << schemaVersion); + YDB_LOG_INFO_CTX(ctx, "TCreateTableUnit::Execute: trying to create table", + {"tabletId", DataShard.TabletID()}, + {"tableId", tableId}, + {"version", schemaVersion}); TUserTable::TPtr info = DataShard.CreateUserTable(txc, schemeTx.GetCreateTable()); DataShard.AddUserTable(tableId, info); diff --git a/ydb/core/tx/datashard/datashard__lock_rows.cpp b/ydb/core/tx/datashard/datashard__lock_rows.cpp index 3bd7cc0f13e..6af12d40373 100644 --- a/ydb/core/tx/datashard/datashard__lock_rows.cpp +++ b/ydb/core/tx/datashard/datashard__lock_rows.cpp @@ -650,13 +650,6 @@ void TDataShard::HandleLockRowsRequest(NEvents::TDataEvents::TEvLockRows::TPtr e return ETxLockRows::Restart; } - if (skipAbsent && (row.Ready == NTable::EReady::Gone || row.RowOp == NTable::ERowOp::Erase)) { - success->Record.AddSkippedAbsentKeys(processedKeys); - runtimeLock.Reset(); - ++processedKeys; - continue; - } - // Undecided volatile transactions will have non-zero VolatileVersion // We don't wait until they are decided and return a modified flag // instead. A subsequent re-read will wait for the decision. @@ -776,6 +769,19 @@ void TDataShard::HandleLockRowsRequest(NEvents::TDataEvents::TEvLockRows::TPtr e } } + // Note: we run the skipAbsent check *after* we check that we already have the lock + // for this key. The purpose of skipAbsent is to allow other transactions to lock and + // insert a row to this key if this transaction is trying to update/delete a row that + // is not there. If we already locked the key, this means that previously in this + // transaction we inserted the row, so it should be able to do other operations + // to it, and we shouldn't skip even if no committed row exists. + if (skipAbsent && (row.Ready == NTable::EReady::Gone || row.RowOp == NTable::ERowOp::Erase)) { + success->Record.AddSkippedAbsentKeys(processedKeys); + runtimeLock.Reset(); + ++processedKeys; + continue; + } + // Don't bother waiting in skipLocked mode when current owner conflicts with us if (skipLocked && currentOwner && !IsCompatibleRowLockMode(currentLockMode, lockMode)) { finishSkippedLocked(); diff --git a/ydb/core/tx/datashard/datashard_ut_lock_rows.cpp b/ydb/core/tx/datashard/datashard_ut_lock_rows.cpp index 3159b51c55c..7c551f0be2b 100644 --- a/ydb/core/tx/datashard/datashard_ut_lock_rows.cpp +++ b/ydb/core/tx/datashard/datashard_ut_lock_rows.cpp @@ -9,6 +9,7 @@ #include <util/digest/multi.h> #include <util/random/fast.h> +#include <util/string/join.h> namespace { struct TRequestId { @@ -2200,35 +2201,41 @@ Y_UNIT_TEST_SUITE(DataShardLockRows) { )"), "<empty>"); + // Lock key 5 by lock 234 + auto req2 = lockRows.SendRequest(lock2, tableId, TKeysBuilder().Add(5).Build()); + lockRows.ExpectResult(req2); + // Try locking keys 1, 2, 3, 4 by lock 234. - auto req2 = lockRows.SendRequest( - lock2, tableId, TKeysBuilder().Add(1).Add(2).Add(3).Add(4).Build(), + auto skippingReq = lockRows.SendRequest( + lock2, tableId, TKeysBuilder().Add(1).Add(2).Add(3).Add(4).Add(5).Build(), [&](NEvents::TDataEvents::TEvLockRows* ev) { ev->Record.MutableSnapshot()->SetStep(snapshot.Step); ev->Record.MutableSnapshot()->SetTxId(snapshot.TxId); ev->Record.SetSkipLocked(true); ev->Record.SetSkipAbsent(true); }); - auto res = lockRows.ExpectResult(req2); + auto res = lockRows.ExpectResult(skippingReq); - // Key 1 was locked - const auto& locked = res->Record.GetLockedKeys(); - UNIT_ASSERT_VALUES_EQUAL(locked.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(locked[0], 0); + // Key 1 was locked because it is present, and key 5 was locked because it was + // previously locked by the same transaction. + UNIT_ASSERT_VALUES_EQUAL( + JoinSeq(",", res->Record.GetLockedKeys()), + "0,4"); // Key 2 was skipped because it was previously locked by req1 - const auto& skippedLocked = res->Record.GetSkippedLockedKeys(); - UNIT_ASSERT_VALUES_EQUAL(skippedLocked.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(skippedLocked[0], 1); + UNIT_ASSERT_VALUES_EQUAL( + JoinSeq(",", res->Record.GetSkippedLockedKeys()), + "1"); // Key 3 was skipped because it was deleted, and key 4 was skipped because it was absent. - const auto& skippedAbsent = res->Record.GetSkippedAbsentKeys(); - UNIT_ASSERT_VALUES_EQUAL(skippedAbsent.size(), 2); - UNIT_ASSERT_VALUES_EQUAL(skippedAbsent[0], 2); - UNIT_ASSERT_VALUES_EQUAL(skippedAbsent[1], 3); + UNIT_ASSERT_VALUES_EQUAL( + JoinSeq(",", res->Record.GetSkippedAbsentKeys()), + "2,3"); // Key 3 was deleted after the snapshot, but it is not in ModifiedKeys because we skipped it. - UNIT_ASSERT_VALUES_EQUAL(res->Record.GetModifiedKeys().size(), 0); + UNIT_ASSERT_VALUES_EQUAL( + JoinSeq(",", res->Record.GetModifiedKeys()), + ""); } } // Y_UNIT_TEST_SUITE(DataShardLockRows) diff --git a/ydb/core/tx/datashard/drop_table_unit.cpp b/ydb/core/tx/datashard/drop_table_unit.cpp index 386b0d727f8..254392e3454 100644 --- a/ydb/core/tx/datashard/drop_table_unit.cpp +++ b/ydb/core/tx/datashard/drop_table_unit.cpp @@ -2,6 +2,8 @@ #include "datashard_pipeline.h" #include "execution_unit_ctors.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -68,8 +70,8 @@ EExecutionStatus TDropTableUnit::Execute(TOperation::TPtr op, if (!schemeTx.HasDropTable()) return EExecutionStatus::Executed; - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, - "Trying to DROP TABLE at " << DataShard.TabletID()); + YDB_LOG_INFO_CTX(ctx, "TDropTableUnit::Execute: trying to drop table", + {"tabletId", DataShard.TabletID()}); ui64 tableId = schemeTx.GetDropTable().GetId_Deprecated(); if (schemeTx.GetDropTable().HasPathId()) { diff --git a/ydb/core/tx/datashard/erase_rows_condition.cpp b/ydb/core/tx/datashard/erase_rows_condition.cpp index f191ca90a11..93144075097 100644 --- a/ydb/core/tx/datashard/erase_rows_condition.cpp +++ b/ydb/core/tx/datashard/erase_rows_condition.cpp @@ -10,6 +10,8 @@ extern "C" { #include <yql/essentials/parser/pg_wrapper/postgresql/src/include/catalog/pg_type_d.h> + +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD } namespace NKikimr { @@ -36,8 +38,8 @@ class TExpirationCondition: public IEraseRowsCondition { TMaybe<TString> GetWallClockDyNumber() const { const auto instantValue = InstantValue(WallClockInstant, Unit); if (!instantValue) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Unsupported unit: " << static_cast<ui32>(Unit)); + YDB_LOG_CRIT("Unsupported", + {"unit", static_cast<ui32>(Unit)}); CannotSerialize = true; return Nothing(); } @@ -46,8 +48,8 @@ class TExpirationCondition: public IEraseRowsCondition { WallClockSerialized = NDyNumber::ParseDyNumberString(strInstant); if (!WallClockSerialized) { CannotSerialize = true; - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Cannot parse DyNumber from: " << strInstant.Quote()); + YDB_LOG_CRIT("Cannot parse DyNumber", + {"from", strInstant}); } return WallClockSerialized; @@ -57,8 +59,8 @@ class TExpirationCondition: public IEraseRowsCondition { const auto& result = NPg::PgNativeBinaryFromNativeText(value, Type.GetPgTypeDesc()); if (result.Error) { CannotSerialize = true; - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Cannot create PG native binary from: " << value.Quote()); + YDB_LOG_CRIT("Cannot create PG native binary", + {"from", value}); } else { WallClockSerialized = std::move(result.Str); } @@ -76,8 +78,8 @@ class TExpirationCondition: public IEraseRowsCondition { case INT8OID: { const auto instantValue = InstantValue(WallClockInstant, Unit); if (!instantValue) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Unsupported unit: " << static_cast<ui32>(Unit)); + YDB_LOG_CRIT("Unsupported", + {"unit", static_cast<ui32>(Unit)}); CannotSerialize = true; return Nothing(); } @@ -87,7 +89,7 @@ class TExpirationCondition: public IEraseRowsCondition { } default: CannotSerialize = true; - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Unsupported PG type"); + YDB_LOG_CRIT("Unsupported PG type"); } return WallClockSerialized; } @@ -133,8 +135,8 @@ class TExpirationCondition: public IEraseRowsCondition { case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: return TInstant::MicroSeconds(value / 1000) <= WallClockInstant; default: - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Unsupported unit: " << static_cast<ui32>(Unit)); + YDB_LOG_CRIT("Unsupported", + {"unit", static_cast<ui32>(Unit)}); return false; } default: diff --git a/ydb/core/tx/datashard/execute_data_tx_unit.cpp b/ydb/core/tx/datashard/execute_data_tx_unit.cpp index f6fd17e7baa..9d305b0afd9 100644 --- a/ydb/core/tx/datashard/execute_data_tx_unit.cpp +++ b/ydb/core/tx/datashard/execute_data_tx_unit.cpp @@ -5,6 +5,8 @@ #include "setup_sys_locks.h" #include "datashard_locks_db.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -122,8 +124,10 @@ EExecutionStatus TExecuteDataTxUnit::Execute(TOperation::TPtr op, else { ui64 consumed = tx->GetDataTx()->GetTxSize() + engine->GetMemoryAllocated(); if (MaybeRequestMoreTxMemory(consumed, txc)) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Operation " << *op << " at " << DataShard.TabletID() - << " requested " << txc.GetRequestedMemory() << " more memory"); + YDB_LOG_TRACE_CTX(ctx, "TExecuteDataTxUnit::Execute: requested more memory", + {"operation", *op}, + {"tabletId", DataShard.TabletID()}, + {"requestedMemory", txc.GetRequestedMemory()}); DataShard.IncCounter(COUNTER_TX_WAIT_RESOURCE); return EExecutionStatus::Restart; @@ -175,10 +179,11 @@ EExecutionStatus TExecuteDataTxUnit::Execute(TOperation::TPtr op, throw; } } catch (const TMemoryLimitExceededException&) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Operation " << *op << " at " << DataShard.TabletID() - << " exceeded memory limit " << txc.GetMemoryLimit() - << " and requests " << txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR - << " more for the next try"); + YDB_LOG_TRACE_CTX(ctx, "TExecuteDataTxUnit::ExecuteDataTx: exceeded memory limit and requests more for the next try", + {"operation", *op}, + {"tabletId", DataShard.TabletID()}, + {"memoryLimit", txc.GetMemoryLimit()}, + {"memoryLimitWithFactor", txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR}); txc.NotEnoughMemory(); DataShard.IncCounter(DataShard.NotEnoughMemoryCounter(txc.GetNotEnoughMemoryCount())); @@ -191,8 +196,9 @@ EExecutionStatus TExecuteDataTxUnit::Execute(TOperation::TPtr op, return EExecutionStatus::Restart; } catch (const TNotReadyTabletException&) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Tablet " << DataShard.TabletID() - << " is not ready for " << *op << " execution"); + YDB_LOG_TRACE_CTX(ctx, "TExecuteDataTxUnit::ExecuteDataTx: tablet is not ready for execution", + {"tabletId", DataShard.TabletID()}, + {"operation", *op}); DataShard.IncCounter(COUNTER_TX_TABLET_NOT_READY); @@ -201,8 +207,9 @@ EExecutionStatus TExecuteDataTxUnit::Execute(TOperation::TPtr op, return EExecutionStatus::Restart; } catch (const TRollbackAndWaitException&) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Tablet " << DataShard.TabletID() - << " needs to wait " << *op << " for dependencies"); + YDB_LOG_TRACE_CTX(ctx, "TExecuteDataTxUnit::ExecuteDataTx: tablet needs to wait for dependencies", + {"tabletId", DataShard.TabletID()}, + {"operation", *op}); tx->GetDataTx()->ResetCollectedChanges(); tx->ReleaseTxData(txc, ctx); @@ -270,15 +277,18 @@ void TExecuteDataTxUnit::ExecuteDataTx(TOperation::TPtr op, switch (engineResult) { case IEngineFlat::EResult::ResultTooBig: - LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, errorMessage); + YDB_LOG_ERROR_CTX(ctx, "TExecuteDataTxUnit::ExecuteDataTx: result too big", + {"errorMessage", errorMessage}); break; case IEngineFlat::EResult::Cancelled: - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, errorMessage); + YDB_LOG_NOTICE_CTX(ctx, "TExecuteDataTxUnit::ExecuteDataTx: execution cancelled", + {"errorMessage", errorMessage}); Y_ENSURE(tx->GetDataTx()->CanCancel()); break; default: if (op->IsReadOnly() || op->IsImmediate()) { - LOG_CRIT_S(ctx, NKikimrServices::TX_DATASHARD, errorMessage); + YDB_LOG_CRIT_CTX(ctx, "TExecuteDataTxUnit::ExecuteDataTx: unexpected execution error", + {"errorMessage", errorMessage}); } else { // TODO: Kill only current datashard tablet. Y_ENSURE(false, "Unexpected execution error in read-write transaction: " @@ -306,15 +316,17 @@ void TExecuteDataTxUnit::ExecuteDataTx(TOperation::TPtr op, op->ChangeRecords() = std::move(tx->GetDataTx()->GetCollectedChanges()); } - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "Executed operation " << *op << " at tablet " << DataShard.TabletID() - << " with status " << result->GetStatus()); + YDB_LOG_TRACE_CTX(ctx, "TExecuteDataTxUnit::ExecuteDataTx: executed operation with status", + {"operation", *op}, + {"tabletId", DataShard.TabletID()}, + {"resultStatus", result->GetStatus()}); auto& counters = tx->GetDataTx()->GetCounters(); - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "Datashard execution counters for " << *op << " at " - << DataShard.TabletID() << ": " << counters.ToString()); + YDB_LOG_TRACE_CTX(ctx, "Datashard execution counters", + {"operation", *op}, + {"tabletId", DataShard.TabletID()}, + {"counters", counters}); KqpUpdateDataShardStatCounters(DataShard, counters); if (tx->GetDataTx()->CollectStats()) { @@ -367,9 +379,10 @@ void TExecuteDataTxUnit::AddLocksToResult(TOperation::TPtr op, const TActorConte for (const auto& lock : locks) { if (lock.IsError()) { - LOG_NOTICE_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Lock is not set for " << *op << " at " << DataShard.TabletID() - << " lock " << lock); + YDB_LOG_NOTICE_CTX(TActivationContext::AsActorContext(), "TExecuteDataTxUnit::AddLocksToResult: lock is not set", + {"operation", *op}, + {"tabletId", DataShard.TabletID()}, + {"lock", lock}); } op->Result()->AddTxLock(lock.LockId, lock.DataShard, lock.Generation, lock.Counter, lock.SchemeShard, lock.PathId, lock.HasWrites); diff --git a/ydb/core/tx/datashard/execute_write_unit.cpp b/ydb/core/tx/datashard/execute_write_unit.cpp index 0bc913a8e3d..0c903223cb3 100644 --- a/ydb/core/tx/datashard/execute_write_unit.cpp +++ b/ydb/core/tx/datashard/execute_write_unit.cpp @@ -126,15 +126,20 @@ public: HandleBreakerLocks(locksBrokenByTx, writeOp->GetTxId(), querySpanId, DataShard.SysLocksTable(), writeResult.Record, ctx, "Write transaction broke other locks", kqpLocks); - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "add locks to result: " << locks.size()); + YDB_LOG_TRACE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "Add locks", + {"result", locks.size()}); for (const auto& lock : locks) { if (lock.IsError()) { - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, "Lock is not set for " << *writeOp << " at " << DataShard.TabletID() << " lock " << lock); + YDB_LOG_NOTICE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::AddLocksToResult: lock is not set", + {"operation", *writeOp}, + {"tabletId", DataShard.TabletID()}, + {"lock", lock}); } writeResult.AddTxLock(lock.LockId, lock.DataShard, lock.Generation, lock.Counter, lock.SchemeShard, lock.PathId, lock.HasWrites); - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "add lock to result: " << writeResult.Record.GetTxLocks().rbegin()->ShortDebugString()); + YDB_LOG_TRACE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "Add lock", + {"result", writeResult.Record.GetTxLocks().rbegin()->ShortDebugString()}); } DataShard.SubscribeNewLocks(ctx); } @@ -149,7 +154,9 @@ public: bool CheckForVolatileReadDependencies(TDataShardUserDb& userDb, TWriteOperation& writeOp, TTransactionContext& txc, const TActorContext& ctx) { if (!userDb.GetVolatileReadDependencies().empty()) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Operation " << writeOp << " at " << DataShard.TabletID() << " aborting because volatile read dependencies"); + YDB_LOG_TRACE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::CheckForVolatileReadDependencies: aborting because volatile read dependencies", + {"writeOp", writeOp}, + {"tabletId", DataShard.TabletID()}); for (ui64 txId : userDb.GetVolatileReadDependencies()) { writeOp.AddVolatileDependency(txId); @@ -201,7 +208,9 @@ public: }; EExecutionStatus OnTabletNotReadyException(TDataShardUserDb& userDb, TWriteOperation& writeOp, size_t operationIndexToPrecharge, TTransactionContext& txc, const TActorContext& ctx) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Tablet " << DataShard.TabletID() << " is not ready for " << writeOp << " execution"); + YDB_LOG_TRACE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::OnTabletNotReadyException: tablet is not ready for execution", + {"tabletId", DataShard.TabletID()}, + {"writeOp", writeOp}); // Precharge if (operationIndexToPrecharge != SIZE_MAX) { @@ -247,7 +256,9 @@ public: } if (userDb.GetSnapshotReadConflict()) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Operation " << writeOp << " at " << DataShard.TabletID() << " aborting. Conflict with another transaction."); + YDB_LOG_TRACE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::OnUniqueConstrainException: aborting, conflict with another transaction", + {"writeOp", writeOp}, + {"tabletId", DataShard.TabletID()}); writeOp.SetError(NKikimrDataEvents::TEvWriteResult::STATUS_LOCKS_BROKEN, "Read conflict with concurrent transaction."); writeOp.GetWriteResult()->Record.MutableTxStats()->SetLocksBrokenAsVictim(1); NDataIntegrity::LogVictimDetected(ctx, DataShard.TabletID(), "Write transaction was a victim of broken locks", @@ -257,7 +268,9 @@ public: FillDeferredBreakerInfo(lockTxId, writeOp.GetWriteResult()->Record.MutableTxStats()); } } else { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Operation " << writeOp << " at " << DataShard.TabletID() << " aborting. Conflict with existing key."); + YDB_LOG_TRACE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::OnUniqueConstrainException: aborting, conflict with existing key", + {"writeOp", writeOp}, + {"tabletId", DataShard.TabletID()}); writeOp.SetError(NKikimrDataEvents::TEvWriteResult::STATUS_CONSTRAINT_VIOLATION, "Conflict with existing key."); } @@ -362,7 +375,9 @@ public: TWriteOperation* writeOp = TWriteOperation::CastWriteOperation(op); const ui64 tabletId = DataShard.TabletID(); - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Executing write operation for " << *op << " at " << tabletId); + YDB_LOG_DEBUG_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::Execute: executing write operation", + {"operation", *op}, + {"tabletId", tabletId}); if (writeOp->GetWriteResult() || op->HasResultSentFlag() || op->IsImmediate() && CheckRejectDataTx(op, ctx)) { return EExecutionStatus::Executed; @@ -473,7 +488,10 @@ public: ui64 consumedMemory = writeTx->GetTxSize(); if (MaybeRequestMoreTxMemory(consumedMemory, txc)) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Operation " << *op << " at " << tabletId << " requested " << txc.GetRequestedMemory() << " more memory"); + YDB_LOG_TRACE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::Execute: requested more memory", + {"operation", *op}, + {"tabletId", tabletId}, + {"requestedMemory", txc.GetRequestedMemory()}); DataShard.IncCounter(COUNTER_TX_WAIT_RESOURCE); return EExecutionStatus::Restart; @@ -481,7 +499,9 @@ public: if (guardLocks.LockTxId) { auto abortLock = [&]() { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Operation " << *op << " at " << tabletId << " aborting because it cannot acquire locks"); + YDB_LOG_TRACE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::Execute: aborting because it cannot acquire locks", + {"operation", *op}, + {"tabletId", tabletId}); writeOp->SetError(NKikimrDataEvents::TEvWriteResult::STATUS_LOCKS_BROKEN, "Operation is aborting because it cannot acquire locks"); writeOp->GetWriteResult()->Record.MutableTxStats()->SetLocksBrokenAsVictim(1); NDataIntegrity::LogVictimDetected(ctx, tabletId, "Write transaction was a victim of broken locks", @@ -536,7 +556,9 @@ public: : KqpValidateLocks(tabletId, sysLocks, kqpLocks, useGenericReadSets, inReadSets); if (!validated) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Operation " << *op << " at " << tabletId << " aborting because locks are not valid"); + YDB_LOG_TRACE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::Execute: aborting because locks are not valid", + {"operation", *op}, + {"tabletId", tabletId}); writeOp->SetError(NKikimrDataEvents::TEvWriteResult::STATUS_LOCKS_BROKEN, "Operation is aborting because locks are not valid"); writeOp->GetWriteResult()->Record.MutableTxStats()->SetLocksBrokenAsVictim(brokenLocks.size()); @@ -593,12 +615,17 @@ public: } } DoUpdateToUserDb(userDb, validatedOperation, txc); - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Executed write operation for " << *writeOp << " at " << DataShard.TabletID() << ", row count=" << validatedOperation.GetMatrix().GetRowCount()); + YDB_LOG_DEBUG_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::Execute: executed write operation for row", + {"operation", *writeOp}, + {"tabletId", DataShard.TabletID()}, + {"count", validatedOperation.GetMatrix().GetRowCount()}); } validatedOperationIndex = SIZE_MAX; DataShard.AddRecentWriteForTli(mvccVersion, guardLocks.QuerySpanId, writeOp->GetTarget().NodeId()); } else { - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, "Skip empty write operation for " << *writeOp << " at " << DataShard.TabletID()); + YDB_LOG_DEBUG_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::Execute: skip empty write operation", + {"operation", *writeOp}, + {"tabletId", DataShard.TabletID()}); } if (CheckForVolatileReadDependencies(userDb, *writeOp, txc, ctx)) @@ -743,7 +770,9 @@ public: return EExecutionStatus::Continue; } - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, "Operation " << *writeOp << " at " << DataShard.TabletID() << " aborting. Conflict with another transaction."); + YDB_LOG_TRACE_CTX_COMP(ctx, NKikimrServices::TX_DATASHARD, "TExecuteWriteUnit::Execute: aborting, write conflict with concurrent transaction", + {"operation", *writeOp}, + {"tabletId", DataShard.TabletID()}); writeOp->SetError(NKikimrDataEvents::TEvWriteResult::STATUS_LOCKS_BROKEN, "Write conflict with concurrent transaction."); writeOp->GetWriteResult()->Record.MutableTxStats()->SetLocksBrokenAsVictim(1); NDataIntegrity::LogVictimDetected(ctx, DataShard.TabletID(), "Write transaction was a victim of broken locks", @@ -763,7 +792,9 @@ public: } catch (const TKeySizeConstraintException&) { writeOp->SetError(NKikimrDataEvents::TEvWriteResult::STATUS_CONSTRAINT_VIOLATION, TStringBuilder() << "Size of key in secondary index is more than " << NLimits::MaxWriteKeySize); txc.DB.RollbackChanges(); - LOG_ERROR_S(ctx, NKikimrDataEvents::TEvWriteResult::STATUS_CONSTRAINT_VIOLATION, "Operation " << *writeOp << " at " << DataShard.TabletID() << " aborting. Size of key of secondary index is too big."); + YDB_LOG_ERROR_CTX_COMP(ctx, NKikimrDataEvents::TEvWriteResult::STATUS_CONSTRAINT_VIOLATION, "TExecuteWriteUnit::Execute: aborting, secondary index key is too big", + {"operation", *writeOp}, + {"tabletId", DataShard.TabletID()}); if (auto status = ensureAbortOutReadSets()) { return *status; diff --git a/ydb/core/tx/datashard/execution_unit.cpp b/ydb/core/tx/datashard/execution_unit.cpp index 1225f85a264..092d63945ad 100644 --- a/ydb/core/tx/datashard/execution_unit.cpp +++ b/ydb/core/tx/datashard/execution_unit.cpp @@ -2,6 +2,8 @@ #include "execution_unit_ctors.h" #include "datashard_impl.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -188,8 +190,8 @@ bool TExecutionUnit::CheckRejectDataTx(TOperation::TPtr op, const TActorContext& ->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, err); } - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, - "Tablet " << DataShard.TabletID() << " rejecting tx due to split"); + YDB_LOG_NOTICE_CTX(ctx, "Tablet rejecting tx due to split", + {"tabletId", DataShard.TabletID()}); op->Abort(); return true; @@ -210,7 +212,7 @@ bool TExecutionUnit::CheckRejectDataTx(TOperation::TPtr op, const TActorContext& BuildResult(op)->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, err); } - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_NOTICE_CTX(ctx, err); op->Abort(); return true; @@ -227,7 +229,7 @@ bool TExecutionUnit::CheckRejectDataTx(TOperation::TPtr op, const TActorContext& ->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, err); } - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_NOTICE_CTX(ctx, err); op->Abort(); return true; @@ -246,8 +248,8 @@ bool TExecutionUnit::CheckRejectDataTx(TOperation::TPtr op, const TActorContext& ->AddError(NKikimrTxDataShard::TError::SHARD_IS_BLOCKED, err); } - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, - "Tablet " << DataShard.TabletID() << " rejecting tx due to changes queue overflow"); + YDB_LOG_NOTICE_CTX(ctx, "Tablet rejecting tx due to changes queue overflow", + {"tabletId", DataShard.TabletID()}); op->Abort(); return true; @@ -265,7 +267,7 @@ bool TExecutionUnit::CheckRejectDataTx(TOperation::TPtr op, const TActorContext& ->AddError(NKikimrTxDataShard::TError::WRONG_SHARD_STATE, err); } - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_NOTICE_CTX(ctx, err); op->Abort(); return true; diff --git a/ydb/core/tx/datashard/export_common.h b/ydb/core/tx/datashard/export_common.h index 13a83923997..68e8784ffb5 100644 --- a/ydb/core/tx/datashard/export_common.h +++ b/ydb/core/tx/datashard/export_common.h @@ -9,24 +9,6 @@ #include <util/generic/map.h> #include <util/generic/maybe.h> -#if defined EXPORT_LOG_T || \ - defined EXPORT_LOG_D || \ - defined EXPORT_LOG_I || \ - defined EXPORT_LOG_N || \ - defined EXPORT_LOG_W || \ - defined EXPORT_LOG_E || \ - defined EXPORT_LOG_C -#error log macro redefinition -#endif - -#define EXPORT_LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) -#define EXPORT_LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::DATASHARD_BACKUP, "[Export] [" << LogPrefix() << "] " << stream) - namespace NKikimr::NDataShard { TMaybe<Ydb::Table::CreateTableRequest> GenYdbScheme( diff --git a/ydb/core/tx/datashard/export_s3_uploader.cpp b/ydb/core/tx/datashard/export_s3_uploader.cpp index a361f3e1de0..b0647da225f 100644 --- a/ydb/core/tx/datashard/export_s3_uploader.cpp +++ b/ydb/core/tx/datashard/export_s3_uploader.cpp @@ -22,6 +22,7 @@ #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/hfunc.h> #include <ydb/library/actors/http/http_proxy.h> +#include <ydb/library/actors/struct_log/text_writer.h> #include <library/cpp/random_provider/random_provider.h> #include <util/generic/buffer.h> @@ -35,6 +36,8 @@ #include <ydb/core/protos/config.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::DATASHARD_BACKUP + namespace NKikimr { namespace NDataShard { @@ -136,23 +139,23 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void Handle(NHttp::TEvHttpProxy::TEvHttpIncomingResponse::TPtr& ev) { const auto& msg = *ev->Get(); - EXPORT_LOG_D("Handle NHttp::TEvHttpProxy::TEvHttpIncomingResponse" - << ": self# " << this->SelfId() - << ", status# " << (msg.Response ? msg.Response->Status : "null") - << ", body# " << (msg.Response ? msg.Response->Body : "null")); + YDB_LOG_DEBUG("[Export]", + {"self", this->SelfId()}, + {"status", (msg.Response ? msg.Response->Status : "null")}, + {"body", (msg.Response ? msg.Response->Body : "null")}); if (!msg.Response || !msg.Response->Status.StartsWith("200")) { - EXPORT_LOG_E("Error at 'GetProxy'" - << ": self# " << this->SelfId() - << ", error# " << msg.GetError()); + YDB_LOG_ERROR("[Export]", + {"self", this->SelfId()}, + {"error", msg.GetError()}); return RetryOrFinish(Aws::S3::S3Error({Aws::S3::S3Errors::SERVICE_UNAVAILABLE, true})); } if (msg.Response->Body.find('<') != TStringBuf::npos) { - EXPORT_LOG_E("Error at 'GetProxy'" - << ": self# " << this->SelfId() - << ", error# " << "invalid body" - << ", body# " << msg.Response->Body); + YDB_LOG_ERROR("[Export] body", + {"self", this->SelfId()}, + {"body", msg.Response->Body}, + {"error", "invalid"}); return RetryOrFinish(Aws::S3::S3Error({Aws::S3::S3Errors::SERVICE_UNAVAILABLE, true})); } @@ -160,9 +163,10 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { ProxyResolved = true; const auto& cfg = GetS3StorageConfig()->GetConfig(); - EXPORT_LOG_N("Using proxy: " - << (cfg.proxyScheme == Aws::Http::Scheme::HTTPS ? "https://" : "http://") - << cfg.proxyHost << ":" << cfg.proxyPort); + YDB_LOG_NOTICE("[Export]", + {"proxy", (cfg.proxyScheme == Aws::Http::Scheme::HTTPS ? "https://" : "http://")}, + {"proxyHost", cfg.proxyHost}, + {"proxyPort", cfg.proxyPort}); Restart(); } @@ -314,9 +318,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void HandleScheme(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - EXPORT_LOG_D("HandleScheme TEvExternalStorage::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); + YDB_LOG_DEBUG("[Export]", + {"result", result}); if (!CheckResult(result, TStringBuf("PutObject (scheme)"))) { return; @@ -338,9 +341,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void HandlePermissions(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - EXPORT_LOG_D("HandleMetadata TEvExternalStorage::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); + YDB_LOG_DEBUG("[Export]", + {"result", result}); if (!CheckResult(result, TStringBuf("PutObject (permissions)"))) { return; @@ -362,9 +364,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void HandleChangefeed(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - EXPORT_LOG_D("HandleChangefeed TEvExternalStorage::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); + YDB_LOG_DEBUG("[Export]", + {"result", result}); if (!CheckResult(result, TStringBuf("PutObject (changefeed)"))) { return; @@ -385,9 +386,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void HandleTopic(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - EXPORT_LOG_D("HandleTopic TEvExternalStorage::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); + YDB_LOG_DEBUG("[Export]", + {"result", result}); if (!CheckResult(result, TStringBuf("PutObject (topic)"))) { return; @@ -409,9 +409,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void HandleMetadata(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - EXPORT_LOG_D("HandleMetadata TEvExternalStorage::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); + YDB_LOG_DEBUG("[Export]", + {"result", result}); if (!CheckResult(result, TStringBuf("PutObject (metadata)"))) { return; @@ -437,9 +436,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void HandleChecksum(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - EXPORT_LOG_D("HandleChecksum TEvExternalStorage::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); + YDB_LOG_DEBUG("[Export]", + {"result", result}); if (!CheckResult(result, TStringBuf("PutObject (checksum)"))) { return; @@ -449,9 +447,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { } void Handle(TEvExportScan::TEvReady::TPtr& ev) { - EXPORT_LOG_D("Handle TEvExportScan::TEvReady" - << ": self# " << this->SelfId() - << ", sender# " << ev->Sender); + YDB_LOG_DEBUG("[Export]", + {"sender", ev->Sender}); Scanner = ev->Sender; @@ -466,16 +463,14 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { } void Handle(TEvBuffer::TPtr& ev) { - EXPORT_LOG_D("Handle TEvExportScan::TEvBuffer" - << ": self# " << this->SelfId() - << ", sender# " << ev->Sender - << ", msg# " << ev->Get()->ToString()); + YDB_LOG_DEBUG("[Export]", + {"sender", ev->Sender}, + {"msg", ev->Get()->ToString()}); if (ev->Sender != Scanner) { - EXPORT_LOG_W("Received buffer from unknown scanner" - << ": self# " << this->SelfId() - << ", sender# " << ev->Sender - << ", scanner# " << Scanner); + YDB_LOG_WARN("[Export]", + {"sender", ev->Sender}, + {"scanner", Scanner}); return; } @@ -509,9 +504,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void HandleData(TEvExternalStorage::TEvPutObjectResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - EXPORT_LOG_D("HandleData TEvExternalStorage::TEvPutObjectResponse" - << ": self# " << this->SelfId() - << ", result# " << result); + YDB_LOG_DEBUG("[Export]", + {"result", result}); if (!CheckResult(result, TStringBuf("PutObject (data)"))) { return; @@ -534,9 +528,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void Handle(TEvDataShard::TEvS3Upload::TPtr& ev) { auto& upload = ev->Get()->Upload; - EXPORT_LOG_D("Handle TEvDataShard::TEvS3Upload" - << ": self# " << this->SelfId() - << ", upload# " << upload); + YDB_LOG_DEBUG("[Export]", + {"upload", upload}); if (!upload) { auto request = Aws::S3::Model::CreateMultipartUploadRequest() @@ -593,9 +586,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void Handle(TEvExternalStorage::TEvCreateMultipartUploadResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - EXPORT_LOG_D("Handle TEvExternalStorage::TEvCreateMultipartUploadResponse" - << ": self# " << this->SelfId() - << ", result# " << result); + YDB_LOG_DEBUG("[Export]", + {"result", result}); if (!CheckResult(result, TStringBuf("CreateMultipartUpload"))) { return; @@ -607,9 +599,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void Handle(TEvExternalStorage::TEvUploadPartResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - EXPORT_LOG_D("Handle TEvExternalStorage::TEvUploadPartResponse" - << ": self# " << this->SelfId() - << ", result# " << result); + YDB_LOG_DEBUG("[Export]", + {"result", result}); if (!CheckResult(result, TStringBuf("UploadPart"))) { return; @@ -638,9 +629,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void Handle(TEvExternalStorage::TEvCompleteMultipartUploadResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - EXPORT_LOG_D("Handle TEvExternalStorage::TEvCompleteMultipartUploadResponse" - << ": self# " << this->SelfId() - << ", result# " << result); + YDB_LOG_DEBUG("[Export]", + {"result", result}); if (result.IsSuccess()) { return PassAway(); @@ -658,7 +648,13 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { UploadId.Clear(); // force getting info after restart Retry(); } else { - Error = TStringBuilder() << LogPrefix() << " error: " << error; + NActors::NStructuredLog::TTextWriter writer; + + TStringBuilder errorBuilder; + writer.Write(errorBuilder, LogPrefix()); + errorBuilder << " error: " << error; + + Error = errorBuilder; PassAway(); } } @@ -666,9 +662,8 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { void Handle(TEvExternalStorage::TEvAbortMultipartUploadResponse::TPtr& ev) { const auto& result = ev->Get()->Result; - EXPORT_LOG_D("Handle TEvExternalStorage::TEvAbortMultipartUploadResponse" - << ": self# " << this->SelfId() - << ", result# " << result); + YDB_LOG_DEBUG("[Export]", + {"result", result}); if (result.IsSuccess()) { return PassAway(); @@ -692,9 +687,9 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { return true; } - EXPORT_LOG_E("Error at '" << marker << "'" - << ": self# " << this->SelfId() - << ", error# " << result); + YDB_LOG_ERROR("[Export]", + {"marker", marker}, + {"error", result}); RetryOrFinish(result.GetError()); return false; @@ -714,17 +709,21 @@ class TS3Uploader: public TActorBootstrapped<TS3Uploader<TSettings>> { if (CanRetry(error)) { Retry(); } else { - Finish(false, TStringBuilder() << LogPrefix() << " error: " << error); + NStructuredLog::TTextWriter writer; + TStringBuilder errorBuilder; + writer.Write(errorBuilder, LogPrefix()); + errorBuilder << " error: " << error; + + Finish(false, errorBuilder); } } void Finish(bool success = true, const TString& error = TString()) { - EXPORT_LOG_I("Finish" - << ": self# " << this->SelfId() - << ", success# " << success - << ", error# " << error - << ", multipart# " << MultiPart - << ", uploadId# " << UploadId); + YDB_LOG_INFO("[Export]", + {"success", success}, + {"error", error}, + {"multipart", MultiPart}, + {"uploadId", UploadId}); if (!success) { Error = error; @@ -767,8 +766,11 @@ public: return NKikimrServices::TActivity::EXPORT_UPLOADER_ACTOR; } - static constexpr TStringBuf LogPrefix() { - return NBackup::NFieldsWrappers::GetStorageName<TSettings>(); + NActors::NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "S3Uploader"}, + {"selfId", this->SelfId()}, + {"storageName", NBackup::NFieldsWrappers::GetStorageName<TSettings>()}); } static TMaybe<THttpResolverConfig> GetHttpResolverConfigSafe( @@ -816,9 +818,9 @@ public: } void Bootstrap() { - EXPORT_LOG_D("Bootstrap" - << ": self# " << this->SelfId() - << ", attempt# " << Attempt); + YDB_LOG_CREATE_CONTEXT(LogPrefix()); + YDB_LOG_DEBUG("[Export]", + {"attempt", Attempt}); if constexpr (!RequiresHttpResolver<TSettings>) { ProxyResolved = true; @@ -834,6 +836,8 @@ public: } STATEFN(StateBase) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateBase"}); switch (ev->GetTypeRewrite()) { hFunc(TEvExportScan::TEvReady, Handle); @@ -843,6 +847,8 @@ public: } STATEFN(StateResolveProxy) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateResolveProxy"}); switch (ev->GetTypeRewrite()) { hFunc(NHttp::TEvHttpProxy::TEvHttpIncomingResponse, Handle); default: @@ -851,6 +857,8 @@ public: } STATEFN(StateUploadScheme) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateUploadScheme"}); switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleScheme); default: @@ -859,6 +867,8 @@ public: } STATEFN(StateUploadPermissions) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateUploadPermissions"}); switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvPutObjectResponse, HandlePermissions); default: @@ -867,6 +877,8 @@ public: } STATEFN(StateUploadChangefeed) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateUploadChangefeed"}); switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleChangefeed); default: @@ -875,6 +887,8 @@ public: } STATEFN(StateUploadTopic) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateUploadTopic"}); switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleTopic); default: @@ -883,6 +897,8 @@ public: } STATEFN(StateUploadMetadata) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateUploadMetadata"}); switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleMetadata); default: @@ -891,6 +907,8 @@ public: } STATEFN(StateUploadChecksum) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateUploadChecksum"}); switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvPutObjectResponse, HandleChecksum); default: @@ -899,6 +917,8 @@ public: } STATEFN(StateUploadData) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateUploadData"}); switch (ev->GetTypeRewrite()) { hFunc(TEvBuffer, Handle); hFunc(TEvDataShard::TEvS3Upload, Handle); diff --git a/ydb/core/tx/datashard/export_scan.cpp b/ydb/core/tx/datashard/export_scan.cpp index ec3b8b4dd22..21de3073690 100644 --- a/ydb/core/tx/datashard/export_scan.cpp +++ b/ydb/core/tx/datashard/export_scan.cpp @@ -10,6 +10,8 @@ #include <util/generic/bitmap.h> #include <util/string/builder.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::DATASHARD_BACKUP + namespace NKikimr { namespace NDataShard { @@ -114,8 +116,7 @@ class TExportScan: private NActors::IActorCallback, public IActorExceptionHandle void Handle(TEvExportScan::TEvReset::TPtr&) { Y_ENSURE(IsReady()); - EXPORT_LOG_D("Handle TEvExportScan::TEvReset" - << ": self# " << SelfId()); + YDB_LOG_DEBUG("[Export]"); Stats.Reset(new TStats); State.Reset(ES_UPLOADER_READY).Reset(ES_BUFFER_SENT).Reset(ES_NO_MORE_DATA); @@ -126,8 +127,7 @@ class TExportScan: private NActors::IActorCallback, public IActorExceptionHandle void Handle(TEvExportScan::TEvFeed::TPtr&) { Y_ENSURE(IsReady()); - EXPORT_LOG_D("Handle TEvExportScan::TEvFeed" - << ": self# " << SelfId()); + YDB_LOG_DEBUG("[Export]"); State.Set(ES_UPLOADER_READY).Reset(ES_BUFFER_SENT); Spent->Alter(true); @@ -139,9 +139,8 @@ class TExportScan: private NActors::IActorCallback, public IActorExceptionHandle void Handle(TEvExportScan::TEvFinish::TPtr& ev) { Y_ENSURE(IsReady()); - EXPORT_LOG_D("Handle TEvExportScan::TEvFinish" - << ": self# " << SelfId() - << ", msg# " << ev->Get()->ToString()); + YDB_LOG_DEBUG("[Export]", + {"msg", ev->Get()->ToString()}); Success = ev->Get()->Success; Error = ev->Get()->Error; @@ -149,8 +148,11 @@ class TExportScan: private NActors::IActorCallback, public IActorExceptionHandle } public: - static constexpr TStringBuf LogPrefix() { - return "scanner"sv; + + NActors::NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "ExportScan"}, + {"selfId", this->SelfId()}); } explicit TExportScan(std::function<IActor*()>&& createUploaderFn, IBuffer::TPtr buffer) @@ -205,7 +207,8 @@ public: if (!Buffer->Collect(row)) { Success = false; Error = Buffer->GetError(); - EXPORT_LOG_E("Error read data from table: " << Error); + YDB_LOG_ERROR("[Export]", + {"error", Error}); return EScan::Final; } @@ -248,6 +251,7 @@ public: } STATEFN(StateWork) { + YDB_LOG_CREATE_CONTEXT(LogPrefix()); switch (ev->GetTypeRewrite()) { hFunc(TEvExportScan::TEvReset, Handle); hFunc(TEvExportScan::TEvFeed, Handle); diff --git a/ydb/core/tx/datashard/finish_propose_unit.cpp b/ydb/core/tx/datashard/finish_propose_unit.cpp index f2ee9e530ce..f715b0e0863 100644 --- a/ydb/core/tx/datashard/finish_propose_unit.cpp +++ b/ydb/core/tx/datashard/finish_propose_unit.cpp @@ -5,6 +5,8 @@ #include "execution_unit_ctors.h" #include "probes.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + LWTRACE_USING(DATASHARD_PROVIDER) namespace NKikimr { @@ -160,15 +162,16 @@ void TFinishProposeUnit::CompleteRequest(TOperation::TPtr op, TDuration duration = TAppData::TimeProvider->Now() - op->GetReceivedAt(); res->Record.SetProposeLatency(duration.MilliSeconds()); - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "Propose transaction complete txid " << op->GetTxId() << " at tablet " - << DataShard.TabletID() << " send to client, exec latency: " - << res->Record.GetExecLatency() << " ms, propose latency: " - << duration.MilliSeconds() << " ms, status: " << res->GetStatus()); + YDB_LOG_TRACE_CTX(ctx, "TFinishProposeUnit::CompleteRequest: propose transaction complete, sending result to client", + {"txId", op->GetTxId()}, + {"tabletId", DataShard.TabletID()}, + {"execLatency", res->Record.GetExecLatency()}, + {"proposeLatency", duration.MilliSeconds()}, + {"status", res->GetStatus()}); TString errors = res->GetError(); if (errors.size()) { - LOG_LOG_S_THROTTLE(DataShard.GetLogThrottler(TDataShard::ELogThrottlerType::FinishProposeUnit_CompleteRequest), ctx, NActors::NLog::PRI_ERROR, NKikimrServices::TX_DATASHARD, + LOG_LOG_S_THROTTLE(DataShard.GetLogThrottler(TDataShard::ELogThrottlerType::FinishProposeUnit_CompleteRequest), ctx, NActors::NLog::PRI_ERROR, NKikimrServices::TX_DATASHARD, "Errors while proposing transaction txid " << op->GetTxId() << " at tablet " << DataShard.TabletID() << " status: " << res->GetStatus() << " errors: " << errors); diff --git a/ydb/core/tx/datashard/finish_propose_write_unit.cpp b/ydb/core/tx/datashard/finish_propose_write_unit.cpp index 7eabf4aa057..9df12524ecd 100644 --- a/ydb/core/tx/datashard/finish_propose_write_unit.cpp +++ b/ydb/core/tx/datashard/finish_propose_write_unit.cpp @@ -6,6 +6,8 @@ #include "execution_unit_ctors.h" #include "probes.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + LWTRACE_USING(DATASHARD_PROVIDER) namespace NKikimr { @@ -157,13 +159,14 @@ void TFinishProposeWriteUnit::CompleteRequest(TOperation::TPtr op, const TActorC TDuration duration = TAppData::TimeProvider->Now() - op->GetReceivedAt(); - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "Propose transaction complete txid " << op->GetTxId() << " at tablet " - << DataShard.TabletID() << " send to client, propose latency: " - << duration.MilliSeconds() << " ms, status: " << res->GetStatus()); + YDB_LOG_TRACE_CTX(ctx, "TFinishProposeWriteUnit::CompleteRequest: propose transaction complete, sending result to client", + {"txId", op->GetTxId()}, + {"tabletId", DataShard.TabletID()}, + {"latency", duration.MilliSeconds()}, + {"status", res->GetStatus()}); if (res->IsError()) { - LOG_LOG_S_THROTTLE(DataShard.GetLogThrottler(TDataShard::ELogThrottlerType::FinishProposeUnit_CompleteRequest), ctx, NActors::NLog::PRI_ERROR, NKikimrServices::TX_DATASHARD, + LOG_LOG_S_THROTTLE(DataShard.GetLogThrottler(TDataShard::ELogThrottlerType::FinishProposeUnit_CompleteRequest), ctx, NActors::NLog::PRI_ERROR, NKikimrServices::TX_DATASHARD, "Errors while proposing transaction txid " << op->GetTxId() << " at tablet " << DataShard.TabletID() << " " << res->GetError()); } @@ -221,8 +224,8 @@ void TFinishProposeWriteUnit::UpdateCounters(const TWriteOperation* writeOp, con } else { if (res->IsError()) { DataShard.IncCounter(COUNTER_WRITE_ERROR); - LOG_LOG_S_THROTTLE(DataShard.GetLogThrottler(TDataShard::ELogThrottlerType::FinishProposeUnit_UpdateCounters), ctx, NActors::NLog::PRI_ERROR, NKikimrServices::TX_DATASHARD, - "Prepare transaction failed. txid " << writeOp->GetTxId() + LOG_LOG_S_THROTTLE(DataShard.GetLogThrottler(TDataShard::ELogThrottlerType::FinishProposeUnit_UpdateCounters), ctx, NActors::NLog::PRI_ERROR, NKikimrServices::TX_DATASHARD, + "Prepare transaction failed. txid " << writeOp->GetTxId() << " at tablet " << DataShard.TabletID() << " errors: " << res->GetError()); } else { DataShard.IncCounter(COUNTER_WRITE_IMMEDIATE); diff --git a/ydb/core/tx/datashard/import_common.h b/ydb/core/tx/datashard/import_common.h index ebe5ea9ed3d..d7c4f73b150 100644 --- a/ydb/core/tx/datashard/import_common.h +++ b/ydb/core/tx/datashard/import_common.h @@ -12,24 +12,6 @@ #include <util/generic/string.h> #include <util/generic/vector.h> -#if defined IMPORT_LOG_T || \ - defined IMPORT_LOG_D || \ - defined IMPORT_LOG_I || \ - defined IMPORT_LOG_N || \ - defined IMPORT_LOG_W || \ - defined IMPORT_LOG_E || \ - defined IMPORT_LOG_C -#error log macro redefinition -#endif - -#define IMPORT_LOG_T(stream) LOG_TRACE_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_D(stream) LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_I(stream) LOG_INFO_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_N(stream) LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_W(stream) LOG_WARN_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_E(stream) LOG_ERROR_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) -#define IMPORT_LOG_C(stream) LOG_CRIT_S(*TlsActivationContext, NKikimrServices::DATASHARD_RESTORE, "[Import] [" << LogPrefix() << "] " << stream) - namespace NKikimr { namespace NDataShard { diff --git a/ydb/core/tx/datashard/import_s3.cpp b/ydb/core/tx/datashard/import_s3.cpp index ec1ffced0bc..e1762b34996 100644 --- a/ydb/core/tx/datashard/import_s3.cpp +++ b/ydb/core/tx/datashard/import_s3.cpp @@ -37,6 +37,8 @@ #include <util/memory/pool.h> #include <util/string/builder.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::DATASHARD_RESTORE + namespace { struct DestroyZCtx { @@ -647,7 +649,8 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void AllocateResource() { - IMPORT_LOG_D("AllocateResource"); + YDB_LOG_DEBUG("[Import] Submitting resource broker task", + {"logPrefix", LogPrefix()}); const auto* appData = AppData(); this->Send(MakeResourceBrokerID(), new TEvResourceBroker::TEvSubmitTask( @@ -662,9 +665,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void Handle(TEvResourceBroker::TEvResourceAllocated::TPtr& ev) { - IMPORT_LOG_I("Handle TEvResourceBroker::TEvResourceAllocated {" - << " TaskId: " << ev->Get()->TaskId - << " }"); + YDB_LOG_INFO("[Import] Resource broker task allocated", + {"logPrefix", LogPrefix()}, + {"taskId", ev->Get()->TaskId}); TaskId = ev->Get()->TaskId; Restart(); @@ -688,8 +691,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void Restart() { - IMPORT_LOG_N("Restart" - << ": attempt# " << Attempt); + YDB_LOG_NOTICE("[Import] Restarting import", + {"logPrefix", LogPrefix()}, + {"attempt", Attempt}); if (const TActorId client = std::exchange(Client, TActorId())) { this->Send(client, new TEvents::TEvPoisonPill()); @@ -702,8 +706,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void HeadObject(const TString& key) { - IMPORT_LOG_D("HeadObject" - << ": key# " << key); + YDB_LOG_DEBUG("[Import] Sending HeadObject request", + {"logPrefix", LogPrefix()}, + {"key", key}); auto request = Model::HeadObjectRequest() .WithKey(key); @@ -712,9 +717,11 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void GetObject(const TString& key, const std::pair<ui64, ui64>& range) { - IMPORT_LOG_D("GetObject" - << ": key# " << key - << ", range# " << range.first << "-" << range.second); + YDB_LOG_DEBUG("[Import] Sending GetObject request", + {"logPrefix", LogPrefix()}, + {"key", key}, + {"range", range.first}, + {"rangeEnd", range.second}); auto request = Model::GetObjectRequest() .WithKey(key) @@ -724,7 +731,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void Handle(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { - IMPORT_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("[Import] Received HeadObject response", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->ToString()}); const auto& result = ev->Get()->Result; if (!result.IsSuccess()) { @@ -733,8 +742,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { case S3Errors::NO_SUCH_KEY: break; default: - IMPORT_LOG_E("Error at 'HeadObject'" - << ": error# " << result); + YDB_LOG_ERROR("[Import] HeadObject request failed", + {"logPrefix", LogPrefix()}, + {"error", result}); return RetryOrFinish(result.GetError()); } @@ -783,7 +793,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { // Encrypted file can not have zero length const TString error = TStringBuilder() << Settings.GetDataKey(DataFormat, CompressionCodec) << ": file is corrupted"; - IMPORT_LOG_E(error); + YDB_LOG_ERROR("[Import] Import data file is corrupted", + {"logPrefix", LogPrefix()}, + {"error", error}); return Finish(false, error); } @@ -796,9 +808,10 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void Handle(TEvDataShard::TEvS3DownloadInfo::TPtr& ev) { - IMPORT_LOG_D("Handle " << ev->Get()->ToString()); - const auto& info = ev->Get()->Info; + YDB_LOG_DEBUG("[Import] Received S3 download info", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->ToString()}); if (!info.DataETag) { this->Send(DataShard, new TEvDataShard::TEvStoreS3DownloadInfo(TxId, { ETag, ProcessedBytes, WrittenBytes, WrittenRows, ProcessedChecksumState, DownloadState @@ -810,8 +823,10 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void ProcessDownloadInfo(const TS3Download& info, const TStringBuf marker, bool loadState = false) { - IMPORT_LOG_N("Process download info at '" << marker << "'" - << ": info# " << info); + YDB_LOG_NOTICE("[Import]", + {"logPrefix", LogPrefix()}, + {"marker", marker}, + {"info", info}); Y_ENSURE(info.DataETag); if (!CheckETag(*info.DataETag, ETag, marker)) { @@ -826,7 +841,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { if (TString restoreErr; !Reader->RestoreFromState(DownloadState, restoreErr)) { const TString error = TStringBuilder() << Settings.GetDataKey(DataFormat, CompressionCodec) << ": failed to restore reader state: " << restoreErr; - IMPORT_LOG_E(error); + YDB_LOG_ERROR("[Import]", + {"logPrefix", LogPrefix()}, + {"error", error}); return Finish(false, error); } } @@ -852,7 +869,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void Handle(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { - IMPORT_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("[Import]", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->ToString()}); auto& msg = *ev->Get(); const auto& result = msg.Result; @@ -866,10 +885,11 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { return; } - IMPORT_LOG_T("Content size" - << ": processed-bytes# " << ProcessedBytes - << ", content-length# " << ContentLength - << ", body-size# " << msg.Body.size()); + YDB_LOG_TRACE("[Import]", + {"logPrefix", LogPrefix()}, + {"processedBytes", ProcessedBytes}, + {"contentLength", ContentLength}, + {"bodySize", msg.Body.size()}); *Counters.BytesReceived += msg.Body.size(); Counters.LatencyRead.Finish(Now()); @@ -880,7 +900,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void HandleChecksum(TEvExternalStorage::TEvHeadObjectResponse::TPtr& ev) { - IMPORT_LOG_D("HandleChecksum " << ev->Get()->ToString()); + YDB_LOG_DEBUG("[Import]", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->ToString()}); const auto& result = ev->Get()->Result; @@ -894,7 +916,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void HandleChecksum(TEvExternalStorage::TEvGetObjectResponse::TPtr& ev) { - IMPORT_LOG_D("HandleChecksum " << ev->Get()->ToString()); + YDB_LOG_DEBUG("[Import]", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->ToString()}); auto& msg = *ev->Get(); const auto& result = msg.Result; @@ -1039,12 +1063,15 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { // --- Direct part import (EnableDataShardDirectPartImport) --- void BeginDirectImport() { - IMPORT_LOG_I("Begin direct part write"); + YDB_LOG_INFO("[Import] Beginning direct import", + {"logPrefix", LogPrefix()}); this->Send(DataShard, new TEvDataShard::TEvS3DirectWriteBegin(TxId, TableInfo.GetId())); } void Handle(TEvDataShard::TEvS3DirectWriteBeginResult::TPtr& ev) { - IMPORT_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("[Import]", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->ToString()}); auto* msg = ev->Get(); if (!msg->Success) { @@ -1105,16 +1132,19 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { info.WrittenRows = WrittenRows; info.ProcessedChecksumState = ProcessedChecksumState; - IMPORT_LOG_I("Finish direct part write" - << ": writtenBytes# " << WrittenBytes - << ", writtenRows# " << WrittenRows); + YDB_LOG_INFO("[Import] Direct import completed", + {"logPrefix", LogPrefix()}, + {"writtenBytes", WrittenBytes}, + {"writtenRows", WrittenRows}); this->Send(DataShard, new TEvDataShard::TEvS3DirectWriteFinish( TxId, TableInfo.GetId(), std::move(result), info)); } void Handle(TEvDataShard::TEvS3DirectWriteFinishResult::TPtr& ev) { - IMPORT_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("[Import]", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->ToString()}); auto* msg = ev->Get(); if (!msg->Success) { @@ -1157,9 +1187,10 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { void UploadRows() { const auto& record = RequestBuilder.GetRecord(); - IMPORT_LOG_I("Upload rows" - << ": count# " << record->RowsSize() - << ", size# " << record->ByteSizeLong()); + YDB_LOG_INFO("[Import]", + {"logPrefix", LogPrefix()}, + {"count", record->RowsSize()}, + {"size", record->ByteSizeLong()}); Counters.LatencyProcess.Finish(Now()); Counters.LatencyWrite.Start(Now()); @@ -1170,7 +1201,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { } void Handle(TEvDataShard::TEvS3UploadRowsResponse::TPtr& ev) { - IMPORT_LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("[Import]", + {"logPrefix", LogPrefix()}, + {"ev", ev->Get()->ToString()}); *Counters.BytesWritten += RequestBuilder.GetCellBytes(); Counters.LatencyWrite.Finish(Now()); @@ -1193,8 +1226,10 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { return true; } - IMPORT_LOG_E("Error at '" << marker << "'" - << ": error# " << result); + YDB_LOG_ERROR("[Import]", + {"logPrefix", LogPrefix()}, + {"marker", marker}, + {"error", result}); RetryOrFinish(result.GetError()); return false; @@ -1210,7 +1245,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { << ": expected '" << expected << "'" << ", got '" << got << "'"; - IMPORT_LOG_E(error); + YDB_LOG_ERROR("[Import]", + {"logPrefix", LogPrefix()}, + {"error", error}); Finish(false, error); return false; @@ -1218,7 +1255,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { bool CheckScheme() { auto finish = [this](const TString& error) -> bool { - IMPORT_LOG_E(error); + YDB_LOG_ERROR("[Import]", + {"logPrefix", LogPrefix()}, + {"error", error}); Finish(false, error); return false; @@ -1278,7 +1317,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { << ": expected '" << ExpectedChecksum << "'" << ", got '" << gotChecksum << "'"; - IMPORT_LOG_E(error); + YDB_LOG_ERROR("[Import]", + {"logPrefix", LogPrefix()}, + {"error", error}); Finish(false, error); return false; @@ -1321,11 +1362,12 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader<TSettings>> { void Finish(bool success = true, const TString& error = TString()) { DownloadInterrupted = true; - IMPORT_LOG_N("Finish" - << ": success# " << success - << ", error# " << error - << ", writtenBytes# " << WrittenBytes - << ", writtenRows# " << WrittenRows); + YDB_LOG_NOTICE("[Import]", + {"logPrefix", LogPrefix()}, + {"success", success}, + {"error", error}, + {"writtenBytes", WrittenBytes}, + {"writtenRows", WrittenRows}); // If a direct part write was reserved but never handed off, drop its barrier // so the uncommitted blobs get garbage collected. @@ -1363,8 +1405,12 @@ public: return NBackup::NFieldsWrappers::GetStorageName<TSettings>(); } - TStringBuf LogPrefix() const { - return LogPrefix_; + NActors::NStructuredLog::TStructuredMessage LogPrefix() { + return YDB_LOG_CREATE_MESSAGE( + {"actorClassName", "S3Downloader"}, + {"selfId", this->SelfId()}, + {"txId", TxId}, + {"storageName", NBackup::NFieldsWrappers::GetStorageName<TSettings>()}); } static TSettings GetSettings(const NKikimrSchemeOp::TRestoreTask& task); @@ -1382,7 +1428,6 @@ public: , CompressionCodec(NBackupRestoreTraits::ECompressionCodec::None) , TableInfo(tableInfo) , Scheme(task.GetTableDescription()) - , LogPrefix_(TStringBuilder() << PartLogPrefix() << ":" << TxId) , Retries(task.GetNumberOfRetries()) , ReadBatchSize(GetReadBatchSize(task)) , ReadBufferSizeLimit(AppData()->DataShardConfig.GetRestoreReadBufferSizeLimit()) @@ -1394,8 +1439,9 @@ public: } void Bootstrap() { - IMPORT_LOG_D("Bootstrap" - << ": attempt# " << Attempt); + YDB_LOG_CREATE_CONTEXT(LogPrefix()); + YDB_LOG_DEBUG("[Import]", + {"attempt", Attempt}); if (!CheckScheme()) { return; @@ -1409,6 +1455,8 @@ public: } STATEFN(StateAllocateResource) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateAllocateResource"}); switch (ev->GetTypeRewrite()) { hFunc(TEvResourceBroker::TEvResourceAllocated, Handle); sFunc(TEvents::TEvPoisonPill, NotifyDied); @@ -1416,6 +1464,8 @@ public: } STATEFN(StateDownloadData) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateDownloadData"}); switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvHeadObjectResponse, Handle); hFunc(TEvExternalStorage::TEvGetObjectResponse, Handle); @@ -1433,6 +1483,8 @@ public: } STATEFN(StateDownloadChecksum) { + YDB_LOG_CREATE_CONTEXT(LogPrefix(), + {"actorState", "StateDownloadChecksum"}); switch (ev->GetTypeRewrite()) { hFunc(TEvExternalStorage::TEvHeadObjectResponse, HandleChecksum); hFunc(TEvExternalStorage::TEvGetObjectResponse, HandleChecksum); @@ -1451,7 +1503,6 @@ private: NBackupRestoreTraits::ECompressionCodec CompressionCodec; const TTableInfo TableInfo; const NKikimrSchemeOp::TTableDescription Scheme; - const TString LogPrefix_; const ui32 Retries; ui32 Attempt = 0; diff --git a/ydb/core/tx/datashard/incr_restore_scan.cpp b/ydb/core/tx/datashard/incr_restore_scan.cpp index 0de81880fd4..05b625a715e 100644 --- a/ydb/core/tx/datashard/incr_restore_scan.cpp +++ b/ydb/core/tx/datashard/incr_restore_scan.cpp @@ -12,6 +12,8 @@ #include <ydb/library/actors/core/actor.h> #include <ydb/library/services/services.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::CHANGE_EXCHANGE + namespace NKikimr::NDataShard { using namespace NActors; @@ -95,7 +97,9 @@ public: } void Start(TEvIncrementalRestoreScan::TEvServe::TPtr& ev) { - LOG_D("Handle TEvIncrementalRestoreScan::TEvServe " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle TEvIncrementalRestoreScan::TEvServe", + {"logPrefix", GetLogPrefix()}, + {"ev", ev->Get()->ToString()}); // Store/update the actorId on each command receipt (handles SchemeShard restarts) OperatorActorId = ev->Sender; @@ -104,7 +108,9 @@ public: } void Handle(NChangeExchange::TEvChangeExchange::TEvRequestRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"logPrefix", GetLogPrefix()}, + {"ev", ev->Get()->ToString()}); TVector<TChangeRecord::TPtr> records(::Reserve(ev->Get()->Records.size())); @@ -118,7 +124,9 @@ public: } void Handle(NChangeExchange::TEvChangeExchange::TEvRemoveRecords::TPtr& ev) { - LOG_D("Handle " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle", + {"logPrefix", GetLogPrefix()}, + {"ev", ev->Get()->ToString()}); for (auto recordId : ev->Get()->Records) { PendingRecords.erase(recordId); @@ -128,7 +136,9 @@ public: } void Handle(TEvIncrementalRestoreScan::TEvFinished::TPtr& ev) { - LOG_D("Handle TEvIncrementalRestoreScan::TEvFinished " << ev->Get()->ToString()); + YDB_LOG_DEBUG("Handle TEvIncrementalRestoreScan::TEvFinished", + {"logPrefix", GetLogPrefix()}, + {"ev", ev->Get()->ToString()}); Driver->Touch(EScan::Final); } @@ -151,7 +161,8 @@ public: } EScan Seek(TLead& lead, ui64) override { - LOG_D("Seek"); + YDB_LOG_DEBUG("Seek", + {"logPrefix", GetLogPrefix()}); if (LastKey) { lead.To(ValueTags, LastKey->GetCells(), ESeek::Upper); @@ -179,7 +190,8 @@ public: } EScan Exhausted() override { - LOG_D("Exhausted"); + YDB_LOG_DEBUG("Exhausted", + {"logPrefix", GetLogPrefix()}); NoMoreData = true; @@ -192,15 +204,19 @@ public: } TAutoPtr<IDestructable> Finish(EStatus status) override { - LOG_D("Finish " << status); + YDB_LOG_DEBUG("Finish", + {"logPrefix", GetLogPrefix()}, + {"status", status}); const bool success = IsScanSuccess(status); const auto endStatus = MapScanStatus(status); if (!success) { // Error propagation: see github.com/ydb-platform/ydb/issues/18797 // DS classifies cause (EndStatus); SS owns retry policy. - LOG_E("IncrementalRestoreScan finished with error status: " << status - << " endStatus=" << static_cast<int>(endStatus)); + YDB_LOG_ERROR("IncrementalRestoreScan finished with error", + {"logPrefix", GetLogPrefix()}, + {"status", status}, + {"endStatus", static_cast<int>(endStatus)}); } TString errorMsg; diff --git a/ydb/core/tx/datashard/incremental_restore_src_actor.cpp b/ydb/core/tx/datashard/incremental_restore_src_actor.cpp index 801cee52f6d..e75ebbbadd2 100644 --- a/ydb/core/tx/datashard/incremental_restore_src_actor.cpp +++ b/ydb/core/tx/datashard/incremental_restore_src_actor.cpp @@ -13,6 +13,8 @@ #include <ydb/library/actors/core/hfunc.h> #include <ydb/library/services/services.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr::NDataShard { using namespace NActors; @@ -120,11 +122,13 @@ public: .SetReadAhead(readAheadLo, readAheadHi) .SetReadPrio(TScanOptions::EReadPrio::Low)); - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, - "TIncrementalRestoreSrcActor[" << SelfId() << "] started scan task " << ScanTaskId - << " SubOpTxId=" << SubOpTxId - << " SrcPathId=" << SrcPathId << " DstPathId=" << DstPathId - << " at tablet " << Self->TabletID()); + YDB_LOG_INFO_CTX(ctx, "TIncrementalRestoreSrcActor: started scan task", + {"selfId", SelfId()}, + {"scanTaskId", ScanTaskId}, + {"subOpTxId", SubOpTxId}, + {"srcPathId", SrcPathId}, + {"dstPathId", DstPathId}, + {"tabletId", Self->TabletID()}); } private: @@ -135,10 +139,11 @@ private: return; } - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, - "TIncrementalRestoreSrcActor[" << SelfId() << "] scan finished" - << " SubOpTxId=" << SubOpTxId << " success=" << msg->Success - << " endStatus=" << static_cast<int>(msg->EndStatus)); + YDB_LOG_INFO_CTX(ctx, "TIncrementalRestoreSrcActor: scan finished", + {"selfId", SelfId()}, + {"subOpTxId", SubOpTxId}, + {"success", msg->Success}, + {"endStatus", static_cast<int>(msg->EndStatus)}); ReplyAndDie(ctx, msg->Success, msg->EndStatus, msg->Error); } @@ -202,12 +207,12 @@ namespace NKikimr::NDataShard { void TDataShard::Handle(TEvDataShard::TEvIncrementalRestoreSrcCreateRequest::TPtr& ev, const TActorContext& ctx) { const auto& rec = ev->Get()->Record; - LOG_INFO_S(ctx, NKikimrServices::TX_DATASHARD, - "TEvIncrementalRestoreSrcCreateRequest received at tablet " << TabletID() - << " OperationId=" << rec.GetOperationId() - << " SubOpTxId=" << rec.GetSubOpTxId() - << " ShardIdx=" << rec.GetShardIdx() - << " SchemeShardGeneration=" << rec.GetSchemeShardGeneration()); + YDB_LOG_INFO_CTX(ctx, "TIncrementalRestoreSrcActor: TEvIncrementalRestoreSrcCreateRequest received", + {"tabletId", TabletID()}, + {"operationId", rec.GetOperationId()}, + {"subOpTxId", rec.GetSubOpTxId()}, + {"shardIdx", rec.GetShardIdx()}, + {"schemeShardGeneration", rec.GetSchemeShardGeneration()}); CreateIncrementalRestoreSrcActor(this, rec); } diff --git a/ydb/core/tx/datashard/key_validator.cpp b/ydb/core/tx/datashard/key_validator.cpp index f71b018e451..afaa91d5467 100644 --- a/ydb/core/tx/datashard/key_validator.cpp +++ b/ydb/core/tx/datashard/key_validator.cpp @@ -7,6 +7,8 @@ #include <ydb/library/actors/core/log.h> #include <ydb/library/services/services.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + using namespace NKikimr; @@ -30,7 +32,9 @@ void TKeyValidator::AddReadRange(const TTableId& tableId, const TVector<NTable:: columnOps.emplace_back(std::move(op)); } - LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::TX_DATASHARD, "-- AddReadRange: " << DebugPrintRange(keyTypes, range, *AppData()->TypeRegistry) << " table: " << tableId); + YDB_LOG_TRACE("Adding read range to key validator", + {"readRange", DebugPrintRange(keyTypes, range, *AppData()->TypeRegistry)}, + {"tableId", tableId}); auto desc = MakeHolder<TKeyDesc>(tableId, range, TKeyDesc::ERowOperation::Read, keyTypes, columnOps, itemsLimit, 0 /* bytesLimit */, reverse); @@ -51,7 +55,9 @@ void TKeyValidator::AddWriteRange(const TTableId& tableId, const TTableRange& ra columnOps.emplace_back(std::move(op)); } - LOG_TRACE_S(*NActors::TlsActivationContext, NKikimrServices::TX_DATASHARD, "-- AddWriteRange: " << DebugPrintRange(keyTypes, range, *AppData()->TypeRegistry) << " table: " << tableId); + YDB_LOG_TRACE("Adding write range to key validator", + {"writeRange", DebugPrintRange(keyTypes, range, *AppData()->TypeRegistry)}, + {"tableId", tableId}); auto rowOp = isPureEraseOp ? TKeyDesc::ERowOperation::Erase : TKeyDesc::ERowOperation::Update; auto desc = MakeHolder<TKeyDesc>(tableId, range, rowOp, keyTypes, columnOps); diff --git a/ydb/core/tx/datashard/memory_state_migration.cpp b/ydb/core/tx/datashard/memory_state_migration.cpp index 680b432ee54..3f515172f3f 100644 --- a/ydb/core/tx/datashard/memory_state_migration.cpp +++ b/ydb/core/tx/datashard/memory_state_migration.cpp @@ -3,6 +3,8 @@ #include <ydb/core/protos/datashard_config.pb.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr::NDataShard { static constexpr size_t MAX_DATASHARD_STATE_CHUNK_SIZE = 8_MB; @@ -102,14 +104,12 @@ private: offset += prevSize; // Try to fail gracefully instead of crashing on unexpected data if (offset < lastOffset) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Received TEvInMemoryStateResponse with checkpoints that go backwards"); + YDB_LOG_CRIT("Received TEvInMemoryStateResponse with checkpoints that go backwards"); Failed(); return; } if (Buffer.size() < offset) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Received TEvInMemoryStateResponse with checkpoints that overflow current buffer"); + YDB_LOG_CRIT("Received TEvInMemoryStateResponse with checkpoints that overflow current buffer"); Failed(); return; } @@ -133,8 +133,7 @@ private: TRopeStream stream(Buffer.Begin(), chunkSize); bool ok = state->ParseFromZeroCopyStream(&stream); if (!ok) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Received TEvInMemoryStateResponse has a chunk that cannot be parsed"); + YDB_LOG_CRIT("Received TEvInMemoryStateResponse has a chunk that cannot be parsed"); Failed(); return; } @@ -184,8 +183,8 @@ private: for (const auto& protoRange : state->GetLockRanges()) { auto* row = Locks.FindPtr(protoRange.GetLockId()); if (!row) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Received lock range for a missing lock " << protoRange.GetLockId()); + YDB_LOG_CRIT("Received lock range for a missing lock", + {"rangeLockId", protoRange.GetLockId()}); Failed(); return; } @@ -197,8 +196,8 @@ private: for (const auto& protoConflict : state->GetLockConflicts()) { auto* row = Locks.FindPtr(protoConflict.GetLockId()); if (!row) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Received lock conflict for a missing lock " << protoConflict.GetLockId()); + YDB_LOG_CRIT("Received lock conflict for a missing lock", + {"conflictLockId", protoConflict.GetLockId()}); Failed(); return; } @@ -207,8 +206,8 @@ private: for (const auto& protoVolatileDep : state->GetLockVolatileDependencies()) { auto* row = Locks.FindPtr(protoVolatileDep.GetLockId()); if (!row) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Received volatile dependency for a missing lock " << protoVolatileDep.GetLockId()); + YDB_LOG_CRIT("Received volatile dependency for a missing lock", + {"volatileDepLockId", protoVolatileDep.GetLockId()}); Failed(); return; } diff --git a/ydb/core/tx/datashard/move_index_unit.cpp b/ydb/core/tx/datashard/move_index_unit.cpp index eb18eaaa6cc..fff42225037 100644 --- a/ydb/core/tx/datashard/move_index_unit.cpp +++ b/ydb/core/tx/datashard/move_index_unit.cpp @@ -2,6 +2,8 @@ #include "datashard_pipeline.h" #include "execution_unit_ctors.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -84,10 +86,10 @@ public: return EExecutionStatus::Restart; } - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "TMoveIndexUnit Execute" - << ": schemeTx# " << schemeTx.DebugString() - << ": changeRecords size# " << op->ChangeRecords().size() - << ", at tablet# " << DataShard.TabletID()); + YDB_LOG_DEBUG("TMoveIndexUnit Execute changeRecords", + {"schemeTx", schemeTx.DebugString()}, + {"size", op->ChangeRecords().size()}, + {"tablet", DataShard.TabletID()}); DataShard.SuspendChangeSender(ctx); diff --git a/ydb/core/tx/datashard/move_table_unit.cpp b/ydb/core/tx/datashard/move_table_unit.cpp index 10d283431cd..f21019293c8 100644 --- a/ydb/core/tx/datashard/move_table_unit.cpp +++ b/ydb/core/tx/datashard/move_table_unit.cpp @@ -2,6 +2,8 @@ #include "datashard_pipeline.h" #include "execution_unit_ctors.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -84,10 +86,10 @@ public: return EExecutionStatus::Restart; } - LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "TMoveTableUnit Execute" - << ": schemeTx# " << schemeTx.DebugString() - << ": changeRecords size# " << op->ChangeRecords().size() - << ", at tablet# " << DataShard.TabletID()); + YDB_LOG_DEBUG("TMoveTableUnit Execute changeRecords", + {"schemeTx", schemeTx.DebugString()}, + {"size", op->ChangeRecords().size()}, + {"tablet", DataShard.TabletID()}); DataShard.SuspendChangeSender(ctx); diff --git a/ydb/core/tx/datashard/operation.cpp b/ydb/core/tx/datashard/operation.cpp index cfa44e8e8f1..a462a31fe90 100644 --- a/ydb/core/tx/datashard/operation.cpp +++ b/ydb/core/tx/datashard/operation.cpp @@ -4,6 +4,8 @@ #include <ydb/library/actors/core/monotonic_provider.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -62,9 +64,11 @@ void TOperation::AddInReadSet(const TReadSetKey &rsKey, auto it = CoverageBuilders().find(std::make_pair(rsKey.From, rsKey.To)); if (it != CoverageBuilders().end()) { if (it->second->AddResult(btList)) { - LOG_TRACE_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Filled readset for " << *this << " from=" << rsKey.From - << " to=" << rsKey.To << "origin=" << rsKey.Origin); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "Filled readset", + {"operation", *this}, + {"from", rsKey.From}, + {"to", rsKey.To}, + {"origin", rsKey.Origin}); InReadSets()[it->first].emplace_back(TRSData{ std::move(readSet), rsKey.Origin }); if (it->second->IsComplete()) { Y_ENSURE(InputDataRef().RemainReadSets > 0, "RemainReadSets counter underflow"); @@ -72,9 +76,11 @@ void TOperation::AddInReadSet(const TReadSetKey &rsKey, } } } else { - LOG_NOTICE_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "Discarded readset for " << *this << " from=" << rsKey.From - << " to=" << rsKey.To << "origin=" << rsKey.Origin); + YDB_LOG_NOTICE_CTX(TActivationContext::AsActorContext(), "Discarded readset", + {"operation", *this}, + {"from", rsKey.From}, + {"to", rsKey.To}, + {"origin", rsKey.Origin}); } } diff --git a/ydb/core/tx/datashard/plan_queue_unit.cpp b/ydb/core/tx/datashard/plan_queue_unit.cpp index 052444c76a1..2a48f149876 100644 --- a/ydb/core/tx/datashard/plan_queue_unit.cpp +++ b/ydb/core/tx/datashard/plan_queue_unit.cpp @@ -2,6 +2,8 @@ #include "datashard_pipeline.h" #include "execution_unit_ctors.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -47,16 +49,14 @@ bool TPlanQueueUnit::IsReadyToExecute(TOperation::TPtr op) const TOperation::TPtr TPlanQueueUnit::FindReadyOperation() const { if (Pipeline.OutOfOrderLimits()) { - LOG_TRACE_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "TPlanQueueUnit at " << DataShard.TabletID() - << " out-of-order limits exceeded"); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "TPlanQueueUnit::FindReadyOperation: out-of-order limits exceeded", + {"tabletId", DataShard.TabletID()}); return nullptr; } if (!OpsInFly.size()) { - LOG_TRACE_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "TPlanQueueUnit at " << DataShard.TabletID() - << " has no attached operations"); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "TPlanQueueUnit::FindReadyOperation: has no attached operations", + {"tabletId", DataShard.TabletID()}); return nullptr; } @@ -65,32 +65,32 @@ TOperation::TPtr TPlanQueueUnit::FindReadyOperation() const auto op = Pipeline.GetNextPlannedOp(step, txId); if (!op) { - LOG_TRACE_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "TPlanQueueUnit at " << DataShard.TabletID() - << " couldn't find next planned operation after [" - << step << ":" << txId << "]"); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "TPlanQueueUnit::FindReadyOperation: couldn't find next planned operation", + {"tabletId", DataShard.TabletID()}, + {"step", step}, + {"txId", txId}); return nullptr; } if (op->IsInProgress()) { - LOG_TRACE_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "TPlanQueueUnit at " << DataShard.TabletID() - << " found next planned operation " << *op << " which is already in progress"); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "TPlanQueueUnit::FindReadyOperation: found next planned operation is already in progress", + {"tabletId", DataShard.TabletID()}, + {"operation", *op}); return nullptr; } if (!Pipeline.CanRunOp(*op)) { - LOG_TRACE_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "TPlanQueueUnit at " << DataShard.TabletID() - << " cannot run found next planned operation " << *op); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "TPlanQueueUnit::FindReadyOperation: cannot run found next planned operation", + {"tabletId", DataShard.TabletID()}, + {"operation", *op}); return nullptr; } if (op->GetCurrentUnit() != Kind) { - LOG_TRACE_S(TActivationContext::AsActorContext(), NKikimrServices::TX_DATASHARD, - "TPlanQueueUnit at " << DataShard.TabletID() - << " found next planned operation " << *op - << " is executing on unit " << op->GetCurrentUnit()); + YDB_LOG_TRACE_CTX(TActivationContext::AsActorContext(), "TPlanQueueUnit::FindReadyOperation: found next planned operation is executing on another unit", + {"tabletId", DataShard.TabletID()}, + {"operation", *op}, + {"currentUnit", op->GetCurrentUnit()}); return nullptr; } diff --git a/ydb/core/tx/datashard/prepare_data_tx_in_rs_unit.cpp b/ydb/core/tx/datashard/prepare_data_tx_in_rs_unit.cpp index dfefa5b5a2f..1e0d5c4d27a 100644 --- a/ydb/core/tx/datashard/prepare_data_tx_in_rs_unit.cpp +++ b/ydb/core/tx/datashard/prepare_data_tx_in_rs_unit.cpp @@ -2,6 +2,8 @@ #include "datashard_pipeline.h" #include "execution_unit_ctors.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -77,11 +79,11 @@ EExecutionStatus TPrepareDataTxInRSUnit::Execute(TOperation::TPtr op, TVector<TRSData>())); } } catch (const TMemoryLimitExceededException &) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "Operation " << *op << " at " << DataShard.TabletID() - << " exceeded memory limit " << txc.GetMemoryLimit() - << " and requests " << txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR - << " more for the next try"); + YDB_LOG_TRACE_CTX(ctx, "TPrepareDataTxInRSUnit::Execute: exceeded memory limit and requests more for the next try", + {"operation", *op}, + {"tabletId", DataShard.TabletID()}, + {"memoryLimit", txc.GetMemoryLimit()}, + {"memoryLimitWithFactor", txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR}); engine->ReleaseUnusedMemory(); txc.RequestMemory(txc.GetMemoryLimit() * MEMORY_REQUEST_FACTOR); @@ -90,9 +92,9 @@ EExecutionStatus TPrepareDataTxInRSUnit::Execute(TOperation::TPtr op, return EExecutionStatus::Restart; } catch (const TNotReadyTabletException&) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "Tablet " << DataShard.TabletID() << " is not ready for " << *op - << " execution"); + YDB_LOG_TRACE_CTX(ctx, "TPrepareDataTxInRSUnit::Execute: tablet is not ready for execution", + {"tabletId", DataShard.TabletID()}, + {"operation", *op}); return EExecutionStatus::Restart; } diff --git a/ydb/core/tx/datashard/read_table_scan.cpp b/ydb/core/tx/datashard/read_table_scan.cpp index d556f802291..628d5707dcb 100644 --- a/ydb/core/tx/datashard/read_table_scan.cpp +++ b/ydb/core/tx/datashard/read_table_scan.cpp @@ -14,6 +14,8 @@ //#include <util/generic/cast.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -439,9 +441,9 @@ public: HFunc(TEvDataShard::TEvGetReadTableScanStateRequest, Handle); IgnoreFunc(TEvInterconnect::TEvNodeConnected); default: - LOG_ERROR(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "TReadTableScan: StateWork unexpected event type: %" PRIx32 " event: %s", - ev->GetTypeRewrite(), ev->ToString().data()); + YDB_LOG_ERROR_CTX(*TlsActivationContext, "TReadTableScan: StateWork unexpected event", + {"type", ev->GetTypeRewrite()}, + {"event", ev->ToString().data()}); } } @@ -456,8 +458,8 @@ private: void Undelivered(TEvents::TEvUndelivered::TPtr &, const TActorContext &ctx) { - LOG_ERROR(ctx, NKikimrServices::TX_DATASHARD, - "TReadTableScan: undelivered event TxId: %" PRIu64, TxId); + YDB_LOG_ERROR_CTX(ctx, "TReadTableScan: undelivered event", + {"txId", TxId}); Error = "cannot reach sink actor"; Driver->Touch(EScan::Final); @@ -465,8 +467,8 @@ private: void Disconnected(TEvInterconnect::TEvNodeDisconnected::TPtr &, const TActorContext &ctx) { - LOG_ERROR(ctx, NKikimrServices::TX_DATASHARD, - "TReadTableScan: disconnect TxId: %" PRIu64, TxId); + YDB_LOG_ERROR_CTX(ctx, "TReadTableScan: disconnect", + {"txId", TxId}); Error = "cannot reach sink actor"; Driver->Touch(EScan::Final); @@ -477,10 +479,10 @@ private: Y_ENSURE(PendingAcks); --PendingAcks; - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Got stream data ack ShardId: " << ShardId - << ", TxId: " << TxId - << ", PendingAcks: " << PendingAcks); + YDB_LOG_DEBUG_CTX(ctx, "Got stream data ack", + {"shardId", ShardId}, + {"txId", TxId}, + {"pendingAcks", PendingAcks}); if (Finished && !PendingAcks) Driver->Touch(EScan::Feed); @@ -488,8 +490,8 @@ private: void Handle(TEvTxProcessing::TEvStreamIsDead::TPtr &ev, const TActorContext &ctx) { - LOG_INFO(ctx, NKikimrServices::TX_DATASHARD, - "TReadTableScan: stream disconnect TxId: %" PRIu64, TxId); + YDB_LOG_INFO_CTX(ctx, "TReadTableScan: stream disconnect", + {"txId", TxId}); Error = "got dead stream notification"; Driver->Touch(EScan::Final); @@ -508,10 +510,10 @@ private: Writer->Reserve(MessageSizeLimit); - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Got quota for read table scan ShardId: " << ShardId - << ", TxId: " << TxId - << ", MessageQuota: " << MessageQuota); + YDB_LOG_DEBUG_CTX(ctx, "Got quota for read table scan", + {"shardId", ShardId}, + {"txId", TxId}, + {"messageQuota", MessageQuota}); CheckQuota(ctx); @@ -653,13 +655,13 @@ private: ++PendingAcks; --MessageQuota; - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Send response data ShardId: " << ShardId - << ", TxId: " << TxId - << ", Size: " << Writer->GetMessageSize() - << ", Rows: " << Writer->GetMessageRows() - << ", PendingAcks: " << PendingAcks - << ", MessageQuota: " << MessageQuota); + YDB_LOG_DEBUG_CTX(ctx, "Send response data", + {"shardId", ShardId}, + {"txId", TxId}, + {"size", Writer->GetMessageSize()}, + {"rows", Writer->GetMessageRows()}, + {"pendingAcks", PendingAcks}, + {"messageQuota", MessageQuota}); if (RowLimit) { RowLimit -= rows; @@ -678,7 +680,8 @@ private: Y_DEBUG_ABORT_UNLESS(DebugCheckKeyInRange(key)); if (!Writer->PutRow(row, Error)) { - LOG_ERROR_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, "Got scan fatal error: " << Error); + YDB_LOG_ERROR("Got scan fatal", + {"error", Error}); IsFatalError = true; return EScan::Final; } @@ -712,10 +715,10 @@ private: ctx.Send(Sink, request.Release()); } - LOG_DEBUG_S(ctx, NKikimrServices::TX_DATASHARD, - "Finish scan ShardId: " << ShardId - << ", TxId: " << TxId - << ", MessageQuota: " << MessageQuota); + YDB_LOG_DEBUG_CTX(ctx, "Finish scan", + {"shardId", ShardId}, + {"txId", TxId}, + {"messageQuota", MessageQuota}); Driver = nullptr; diff --git a/ydb/core/tx/datashard/read_table_scan_unit.cpp b/ydb/core/tx/datashard/read_table_scan_unit.cpp index f4bd215570d..74439b22222 100644 --- a/ydb/core/tx/datashard/read_table_scan_unit.cpp +++ b/ydb/core/tx/datashard/read_table_scan_unit.cpp @@ -3,6 +3,8 @@ #include "execution_unit_ctors.h" #include "read_table_scan.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -153,9 +155,11 @@ EExecutionStatus TReadTableScanUnit::Execute(TOperation::TPtr op, if (op->HasScanResult()) { auto *result = CheckedCast<TReadTableProd*>(op->ScanResult().Get()); - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "ReadTable scan complete for " << *op << " at " - << DataShard.TabletID() << " error: " << result->Error << ", IsFatalError: " << result->IsFatalError); + YDB_LOG_TRACE_CTX(ctx, "ReadTable scan complete", + {"operation", *op}, + {"tabletId", DataShard.TabletID()}, + {"error", result->Error}, + {"isFatalError", result->IsFatalError}); tx->SetScanTask(0); @@ -200,9 +204,9 @@ void TReadTableScanUnit::ProcessEvent(TAutoPtr<NActors::IEventHandle> &ev, IgnoreFunc(TEvTxProcessing::TEvStreamClearancePending); IgnoreFunc(TEvTxProcessing::TEvStreamClearanceResponse); default: - LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, - "TReadTableScanUnit::ProcessEvent unhandled event type: " << ev->GetTypeRewrite() - << " event: " << ev->ToString()); + YDB_LOG_ERROR_CTX(ctx, "TReadTableScanUnit::ProcessEvent unhandled event", + {"type", ev->GetTypeRewrite()}, + {"event", ev->ToString()}); Y_DEBUG_ABORT("unexpected event %" PRIu64, (ui64)ev->GetTypeRewrite()); } } @@ -237,7 +241,8 @@ void TReadTableScanUnit::Abort(const TString &err, tx->SetScanTask(0); } - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_NOTICE_CTX(ctx, "TReadTableScanUnit::Abort: aborting operation", + {"errorMessage", err}); op->ResetWaitingForScanFlag(); } diff --git a/ydb/core/tx/datashard/truncate_unit.cpp b/ydb/core/tx/datashard/truncate_unit.cpp index cc021fbbb79..4263c005c43 100644 --- a/ydb/core/tx/datashard/truncate_unit.cpp +++ b/ydb/core/tx/datashard/truncate_unit.cpp @@ -4,6 +4,8 @@ #include "setup_sys_locks.h" #include "datashard_locks_db.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -50,18 +52,20 @@ EExecutionStatus TTruncateUnit::Execute( const auto version = truncate.GetTableSchemaVersion(); Y_ENSURE(version); - LOG_TRACE_S(actorCtx, NKikimrServices::TX_DATASHARD, - "TTruncateUnit::Execute. Changing SchemaVersion. TableId = " << pathId.LocalPathId << "; " - << " New SchemaVersion = " << version << "; " - << " TxId = " << op->GetTxId() << "."); + YDB_LOG_TRACE_CTX(actorCtx, "TTruncateUnit::Execute: changing schema version", + {"localPathId", pathId.LocalPathId}, + {"version", version}, + {"txId", op->GetTxId()}); auto tableId = pathId.LocalPathId; Y_ENSURE(DataShard.GetUserTables().contains(tableId)); auto localTid = DataShard.GetUserTables().at(tableId)->LocalTid; - LOG_DEBUG_S(actorCtx, NKikimrServices::TX_DATASHARD, - "TTruncateUnit::Execute - About to TRUNCATE TABLE at " << DataShard.TabletID() - << " tableId# " << tableId << " localTid# " << localTid << " TxId = " << op->GetTxId()); + YDB_LOG_DEBUG_CTX(actorCtx, "TTruncateUnit::Execute: about to truncate table", + {"tabletId", DataShard.TabletID()}, + {"tableId", tableId}, + {"localTid", localTid}, + {"txId", op->GetTxId()}); // break locks TDataShardLocksDb locksDb(DataShard, txc); @@ -105,9 +109,9 @@ EExecutionStatus TTruncateUnit::Execute( DataShard.SysLocksTable().ApplyLocks(); DataShard.SubscribeNewLocks(actorCtx); - LOG_DEBUG_S(actorCtx, NKikimrServices::TX_DATASHARD, - "TTruncateUnit::Execute - Finished successfully. TableId = " << tableId - << " TxId = " << op->GetTxId() << " - Operation COMPLETED"); + YDB_LOG_DEBUG_CTX(actorCtx, "TTruncateUnit::Execute: finished successfully", + {"tableId", tableId}, + {"txId", op->GetTxId()}); return EExecutionStatus::DelayCompleteNoMoreRestarts; } diff --git a/ydb/core/tx/datashard/ut_truncate/datashard_ut_truncate.cpp b/ydb/core/tx/datashard/ut_truncate/datashard_ut_truncate.cpp index 8d72c8451ff..40ceb6f9300 100644 --- a/ydb/core/tx/datashard/ut_truncate/datashard_ut_truncate.cpp +++ b/ydb/core/tx/datashard/ut_truncate/datashard_ut_truncate.cpp @@ -23,7 +23,6 @@ public: Server = new TServer(serverSettings); Runtime = Server->GetRuntime(); - Runtime->GetAppData(0).FeatureFlags.SetEnableTruncateTable(true); if (enableDebugLogs) { Runtime->SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); diff --git a/ydb/core/tx/datashard/validate_row_condition.cpp b/ydb/core/tx/datashard/validate_row_condition.cpp index 30192439440..32398c59cb7 100644 --- a/ydb/core/tx/datashard/validate_row_condition.cpp +++ b/ydb/core/tx/datashard/validate_row_condition.cpp @@ -13,6 +13,8 @@ #include <ydb/core/tx/datashard/build_index/common_helper.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::BUILD_INDEX + namespace NKikimr::NDataShard { class TValidateRowConditionScan final : public TActor<TValidateRowConditionScan>, public NTable::IScan { @@ -40,10 +42,10 @@ public: columnNames.push_back(col); } ScanTags = BuildTags(tableInfo, std::move(columnNames)); - LOG_I("Create TValidateRowConditionScan" - << " id# " << Request.GetId() - << " tabletId# " << TabletId - << " notNullColumns# " << Request.NotNullColumnsSize()); + YDB_LOG_INFO("Create TValidateRowConditionScan", + {"id", Request.GetId()}, + {"tabletId", TabletId}, + {"notNullColumns", Request.NotNullColumnsSize()}); } ~TValidateRowConditionScan() final = default; @@ -99,21 +101,21 @@ public: } if (Status == NKikimrSetColumnConstraint::EValidateStatus::DONE && IsValid) { - LOG_N("TValidateRowConditionScan: Done (valid)" - << " id# " << Request.GetId() - << " tabletId# " << TabletId - << " scanStatus# " << (int)scanStatus); + YDB_LOG_NOTICE("TValidateRowConditionScan: Done (valid)", + {"id", Request.GetId()}, + {"tabletId", TabletId}, + {"scanStatus", (int)scanStatus}); } else if (!IsValid) { - LOG_N("TValidateRowConditionScan: Done (invalid, NULL found)" + LOG_NOTICE_S(*TlsActivationContext, NKikimrServices::BUILD_INDEX,"TValidateRowConditionScan: Done (invalid, NULL found)" << " id# " << Request.GetId() << " tabletId# " << TabletId << " scanStatus# " << (int)scanStatus); } else { - LOG_E("TValidateRowConditionScan: Failed" - << " id# " << Request.GetId() - << " tabletId# " << TabletId - << " buildStatus# " << (int)Status - << " scanStatus# " << (int)scanStatus); + YDB_LOG_ERROR("TValidateRowConditionScan: Failed", + {"id", Request.GetId()}, + {"tabletId", TabletId}, + {"buildStatus", (int)Status}, + {"scanStatus", (int)scanStatus}); } TActivationContext::Send(new IEventHandle(Sender, SelfId(), response.Release())); @@ -175,16 +177,16 @@ void TDataShard::HandleSafe(TEvDataShard::TEvValidateRowConditionRequest::TPtr& auto rowVersion = GetMvccTxVersion(EMvccTxMode::ReadOnly); TScanRecord::TSeqNo seqNo = {record.GetSeqNoGeneration(), record.GetSeqNoRound()}; - LOG_D("HandleSafe TEvValidateRowConditionRequest" - << " id# " << id - << " tabletId# " << record.GetTabletId() - << " ownerId# " << record.GetOwnerId() - << " pathId# " << record.GetPathId() - << " notNullColumns# " << record.NotNullColumnsSize() - << " rowVersion# " << rowVersion); + YDB_LOG_DEBUG("HandleSafe TEvValidateRowConditionRequest", + {"id", id}, + {"tabletId", record.GetTabletId()}, + {"ownerId", record.GetOwnerId()}, + {"pathId", record.GetPathId()}, + {"notNullColumns", record.NotNullColumnsSize()}, + {"rowVersion", rowVersion}); if (VolatileTxManager.HasVolatileTxsAtSnapshot(rowVersion)) { - LOG_D("HandleSafe TEvValidateRowConditionRequest: waiting for volatile txs" + LOG_DEBUG_S(*TlsActivationContext, NKikimrServices::BUILD_INDEX,"HandleSafe TEvValidateRowConditionRequest: waiting for volatile txs" << " id# " << id << " rowVersion# " << rowVersion); VolatileTxManager.AttachWaitingSnapshotEvent(rowVersion, std::unique_ptr<IEventHandle>(ev.Release())); @@ -205,40 +207,40 @@ void TDataShard::HandleSafe(TEvDataShard::TEvValidateRowConditionRequest::TPtr& }; if (record.GetTabletId() != TabletID()) { - LOG_E("HandleSafe TEvValidateRowConditionRequest: wrong shard" - << " id# " << id - << " expected# " << TabletID() - << " got# " << record.GetTabletId()); + YDB_LOG_ERROR("HandleSafe TEvValidateRowConditionRequest: wrong shard", + {"id", id}, + {"expected", TabletID()}, + {"got", record.GetTabletId()}); sendResponse(NKikimrSetColumnConstraint::EValidateStatus::BAD_REQUEST, TStringBuilder() << "Wrong shard " << record.GetTabletId() << " this is " << TabletID()); return; } const auto tableId = TTableId(record.GetOwnerId(), record.GetPathId()); if (!GetUserTables().contains(tableId.PathId.LocalPathId)) { - LOG_E("HandleSafe TEvValidateRowConditionRequest: unknown table" - << " id# " << id - << " localPathId# " << tableId.PathId.LocalPathId); + YDB_LOG_ERROR("HandleSafe TEvValidateRowConditionRequest: unknown table", + {"id", id}, + {"localPathId", tableId.PathId.LocalPathId}); sendResponse(NKikimrSetColumnConstraint::EValidateStatus::BAD_REQUEST, TStringBuilder() << "Unknown table id: " << tableId.PathId.LocalPathId); return; } if (!IsStateActive()) { - LOG_E("HandleSafe TEvValidateRowConditionRequest: shard not active" - << " id# " << id - << " tabletId# " << TabletID()); + YDB_LOG_ERROR("HandleSafe TEvValidateRowConditionRequest: shard not active", + {"id", id}, + {"tabletId", TabletID()}); sendResponse(NKikimrSetColumnConstraint::EValidateStatus::BAD_REQUEST, TStringBuilder() << "Shard " << TabletID() << " is not ready for requests"); return; } const auto& userTable = *GetUserTables().at(tableId.PathId.LocalPathId); - LOG_I("HandleSafe TEvValidateRowConditionRequest: starting scan" - << " id# " << id - << " tabletId# " << TabletID() - << " localTid# " << userTable.LocalTid); + YDB_LOG_INFO("HandleSafe TEvValidateRowConditionRequest: starting scan", + {"id", id}, + {"tabletId", TabletID()}, + {"localTid", userTable.LocalTid}); auto scan = new TValidateRowConditionScan(record, ev->Sender, TabletID(), userTable, GetScanManager()); StartScan(this, scan, id, seqNo, rowVersion, userTable.LocalTid); } -} // namespace NKikimr::NDataShard
\ No newline at end of file +} // namespace NKikimr::NDataShard diff --git a/ydb/core/tx/datashard/volatile_tx.cpp b/ydb/core/tx/datashard/volatile_tx.cpp index 7aa07e476eb..858411ff954 100644 --- a/ydb/core/tx/datashard/volatile_tx.cpp +++ b/ydb/core/tx/datashard/volatile_tx.cpp @@ -2,6 +2,8 @@ #include "datashard_impl.h" #include <library/cpp/resource/resource.h> +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr::NDataShard { class TVolatileTxPersistence { @@ -65,8 +67,10 @@ namespace NKikimr::NDataShard { txc.DB.CommitTx(tid, commitTxId, info->Version); Self->GetConflictsCache().GetTableCache(tid).RemoveUncommittedWrites(commitTxId, txc.DB); } else if (txc.DB.HasRemovedTx(tid, commitTxId)) { - LOG_CRIT_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Committing removed changes txId# " << commitTxId << " tid# " << tid << " shard# " << Self->TabletID()); + YDB_LOG_CRIT("Committing removed changes", + {"txId", commitTxId}, + {"tid", tid}, + {"shard", Self->TabletID()}); Self->IncCounter(COUNTER_REMOVED_COMMITTED_TXS); } } @@ -849,8 +853,10 @@ namespace NKikimr::NDataShard { ui64 dstTabletId = record.GetTabletDest(); if (dstTabletId != Self->TabletID()) { - LOG_WARN_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Unexpected readset from " << srcTabletId << " to " << dstTabletId << " at tablet " << Self->TabletID()); + YDB_LOG_WARN("Unexpected readset from participant", + {"srcTabletId", srcTabletId}, + {"dstTabletId", dstTabletId}, + {"tabletId", Self->TabletID()}); return true; } @@ -868,9 +874,10 @@ namespace NKikimr::NDataShard { if (record.GetFlags() & NKikimrTx::TEvReadSet::FLAG_NO_DATA) { Y_ENSURE(!(record.GetFlags() & NKikimrTx::TEvReadSet::FLAG_EXPECT_READSET), "Unexpected FLAG_EXPECT_READSET + FLAG_NO_DATA in ProcessReadSet"); - LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Processed readset without data from " << srcTabletId << " to " << dstTabletId - << " at tablet " << Self->TabletID()); + YDB_LOG_TRACE("TVolatileTxManager::ProcessReadSet: processed readset without data", + {"srcTabletId", srcTabletId}, + {"dstTabletId", dstTabletId}, + {"tabletId", Self->TabletID()}); return false; } @@ -879,17 +886,21 @@ namespace NKikimr::NDataShard { Y_ENSURE(ok, "Failed to parse readset from " << srcTabletId << " to " << dstTabletId); if (data.GetDecision() != NKikimrTx::TReadSetData::DECISION_COMMIT) { - LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Processed readset with decision " << ui32(data.GetDecision()) << " from " - << srcTabletId << " to " << dstTabletId << " at tablet " << Self->TabletID()); + YDB_LOG_TRACE("TVolatileTxManager::ProcessReadSet: processed readset with decision", + {"decision", ui32(data.GetDecision())}, + {"srcTabletId", srcTabletId}, + {"dstTabletId", dstTabletId}, + {"tabletId", Self->TabletID()}); return false; } if (record.GetStep() != info->Version.Step) { - LOG_TRACE_S(*TlsActivationContext, NKikimrServices::TX_DATASHARD, - "Processed readset from " << srcTabletId << " to " << dstTabletId - << " with step " << record.GetStep() << " expecting " << info->Version.Step - << ", treating like abort due to divergence at tablet " << Self->TabletID()); + YDB_LOG_TRACE("TVolatileTxManager::ProcessReadSet: readset step mismatch, treating like abort due to divergence", + {"srcTabletId", srcTabletId}, + {"dstTabletId", dstTabletId}, + {"receivedStep", record.GetStep()}, + {"expectedStep", info->Version.Step}, + {"tabletId", Self->TabletID()}); return false; } diff --git a/ydb/core/tx/datashard/wait_for_stream_clearance_unit.cpp b/ydb/core/tx/datashard/wait_for_stream_clearance_unit.cpp index 7280f327cc4..11059ee809c 100644 --- a/ydb/core/tx/datashard/wait_for_stream_clearance_unit.cpp +++ b/ydb/core/tx/datashard/wait_for_stream_clearance_unit.cpp @@ -2,6 +2,8 @@ #include "datashard_pipeline.h" #include "execution_unit_ctors.h" +#define YDB_LOG_THIS_FILE_COMPONENT NKikimrServices::TX_DATASHARD + namespace NKikimr { namespace NDataShard { @@ -94,9 +96,10 @@ EExecutionStatus TWaitForStreamClearanceUnit::Execute(TOperation::TPtr op, op->SetWaitingForStreamClearanceFlag(); op->SetProcessDisconnectsFlag(); - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "Requested stream clearance from " << tx->GetStreamSink() - << " for " << *op << " at " << DataShard.TabletID()); + YDB_LOG_TRACE_CTX(ctx, "Requested stream clearance", + {"streamSink", tx->GetStreamSink()}, + {"operation", *op}, + {"tabletId", DataShard.TabletID()}); } while (op->HasPendingInputEvents()) { @@ -123,9 +126,9 @@ void TWaitForStreamClearanceUnit::ProcessEvent(TAutoPtr<NActors::IEventHandle> & OHFunc(TEvents::TEvUndelivered, Handle); IgnoreFunc(TEvTxProcessing::TEvStreamClearancePending); default: - LOG_ERROR_S(ctx, NKikimrServices::TX_DATASHARD, - "TWaitForStreamClearanceUnit::ProcessEvent unhandled event type: " << ev->GetTypeRewrite() - << " event: " << ev->ToString()); + YDB_LOG_ERROR_CTX(ctx, "TWaitForStreamClearanceUnit::ProcessEvent unhandled event", + {"type", ev->GetTypeRewrite()}, + {"event", ev->ToString()}); Y_DEBUG_ABORT("unexpected event %" PRIu64, (ui64)ev->GetTypeRewrite()); } } @@ -152,8 +155,9 @@ void TWaitForStreamClearanceUnit::Handle(TEvTxProcessing::TEvStreamClearanceResp { if (op->IsWaitingForStreamClearance()) { if (ev->Get()->Record.GetCleared()) { - LOG_TRACE_S(ctx, NKikimrServices::TX_DATASHARD, - "Got stream clearance for " << *op << " at " << DataShard.TabletID()); + YDB_LOG_TRACE_CTX(ctx, "Got stream clearance", + {"operation", *op}, + {"tabletId", DataShard.TabletID()}); op->ResetWaitingForStreamClearanceFlag(); } else { Abort(TStringBuilder() << "Got stream clearance reject for " << *op @@ -198,7 +202,8 @@ void TWaitForStreamClearanceUnit::Abort(const TString &err, tx->SetScanSnapshotId(0); } - LOG_NOTICE_S(ctx, NKikimrServices::TX_DATASHARD, err); + YDB_LOG_NOTICE_CTX(ctx, "TWaitForStreamClearanceUnit::Abort: aborting operation", + {"errorMessage", err}); op->ResetWaitingForStreamClearanceFlag(); } diff --git a/ydb/core/tx/long_tx_service/long_tx_service_impl.cpp b/ydb/core/tx/long_tx_service/long_tx_service_impl.cpp index ac7ff42323a..802283ebee1 100644 --- a/ydb/core/tx/long_tx_service/long_tx_service_impl.cpp +++ b/ydb/core/tx/long_tx_service/long_tx_service_impl.cpp @@ -1796,7 +1796,7 @@ void TLongTxServiceActor::Handle(TEvPrivate::TEvRunDeadlockDetection::TPtr& ev) continue; } - YDB_LOG_DEBUG("Breaking the wait edge", + YDB_LOG_WARN("Breaking the wait edge", {"logPrefix", LogPrefix}, {"id", edge.Id}, {"awaiter", edge.Awaiter.LockInfo(SelfId())}, diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_truncate_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_truncate_table.cpp index ba5a9c9bbac..5fdc4740d12 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_truncate_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_truncate_table.cpp @@ -228,11 +228,6 @@ public: result.Reset(new TEvSchemeShard::TEvModifySchemeTransactionResult( NKikimrScheme::StatusAccepted, ui64(OperationId.GetTxId()), ui64(ssId))); - if (!AppData()->FeatureFlags.GetEnableTruncateTable()) { - result->SetError(NKikimrScheme::StatusPreconditionFailed, "TRUNCATE TABLE statement is not supported"); - return result; - } - const auto& truncateTableOp = Transaction.GetTruncateTable(); const auto stringTablePath = NKikimr::JoinPath({Transaction.GetWorkingDir(), truncateTableOp.GetTableName()}); TPath tablePath = TPath::Resolve(stringTablePath, context.SS); diff --git a/ydb/core/tx/schemeshard/ut_background_cleaning/ut_background_cleaning.cpp b/ydb/core/tx/schemeshard/ut_background_cleaning/ut_background_cleaning.cpp index eb5d37b7158..a538e345811 100644 --- a/ydb/core/tx/schemeshard/ut_background_cleaning/ut_background_cleaning.cpp +++ b/ydb/core/tx/schemeshard/ut_background_cleaning/ut_background_cleaning.cpp @@ -301,11 +301,7 @@ Y_UNIT_TEST_SUITE(TSchemeshardBackgroundCleaningTest) { options.FinalEvents.emplace_back(TEvInterconnect::EvNodeDisconnected); runtime.DispatchEvents(options); - { - TDispatchOptions options; - options.FinalEvents.emplace_back(TEvInterconnect::EvNodeConnected); - runtime.DispatchEvents(options); - } + env.SimulateSleep(runtime, TDuration::Seconds(50)); CheckTable(runtime, "/MyRoot/tmp/TempTable1", TTestTxConfig::SchemeShard, false); CheckTable(runtime, "/MyRoot/tmp/TempTable2", TTestTxConfig::SchemeShard, false); diff --git a/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp b/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp index c0215f34965..706407d5c9e 100644 --- a/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp +++ b/ydb/core/tx/schemeshard/ut_set_column_constraint/ut_set_column_constraint.cpp @@ -608,6 +608,78 @@ Y_UNIT_TEST_SUITE(SetNotNullTest) { TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); } + Y_UNIT_TEST(DatashardRebootLostMessage) { + TTestBasicRuntime runtime; + runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NActors::NLog::PRI_TRACE); + + TTestEnv env(runtime); + + ui64 txId = 100; + + TString root = "/MyRoot"; + TString tablePath = root + "/Table"; + + TestCreateTable(runtime, ++txId, root, R"( + Name: "Table" + Columns { Name: "key" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + )"); + env.TestWaitNotification(runtime, txId); + + ui64 datashardTabletId = 0; + bool firstRequestDropped = false; + auto prevObserver = runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& ev) { + if (!firstRequestDropped && + ev->GetTypeRewrite() == TEvDataShard::TEvValidateRowConditionRequest::EventType) + { + datashardTabletId = ev->Get<TEvDataShard::TEvValidateRowConditionRequest>()->Record.GetTabletId(); + firstRequestDropped = true; + return TTestActorRuntime::EEventAction::DROP; + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + ui64 setConstraintTxId = ++txId; + auto response = TestSetColumnConstraint( + runtime, setConstraintTxId, + TTestTxConfig::SchemeShard, + root, + tablePath, + {"value"}); + + Cerr << "SET COLUMN CONSTRAINT RESPONSE: " << response.ShortDebugString() << Endl; + + UNIT_ASSERT_VALUES_EQUAL_C( + response.GetStatus(), + Ydb::StatusIds::SUCCESS, + response.ShortDebugString()); + + runtime.WaitFor("first validate request", [&] { + return firstRequestDropped; + }); + + UNIT_ASSERT_C(datashardTabletId != 0, "Failed to capture DataShard tabletId"); + + RebootTablet(runtime, datashardTabletId, runtime.AllocateEdgeActor()); + + env.TestWaitNotification(runtime, setConstraintTxId, TTestTxConfig::SchemeShard); + + TestCheckColumnsNotNull(runtime, tablePath, {{"value", true}}); + + { + TVector<TCell> cells = { + TCell::Make((ui32)1), TCell() + }; + + WriteOp(runtime, TTestTxConfig::SchemeShard, ++txId, tablePath, + 0, NKikimrDataEvents::TEvWrite::TOperation::OPERATION_UPSERT, + {1, 2}, TSerializedCellMatrix(cells, 1, 2), false); + } + } + Y_UNIT_TEST(AlreadyNotNull) { TTestBasicRuntime runtime; runtime.SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE); diff --git a/ydb/core/tx/schemeshard/ut_truncate_table_reboots/ut_truncate_table_reboots.cpp b/ydb/core/tx/schemeshard/ut_truncate_table_reboots/ut_truncate_table_reboots.cpp index c9be94604a9..d63b4ab1ad9 100644 --- a/ydb/core/tx/schemeshard/ut_truncate_table_reboots/ut_truncate_table_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_truncate_table_reboots/ut_truncate_table_reboots.cpp @@ -14,8 +14,6 @@ Y_UNIT_TEST_SUITE(TruncateTableReboots) { t.Run([&](TTestActorRuntime& runtime, bool& activeZone) { { TInactiveZone inactive(activeZone); - - runtime.GetAppData().FeatureFlags.SetEnableTruncateTable(true); TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( Name: "TestTable" @@ -63,8 +61,6 @@ Y_UNIT_TEST_SUITE(TruncateTableReboots) { { TInactiveZone inactive(activeZone); - runtime.GetAppData().FeatureFlags.SetEnableTruncateTable(true); - TestCreateTable(runtime, ++t.TxId, "/MyRoot", R"( Name: "PartitionedTable" Columns { Name: "id" Type: "Uint64" } diff --git a/ydb/core/tx/schemeshard/ut_truncate_table_simple/ut_truncate_table_simple.cpp b/ydb/core/tx/schemeshard/ut_truncate_table_simple/ut_truncate_table_simple.cpp index fbea85ca65b..32dc2f611ad 100644 --- a/ydb/core/tx/schemeshard/ut_truncate_table_simple/ut_truncate_table_simple.cpp +++ b/ydb/core/tx/schemeshard/ut_truncate_table_simple/ut_truncate_table_simple.cpp @@ -14,8 +14,6 @@ Y_UNIT_TEST_SUITE(TruncateTable) { TTestEnv env(runtime); ui64 txId = 100; - runtime.GetAppData().FeatureFlags.SetEnableTruncateTable(true); - TestCreateTable(runtime, ++txId, "/MyRoot", R"( Name: "TestTable" Columns { Name: "id" Type: "Uint64" } @@ -75,8 +73,6 @@ Y_UNIT_TEST_SUITE(TruncateTable) { TTestEnv env(runtime); ui64 txId = 100; - runtime.GetAppData().FeatureFlags.SetEnableTruncateTable(true); - TestCreateTable(runtime, ++txId, "/MyRoot", R"( Name: "TestTable" Columns { Name: "id" Type: "Uint64" } @@ -137,8 +133,6 @@ Y_UNIT_TEST_SUITE(TruncateTable) { ui64 txId = 100; runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); - runtime.GetAppData().FeatureFlags.SetEnableTruncateTable(true); - TestCreateTable(runtime, ++txId, "/MyRoot", R"( Name: "TestTable" Columns { Name: "id" Type: "Uint64" } @@ -202,8 +196,6 @@ Y_UNIT_TEST_SUITE(TruncateTable) { TTestEnv env(runtime); ui64 txId = 100; - runtime.GetAppData().FeatureFlags.SetEnableTruncateTable(true); - TestTruncateTable(runtime, ++txId, "/MyRoot", "NonExistentTable", {NKikimrScheme::StatusPathDoesNotExist}); env.TestWaitNotification(runtime, txId); @@ -214,8 +206,6 @@ Y_UNIT_TEST_SUITE(TruncateTable) { TTestEnv env(runtime); ui64 txId = 100; - runtime.GetAppData().FeatureFlags.SetEnableTruncateTable(true); - TestCreateTable(runtime, ++txId, "/MyRoot", R"( Name: "TestTable" Columns { Name: "id" Type: "Uint64" } @@ -246,8 +236,6 @@ Y_UNIT_TEST_SUITE(TruncateTable) { TTestEnv env(runtime); ui64 txId = 100; - runtime.GetAppData().FeatureFlags.SetEnableTruncateTable(true); - TestCreateTable(runtime, ++txId, "/MyRoot", R"( Name: "TestTable" Columns { Name: "id" Type: "Uint64" } @@ -357,8 +345,6 @@ Y_UNIT_TEST_SUITE(TruncateTable) { TTestEnv env(runtime); ui64 txId = 100; - runtime.GetAppData().FeatureFlags.SetEnableTruncateTable(true); - TestCreateTable(runtime, ++txId, "/MyRoot", R"( Name: "TestTable" Columns { Name: "id" Type: "Uint64" } diff --git a/ydb/core/util/actorsys_test/testactorsys.h b/ydb/core/util/actorsys_test/testactorsys.h index 7c76a4db0be..7f609d51b11 100644 --- a/ydb/core/util/actorsys_test/testactorsys.h +++ b/ydb/core/util/actorsys_test/testactorsys.h @@ -314,7 +314,7 @@ public: IExecutorPool *pool = CreateTestExecutorPool(nodeId); setup->Executors[0].Reset(pool); #if !defined(_msan_enabled_) - auto memPool = NInterconnect::NRdma::CreateDummyMemPool(); + auto memPool = NInterconnect::NRdma::CreateDummyMemPool(/*emulateRegistration=*/true); setup->RcBufAllocator = std::make_shared<TRdmaAllocatorWithFallback>(memPool); #endif diff --git a/ydb/core/ymq/actor/actor.h b/ydb/core/ymq/actor/actor.h index 7b4c4c142fc..9b06ce534fa 100644 --- a/ydb/core/ymq/actor/actor.h +++ b/ydb/core/ymq/actor/actor.h @@ -11,7 +11,7 @@ public: virtual ~IReplyCallback() = default; virtual void DoSendReply(const NKikimrClient::TSqsResponse& resp) = 0; - virtual void OnIamAuthError() {} + virtual void OnIamAuthSuccess() {} }; class IPingReplyCallback { diff --git a/ydb/core/ymq/actor/auth_multi_factory.cpp b/ydb/core/ymq/actor/auth_multi_factory.cpp index 43160aed0f2..3d4c84b5939 100644 --- a/ydb/core/ymq/actor/auth_multi_factory.cpp +++ b/ydb/core/ymq/actor/auth_multi_factory.cpp @@ -545,12 +545,12 @@ void TCloudAuthRequestProxy::DoReply() { void TCloudAuthRequestProxy::SetError(const TErrorClass& errorClass, const TString& message) { auto* error = MakeMutableError(); ::NKikimr::NSQS::MakeError(error, errorClass, Sprintf("%s Request id to report: %s.", message.c_str(), RequestId_.c_str())); - if (Callback_) { - Callback_->OnIamAuthError(); - } } void TCloudAuthRequestProxy::OnSuccessfulAuth() { + if (Callback_) { + Callback_->OnIamAuthSuccess(); + } #define SQS_REQUEST_CASE(action) \ ProposeStaticCreds(*RequestHolder_->Y_CAT(Mutable, action)()); diff --git a/ydb/core/ymq/http/http.cpp b/ydb/core/ymq/http/http.cpp index 9e64822546d..715d71c746e 100644 --- a/ydb/core/ymq/http/http.cpp +++ b/ydb/core/ymq/http/http.cpp @@ -77,7 +77,7 @@ public: response.FolderId = resp.GetFolderId(); response.IsFifo = resp.GetIsFifo(); response.ResourceId = resp.GetResourceId(); - response.SkipMetering = IamAuthFailed_; + response.SkipMetering = !IamAuthSuccess_; for (const auto& tag : resp.GetQueueTags()) { response.QueueTags[tag.GetKey()] = tag.GetValue(); } @@ -85,8 +85,8 @@ public: Request_->SendResponse(response); } - void OnIamAuthError() override { - IamAuthFailed_ = true; + void OnIamAuthSuccess() override { + IamAuthSuccess_ = true; } private: @@ -107,7 +107,7 @@ private: private: THttpRequest* const Request_; const TSqsRequest RequestParams_; - bool IamAuthFailed_ = false; + bool IamAuthSuccess_ = false; }; class TPingHttpCallback : public IPingReplyCallback { diff --git a/ydb/library/actors/interconnect/rdma/mem_pool.cpp b/ydb/library/actors/interconnect/rdma/mem_pool.cpp index 2590f8d6b77..8e361a17b66 100644 --- a/ydb/library/actors/interconnect/rdma/mem_pool.cpp +++ b/ydb/library/actors/interconnect/rdma/mem_pool.cpp @@ -476,8 +476,8 @@ namespace NInterconnect::NRdma { class TMemPoolBase: public IMemPool { public: - TMemPoolBase(size_t maxChunk, NMonitoring::TDynamicCounterPtr counter) - : Ctxs(GetAllCtxs()) + TMemPoolBase(size_t maxChunk, NMonitoring::TDynamicCounterPtr counter, bool emulateRegistration = false) + : Ctxs(emulateRegistration ? NInterconnect::NRdma::NLinkMgr::TCtxsMap{} : GetAllCtxs()) , MaxChunk(maxChunk) , Alignment(NSystemInfo::GetPageSize()) { @@ -629,8 +629,8 @@ namespace NInterconnect::NRdma { class TDummyMemPool: public TMemPoolBase { public: - TDummyMemPool() - : TMemPoolBase(-1, MakeCounters(nullptr)) + explicit TDummyMemPool(bool emulateRegistration = false) + : TMemPoolBase(-1, MakeCounters(nullptr), emulateRegistration) {} TMemRegionPtr AllocImpl(int size, ui32) noexcept override { @@ -961,8 +961,21 @@ namespace NInterconnect::NRdma { thread_local TSlotMemPool::TSlotMemPoolCache TSlotMemPool::LocalCache; - std::shared_ptr<IMemPool> CreateDummyMemPool() noexcept { - auto* pool = Singleton<TDummyMemPool>(); + namespace { + struct TDummyMemPoolRealTag : public TDummyMemPool { + TDummyMemPoolRealTag() : TDummyMemPool(/*emulateRegistration=*/false) {} + }; + struct TDummyMemPoolEmulatedTag : public TDummyMemPool { + TDummyMemPoolEmulatedTag() : TDummyMemPool(/*emulateRegistration=*/true) {} + }; + } + + std::shared_ptr<IMemPool> CreateDummyMemPool(bool emulateRegistration) noexcept { + if (emulateRegistration) { + auto* pool = Singleton<TDummyMemPoolEmulatedTag>(); + return std::shared_ptr<TDummyMemPool>(pool, [](TDummyMemPool*) {}); + } + auto* pool = Singleton<TDummyMemPoolRealTag>(); return std::shared_ptr<TDummyMemPool>(pool, [](TDummyMemPool*) {}); } diff --git a/ydb/library/actors/interconnect/rdma/mem_pool.h b/ydb/library/actors/interconnect/rdma/mem_pool.h index badf8d435c0..73ec6c895fd 100644 --- a/ydb/library/actors/interconnect/rdma/mem_pool.h +++ b/ydb/library/actors/interconnect/rdma/mem_pool.h @@ -108,6 +108,6 @@ namespace NInterconnect::NRdma { virtual void Tick(NMonotonic::TMonotonic time) noexcept = 0; }; - std::shared_ptr<IMemPool> CreateDummyMemPool() noexcept; + std::shared_ptr<IMemPool> CreateDummyMemPool(bool emulateRegistration = false) noexcept; std::shared_ptr<IMemPool> CreateSlotMemPool(NMonitoring::TDynamicCounters* counters, std::optional<TMemPoolSettings> settings) noexcept; } diff --git a/ydb/library/actors/testlib/test_runtime.cpp b/ydb/library/actors/testlib/test_runtime.cpp index dbd4c2f5877..4facc584545 100644 --- a/ydb/library/actors/testlib/test_runtime.cpp +++ b/ydb/library/actors/testlib/test_runtime.cpp @@ -1774,7 +1774,7 @@ namespace NActors { } if (UseRdmaAllocator) { - auto memPool = NInterconnect::NRdma::CreateDummyMemPool(); + auto memPool = NInterconnect::NRdma::CreateDummyMemPool(/*emulateRegistration=*/true); setup->RcBufAllocator = std::make_shared<TRdmaAllocatorWithFallback>(memPool); } diff --git a/ydb/library/actors/util/cpu_topology_ut.cpp b/ydb/library/actors/util/cpu_topology_ut.cpp index da275025554..ba601a8305d 100644 --- a/ydb/library/actors/util/cpu_topology_ut.cpp +++ b/ydb/library/actors/util/cpu_topology_ut.cpp @@ -90,7 +90,7 @@ public: }; TFsPath SnapshotSystemRoot(const TString& name) { - return TFsPath(ArcadiaSourceRoot()) / "ydb/library/actors/util/testdata/cpu_topology" / name / "sys/devices/system"; + return TFsPath(SRC_("testdata/cpu_topology")) / name / "sys/devices/system"; } TCpuTopology LoadSnapshot(const TString& name) { diff --git a/ydb/library/yql/dq/comp_nodes/dq_scalar_hash_join.cpp b/ydb/library/yql/dq/comp_nodes/dq_scalar_hash_join.cpp index a3f66abda3c..47c8a30ce26 100644 --- a/ydb/library/yql/dq/comp_nodes/dq_scalar_hash_join.cpp +++ b/ydb/library/yql/dq/comp_nodes/dq_scalar_hash_join.cpp @@ -10,6 +10,9 @@ #include <ydb/library/yql/dq/comp_nodes/dq_join_common.h> #include <ydb/library/yql/dq/comp_nodes/hash_join_utils/scalar_layout_converter.h> +#include <algorithm> +#include <numeric> + namespace NKikimr::NMiniKQL { namespace { @@ -21,18 +24,22 @@ struct TDqScalarJoinMetadata { TSides<TVector<ui32>> KeyColumns; TVector<TType*> ResultItemTypes; TDqJoinImplRenames Renames; + EJoinKind Kind; + TSides<TVector<TType*>> UserTypes; + TSides<TVector<int>> ColumnPermutation; }; class TScalarPackedTupleSource : public NNonCopyable::TMoveOnly { public: TScalarPackedTupleSource(TComputationContext& ctx, IComputationWideFlowNode* flow, IScalarLayoutConverter* converter, - int columns) + int columns, const TVector<int>& columnPermutation) : Ctx_(&ctx) , Flow_(flow) , Buff_(columns) , Pointers_(columns) , Converter_(converter) , Columns_(columns) + , ColumnPermutation_(columnPermutation) { for (int index = 0; index < columns; ++index) { Pointers_[index] = &Buff_[index]; @@ -66,8 +73,14 @@ public: } return Yield{}; case EFetchResult::One: { - for (int i = 0; i < Columns_; ++i) { - BatchValues_.push_back(Buff_[i]); + if (ColumnPermutation_.empty()) { + for (int i = 0; i < Columns_; ++i) { + BatchValues_.push_back(Buff_[i]); + } + } else { + for (int i = 0; i < Columns_; ++i) { + BatchValues_.push_back(Buff_[ColumnPermutation_[i]]); + } } ++BatchCount_; // loop to accumulate more until batch full or finish @@ -95,84 +108,110 @@ private: TMKQLVector<NYql::NUdf::TUnboxedValue*> Pointers_; IScalarLayoutConverter* Converter_; int Columns_; + TVector<int> ColumnPermutation_; static constexpr int BatchSize_ = 1024; TMKQLVector<NYql::NUdf::TUnboxedValue> BatchValues_; int BatchCount_ = 0; }; +template <EJoinKind Kind> struct TRenamesScalarOutput : NNonCopyable::TMoveOnly { struct TFlushResult { TVector<NUdf::TUnboxedValue> Buffer; TSides<TPackResult> Packs; }; + struct Empty {}; + using BuildNullIfNeeded = std::conditional_t<Kind == EJoinKind::Left, TPackResult, Empty>; + TRenamesScalarOutput(const TDqScalarJoinMetadata* meta, TSides<IScalarLayoutConverter*> converters) : Renames_(&meta->Renames) , Converters_(converters) , BuildWidth_(std::ssize(meta->InputTypes.Build)) , ProbeWidth_(std::ssize(meta->InputTypes.Probe)) - {} + { + if constexpr (!std::is_same_v<decltype(Nulls_), Empty>) { + TMKQLVector<NUdf::TUnboxedValue> nulls(BuildWidth_); + Converters_.Build->Pack(nulls.data(), Nulls_); + } + } int Columns() const { return Renames_->size(); } i64 SizeTuples() const { - return Output_.NItems; + return Output_.Probe.NTuples; } auto MakeConsumeFn() { - return [this](TSides<TSingleTuple> tuples) { - for(ESide side: EachSide) { - Converters_.SelectSide(side)->GetTupleLayout()->TupleDeepCopy( - tuples.SelectSide(side).PackedData, tuples.SelectSide(side).OverflowBegin, - Output_.Data.SelectSide(side).PackedTuples, Output_.Data.SelectSide(side).Overflow); + struct ConsumeFn { + TRenamesScalarOutput& Self; + + void operator()(TSides<TSingleTuple> tuples) { + for (ESide side : EachSide) { + Self.Output_.SelectSide(side).AppendTuple( + tuples.SelectSide(side), Self.Converters_.SelectSide(side)->GetTupleLayout()); + } + } + + void operator()(TSingleTuple tuple) { + if constexpr (Kind == EJoinKind::Left) { + TSingleTuple null{.PackedData = Self.Nulls_.PackedTuples.data(), + .OverflowBegin = Self.Nulls_.Overflow.data()}; + this->operator()(TSides<TSingleTuple>{.Build = null, .Probe = tuple}); + } else if constexpr (SemiOrOnlyJoin(Kind)) { + Self.Output_.Probe.AppendTuple(tuple, Self.Converters_.Probe->GetTupleLayout()); + } } - Output_.NItems++; }; + return ConsumeFn{*this}; } TFlushResult Flush() { TFlushResult res; - res.Packs.Build.NTuples = Output_.NItems; - res.Packs.Build.PackedTuples = std::move(Output_.Data.Build.PackedTuples); - res.Packs.Build.Overflow = std::move(Output_.Data.Build.Overflow); + const i64 nItems = Output_.Probe.NTuples; + res.Packs.Build = std::move(Output_.Build); + res.Packs.Probe = std::move(Output_.Probe); - res.Packs.Probe.NTuples = Output_.NItems; - res.Packs.Probe.PackedTuples = std::move(Output_.Data.Probe.PackedTuples); - res.Packs.Probe.Overflow = std::move(Output_.Data.Probe.Overflow); + res.Buffer.reserve(nItems * Columns()); - res.Buffer.reserve(Output_.NItems * Columns()); - TMKQLVector<NUdf::TUnboxedValue> buildValues(BuildWidth_); - TMKQLVector<NUdf::TUnboxedValue> probeValues(ProbeWidth_); - - for (i64 tupleIndex = 0; tupleIndex < Output_.NItems; ++tupleIndex) { - Converters_.Build->Unpack(res.Packs.Build, tupleIndex, buildValues.data()); - Converters_.Probe->Unpack(res.Packs.Probe, tupleIndex, probeValues.data()); - for (auto rename : *Renames_) { - if (rename.Side == ESide::Build) { - res.Buffer.push_back(buildValues[rename.Index]); - } else { + if constexpr (LeftSemiOrOnly(Kind)) { + TMKQLVector<NUdf::TUnboxedValue> probeValues(ProbeWidth_); + for (i64 tupleIndex = 0; tupleIndex < nItems; ++tupleIndex) { + Converters_.Probe->Unpack(res.Packs.Probe, tupleIndex, probeValues.data()); + for (auto rename : *Renames_) { + MKQL_ENSURE(rename.Side == ESide::Probe, + "renames in Semi or Only Left Join shouldn't contain columns from right side"); res.Buffer.push_back(probeValues[rename.Index]); } } + } else { + TMKQLVector<NUdf::TUnboxedValue> buildValues(BuildWidth_); + TMKQLVector<NUdf::TUnboxedValue> probeValues(ProbeWidth_); + for (i64 tupleIndex = 0; tupleIndex < nItems; ++tupleIndex) { + Converters_.Build->Unpack(res.Packs.Build, tupleIndex, buildValues.data()); + Converters_.Probe->Unpack(res.Packs.Probe, tupleIndex, probeValues.data()); + for (auto rename : *Renames_) { + if (rename.Side == ESide::Build) { + res.Buffer.push_back(buildValues[rename.Index]); + } else { + res.Buffer.push_back(probeValues[rename.Index]); + } + } + } } - Output_.NItems = 0; return res; } private: - struct TuplePairs { - i64 NItems = 0; - TSides<TPackResult> Data; - }; - - TuplePairs Output_; const TDqJoinImplRenames* Renames_; TSides<IScalarLayoutConverter*> Converters_; const int BuildWidth_; const int ProbeWidth_; + TSides<TPackResult> Output_; + BuildNullIfNeeded Nulls_; }; template <EJoinKind Kind> @@ -199,7 +238,7 @@ public: private: class TStreamState : public TComputationValue<TStreamState> { using TBase = TComputationValue<TStreamState>; - using JoinType = NJoinPackedTuples::THybridHashJoin<TScalarPackedTupleSource, TestStorageSettings, EJoinKind::Inner>; + using JoinType = NJoinPackedTuples::THybridHashJoin<TScalarPackedTupleSource, TestStorageSettings, Kind>; public: TStreamState(TMemoryUsageInfo* memInfo, TComputationContext& ctx, TSides<IComputationWideFlowNode*> flows, @@ -210,9 +249,11 @@ private: , JoinCtx_(&ctx) , Join_(TSides<TScalarPackedTupleSource>{ .Build = {ctx, flows.Build, Converters_.Build.get(), - static_cast<int>(std::ssize(Meta_->InputTypes.Build))}, + static_cast<int>(std::ssize(Meta_->InputTypes.Build)), + Meta_->ColumnPermutation.Build}, .Probe = {ctx, flows.Probe, Converters_.Probe.get(), - static_cast<int>(std::ssize(Meta_->InputTypes.Probe))}}, + static_cast<int>(std::ssize(Meta_->InputTypes.Probe)), + Meta_->ColumnPermutation.Probe}}, ctx, "ScalarHashJoinPacked", TSides<const NPackedTuple::TTupleLayout*>{.Build = Converters_.Build->GetTupleLayout(), .Probe = Converters_.Probe->GetTupleLayout()}) @@ -286,8 +327,8 @@ private: TSides<std::unique_ptr<IScalarLayoutConverter>> Converters_; TComputationContext* JoinCtx_; JoinType Join_; - TRenamesScalarOutput Output_; - std::optional<TRenamesScalarOutput::TFlushResult> Buffer_; + TRenamesScalarOutput<Kind> Output_; + std::optional<typename TRenamesScalarOutput<Kind>::TFlushResult> Buffer_; size_t BufferPos_ = 0; const int Threshold_ = 10000; }; @@ -296,13 +337,13 @@ private: TSides<std::unique_ptr<IScalarLayoutConverter>> converters; TTypeInfoHelper helper; for(ESide side: EachSide) { - TVector<NPackedTuple::EColumnRole> roles(std::ssize(Meta_->InputTypes.SelectSide(side)), + TVector<NPackedTuple::EColumnRole> roles(std::ssize(Meta_->UserTypes.SelectSide(side)), NPackedTuple::EColumnRole::Payload); for (int column : Meta_->KeyColumns.SelectSide(side)) { roles[column] = NPackedTuple::EColumnRole::Key; } converters.SelectSide(side) = - MakeScalarLayoutConverter(helper, Meta_->InputTypes.SelectSide(side), roles, ctx.HolderFactory); + MakeScalarLayoutConverter(helper, Meta_->UserTypes.SelectSide(side), roles, ctx.HolderFactory); } state = ctx.HolderFactory.Create<TStreamState>(ctx, Flows_, std::move(converters), Meta_.get()); @@ -355,6 +396,7 @@ IComputationWideFlowNode* WrapDqScalarHashJoin(TCallable& callable, const TCompu const auto joinKindNode = callable.GetInput(2); const auto rawKind = AS_VALUE(TDataLiteral, joinKindNode)->AsValue().Get<ui32>(); const auto joinKind = GetJoinKind(rawKind); + meta.Kind = joinKind; const auto leftKeyColumnsLiteral = callable.GetInput(3); const auto leftKeyColumnsTuple = AS_VALUE(TTupleLiteral, leftKeyColumnsLiteral); @@ -376,7 +418,6 @@ IComputationWideFlowNode* WrapDqScalarHashJoin(TCallable& callable, const TCompu const auto rightFlow = dynamic_cast<IComputationWideFlowNode*>(LocateNode(ctx.NodeLocator, callable, 1)); MKQL_ENSURE(leftFlow, "Expected WideFlow as a left input"); MKQL_ENSURE(rightFlow, "Expected WideFlow as a right input"); - MKQL_ENSURE(joinKind == EJoinKind::Inner, "Only inner is supported, see gh#26780 for details."); TDqUserRenames userRenames = FromGraceFormat(TGraceJoinRenames::FromRuntimeNodes(callable.GetInput(5), callable.GetInput(6))); @@ -387,8 +428,81 @@ IComputationWideFlowNode* WrapDqScalarHashJoin(TCallable& callable, const TCompu meta.Renames.push_back({.Index = rename.Index, .Side = side}); } - return new TScalarHashJoinWrapper<EJoinKind::Inner>(ctx.Mutables, std::move(meta), - {.Build = rightFlow, .Probe = leftFlow}); + for (ESide side : EachSide) { + auto& keyColumns = meta.KeyColumns.SelectSide(side); + const int numDataCols = std::ssize(meta.InputTypes.SelectSide(side)); + const int numKeys = std::ssize(keyColumns); + + bool needsReorder = false; + for (int i = 0; i < numKeys; ++i) { + if (keyColumns[i] != static_cast<ui32>(i)) { + needsReorder = true; + break; + } + } + if (!needsReorder) { + continue; + } + + TVector<int> perm(numDataCols); + std::iota(perm.begin(), perm.end(), 0); + for (int i = 0; i < numKeys; ++i) { + const int keyColumn = static_cast<int>(keyColumns[i]); + MKQL_ENSURE(keyColumn >= 0 && keyColumn < numDataCols, + Sprintf("key column index %i on %s side is out of range [0, %i)", keyColumn, + AsString(side), numDataCols)); + auto it = std::find(perm.begin() + i, perm.end(), keyColumn); + MKQL_ENSURE(it != perm.end(), + Sprintf("key column index %i on %s side is duplicated or could not be placed", + keyColumn, AsString(side))); + std::swap(perm[i], *it); + } + + meta.ColumnPermutation.SelectSide(side) = perm; + + const auto origTypes = meta.InputTypes.SelectSide(side); + for (int i = 0; i < numDataCols; ++i) { + meta.InputTypes.SelectSide(side)[i] = origTypes[perm[i]]; + } + + TVector<int> inv(numDataCols); + for (int i = 0; i < numDataCols; ++i) { + inv[perm[i]] = i; + } + for (auto& rename : meta.Renames) { + if (rename.Side == side) { + rename.Index = inv[rename.Index]; + } + } + + for (int i = 0; i < numKeys; ++i) { + keyColumns[i] = i; + } + } + + for (ESide side : EachSide) { + for (TType* thisType : meta.InputTypes.SelectSide(side)) { + if (joinKind == EJoinKind::Left && side == ESide::Build && !thisType->IsOptional()) { + meta.UserTypes.SelectSide(side).push_back(TOptionalType::Create(thisType, ctx.Env)); + } else { + meta.UserTypes.SelectSide(side).push_back(thisType); + } + } + } + + using enum EJoinKind; + const TSides<IComputationWideFlowNode*> flows{.Build = rightFlow, .Probe = leftFlow}; + if (joinKind == Inner) { + return new TScalarHashJoinWrapper<Inner>(ctx.Mutables, std::move(meta), flows); + } else if (joinKind == LeftOnly) { + return new TScalarHashJoinWrapper<LeftOnly>(ctx.Mutables, std::move(meta), flows); + } else if (joinKind == LeftSemi) { + return new TScalarHashJoinWrapper<LeftSemi>(ctx.Mutables, std::move(meta), flows); + } else if (joinKind == Left) { + return new TScalarHashJoinWrapper<Left>(ctx.Mutables, std::move(meta), flows); + } else { + MKQL_ENSURE(false, "unsupported join type in scalar hash join, see gh#26780 for details."); + } } } // namespace NKikimr::NMiniKQL diff --git a/ydb/library/yql/dq/comp_nodes/ut/dq_hash_join_ut.cpp b/ydb/library/yql/dq/comp_nodes/ut/dq_hash_join_ut.cpp index be76fe5a029..369517c4980 100644 --- a/ydb/library/yql/dq/comp_nodes/ut/dq_hash_join_ut.cpp +++ b/ydb/library/yql/dq/comp_nodes/ut/dq_hash_join_ut.cpp @@ -1038,24 +1038,24 @@ Y_UNIT_TEST_SUITE(TDqHashJoinBasicTest) { Test(EmptyRightInnerTestData(), BlockJoin); } - Y_UNIT_TEST(TestLeftKind) { - Test(LeftJoinTestData(), true); + Y_UNIT_TEST_TWIN(TestLeftKind, BlockJoin) { + Test(LeftJoinTestData(), BlockJoin); } - Y_UNIT_TEST(TestLeftJoinWithMatches) { - Test(LeftJoinWithMatchesTestData(), true); + Y_UNIT_TEST_TWIN(TestLeftJoinWithMatches, BlockJoin) { + Test(LeftJoinWithMatchesTestData(), BlockJoin); } - Y_UNIT_TEST(TestLeftJoinSpilling) { - Test(LeftJoinSpillingTestData(), true); + Y_UNIT_TEST_TWIN(TestLeftJoinSpilling, BlockJoin) { + Test(LeftJoinSpillingTestData(), BlockJoin); } - Y_UNIT_TEST(TestLeftJoinSpillingTwoKeys) { - Test(LeftJoinSpillingTwoKeysTestData(), true); + Y_UNIT_TEST_TWIN(TestLeftJoinSpillingTwoKeys, BlockJoin) { + Test(LeftJoinSpillingTwoKeysTestData(), BlockJoin); } - Y_UNIT_TEST(TestLeftJoinSpillingMultiKey) { - Test(LeftJoinSpillingMultiKeyTestData(), true); + Y_UNIT_TEST_TWIN(TestLeftJoinSpillingMultiKey, BlockJoin) { + Test(LeftJoinSpillingMultiKeyTestData(), BlockJoin); } Y_UNIT_TEST(TestLeftKindLeftIsBuild) { @@ -1090,8 +1090,8 @@ Y_UNIT_TEST_SUITE(TDqHashJoinBasicTest) { Test(LargeBothSidesInnerSpillingTestData(), true); } - Y_UNIT_TEST(TestLargeBothSidesLeftSpilling) { - Test(LargeBothSidesLeftSpillingTestData(), true); + Y_UNIT_TEST_TWIN(TestLargeBothSidesLeftSpilling, BlockJoin) { + Test(LargeBothSidesLeftSpillingTestData(), BlockJoin); } Y_UNIT_TEST(TestSlicedBlocksInnerSpilling) { @@ -1122,16 +1122,16 @@ Y_UNIT_TEST_SUITE(TDqHashJoinBasicTest) { // Test(ExclusionTestData(), BlockJoin); // } - Y_UNIT_TEST(TestLeftSemiKind) { - Test(LeftSemiTestData(),true); + Y_UNIT_TEST_TWIN(TestLeftSemiKind, BlockJoin) { + Test(LeftSemiTestData(), BlockJoin); } // Y_UNIT_TEST_TWIN(TestRightSemiKind, BlockJoin) { // Test(RightSemiTestData(), BlockJoin); // } - Y_UNIT_TEST(TestLeftOnlyKind) { - Test(LeftOnlyTestData(), true); + Y_UNIT_TEST_TWIN(TestLeftOnlyKind, BlockJoin) { + Test(LeftOnlyTestData(), BlockJoin); } // Y_UNIT_TEST_TWIN(TestRightOnlyKind, BlockJoin) { @@ -1141,12 +1141,12 @@ Y_UNIT_TEST_SUITE(TDqHashJoinBasicTest) { Test(InnerJoinRenamesTestData(), BlockJoin); } - Y_UNIT_TEST(TestSwappedKeyColumnsInner) { - Test(SwappedKeyColumnsInnerTestData(), true); + Y_UNIT_TEST_TWIN(TestSwappedKeyColumnsInner, BlockJoin) { + Test(SwappedKeyColumnsInnerTestData(), BlockJoin); } - Y_UNIT_TEST(TestSwappedKeyColumnsLeftSemi) { - Test(SwappedKeyColumnsLeftSemiTestData(), true); + Y_UNIT_TEST_TWIN(TestSwappedKeyColumnsLeftSemi, BlockJoin) { + Test(SwappedKeyColumnsLeftSemiTestData(), BlockJoin); } Y_UNIT_TEST(TestBlockJoinScalarColumn) { diff --git a/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp b/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp index 6ab97a23a37..9b529c720e6 100644 --- a/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp +++ b/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp @@ -507,4 +507,247 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) { runtime.GrabEdgeEventRethrow<NActors::TEvents::TEvWakeup>(edge); } + class TMockStructuredTokenCredentialsFactory : public NYql::IStructuredTokenCredentialsFactory { + public: + TMockStructuredTokenCredentialsFactory(const std::string yqlToken, const TVector<bool>& pattern) + : YqlToken(yqlToken) + , Pattern(pattern) + { + } + + private: + class TCredentialsProviderFactory: public NYdb::ICredentialsProviderFactory { + private: + class TCredentialsProvider : public NYdb::ICredentialsProvider { + public: + TCredentialsProvider(const std::string yqlToken, const TVector<bool>& pattern) + : YqlToken(yqlToken) + , Pattern(pattern) + { + } + + std::string GetAuthInfo() const override final { + if (Invocation == Pattern.size() || Pattern[Invocation++]) { + return YqlToken; + } + throw yexception() << "Uh, oh"; + } + + bool IsValid() const override final { + return true; + } + + private: + std::string YqlToken; + TVector<bool> Pattern; + + mutable ui32 Invocation = 0; + }; + public: + TCredentialsProviderFactory(const std::string yqlToken, const TVector<bool>& pattern) + : YqlToken(yqlToken) + , Pattern(pattern) + { + } + + std::shared_ptr<NYdb::ICredentialsProvider> CreateProvider() const override { + return std::make_shared<TCredentialsProvider>(YqlToken, Pattern); + } + private: + std::string YqlToken; + TVector<bool> Pattern; + }; + std::shared_ptr<NYdb::ICredentialsProviderFactory> Create(const TString& /*structuredToken*/, bool /*addBearer*/) override { + return std::make_shared<TCredentialsProviderFactory>(YqlToken, Pattern); + } + std::string YqlToken; + TVector<bool> Pattern; + }; + + std::shared_ptr<NYql::IStructuredTokenCredentialsFactory> CreateMockStructuredTokenCredentialsFactory(const std::string yqlToken, const TVector<bool>& pattern) { + return std::make_shared<TMockStructuredTokenCredentialsFactory>(yqlToken, pattern); + } + + Y_UNIT_TEST(LookupWithAuthErrors) { + auto alloc = std::make_shared<NKikimr::NMiniKQL::TScopedAlloc>(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), true, false); + NKikimr::NMiniKQL::TMemoryUsageInfo memUsage("TestMemUsage"); + NKikimr::NMiniKQL::THolderFactory holderFactory(alloc->Ref(), memUsage); + + auto loggerConfig = NYql::NProto::TLoggingConfig(); + loggerConfig.set_allcomponentslevel(::NYql::NProto::TLoggingConfig_ELevel::TLoggingConfig_ELevel_TRACE); + NYql::NLog::InitLogger(loggerConfig, false); + + TTestActorRuntimeBase runtime(1, 1, true); + runtime.Initialize(); + auto edge = runtime.AllocateEdgeActor(); + + NYql::TGenericDataSourceInstance dsi; + dsi.Setkind(NYql::EGenericDataSourceKind::YDB); + dsi.mutable_endpoint()->Sethost("some_host"); + dsi.mutable_endpoint()->Setport(2135); + dsi.Setdatabase("some_db"); + dsi.Setuse_tls(true); + dsi.set_protocol(::NYql::EGenericProtocol::NATIVE); + auto token = dsi.mutable_credentials()->mutable_token(); + token->Settype("IAM"); + token->Setvalue("token_value"); + + auto connectorMock = std::make_shared<NYql::NConnector::NTest::TConnectorClientMock>(); + + // clang-format off + // step 1: ListSplits + { + ::testing::InSequence seq; + // expected sequence: auth failure, retry, auth ok, ListSplits, auth failure, retry, auth ok, ListSplits, auth ok, ReadSplits + for (ui32 stage = 0; stage < 2; ++stage) { + auto listBuilder = connectorMock->ExpectListSplits(); + listBuilder + .Select() + .DataSourceInstance(dsi) + .What() + .Column("id", Ydb::Type::UINT64) + .NullableColumn("optional_id", Ydb::Type::UINT64) + .NullableColumn("string_value", Ydb::Type::STRING) + .Done() + .Table("lookup_test") + .Where() + .Filter() + .Disjunction() + .Operand() + .Conjunction() + .Operand().Equal().Column("id").Value<ui64>(2).Done().Done() + .Operand().Equal().Column("optional_id").OptionalValue<ui64>(102).Done().Done() + .Done() + .Done() + .Operand() + .Conjunction() + .Operand().Equal().Column("id").Value<ui64>(1).Done().Done() + .Operand().Equal().Column("optional_id").OptionalValue<ui64>(101).Done().Done() + .Done() + .Done() + .Operand() + .Conjunction() + .Operand().Equal().Column("id").Value<ui64>(0).Done().Done() + .Operand().Equal().Column("optional_id").OptionalValue<ui64>(100).Done().Done() + .Done() + .Done() + .Operand() + .Conjunction() + .Operand().Equal().Column("id").Value<ui64>(2).Done().Done() + .Operand().Equal().Column("optional_id").OptionalValue<ui64>(102).Done().Done() + .Done() + .Done() + .Done() + .Done() + .Done() + .Done() + .MaxSplitCount(1) + .Result() + .AddResponse(NewSuccess()) + .Description("Actual split info is not important") + ; + } + + { + auto readBuilder = connectorMock->ExpectReadSplits(); + readBuilder + .DataSourceInstance(dsi) + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY) + .Split() + .Description("Actual split info is not important") + .Done() + .Result() + .AddResponse( + MakeRecordBatch( + MakeArray<arrow::UInt64Builder, uint64_t>("id", {0, 1, 2}, arrow::uint64()), + MakeArray<arrow::UInt64Builder, uint64_t>("optional_id", {100, 101, 103}, arrow::uint64()), // the last value is intentionally wrong + MakeArray<arrow::StringBuilder, std::string>("string_value", {"a", "b", "c"}, arrow::utf8()) + ), + NewSuccess() + ) + ; + } + } + // clang-format on + + NYql::Generic::TLookupSource lookupSourceSettings; + *lookupSourceSettings.mutable_data_source_instance() = dsi; + lookupSourceSettings.Settable("lookup_test"); + lookupSourceSettings.SetTokenName("test_token"); + + google::protobuf::Any packedLookupSource; + Y_ABORT_UNLESS(packedLookupSource.PackFrom(lookupSourceSettings)); + + auto lookupActorFactory = [&holderFactory, connectorMock = std::move(connectorMock), lookupSourceSettings = std::move(lookupSourceSettings)](const NActors::TActorId& caller, std::shared_ptr<NKikimr::NMiniKQL::TScopedAlloc>& alloc, NKikimr::NMiniKQL::TTypeEnvironment& typeEnv) mutable { + NKikimr::NMiniKQL::TTypeBuilder typeBuilder(typeEnv); + + NKikimr::NMiniKQL::TStructTypeBuilder keyTypeBuilder{typeEnv}; + keyTypeBuilder.Add("id", typeBuilder.NewDataType(NYql::NUdf::EDataSlot::Uint64, false)); + keyTypeBuilder.Add("optional_id", typeBuilder.NewDataType(NYql::NUdf::EDataSlot::Uint64, true)); + NKikimr::NMiniKQL::TStructTypeBuilder outputypeBuilder{typeEnv}; + outputypeBuilder.Add("string_value", typeBuilder.NewDataType(NYql::NUdf::EDataSlot::String, true)); + + auto guard = Guard(*alloc.get()); + auto keyTypeHelper = std::make_shared<NYql::NDq::IDqAsyncLookupSource::TKeyTypeHelper>(keyTypeBuilder.Build()); + + TVector<bool> pattern { false, true, false, true, true }; + pattern.resize(128); + + auto [lookupSource, actor] = NYql::NDq::CreateGenericLookupActor( + std::move(connectorMock), + CreateMockStructuredTokenCredentialsFactory("token_value", pattern), + caller, + nullptr, + alloc, + keyTypeHelper, + std::move(lookupSourceSettings), + keyTypeBuilder.Build(), + outputypeBuilder.Build(), + typeEnv, + holderFactory, + 1'000'000, + {{"test_token", "{\"sa_id\": \"arbitrary\", \"sa_id_signature\": \"junk\"}"}}); + + auto request = std::make_shared<NYql::NDq::IDqAsyncLookupSource::TUnboxedValueMap>(3, keyTypeHelper->GetValueHash(), keyTypeHelper->GetValueEqual()); + for (size_t i = 0; i != 3; ++i) { + NYql::NUdf::TUnboxedValue* keyItems; + auto key = holderFactory.CreateDirectArrayHolder(2, keyItems); + keyItems[0] = NYql::NUdf::TUnboxedValuePod(ui64(i)); + keyItems[1] = NYql::NUdf::TUnboxedValuePod(ui64(100 + i)); + request->emplace(std::move(key), NYql::NUdf::TUnboxedValue{}); + } + + return std::pair { actor, std::move(request) }; + }; + auto callback = [&holderFactory](std::shared_ptr<NKikimr::NMiniKQL::TScopedAlloc>& alloc, + NYql::NDq::IDqAsyncLookupSource::TEvLookupResult::TPtr& ev) { + auto guard2 = Guard(*alloc.get()); + auto lookupResult = ev->Get()->Result.lock(); + UNIT_ASSERT(lookupResult); + + UNIT_ASSERT_EQUAL(3, lookupResult->size()); + { + const auto* v = lookupResult->FindPtr(CreateStructValue(holderFactory, {0, 100})); + UNIT_ASSERT(v); + NYql::NUdf::TUnboxedValue val = v->GetElement(0); + UNIT_ASSERT(val.AsStringRef() == TStringBuf("a")); + } + { + const auto* v = lookupResult->FindPtr(CreateStructValue(holderFactory, {1, 101})); + UNIT_ASSERT(v); + NYql::NUdf::TUnboxedValue val = v->GetElement(0); + UNIT_ASSERT(val.AsStringRef() == TStringBuf("b")); + } + { + const auto* v = lookupResult->FindPtr(CreateStructValue(holderFactory, {2, 102})); + UNIT_ASSERT(v); + UNIT_ASSERT(!*v); + } + }; + + auto callLookupActor = new TCallLookupActor(std::move(alloc), std::move(lookupActorFactory), std::move(callback), edge); + runtime.Register(callLookupActor); + runtime.GrabEdgeEventRethrow<NActors::TEvents::TEvWakeup>(edge); + } + } // Y_UNIT_TEST_SUITE(GenericProviderLookupActor) diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_base_actor.h b/ydb/library/yql/providers/generic/actors/yql_generic_base_actor.h index bccb566bea5..d7ae418bd10 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_base_actor.h +++ b/ydb/library/yql/providers/generic/actors/yql_generic_base_actor.h @@ -24,6 +24,7 @@ namespace NYql::NDq { EvReadSplitsFinished, EvError, EvRetry, + EvGotCredentials, EvEnd }; @@ -99,6 +100,17 @@ namespace NYql::NDq { TEvState State; }; + struct TEvGotCredentials: NActors::TEventLocal<TEvGotCredentials, EvGotCredentials> { + explicit TEvGotCredentials(TGenericCredentials credentials, TEvState state = {}) + : Credentials(std::move(credentials)) + , State(std::move(state)) + { + } + + TGenericCredentials Credentials; + TEvState State; + }; + protected: // TODO move common logic here }; diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_credentials_provider.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_credentials_provider.cpp index 93c34dd5482..dce965f4177 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_credentials_provider.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_credentials_provider.cpp @@ -1,5 +1,6 @@ #include "yql_generic_credentials_provider.h" +#include <ydb/library/yql/providers/generic/actors/yql_generic_helpers.h> #include <ydb/library/yql/providers/generic/proto/source.pb.h> #include <yql/essentials/providers/common/structured_token/yql_token_builder.h> #include <yql/essentials/utils/log/log.h> @@ -20,33 +21,30 @@ namespace NYql::NDq { } auto credentialsProviderFactory = credentialsFactory->Create(structuredTokenJSON, false); + CredentialsProvider_ = credentialsProviderFactory->CreateProvider(); } - TString TGenericCredentialsProvider::FillCredentials(TGenericDataSourceInstance& dsi) const { + NThreading::TFuture<TGenericCredentials> TGenericCredentialsProvider::AsyncCredentials() const { // 1. If basic auth creds have been provided, use it if (BasicAuthCredentials_) { - auto basic = dsi.mutable_credentials()->mutable_basic(); + TGenericCredentials credentials; + auto basic = credentials.mutable_basic(); *basic->mutable_username() = BasicAuthCredentials_->Username; *basic->mutable_password() = BasicAuthCredentials_->Password; - return {}; - } - - *dsi.mutable_credentials()->mutable_token()->mutable_type() = "IAM"; - - // 3. Otherwise use credentials provider to get token from Token Accessor - Y_ENSURE(CredentialsProvider_, "CredentialsProvider is not initialized"); - - std::string iamToken; - try { - iamToken = CredentialsProvider_->GetAuthInfo(); - } catch (const std::exception& e) { - YQL_CLOG(ERROR, ProviderGeneric) << "FillCredentials: " << e.what(); - return TString(e.what()); + return NThreading::MakeFuture(credentials); } - - *dsi.mutable_credentials()->mutable_token()->mutable_value() = std::move(iamToken); - return {}; + // 2. Otherwise use credentials provider to get token from Token Accessor + Y_ENSURE(CredentialsProvider_); + return CredentialsProvider_->GetAuthInfoAsync() + .Apply([](const NThreading::TFuture<std::string>& future) { + auto iamToken = ExtractFromConstFuture(future); + TGenericCredentials credentials; + auto& token = *credentials.mutable_token(); + *token.mutable_type() = "IAM"; + *token.mutable_value() = std::move(iamToken); + return credentials; + }); } TGenericCredentialsProvider::TPtr diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_credentials_provider.h b/ydb/library/yql/providers/generic/actors/yql_generic_credentials_provider.h index 83b3e540aa9..d9eb7d54523 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_credentials_provider.h +++ b/ydb/library/yql/providers/generic/actors/yql_generic_credentials_provider.h @@ -17,10 +17,10 @@ namespace NYql::NDq { const TString& structuredToken, const IStructuredTokenCredentialsFactory::TPtr& credentialsFactory); - // FillCredentials sets the credentials to access the remote datasource into the DataSourceInstance object. + // Produce generic credentials asynchronously // It can be either IAM-token or login + password for basic auth. - // Returns string containing error, if it happened. - TString FillCredentials(NYql::TGenericDataSourceInstance& dsi) const; + // Set exception in future on errors + NThreading::TFuture<TGenericCredentials> AsyncCredentials() const; private: struct BasicAuthCredentials { diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_helpers.h b/ydb/library/yql/providers/generic/actors/yql_generic_helpers.h new file mode 100644 index 00000000000..3266fac0dd5 --- /dev/null +++ b/ydb/library/yql/providers/generic/actors/yql_generic_helpers.h @@ -0,0 +1,17 @@ +#pragma once + +#include <library/cpp/threading/future/core/future.h> + +namespace { + +template <typename T> +T ExtractFromConstFuture(const NThreading::TFuture<T>& f) { + // We want to avoid making a copy of data stored in a future. + // But there is no direct way to extract data from a const future + // So, we make a copy of the future, that is cheap. Then, extract the value from this copy. + // It destructs the value in the original future, but this trick is legal and documented here: + // https://docs.yandex-team.ru/arcadia-cpp/cookbook/concurrency + return NThreading::TFuture<T>(f).ExtractValueSync(); +} + +} // namespace diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp index f5eb51c2ae1..bac77c67a02 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp @@ -13,6 +13,7 @@ #include <ydb/library/yql/dq/runtime/dq_arrow_helpers.h> #include <yql/essentials/minikql/computation/mkql_computation_node_holders.h> #include <yql/essentials/minikql/mkql_node_builder.h> +#include <ydb/library/yql/providers/generic/actors/yql_generic_helpers.h> #include <ydb/library/yql/providers/generic/proto/source.pb.h> #include <ydb/library/yql/providers/generic/connector/libcpp/error.h> #include <ydb/library/yql/providers/generic/connector/libcpp/utils.h> @@ -45,16 +46,6 @@ namespace NYql::NDq { return resultTypeBuilder.Build(); } - template <typename T> - T ExtractFromConstFuture(const NThreading::TFuture<T>& f) { - // We want to avoid making a copy of data stored in a future. - // But there is no direct way to extract data from a const future - // So, we make a copy of the future, that is cheap. Then, extract the value from this copy. - // It destructs the value in the original future, but this trick is legal and documented here: - // https://docs.yandex-team.ru/arcadia-cpp/cookbook/concurrency - return NThreading::TFuture<T>(f).ExtractValueSync(); - } - using ILookupRetryPolicy = IRetryPolicy<const NYdbGrpc::TGrpcStatus&>; using ILookupRetryState = ILookupRetryPolicy::IRetryState; struct TLookupState { @@ -77,6 +68,10 @@ namespace NYql::NDq { using TBase = TGenericBaseActor<TGenericLookupActor, TLookupState::TPtr>; struct TEvLookupRetry : NActors::TEventLocal<TEvLookupRetry, EvRetry> { + explicit TEvLookupRetry(TLookupState::TPtr state) + : State(std::move(state)) + {} + TLookupState::TPtr State; }; @@ -162,7 +157,7 @@ namespace NYql::NDq { public: void Bootstrap() { - auto dsi = LookupSource.data_source_instance(); + const auto& dsi = LookupSource.data_source_instance(); YQL_CLOG(INFO, ProviderGeneric) << "New generic proivider lookup source actor(ActorId=" << SelfId() << ") for" << " kind=" << NYql::EGenericDataSourceKind_Name(dsi.kind()) << ", endpoint=" << dsi.endpoint().ShortDebugString() @@ -194,6 +189,7 @@ namespace NYql::NDq { private: // events STRICT_STFUNC_EXC(StateFunc, hFunc(TEvLookupRequest, Handle) + hFunc(TEvGotCredentials, Handle) hFunc(TEvListSplitsIterator, Handle) hFunc(TEvListSplitsPart, Handle) hFunc(TEvReadSplitsIterator, Handle) @@ -233,28 +229,33 @@ namespace NYql::NDq { NConnector::NApi::TReadSplitsRequest readRequest; *readRequest.mutable_data_source_instance() = LookupSource.data_source_instance(); - auto error = CredentialsProvider->FillCredentials(*readRequest.mutable_data_source_instance()); - if (error) { - SendError(TActivationContext::ActorSystem(), SelfId(), std::move(error)); - return; - } - *readRequest.add_splits() = std::move(split); readRequest.Setformat(NConnector::NApi::TReadSplitsRequest_EFormat::TReadSplitsRequest_EFormat_ARROW_IPC_STREAMING); readRequest.set_filtering(ev->Get()->State->FullscanLimit > 0 ? NConnector::NApi::TReadSplitsRequest::FILTERING_OPTIONAL : NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY); - Connector->ReadSplits(readRequest, RequestTimeout).Subscribe([ + CredentialsProvider->AsyncCredentials().Apply([ + connector = Connector, + readRequest = std::move(readRequest), + requestTimeout = RequestTimeout + ](const NThreading::TFuture<TGenericCredentials>& future) mutable { + *readRequest.mutable_data_source_instance()->mutable_credentials() = ExtractFromConstFuture(future); + return connector->ReadSplits(readRequest, requestTimeout); + }).Subscribe([ actorSystem = TActivationContext::ActorSystem(), selfId = SelfId(), state = std::move(ev->Get()->State) ](const NConnector::TReadSplitsStreamIteratorAsyncResult& asyncResult) { - YQL_CLOG(DEBUG, ProviderGeneric) << "ActorId=" << selfId << " Got ReadSplitsStreamIterator from Connector"; - auto result = ExtractFromConstFuture(asyncResult); - if (result.Status.Ok()) { - auto ev = new TEvReadSplitsIterator(std::move(result.Iterator)); - ev->State = std::move(state); - actorSystem->Send(new NActors::IEventHandle(selfId, selfId, ev)); - } else { - SendRetryOrError(actorSystem, selfId, result.Status, state); + try { + YQL_CLOG(DEBUG, ProviderGeneric) << "ActorId=" << selfId << " Got ReadSplitsStreamIterator from Connector"; + auto result = ExtractFromConstFuture(asyncResult); + if (result.Status.Ok()) { + auto ev = new TEvReadSplitsIterator(std::move(result.Iterator)); + ev->State = std::move(state); + actorSystem->Send(new NActors::IEventHandle(selfId, selfId, ev)); + } else { + SendRetryOrError(actorSystem, selfId, result.Status, state); + } + } catch (std::exception& ex) { + SendRetryOrError(actorSystem, selfId, NYdbGrpc::TGrpcStatus(grpc::StatusCode::UNAVAILABLE, ex.what()), std::move(state)); } }); } @@ -365,16 +366,32 @@ namespace NYql::NDq { SendRequest(state); } - // must be called with bound Alloc void SendRequest(TLookupState::TPtr state) { + CredentialsProvider->AsyncCredentials().Subscribe([ + actorSystem = TActivationContext::ActorSystem(), + selfId = SelfId(), + state = std::move(state) + ](const NThreading::TFuture<TGenericCredentials>& future) mutable { + try { + actorSystem->Send( + selfId, + new TEvGotCredentials(ExtractFromConstFuture(future), std::move(state))); + } catch (std::exception& ex) { + SendRetryOrError(actorSystem, selfId, NYdbGrpc::TGrpcStatus(grpc::StatusCode::UNAVAILABLE, ex.what()), std::move(state)); + } + }); + } + + void Handle(TEvGotCredentials::TPtr ev) { auto startCycleCount = GetCycleCountFast(); + auto state = std::move(ev->Get()->State); NConnector::NApi::TListSplitsRequest splitRequest; - auto error = FillSelect(*splitRequest.add_selects(), state); + auto error = FillSelect(*splitRequest.add_selects(), state, std::move(ev->Get()->Credentials)); if (error) { SendError(TActivationContext::ActorSystem(), SelfId(), std::move(error)); return; - }; + } splitRequest.Setmax_split_count(1); Connector->ListSplits(splitRequest, RequestTimeout).Subscribe([ @@ -525,9 +542,8 @@ namespace NYql::NDq { auto nextRetry = state->RetryState->GetNextRetryDelay(status); if (nextRetry) { YQL_CLOG(WARN, ProviderGeneric) << "ActorId=" << selfId << " Got retrievable GRPC Error from Connector: " << status.ToDebugString() << ", retry scheduled in " << *nextRetry; - auto ev = new TEvLookupRetry(); - ev->State = std::move(state); - actorSystem->Schedule(*nextRetry, new IEventHandle(selfId, selfId, ev)); + actorSystem->Schedule(*nextRetry, + new IEventHandle(selfId, selfId, new TEvLookupRetry(std::move(state)))); return; } SendError(actorSystem, selfId, NConnector::ErrorFromGRPCStatus(status)); @@ -582,14 +598,10 @@ namespace NYql::NDq { } } - // must be called with bound Alloc - TString FillSelect(NConnector::NApi::TSelect& select, TLookupState::TPtr state) { + TString FillSelect(NConnector::NApi::TSelect& select, TLookupState::TPtr state, TGenericCredentials&& credentials) { auto dsi = LookupSource.data_source_instance(); - auto error = CredentialsProvider->FillCredentials(dsi); - if (error) { - return error; - } - *select.mutable_data_source_instance() = dsi; + *dsi.mutable_credentials() = std::move(credentials); + *select.mutable_data_source_instance() = std::move(dsi); for (ui32 i = 0; i != SelectResultType->GetMembersCount(); ++i) { auto c = select.mutable_what()->add_items()->mutable_column(); @@ -607,6 +619,7 @@ namespace NYql::NDq { } NConnector::NApi::TPredicate::TDisjunction disjunction; + auto guard = Guard(*Alloc); auto request = state->Request.lock(); if (!request) { YQL_CLOG(DEBUG, ProviderGeneric) << "ActorId=" << SelfId() << " FillSelect: parent MIA"; diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp index e29e6cbbaf7..01d7b4ca4d8 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_read_actor.cpp @@ -9,6 +9,7 @@ #include <ydb/library/actors/core/events.h> #include <ydb/library/actors/core/hfunc.h> #include <ydb/library/actors/core/log.h> +#include <ydb/library/yql/providers/generic/actors/yql_generic_helpers.h> #include <ydb/library/yql/providers/generic/connector/api/service/protos/connector.pb.h> #include <ydb/library/yql/providers/generic/connector/libcpp/error.h> #include <ydb/library/yql/providers/generic/connector/libcpp/utils.h> @@ -24,20 +25,6 @@ namespace NYql::NDq { using namespace NActors; - namespace { - - template <typename T> - T ExtractFromConstFuture(const NThreading::TFuture<T>& f) { - // We want to avoid making a copy of data stored in a future. - // But there is no direct way to extract data from a const future - // So, we make a copy of the future, that is cheap. Then, extract the value from this copy. - // It destructs the value in the original future, but this trick is legal and documented here: - // https://docs.yandex-team.ru/arcadia-cpp/cookbook/concurrency - return NThreading::TFuture<T>(f).ExtractValueSync(); - } - - } // namespace - class TGenericReadActor: public TGenericBaseActor<TGenericReadActor>, public IDqComputeActorAsyncInput { public: TGenericReadActor( @@ -71,14 +58,22 @@ namespace NYql::NDq { void Bootstrap() { Become(&TGenericReadActor::StateFunc); - auto issue = InitSplitsReading(); - if (issue) { - return NotifyComputeActorWithIssue( - TActivationContext::ActorSystem(), - ComputeActorId_, - InputIndex_, - std::move(*issue)); - }; + TokenProvider_->AsyncCredentials().Subscribe([ + computeActorId = ComputeActorId_, + inputIndex = InputIndex_, + actorSystem = TActivationContext::ActorSystem(), + selfId = SelfId() + ](const NThreading::TFuture<TGenericCredentials>& future) { + try { + actorSystem->Send(selfId, new TEvGotCredentials(ExtractFromConstFuture(future))); + } catch (std::exception& ex) { + // GetAuthInfoAsync handles retries internally + NConnector::NApi::TError error; + error.set_status(Ydb::StatusIds::UNAUTHORIZED); + error.set_message(ex.what()); + NotifyComputeActorWithError(actorSystem, computeActorId, inputIndex, error); + } + }); } static constexpr char ActorName[] = "GENERIC_READ_ACTOR"; @@ -89,19 +84,21 @@ namespace NYql::NDq { hFunc(TEvReadSplitsIterator, Handle); hFunc(TEvReadSplitsPart, Handle); hFunc(TEvReadSplitsFinished, Handle); + hFunc(TEvGotCredentials, Handle); , ExceptionFunc(std::exception, HandleException) ) // clang-format on - // ReadSplits - TMaybe<TIssue> InitSplitsReading() { + void Handle(TEvGotCredentials::TPtr& ev) { + const auto& credentials = ev->Get()->Credentials; + YQL_CLOG(DEBUG, ProviderGeneric) << "Start splits reading"; if (Partitions_.empty()) { YQL_CLOG(WARN, ProviderGeneric) << "Got empty list of partitions"; ReadSplitsFinished_ = true; NotifyComputeActorWithData(); - return Nothing(); + return; } // Prepare ReadSplits request. For the sake of simplicity, @@ -124,10 +121,7 @@ namespace NYql::NDq { dstSplit->set_description(srcSplit.description()); // Assign actual IAM token to a split - auto error = TokenProvider_->FillCredentials(*dstSplit->mutable_select()->mutable_data_source_instance()); - if (error) { - return TIssue(std::move(error)); - } + *dstSplit->mutable_select()->mutable_data_source_instance()->mutable_credentials() = credentials; } } @@ -143,8 +137,6 @@ namespace NYql::NDq { TEvReadSplitsIterator>( actorSystem, selfId, computeActorId, inputIndex, future); }); - - return Nothing(); } void Handle(TEvReadSplitsIterator::TPtr& ev) { diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_describe_table.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_describe_table.cpp index 38429b28be4..2f813e0547b 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_describe_table.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_describe_table.cpp @@ -15,6 +15,7 @@ #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h> #include <yql/essentials/ast/yql_type_string.h> #include <yql/essentials/ast/yql_expr.h> +#include <ydb/library/yql/providers/generic/actors/yql_generic_helpers.h> #include <ydb/library/yql/providers/generic/expr_nodes/yql_generic_expr_nodes.h> #include <ydb/library/yql/providers/generic/connector/libcpp/error.h> #include <ydb/library/yql/providers/generic/connector/libcpp/client.h> @@ -62,7 +63,7 @@ private: TIssues FillDescribeTableRequest(NConnector::NApi::TDescribeTableRequest& request, const TGenericClusterConfig& clusterConfig, const TString& tablePath); - void FillCredentials(NConnector::NApi::TDescribeTableRequest& request, + NThreading::TFuture<NConnector::NApi::TDescribeTableRequest> FillCredentials(NConnector::NApi::TDescribeTableRequest&& request, const TGenericClusterConfig& clusterConfig); void FillTypeMappingSettings(NConnector::NApi::TDescribeTableRequest& request); @@ -77,7 +78,7 @@ TGenericDescribeTableTransformer::TGenericDescribeTableTransformer(TGenericState : State_(std::move(state)) { } -void TGenericDescribeTableTransformer::FillCredentials(NConnector::NApi::TDescribeTableRequest& request, +NThreading::TFuture<NConnector::NApi::TDescribeTableRequest> TGenericDescribeTableTransformer::FillCredentials(NConnector::NApi::TDescribeTableRequest&& request, const TGenericClusterConfig& clusterConfig) { auto dsi = request.mutable_data_source_instance(); @@ -85,7 +86,7 @@ void TGenericDescribeTableTransformer::FillCredentials(NConnector::NApi::TDescri // connector will use Basic Auth to access external data sources. if (clusterConfig.GetCredentials().Hasbasic()) { *dsi->mutable_credentials() = clusterConfig.GetCredentials(); - return; + return NThreading::MakeFuture(std::move(request)); } // If there are no Basic Auth parameters, two options can be considered: @@ -95,7 +96,7 @@ void TGenericDescribeTableTransformer::FillCredentials(NConnector::NApi::TDescri if (iamToken) { *dsi->mutable_credentials()->mutable_token()->mutable_value() = iamToken; *dsi->mutable_credentials()->mutable_token()->mutable_type() = "IAM"; - return; + return NThreading::MakeFuture(std::move(request)); } // 2. Client provided other creds (service account, iam delegated cloud auth,...) that must be converted into IAM-token @@ -119,11 +120,15 @@ void TGenericDescribeTableTransformer::FillCredentials(NConnector::NApi::TDescri .emplace(clusterConfig.name(), credentialsProviderFactory->CreateProvider()) .first; } - - iamToken = providersIt->second->GetAuthInfo(); - Y_ENSURE(iamToken, "empty IAM token"); - *dsi->mutable_credentials()->mutable_token()->mutable_value() = iamToken; - *dsi->mutable_credentials()->mutable_token()->mutable_type() = "IAM"; + return providersIt->second->GetAuthInfoAsync() + .Apply([request=std::move(request)](const NThreading::TFuture<std::string>& future) mutable { + TString iamToken = ExtractFromConstFuture(future); + auto dsi = request.mutable_data_source_instance(); + Y_ENSURE(iamToken, "empty IAM token"); + *dsi->mutable_credentials()->mutable_token()->mutable_value() = std::move(iamToken); + *dsi->mutable_credentials()->mutable_token()->mutable_type() = "IAM"; + return std::move(request); + }); } template <typename T> @@ -413,36 +418,55 @@ TIssues TGenericDescribeTableTransformer::DescribeTableFromConnector(const TGene auto promise = NThreading::NewPromise(); handles.emplace_back(promise.GetFuture()); - desc->DataSourceInstance = request.data_source_instance(); - Y_ENSURE(State_->GenericClient); + FillCredentials(std::move(request), it->second).Subscribe([ + tableAddress, + desc = std::move(desc), + promise = std::move(promise), + client = State_->GenericClient, + timeout = State_->Configuration->DescribeTableTimeout + ](const NThreading::TFuture<NConnector::NApi::TDescribeTableRequest>& future) mutable { + try { + Y_ENSURE(client); + auto request = ExtractFromConstFuture(future); + desc->DataSourceInstance = request.data_source_instance(); - State_->GenericClient->DescribeTable(request, State_->Configuration->DescribeTableTimeout).Subscribe( - [desc, tableAddress, promise, client = State_->GenericClient](const NConnector::TDescribeTableAsyncResult& f1) mutable { - NConnector::TDescribeTableAsyncResult f2(f1); - auto result = f2.ExtractValueSync(); + client->DescribeTable(request, timeout).Subscribe([desc, tableAddress, promise](const NConnector::TDescribeTableAsyncResult& future) mutable { + try { + auto result = ExtractFromConstFuture(future); - // Check transport error - if (!result.Status.Ok()) { - desc->Issues.AddIssue(TStringBuilder() - << "Call DescribeTable for table " << tableAddress.ToString() << ": " - << result.Status.ToDebugString()); - promise.SetValue(); - return; - } + // Check transport error + if (!result.Status.Ok()) { + desc->Issues.AddIssue(TStringBuilder() + << "Call DescribeTable for table " << tableAddress.ToString() << ": " + << result.Status.ToDebugString()); + promise.SetValue(); + return; + } - // Check logical error - if (!NConnector::IsSuccess(*result.Response)) { - desc->Issues.AddIssues(NConnector::ErrorToIssues( - result.Response->error(), - TStringBuilder() << "Call DescribeTable for table " << tableAddress.ToString() << ": ")); + // Check logical error + if (!NConnector::IsSuccess(*result.Response)) { + desc->Issues.AddIssues(NConnector::ErrorToIssues( + result.Response->error(), + TStringBuilder() << "Call DescribeTable for table " << tableAddress.ToString() << ": ")); + promise.SetValue(); + return; + } + + // Preserve schema for the further usage + desc->Schema = result.Response->schema(); + promise.SetValue(); + } catch (const std::exception&) { + desc->Issues.AddIssue( + NYql::TIssue(TStringBuilder() << "Call DescribeTable for table " << tableAddress.ToString() << ": " << CurrentExceptionMessage())); + promise.SetValue(); + } + }); + } catch (const std::exception&) { + desc->Issues.AddIssue( + NYql::TIssue(TStringBuilder() << "Call FillCredentials for table " << tableAddress.ToString() << ": " << CurrentExceptionMessage())); promise.SetValue(); - return; } - - // Preserve schema for the further usage - desc->Schema = result.Response->schema(); - promise.SetValue(); }); return {}; @@ -459,7 +483,6 @@ TIssues TGenericDescribeTableTransformer::FillDescribeTableRequest(NConnector::N dsi->set_use_tls(clusterConfig.GetUseSsl()); dsi->set_protocol(clusterConfig.GetProtocol()); - FillCredentials(request, clusterConfig); FillTypeMappingSettings(request); auto issues = FillDataSourceOptions(request, clusterConfig); diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_list_splits.cpp b/ydb/library/yql/providers/generic/provider/yql_generic_list_splits.cpp index 7a341d7a41b..8633032f11a 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_list_splits.cpp +++ b/ydb/library/yql/providers/generic/provider/yql_generic_list_splits.cpp @@ -15,6 +15,7 @@ #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h> #include <yql/essentials/ast/yql_type_string.h> #include <yql/essentials/ast/yql_expr.h> +#include <ydb/library/yql/providers/generic/actors/yql_generic_helpers.h> #include <ydb/library/yql/providers/generic/provider/yql_generic_utils.h> #include <ydb/library/yql/providers/generic/provider/yql_generic_predicate_pushdown.h> #include <ydb/library/yql/providers/generic/expr_nodes/yql_generic_expr_nodes.h> @@ -191,9 +192,8 @@ TIssues TGenericListSplitTransformer::ListSplitsFromConnector(const TListSplitRe Y_ENSURE(State_->GenericClient); State_->GenericClient->ListSplits(request).Subscribe([desc, promise, data] - (const NConnector::TListSplitsStreamIteratorAsyncResult f3) mutable { - NConnector::TListSplitsStreamIteratorAsyncResult f4(f3); - auto streamIterResult = f4.ExtractValueSync(); + (const NConnector::TListSplitsStreamIteratorAsyncResult& future) mutable { + auto streamIterResult = ExtractFromConstFuture(future); // Check transport error if (!streamIterResult.Status.Ok()) { @@ -211,9 +211,8 @@ TIssues TGenericListSplitTransformer::ListSplitsFromConnector(const TListSplitRe // Pass drainer to the callback because we want him to stay alive until the callback is called drainer->Run().Subscribe([desc, promise, data, drainer] - (const NThreading::TFuture<NConnector::TListSplitsStreamIteratorDrainer::TBuffer>& f5) mutable { - NThreading::TFuture<NConnector::TListSplitsStreamIteratorDrainer::TBuffer> f6(f5); - auto drainerResult = f6.ExtractValueSync(); + (const NThreading::TFuture<NConnector::TListSplitsStreamIteratorDrainer::TBuffer>& future) mutable { + auto drainerResult = ExtractFromConstFuture(future); // check transport and logical errors if (drainerResult.Issues) { diff --git a/ydb/library/yql/providers/generic/provider/yql_generic_state.h b/ydb/library/yql/providers/generic/provider/yql_generic_state.h index 205d96ab2e7..fea1fae0d0d 100644 --- a/ydb/library/yql/providers/generic/provider/yql_generic_state.h +++ b/ydb/library/yql/providers/generic/provider/yql_generic_state.h @@ -128,7 +128,8 @@ namespace NYql { // key - cluster name, value - TCredentialsProviderPtr // It's important to cache credentials providers, because they make IO - // (synchronous call via Token Accessor client) during the construction. + // (e.g. synchronous call via Token Accessor client) during the construction. + // TODO: reconsider cache usefulness; TokenAccessor is part of deprecated yqv1, IAM cloud delegated auth (which also uses IO) shares singleton instance internally, "simple" providers are inexpensive std::unordered_map<TString, NYdb::TCredentialsProviderPtr> CredentialProviders; IStructuredTokenCredentialsFactory::TPtr CredentialsFactory; diff --git a/ydb/services/workload_manager/ut/common/workload_service_ut_common.cpp b/ydb/services/workload_manager/ut/common/workload_service_ut_common.cpp index c814295509e..08aacab52c8 100644 --- a/ydb/services/workload_manager/ut/common/workload_service_ut_common.cpp +++ b/ydb/services/workload_manager/ut/common/workload_service_ut_common.cpp @@ -646,26 +646,7 @@ public: } void WaitForClassifierPropagation() const override { - const auto nodeIndex = 0; - auto* runtime = GetRuntime(); - const ui32 nodeId = runtime->GetNodeId(nodeIndex); - const TActorId edgeActor = runtime->AllocateEdgeActor(nodeIndex); - - runtime->Send( - NMetadata::NProvider::MakeServiceId(nodeId), - edgeActor, - new NMetadata::NProvider::TEvAskSnapshot(std::make_shared<TResourcePoolClassifierSnapshotsFetcher>()), - nodeIndex); - - const auto response = runtime->GrabEdgeEvent<NMetadata::NProvider::TEvRefreshSubscriberData>( - edgeActor, FUTURE_WAIT_TIMEOUT); - UNIT_ASSERT_C(response, "Timed out waiting for resource pool classifier snapshot refresh"); - - runtime->Send( - NKqp::MakeKqpProxyID(nodeId), - edgeActor, - new NMetadata::NProvider::TEvRefreshSubscriberData(response->Get()->GetSnapshot()), - nodeIndex); + NWorkloadManager::WaitForClassifierPropagation(*GetRuntime()); } void StopWorkloadService(ui64 nodeIndex = 0) const override { @@ -937,4 +918,26 @@ void WaitForClassifierSuccess(TIntrusivePtr<IYdbSetup> ydb, const TQueryRunnerSe } +void WaitForClassifierPropagation(TTestActorRuntime& runtime, ui32 nodeIndex) { + const ui32 nodeId = runtime.GetNodeId(nodeIndex); + const TActorId edgeActor = runtime.AllocateEdgeActor(nodeIndex); + + runtime.Send( + NMetadata::NProvider::MakeServiceId(nodeId), + edgeActor, + new NMetadata::NProvider::TEvAskSnapshot( + std::make_shared<TResourcePoolClassifierSnapshotsFetcher>()), + nodeIndex); + + const auto response = runtime.GrabEdgeEvent<NMetadata::NProvider::TEvRefreshSubscriberData>( + edgeActor, FUTURE_WAIT_TIMEOUT); + UNIT_ASSERT_C(response, "Timed out waiting for resource pool classifier snapshot refresh"); + + runtime.Send( + NKqp::MakeKqpProxyID(nodeId), + edgeActor, + new NMetadata::NProvider::TEvRefreshSubscriberData(response->Get()->GetSnapshot()), + nodeIndex); +} + } // NKikimr::NWorkloadManager diff --git a/ydb/services/workload_manager/ut/common/workload_service_ut_common.h b/ydb/services/workload_manager/ut/common/workload_service_ut_common.h index a5a1a5571b4..e59b9fe5b06 100644 --- a/ydb/services/workload_manager/ut/common/workload_service_ut_common.h +++ b/ydb/services/workload_manager/ut/common/workload_service_ut_common.h @@ -201,4 +201,6 @@ struct TSampleQueries { }; +void WaitForClassifierPropagation(TTestActorRuntime& runtime, ui32 nodeIndex = 0); + } // namespace NKikimr::NWorkloadManager diff --git a/ydb/tests/compatibility/streaming/test_watermarks.py b/ydb/tests/compatibility/streaming/test_watermarks.py new file mode 100644 index 00000000000..8dd7ca4f820 --- /dev/null +++ b/ydb/tests/compatibility/streaming/test_watermarks.py @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- +import logging +import os +import pytest +import time +from typing import Generator, Self + +from ydb.tests.library.compatibility.fixtures import MixedClusterFixture, RestartToAnotherVersionFixture, RollingUpgradeAndDowngradeFixture +from ydb.tests.library.harness.util import LogLevels +from ydb.tests.library.test_meta import link_test_case +from ydb.tests.fq.streaming_common.common import YdbClient + +logger = logging.getLogger(__name__) + + +class StreamingTestBase: + def setup_cluster(self: Self) -> Generator[None, None, None]: + logger.debug(f"setup_cluster, versions {self.versions}") + + if min(self.versions) < (26, 2): + logger.debug("skip test, only available since 26-2") + pytest.skip("Only available since 26-2") + + extra_feature_flags = [ + "enable_external_data_sources", + "enable_streaming_queries", + "enable_shared_reading_in_streaming_queries", + ] + + os.environ["YDB_TEST_DEFAULT_CHECKPOINTING_PERIOD_MS"] = "200" + os.environ["YDB_TEST_LEASE_DURATION_SEC"] = "15" + for _ in super().setup_cluster( + disabled_feature_flags=["enable_drain_on_shutdown"], + extra_feature_flags=extra_feature_flags, + additional_log_configs={ + 'KQP_COMPUTE': LogLevels.TRACE, + 'STREAMS_CHECKPOINT_COORDINATOR': LogLevels.TRACE, + 'STREAMS_STORAGE_SERVICE': LogLevels.TRACE, + 'FQ_ROW_DISPATCHER': LogLevels.TRACE, + 'KQP_PROXY': LogLevels.DEBUG, + 'KQP_EXECUTER': LogLevels.DEBUG, + }, + table_service_config={ + "enable_watermarks": True, + "enable_watermarks_advanced": True, + }, + ): + self.ydb_client = YdbClient(self.driver) + try: + yield + finally: + self.ydb_client.stop() + + def change_cluster_version(self: Self) -> None: + self.ydb_client.stop() + super().change_cluster_version() + self.ydb_client = YdbClient(self.driver) + + def roll(self: Self) -> Generator[None, None, None]: + self.ydb_client.stop() + + for result in super().roll(): + self.ydb_client = YdbClient(self.driver) + yield result + self.ydb_client.stop() + + self.ydb_client = YdbClient(self.driver) + + def create_objects(self: Self, external: bool) -> None: + logger.debug("create_objects") + self.input_topic = 'streaming_recipe/input_topic' + self.output_topic = 'streaming_recipe/output_topic' + self.consumer_name = 'consumer_name' + self.ydb_client.query(f""" + CREATE TOPIC `{self.input_topic}`; + CREATE TOPIC `{self.output_topic}` (CONSUMER {self.consumer_name}); + """) + + self.ydb_client.query(""" + CREATE TABLE table_name ( + key Utf8, + value Utf8, + PRIMARY KEY (key) + ); + """) + + self.ydb_client.query(""" + UPSERT INTO table_name (key, value) VALUES ('key1', 'value1'); + """) + + if external: + self.create_external_data_source() + self.input_object = f"`source_name`.`{self.input_topic}`" + self.output_object = f"`source_name`.`{self.output_topic}`" + else: + self.input_object = f"`{self.input_topic}`" + self.output_object = f"`{self.output_topic}`" + + def create_external_data_source(self: Self) -> None: + logger.debug("create_external_data_source") + endpoint = f"localhost:{self.cluster.nodes[1].port}" + self.ydb_client.create_external_data_source("source_name", endpoint, self.database_path) + + def create_streaming_query(self: Self) -> None: + logger.debug("create_streaming_query") + self.ydb_client.query(f""" + CREATE STREAMING QUERY `my_queries/query_name` AS DO BEGIN + $precompute_data = SELECT value FROM table_name LIMIT 1; + + $input = ( + SELECT + * + FROM + {self.input_object} WITH ( + FORMAT = 'json_each_row', + SCHEMA (time String NOT NULL, level String NOT NULL, host String NOT NULL), + WATERMARK = CAST(time AS Timestamp) - Interval('PT1M') + ) + ); + + $output = ( + SELECT + host, + COUNT(*) AS error_count, + CAST(HOP_START() AS String) AS ts + FROM + $input + WHERE + level == 'error' + GROUP BY + host, + HoppingWindow(CAST(time AS Timestamp), 'PT600S', 'PT600S') + ); + + INSERT INTO {self.output_object} + SELECT ToBytes(Unwrap(Yson::SerializeJson(Yson::From(TableRow())))) || Unwrap($precompute_data) + FROM $output; + END DO; + """) + + def create_simple_streaming_query(self: Self) -> None: + logger.debug("create_simple_streaming_query") + self.ydb_client.query(f""" + CREATE STREAMING QUERY `my_queries/query_name` AS DO BEGIN + $precompute_data = SELECT value FROM table_name LIMIT 1; + + $input = ( + SELECT + * + FROM + {self.input_object} WITH ( + FORMAT = 'json_each_row', + SCHEMA (time String NOT NULL, level String NOT NULL, host String NOT NULL), + WATERMARK = CAST(time AS Timestamp) - Interval('PT1M') + ) + ); + + INSERT INTO {self.output_object} + SELECT ToBytes(Unwrap(Yson::SerializeJson(Yson::From(TableRow())))) || Unwrap($precompute_data) + FROM $input; + END DO; + """) + + def do_write_read(self: Self, input_data: list[str], expected: list[str]) -> None: + logger.debug("do_write_read") + time.sleep(2) + + logger.debug("write data to stream") + self.ydb_client.topic_write(self.input_topic, input_data) + + logger.debug("read data from stream") + actual = self.ydb_client.topic_read_until(self.output_topic, self.consumer_name, len(expected)) + if len(actual) != len(expected): + actual = actual[-len(expected):] # deduplication disabled + assert sorted(actual) == sorted(expected) + + def do_test_part1(self: Self) -> None: + suffix = 'value1' + input_data = [ + '{"time": "2025-01-01T00:00:00.000000Z", "level": "error", "host": "host-1"}', + '{"time": "2025-01-01T00:04:00.000000Z", "level": "error", "host": "host-2"}', + '{"time": "2025-01-01T00:08:00.000000Z", "level": "error", "host": "host-1"}', + '{"time": "2025-01-01T00:12:00.000000Z", "level": "error", "host": "host-2"}', + '{"time": "2025-01-01T00:12:00.000000Z", "level": "error", "host": "host-1"}', + ] + expected = [ + '{"error_count":1,"host":"host-2","ts":"2025-01-01T00:00:00Z"}' + suffix, + '{"error_count":2,"host":"host-1","ts":"2025-01-01T00:00:00Z"}' + suffix, + ] + self.do_write_read(input_data, expected) + + def do_test_part2(self: Self) -> None: + suffix = 'value1' + input_data = [ + '{"time": "2025-01-01T00:15:00.000000Z", "level": "error", "host": "host-2"}', + '{"time": "2025-01-01T00:22:00.000000Z", "level": "error", "host": "host-1"}', + '{"time": "2025-01-01T00:22:00.000000Z", "level": "error", "host": "host-2"}', + ] + expected = [ + '{"error_count":2,"host":"host-2","ts":"2025-01-01T00:10:00Z"}' + suffix, + '{"error_count":1,"host":"host-1","ts":"2025-01-01T00:10:00Z"}' + suffix, + ] + self.do_write_read(input_data, expected) + + +class TestWatermarksMixedCluster(StreamingTestBase, MixedClusterFixture): + @pytest.fixture(autouse=True, scope="function") + def setup(self: Self) -> Generator[None, None, None]: + yield from self.setup_cluster() + + @link_test_case("#28606") + @pytest.mark.parametrize("external", [True, False]) + def test_mixed_cluster(self: Self, external: bool) -> None: + self.create_objects(external) + self.create_streaming_query() + self.do_test_part1() + self.do_test_part2() + + +class TestWatermarksRestartToAnotherVersion(StreamingTestBase, RestartToAnotherVersionFixture): + @pytest.fixture(autouse=True, scope="function") + def setup(self: Self) -> Generator[None, None, None]: + yield from self.setup_cluster() + + @link_test_case("#28606") + @pytest.mark.parametrize("external", [True, False]) + def test_restart_to_another_version(self: Self, external: bool) -> None: + self.create_objects(external) + self.create_streaming_query() + self.do_test_part1() + self.change_cluster_version() + self.do_test_part2() + + +class TestWatermarksRollingUpgradeAndDowngrade(StreamingTestBase, RollingUpgradeAndDowngradeFixture): + @pytest.fixture(autouse=True, scope="function") + def setup(self: Self) -> Generator[None, None, None]: + yield from self.setup_cluster() + + @link_test_case("#28606") + @pytest.mark.parametrize("external", [True, False]) + def test_rolling_upgrade(self: Self, external: bool) -> None: + self.create_objects(external) + self.create_simple_streaming_query() + suffix = 'value1' + + for i, _ in enumerate(self.roll()): + input_data = [ + f'{{"time": "2025-01-01T00:15:00.000000Z", "level": "error", "host": "host-{i}"}}', + ] + expected = [ + f'{{"host":"host-{i}","level":"error","time":"2025-01-01T00:15:00.000000Z"}}' + suffix, + ] + self.do_write_read(input_data, expected) + time.sleep(0.5) diff --git a/ydb/tests/compatibility/streaming/ya.make b/ydb/tests/compatibility/streaming/ya.make index 434fe7c873f..623635cf738 100644 --- a/ydb/tests/compatibility/streaming/ya.make +++ b/ydb/tests/compatibility/streaming/ya.make @@ -9,10 +9,12 @@ SPLIT_FACTOR(10) TEST_SRCS( test_scalar_topic_write.py test_streaming.py + test_watermarks.py ) SIZE(LARGE) REQUIREMENTS(cpu:4) +REQUIREMENTS(ram:16) INCLUDE(${ARCADIA_ROOT}/ydb/tests/large.inc) DEPENDS( @@ -22,6 +24,7 @@ DEPENDS( PEERDIR( contrib/python/boto3 + ydb/tests/fq/streaming_common ydb/tests/library ydb/tests/library/compatibility ydb/tests/library/test_meta diff --git a/ydb/tests/datashard/set_column_constraint/concurrency/test_set_not_null_concurrency.py b/ydb/tests/datashard/set_column_constraint/concurrency/test_set_not_null_concurrency.py new file mode 100644 index 00000000000..2828b30f43d --- /dev/null +++ b/ydb/tests/datashard/set_column_constraint/concurrency/test_set_not_null_concurrency.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +import concurrent.futures +import random +import threading +import time + +import pytest +import ydb + +from ydb.tests.sql.lib.test_base import TestBase + + +_RETRIABLE_NAMES = [ + "Aborted", + "Overloaded", + "Unavailable", + "BadSession", + "SessionBusy", + "SchemeError", + "Undetermined", + "Timeout", +] + +RETRIABLE_ERRORS = tuple( + getattr(ydb.issues, name) for name in _RETRIABLE_NAMES if hasattr(ydb.issues, name) +) + + +def _is_set_not_null_in_progress(err) -> bool: + return "is currently in progress" in str(err) + + +class TestSetNotNullConcurrency(TestBase): + @classmethod + def get_extra_feature_flags(cls): + return super().get_extra_feature_flags() + ["enable_set_column_constraint"] + + def _create_table(self, table_name: str): + self.query(f""" + CREATE TABLE `{table_name}` ( + id Uint64 NOT NULL, + value Int64, + PRIMARY KEY (id) + ) + """) + + def _count_nulls(self, table_name: str) -> int: + result = self.query(f"SELECT COUNT(*) AS cnt FROM `{table_name}` WHERE value IS NULL") + return result[0]["cnt"] + + def _row_count(self, table_name: str) -> int: + result = self.query(f"SELECT COUNT(*) AS cnt FROM `{table_name}`") + return result[0]["cnt"] + + def _upsert_range(self, table_name: str, begin: int, end: int): + values = ", ".join(f"({i}, {i * 10 + 1})" for i in range(begin, end)) + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES {values}") + + def test_set_not_null_with_concurrent_upserts(self): + table_name = f"{self.table_path}_concurrency" + self._create_table(table_name) + + seed_count = 200 + self._upsert_range(table_name, 0, seed_count) + assert self._count_nulls(table_name) == 0 + + duration = 20 + deadline = time.time() + duration + stop = threading.Event() + errors = [] + + next_key = seed_count + next_key_lock = threading.Lock() + + def upsert_worker(): + nonlocal next_key + while not stop.is_set() and time.time() < deadline: + try: + with next_key_lock: + key = next_key + next_key += 1 + self.query( + f"UPSERT INTO `{table_name}` (id, value) VALUES ({key}, {random.randint(1, 10**9)})" + ) + existing = random.randint(0, key) + self.query( + f"UPSERT INTO `{table_name}` (id, value) VALUES ({existing}, {random.randint(1, 10**9)})" + ) + except RETRIABLE_ERRORS: + continue + except ydb.issues.Error as e: + if _is_set_not_null_in_progress(e): + continue + errors.append(("upsert", str(e))) + return + + alter_cycles = 0 + + def alter_worker(): + nonlocal alter_cycles + while not stop.is_set() and time.time() < deadline: + try: + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value SET NOT NULL") + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value DROP NOT NULL") + alter_cycles += 1 + except ydb.issues.PreconditionFailed as e: + errors.append(("set_not_null_precondition", str(e))) + return + except RETRIABLE_ERRORS: + continue + except ydb.issues.Error as e: + errors.append(("alter", str(e))) + return + time.sleep(0.6) + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + futures = [executor.submit(upsert_worker) for _ in range(4)] + futures.append(executor.submit(alter_worker)) + + concurrent.futures.wait(futures, timeout=duration + 60) + stop.set() + for future in futures: + future.result(timeout=10) + + assert not errors, f"Unexpected errors during concurrent execution: {errors}" + assert alter_cycles > 0, "SET NOT NULL / DROP NOT NULL never completed a cycle" + + final_count = self._row_count(table_name) + assert final_count > seed_count, f"expected more than {seed_count} rows, got {final_count}" + + assert self._count_nulls(table_name) == 0 + + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value SET NOT NULL") + assert self._count_nulls(table_name) == 0 + with pytest.raises(ydb.issues.Error): + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (999999999, NULL)") + + def test_set_not_null_rejects_concurrent_null_upserts(self): + table_name = f"{self.table_path}_null_race" + self._create_table(table_name) + self._upsert_range(table_name, 0, 100) + + stop = threading.Event() + errors = [] + + def null_upsert_worker(): + key = 1000 + while not stop.is_set(): + try: + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES ({key}, NULL)") + key += 1 + except RETRIABLE_ERRORS: + continue + except ydb.issues.Error: + continue + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + writer = executor.submit(null_upsert_worker) + try: + for _ in range(15): + try: + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value SET NOT NULL") + assert self._count_nulls(table_name) == 0 + with pytest.raises(ydb.issues.Error): + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (2000, NULL)") + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value DROP NOT NULL") + except ydb.issues.PreconditionFailed: + pass + except RETRIABLE_ERRORS: + continue + time.sleep(0.6) + finally: + stop.set() + writer.result(timeout=10) + + assert not errors, f"Unexpected errors: {errors}" diff --git a/ydb/tests/datashard/set_column_constraint/concurrency/ya.make b/ydb/tests/datashard/set_column_constraint/concurrency/ya.make new file mode 100644 index 00000000000..1b8b8765b8c --- /dev/null +++ b/ydb/tests/datashard/set_column_constraint/concurrency/ya.make @@ -0,0 +1,24 @@ +PY3TEST() +INCLUDE(${ARCADIA_ROOT}/ydb/tests/harness_dep.inc) + +TEST_SRCS( + test_set_not_null_concurrency.py +) + +IF (SANITIZER_TYPE) + REQUIREMENTS(cpu:2) +ENDIF() + +FORK_SUBTESTS() +FORK_TEST_FILES() + +PEERDIR( + ydb/tests/datashard/lib + ydb/tests/sql/lib +) + +DEPENDS( + ydb/apps/ydb +) + +END() diff --git a/ydb/tests/datashard/set_column_constraint/functional/test_set_not_null.py b/ydb/tests/datashard/set_column_constraint/functional/test_set_not_null.py new file mode 100644 index 00000000000..5c508c768ed --- /dev/null +++ b/ydb/tests/datashard/set_column_constraint/functional/test_set_not_null.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +import pytest +import ydb + +from ydb.tests.datashard.lib.types_of_variables import ( + pk_types, + non_pk_types, + type_to_literal_lambda, + cleanup_type_name, +) +from ydb.tests.sql.lib.test_base import TestBase + + +class TestSetNotNull(TestBase): + ALL_TYPES = {**pk_types, **non_pk_types} + + @classmethod + def get_extra_feature_flags(cls): + return super().get_extra_feature_flags() + ["enable_set_column_constraint"] + + def _create_table(self, table_name: str, value_type: str = "Int64"): + self.query(f""" + CREATE TABLE `{table_name}` ( + id Uint64 NOT NULL, + value {value_type}, + PRIMARY KEY (id) + ) + """) + + def _count_nulls(self, table_name: str) -> int: + result = self.query(f"SELECT COUNT(*) AS cnt FROM `{table_name}` WHERE value IS NULL") + return result[0]["cnt"] + + def _row_count(self, table_name: str) -> int: + result = self.query(f"SELECT COUNT(*) AS cnt FROM `{table_name}`") + return result[0]["cnt"] + + def test_set_not_null_success(self): + table_name = f"{self.table_path}_success" + self._create_table(table_name) + + for i in range(1, 11): + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES ({i}, {i * 10})") + + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value SET NOT NULL") + + assert self._count_nulls(table_name) == 0 + assert self._row_count(table_name) == 10 + + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (100, 42)") + assert self._row_count(table_name) == 11 + + with pytest.raises(ydb.issues.Error): + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (200, NULL)") + + with pytest.raises(ydb.issues.Error): + self.query(f"UPSERT INTO `{table_name}` (id) VALUES (300)") + + assert self._count_nulls(table_name) == 0 + + def test_set_not_null_rejects_existing_nulls(self): + table_name = f"{self.table_path}_existing_nulls" + self._create_table(table_name) + + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (1, 10), (2, 20)") + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (3, NULL)") + + with pytest.raises(ydb.issues.PreconditionFailed): + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value SET NOT NULL") + + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (4, NULL)") + assert self._count_nulls(table_name) == 2 + + def test_drop_not_null(self): + table_name = f"{self.table_path}_drop" + self._create_table(table_name) + + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (1, 10), (2, 20)") + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value SET NOT NULL") + + with pytest.raises(ydb.issues.Error): + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (3, NULL)") + + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value DROP NOT NULL") + + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (3, NULL)") + assert self._count_nulls(table_name) == 1 + + def test_set_not_null_idempotent_cycle(self): + table_name = f"{self.table_path}_cycle" + self._create_table(table_name) + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (1, 10)") + + for _ in range(3): + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value SET NOT NULL") + with pytest.raises(ydb.issues.Error): + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (2, NULL)") + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value DROP NOT NULL") + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (2, NULL)") + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (2, 20)") + + assert self._count_nulls(table_name) == 0 + + @pytest.mark.parametrize("type_name", sorted(ALL_TYPES.keys())) + def test_set_not_null_all_types(self, type_name: str): + table_name = f"{self.table_path}_type_{cleanup_type_name(type_name)}" + self._create_table(table_name, value_type=type_name) + + make_literal = type_to_literal_lambda[type_name] + for i in range(1, 4): + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES ({i}, {make_literal(i)})") + + self.query(f"ALTER TABLE `{table_name}` ALTER COLUMN value SET NOT NULL") + assert self._count_nulls(table_name) == 0 + + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (4, {make_literal(4)})") + assert self._row_count(table_name) == 4 + + with pytest.raises(ydb.issues.Error): + self.query(f"UPSERT INTO `{table_name}` (id, value) VALUES (5, NULL)") + + assert self._count_nulls(table_name) == 0 diff --git a/ydb/tests/datashard/set_column_constraint/functional/ya.make b/ydb/tests/datashard/set_column_constraint/functional/ya.make new file mode 100644 index 00000000000..4dbbbff400f --- /dev/null +++ b/ydb/tests/datashard/set_column_constraint/functional/ya.make @@ -0,0 +1,20 @@ +PY3TEST() +INCLUDE(${ARCADIA_ROOT}/ydb/tests/harness_dep.inc) + +TEST_SRCS( + test_set_not_null.py +) + +FORK_SUBTESTS() +FORK_TEST_FILES() + +PEERDIR( + ydb/tests/datashard/lib + ydb/tests/sql/lib +) + +DEPENDS( + ydb/apps/ydb +) + +END() diff --git a/ydb/tests/datashard/set_column_constraint/ya.make b/ydb/tests/datashard/set_column_constraint/ya.make new file mode 100644 index 00000000000..23aaaf450dc --- /dev/null +++ b/ydb/tests/datashard/set_column_constraint/ya.make @@ -0,0 +1,4 @@ +RECURSE( + functional + concurrency +) diff --git a/ydb/tests/datashard/truncate/concurrency/test_truncate_table_concurrency.py b/ydb/tests/datashard/truncate/concurrency/test_truncate_table_concurrency.py index 646b137fc24..8b6748d1d58 100644 --- a/ydb/tests/datashard/truncate/concurrency/test_truncate_table_concurrency.py +++ b/ydb/tests/datashard/truncate/concurrency/test_truncate_table_concurrency.py @@ -14,7 +14,6 @@ class TestTruncateTableConcurrency(TestBase): @classmethod def get_extra_feature_flags(cls): return [ - "enable_truncate_table", "enable_fulltext_index", ] diff --git a/ydb/tests/datashard/truncate/functional/test_truncate_table_fulltext_index.py b/ydb/tests/datashard/truncate/functional/test_truncate_table_fulltext_index.py index 655e3714a92..f1a5ac93340 100644 --- a/ydb/tests/datashard/truncate/functional/test_truncate_table_fulltext_index.py +++ b/ydb/tests/datashard/truncate/functional/test_truncate_table_fulltext_index.py @@ -8,7 +8,7 @@ CLUSTER_CONFIG = dict( 'TX_DATASHARD': LogLevels.DEBUG, 'KQP_PROXY': LogLevels.DEBUG, }, - extra_feature_flags=["enable_fulltext_index", "enable_fulltext_index_prefix", "enable_truncate_table"] + extra_feature_flags=["enable_fulltext_index", "enable_fulltext_index_prefix"] ) diff --git a/ydb/tests/datashard/ya.make b/ydb/tests/datashard/ya.make index 06f8004dd5b..7be9a70911c 100644 --- a/ydb/tests/datashard/ya.make +++ b/ydb/tests/datashard/ya.make @@ -11,6 +11,7 @@ RECURSE( select secondary_index s3 + set_column_constraint truncate ttl vector_index diff --git a/ydb/tests/fq/solomon/scalar_write/test_scalar_solomon_write.py b/ydb/tests/fq/solomon/scalar_write/test_scalar_solomon_write.py index 068157fceaf..c9ec9d6c5af 100644 --- a/ydb/tests/fq/solomon/scalar_write/test_scalar_solomon_write.py +++ b/ydb/tests/fq/solomon/scalar_write/test_scalar_solomon_write.py @@ -607,8 +607,7 @@ class TestScalarSolomonWriteInYdb(SolomonTestBase): restricted_source = entity_name("restricted_source") self.create_source(kikimr, restricted_source) - test_client = YdbClient(kikimr.endpoint.endpoint, kikimr.endpoint.database, "test@builtin") - test_client.wait_connection() + test_client = YdbClient.from_driver_config(kikimr.endpoint.endpoint, kikimr.endpoint.database, "test@builtin") try: self._expect_error( kikimr, diff --git a/ydb/tests/fq/streaming/test_scalar_topic_write.py b/ydb/tests/fq/streaming/test_scalar_topic_write.py index c2e3810c777..fb48ddcb2c4 100644 --- a/ydb/tests/fq/streaming/test_scalar_topic_write.py +++ b/ydb/tests/fq/streaming/test_scalar_topic_write.py @@ -336,10 +336,8 @@ class TestScalarTopicWriteInYdb(StreamingTestBase): ) try: - external_client = YdbClient(endpoint.endpoint, endpoint.database) - test_client = YdbClient(kikimr.endpoint.endpoint, kikimr.endpoint.database, "test@builtin") - external_client.wait_connection() - test_client.wait_connection() + external_client = YdbClient.from_driver_config(endpoint.endpoint, endpoint.database) + test_client = YdbClient.from_driver_config(kikimr.endpoint.endpoint, kikimr.endpoint.database, "test@builtin") test_secret_name = entity_name("test_secret") test_source_name = entity_name("test_target_source") diff --git a/ydb/tests/fq/streaming/test_udfs.py b/ydb/tests/fq/streaming/test_udfs.py index 58809a5eb24..f4a4246a257 100644 --- a/ydb/tests/fq/streaming/test_udfs.py +++ b/ydb/tests/fq/streaming/test_udfs.py @@ -199,8 +199,7 @@ END DO time.sleep(5) second_node = list(kikimr_udfs.cluster.nodes.values())[1] - kikimr_udfs.ydb_client = YdbClient(database=kikimr_udfs.endpoint.database, endpoint=f"grpc://{second_node.host}:{second_node.port}", enable_discovery=False) - kikimr_udfs.ydb_client.wait_connection() + kikimr_udfs.ydb_client = YdbClient.from_driver_config(database=kikimr_udfs.endpoint.database, endpoint=f"grpc://{second_node.host}:{second_node.port}", enable_discovery=False) logger.info("Checking query state after restart") validate_query(precompute_sql, tests_count, status=["SUSPENDED", "FAILED", "STARTING", "RUNNING"], check_issues=False, retry_count=[0, 1]) diff --git a/ydb/tests/fq/streaming/test_watermarks.py b/ydb/tests/fq/streaming/test_watermarks.py index ef14836890c..92264cad8c3 100644 --- a/ydb/tests/fq/streaming/test_watermarks.py +++ b/ydb/tests/fq/streaming/test_watermarks.py @@ -1,125 +1,64 @@ import datetime +import json import logging import pytest import time -from typing import Callable +from typing import Callable, Self -from ydb.tests.fq.streaming_common.common import Kikimr, StreamingTestBase +from ydb.tests.fq.streaming_common.common import Kikimr, StreamingTestBase, YdbClient +from ydb.tests.library.test_meta import link_test_case logger = logging.getLogger(__name__) +DEFAULT_INITIAL_TS = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) class TestWatermarksInYdb(StreamingTestBase): - @pytest.mark.parametrize("shared_reading", [False, True], ids=["no_shared", "shared"]) - @pytest.mark.parametrize("tasks", [1, 2]) - @pytest.mark.parametrize("local_topics", [True, False]) - def test_watermarks(self: StreamingTestBase, kikimr: Kikimr, entity_name: Callable[[str], str], shared_reading: bool, tasks: int, local_topics: bool) -> None: - if local_topics and shared_reading: - pytest.skip("Shared reading is not supported for local topics: YQ-5036") - - endpoint = self.get_endpoint(kikimr, local_topics) - query_name = f"test_watermarks_{shared_reading}{tasks}{local_topics}" - source_name = entity_name(query_name) - self.init_topics(source_name, partitions_count=tasks, endpoint=endpoint) - self.create_source(kikimr, source_name, shared_reading) + idle_timeout_seconds = 5 - cluster = f"{source_name}." if not local_topics else "" - idleness_clause = ', WATERMARK_IDLE_TIMEOUT = "PT5S"' if tasks > 1 else '' - - sql = f''' - CREATE STREAMING QUERY `{query_name}` AS DO BEGIN - PRAGMA ydb.MaxTasksPerStage = '{tasks}'; - - $input = ( - SELECT - input.*, - CAST(ts AS Timestamp) AS event_time, - FROM - {cluster}{self.input_topic} WITH ( - FORMAT = json_each_row, - SCHEMA (ts String, pass Uint64), - WATERMARK = CAST(ts AS Timestamp) - Interval('PT5S') - {idleness_clause} - ) AS input - ); + @staticmethod + def _event( + seconds: int, + event_id: str, + filter: bool = False, + initial_ts: datetime.datetime = DEFAULT_INITIAL_TS, + ) -> str: + event_time = initial_ts + datetime.timedelta(seconds=seconds) + return json.dumps({ + "ts": event_time.isoformat().replace("+00:00", "Z"), + "pass": 0 if filter else 1, + "id": event_id, + }) - $hop = ( - SELECT - CAST(HOP_END() AS String) AS event_time, - AGGREGATE_LIST(ts) AS ts - FROM - $input - WHERE - pass > 0 - GROUP BY - HoppingWindow(CAST(event_time AS Timestamp), 'PT1S', 'PT1S') - ); - - $output = ( - SELECT - CAST(HOP_END() AS String) AS event_time, - AGGREGATE_LIST(ts) AS ts - FROM - $hop - GROUP BY - HoppingWindow(CAST(event_time AS Timestamp), 'PT1S', 'PT1S') - ); - - INSERT INTO {cluster}{self.output_topic} - SELECT ToBytes(Unwrap(Yson::SerializeJson(Yson::From(ts)))) - FROM $output; - END DO; - ''' - kikimr.ydb_client.query(sql) - self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") - - self.write_stream( - data=[ - '{"ts": "1970-01-01T00:00:40Z", "pass": 1}', - '{"ts": "1970-01-01T00:00:50Z", "pass": 1}', - '{"ts": "1970-01-01T00:01:00Z", "pass": 0}', - ], - endpoint=endpoint, - partition_key=b'1', + def _create_query( + self, + kikimr: Kikimr, + entity_name: Callable[[str], str], + scenario: str, + local_topics: bool, + shared_reading: bool, + tasks: int = 2, + partitions_count: int | None = None, + idle_timeout_seconds: int | None = None, + settings: dict[str, str] = {}, + input_parsing: bool = False, + cascade_hopping: bool = False, + ) -> str: + query_name = entity_name(scenario) + partitions_count = partitions_count or tasks + idle_timeout_seconds = idle_timeout_seconds or self.idle_timeout_seconds + input_name, output_name, _ = self.get_io_names( + kikimr, query_name, local_topics, entity_name, partitions_count=partitions_count, shared=shared_reading ) - if shared_reading and tasks > 1: - time.sleep(10) # leave a bit more time to fire up idle timeout - - expected = [ - '[["1970-01-01T00:00:40Z"]]', - '[["1970-01-01T00:00:50Z"]]', - ] - actual = self.read_stream(len(expected), topic_path=self.output_topic, endpoint=endpoint) - assert sorted(actual) == expected - - sql = f'''DROP STREAMING QUERY `{query_name}`;''' - kikimr.ydb_client.query(sql) - - @pytest.mark.parametrize("shared_reading", [False, True], ids=["no_shared", "shared"]) - @pytest.mark.parametrize("tasks", [1, 2]) - @pytest.mark.parametrize("local_topics", [True, False]) - def test_wm_after_parsing(self: StreamingTestBase, kikimr: Kikimr, entity_name: Callable[[str], str], shared_reading: bool, tasks: int, local_topics: bool) -> None: - if shared_reading: - pytest.skip("Shared reading is not supported for watermarks after parsing yet") - - endpoint = self.get_endpoint(kikimr, local_topics) - query_name = f"test_wm_after_parsing_{shared_reading}{tasks}{local_topics}" - source_name = entity_name(query_name) - self.init_topics(source_name, partitions_count=tasks, endpoint=endpoint) - self.create_source(kikimr, source_name, shared_reading) - - cluster = f"{source_name}." if not local_topics else "" - idleness_clause = ', WATERMARK_IDLE_TIMEOUT = "PT5S"' if tasks > 1 else '' - - sql = f''' - CREATE STREAMING QUERY `{query_name}` AS DO BEGIN - PRAGMA ydb.MaxTasksPerStage = '{tasks}'; + settings_str = f"WITH ({', '.join(f'{k} = {v}' for k, v in settings.items())})" if settings else "" + idleness_clause = f', WATERMARK_IDLE_TIMEOUT = "PT{idle_timeout_seconds}S"' if partitions_count > 1 else '' + input = ( + f''' $input = ( SELECT - Yson::ConvertTo(Yson::ParseJson(line), Struct<ts: String, pass: Uint64>) AS row + Yson::ConvertTo(Yson::ParseJson(line), Struct<ts: String, pass: Uint64, id: String>) AS row FROM - {cluster}{self.input_topic} + {input_name} FLATTEN LIST BY ( String::SplitToList(Data, '.') AS line ) @@ -130,7 +69,8 @@ class TestWatermarksInYdb(StreamingTestBase): $input = ( SELECT ts, - pass + pass, + id FROM $input FLATTEN COLUMNS @@ -138,222 +78,476 @@ class TestWatermarksInYdb(StreamingTestBase): $input = ( SELECT - ts, - pass, CAST(ts AS Timestamp) AS event_time, + pass, + id FROM $input WITH ( WATERMARK = CAST(ts AS Timestamp) - Interval('PT5S') {idleness_clause} ) AS input ); - - $output = ( + ''' + if input_parsing + else f''' + $input = ( SELECT - AGGREGATE_LIST(ts) AS ts + CAST(ts AS Timestamp) AS event_time, + pass, + id + FROM + {input_name} WITH ( + FORMAT = json_each_row, + SCHEMA (ts String, pass Uint64, id String), + WATERMARK = CAST(ts AS Timestamp) - Interval('PT5S') + {idleness_clause} + ) + ); + ''' + ) + process = ''' + $process = ( + SELECT + HOP_END() AS event_time, + AGGREGATE_LIST(id) AS id + FROM + $input + WHERE + pass > 0 + GROUP BY + HoppingWindow(event_time, 'PT1S', 'PT1S') + ); + ''' if cascade_hopping else ''' + $process = ( + SELECT + event_time, + id FROM $input WHERE pass > 0 + ); + ''' + kikimr.ydb_client.query(f''' + CREATE STREAMING QUERY `{query_name}` {settings_str} AS DO BEGIN + PRAGMA ydb.MaxTasksPerStage = '{tasks}'; + + {input} + + {process} + + $output = ( + SELECT + HOP_END() AS event_time, + AGGREGATE_LIST(id) AS id + FROM + $process GROUP BY - HoppingWindow(CAST(event_time AS Timestamp), 'PT1S', 'PT1S') + HoppingWindow(event_time, 'PT1S', 'PT1S') ); - INSERT INTO {cluster}{self.output_topic} - SELECT ToBytes(Unwrap(Yson::SerializeJson(Yson::From(ts)))) + INSERT INTO {output_name} + SELECT ToBytes(Unwrap(Yson::SerializeJson(Yson::From(id)))) FROM $output; END DO; - ''' - kikimr.ydb_client.query(sql) + ''') self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") + return query_name - self.write_stream( - data=[ - '{"ts": "1970-01-01T00:00:40Z", "pass": 1}..{"ts": "1970-01-01T00:00:50Z", "pass": 1}..{"ts": "1970-01-01T00:01:00Z", "pass": 0}', - ], - endpoint=endpoint, - partition_key=b'1', - ) + def _write_topic( + self, + ydb_client: YdbClient, + messages: list[str], + partition_id: int = 0, + ) -> None: + ydb_client.topic_write(self.input_topic, messages, partition_id=partition_id) + + def _wait_for_idle(self, shared_reading: bool, tasks: int) -> None: if shared_reading and tasks > 1: - time.sleep(10) # leave a bit more time to fire up idle timeout + # Allow idle timeout to fire in shared reading. + time.sleep(2 * self.idle_timeout_seconds) - expected = [ - '["1970-01-01T00:00:40Z"]', - '["1970-01-01T00:00:50Z"]', - ] - actual = self.read_stream(len(expected), topic_path=self.output_topic, endpoint=endpoint) - assert sorted(actual) == expected + def _wait_for_shared_reading_start(self, shared_reading: bool) -> None: + if shared_reading: + # Allow shared-reading workers to start consuming partitions. + time.sleep(self.idle_timeout_seconds + 1) + + def _read_topic(self, ydb_client: YdbClient, messages_count: int) -> list[str]: + return ydb_client.topic_read(self.output_topic, self.consumer_name, messages_count) + + def _read_topic_check_rows(self, ydb_client: YdbClient, expected: list[str]) -> None: + actual = [] + while len(actual) < len(expected): + actual.extend(json.loads(self._read_topic(ydb_client, 1)[0])) + assert sorted(actual) == sorted(expected) - sql = f'''DROP STREAMING QUERY `{query_name}`;''' - kikimr.ydb_client.query(sql) + def _read_topic_check(self, ydb_client: YdbClient, expected: list[str]) -> None: + actual = self._read_topic(ydb_client, len(expected)) + assert actual == expected + def _drop_query(self, kikimr: Kikimr, query_name: str) -> None: + kikimr.ydb_client.query(f"DROP STREAMING QUERY `{query_name}`;") + + @link_test_case("#28595") + @pytest.mark.parametrize("local_topics", [True, False]) @pytest.mark.parametrize("shared_reading", [False, True], ids=["no_shared", "shared"]) @pytest.mark.parametrize("tasks", [1, 2]) + def test_watermarks( + self: Self, + kikimr: Kikimr, + entity_name: Callable[[str], str], + local_topics: bool, + shared_reading: bool, + tasks: int, + ) -> None: + ydb_client = self.get_ydb_client(kikimr, local_topics) + query_name = f"wm_{shared_reading}{tasks}{local_topics}" + query_name = self._create_query(kikimr, entity_name, query_name, local_topics, shared_reading, tasks) + + try: + self._write_topic( + ydb_client, + [ + self._event(40, "40"), + self._event(50, "50"), + self._event(60, "60", filter=True), + ], + ) + self._wait_for_idle(shared_reading, tasks) + + expected = ['["40"]', '["50"]'] + self._read_topic_check(ydb_client, expected) + finally: + self._drop_query(kikimr, query_name) + + @link_test_case("#28599") @pytest.mark.parametrize("local_topics", [True, False]) - def test_early_events_are_dropped(self: StreamingTestBase, kikimr: Kikimr, entity_name: Callable[[str], str], shared_reading: bool, tasks: int, local_topics: bool) -> None: - if local_topics and shared_reading: - pytest.skip("Shared reading is not supported for local topics: YQ-5036") + @pytest.mark.parametrize("shared_reading", [False, True], ids=["no_shared", "shared"]) + @pytest.mark.parametrize("tasks", [1, 2]) + def test_cascade_hopping_window( + self: Self, + kikimr: Kikimr, + entity_name: Callable[[str], str], + local_topics: bool, + shared_reading: bool, + tasks: int, + ) -> None: + ydb_client = self.get_ydb_client(kikimr, local_topics) + query_name = f"cascade_hopping_window_{shared_reading}{tasks}{local_topics}" + query_name = self._create_query( + kikimr, entity_name, query_name, local_topics, shared_reading, tasks, cascade_hopping=True + ) - endpoint = self.get_endpoint(kikimr, local_topics) - query_name = f"wm_early_{shared_reading}{tasks}{local_topics}" - source_name = entity_name(query_name) - self.init_topics(source_name, partitions_count=tasks, endpoint=endpoint) - self.create_source(kikimr, source_name, shared_reading) + try: + self._write_topic( + ydb_client, + [ + self._event(40, "40"), + self._event(50, "50"), + self._event(60, "60", filter=True), + ], + ) + self._wait_for_idle(shared_reading, tasks) - cluster = f"{source_name}." if not local_topics else "" - idleness_clause = ', WATERMARK_IDLE_TIMEOUT = "PT5S"' if tasks > 1 else '' + expected = ['[["40"]]', '[["50"]]'] + self._read_topic_check(ydb_client, expected) + finally: + self._drop_query(kikimr, query_name) - sql = f''' - CREATE STREAMING QUERY `{query_name}` AS DO BEGIN - PRAGMA ydb.MaxTasksPerStage = '{tasks}'; + @link_test_case("#28600") + @pytest.mark.parametrize("local_topics", [True, False]) + @pytest.mark.parametrize("shared_reading", [False, True], ids=["no_shared", "shared"]) + def test_idle_partition_gt_timeout( + self: Self, + kikimr: Kikimr, + entity_name: Callable[[str], str], + local_topics: bool, + shared_reading: bool, + ) -> None: + idle_timeout_seconds = 10 + ydb_client = self.get_ydb_client(kikimr, local_topics) + query_name = f"idle_partition_gt_timeout_{shared_reading}{local_topics}" + query_name = self._create_query( + kikimr, entity_name, query_name, local_topics, shared_reading, + tasks=1, partitions_count=2, idle_timeout_seconds=idle_timeout_seconds, + ) + self._wait_for_shared_reading_start(shared_reading) - $input = ( - SELECT - input.*, - CAST(ts AS Timestamp) AS event_time, - FROM - {cluster}{self.input_topic} WITH ( - FORMAT = json_each_row, - SCHEMA (ts String, pass Uint64), - WATERMARK = CAST(ts AS Timestamp) - Interval('PT5S') - {idleness_clause} - ) AS input - ); + try: + self._write_topic(ydb_client, [self._event(0, "fst-0")], partition_id=0) + self._write_topic(ydb_client, [self._event(0, "snd-0")], partition_id=1) - $output = ( - SELECT - CAST(HOP_END() AS String) AS event_time, - AGGREGATE_LIST(ts) AS ts - FROM $input - WHERE - pass > 0 - GROUP BY - HoppingWindow(CAST(event_time AS Timestamp), 'PT1S', 'PT1S') - ); + # Start measuring idleness only after both partitions consume the initial events. + self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") - INSERT INTO {cluster}{self.output_topic} - SELECT ToBytes(Unwrap(Yson::SerializeJson(Yson::From(ts)))) - FROM $output; - END DO; - ''' - kikimr.ydb_client.query(sql) - self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") + # Keep the first partition active while the second approaches idle timeout. + time.sleep(idle_timeout_seconds / 2 + 1) + self._write_topic(ydb_client, [self._event(10, "fst-10")], partition_id=0) + + # Ensure this event keeps the first partition active before the next interval starts. + self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") + + # Let the second partition exceed idle timeout without idling the first. + time.sleep(idle_timeout_seconds / 2 + 1) + self._write_topic(ydb_client, [self._event(20, "snd-20")], partition_id=1) - now = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0) + # Ensure the second-partition event is processed before advancing the first watermark. + self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") + self._write_topic(ydb_client, [self._event(20, "fst-20")], partition_id=0) + self._write_topic(ydb_client, [self._event(30, "fst-30")], partition_id=0) - def time_format(time: datetime.datetime, delta: int) -> str: - return (time + datetime.timedelta(minutes=delta)).isoformat().replace("+00:00", "Z") + # Advance the second partition too, so its event at 20 closes its window. + self._write_topic(ydb_client, [self._event(30, "snd-30", filter=True)], partition_id=1) + self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") - self.write_stream( - data=[ - f'{{"ts": "{time_format(now, 0)}", "pass": 1}}', - f'{{"ts": "{time_format(now, 10)}", "pass": 1}}', - f'{{"ts": "{time_format(now, 4)}", "pass": 0}}', - ], - endpoint=endpoint, - partition_key=b'1', + expected = ["fst-0", "snd-0", "fst-10", "fst-20", "snd-20"] + self._read_topic_check_rows(ydb_client, expected) + finally: + self._drop_query(kikimr, query_name) + + @link_test_case("#28601") + @pytest.mark.parametrize("local_topics", [True, False]) + @pytest.mark.parametrize("shared_reading", [False, True], ids=["no_shared", "shared"]) + def test_idle_partition_lt_timeout( + self: Self, + kikimr: Kikimr, + entity_name: Callable[[str], str], + local_topics: bool, + shared_reading: bool, + ) -> None: + idle_timeout_seconds = 20 + ydb_client = self.get_ydb_client(kikimr, local_topics) + query_name = f"idle_partition_lt_timeout_{shared_reading}{local_topics}" + query_name = self._create_query( + kikimr, entity_name, query_name, local_topics, shared_reading, + tasks=1, partitions_count=2, idle_timeout_seconds=idle_timeout_seconds, ) - if shared_reading and tasks > 1: - time.sleep(10) # leave a bit more time to fire up idle timeout + self._wait_for_shared_reading_start(shared_reading) + + try: + self._write_topic(ydb_client, [self._event(0, "fst-0")], partition_id=0) + self._write_topic(ydb_client, [self._event(0, "snd-0")], partition_id=1) + self._write_topic(ydb_client, [self._event(10, "fst-10")], partition_id=0) + + # Keep the second partition below idle timeout. + time.sleep(self.idle_timeout_seconds - 1) + self._write_topic(ydb_client, [self._event(10, "snd-10")], partition_id=1) + self._write_topic(ydb_client, [self._event(20, "fst-20")], partition_id=0) + self._write_topic(ydb_client, [self._event(20, "snd-20")], partition_id=1) + + expected = ["fst-0", "snd-0", "fst-10", "snd-10"] + self._read_topic_check_rows(ydb_client, expected) + finally: + self._drop_query(kikimr, query_name) + + @link_test_case("#28602") + @pytest.mark.parametrize("local_topics", [True, False]) + @pytest.mark.parametrize("shared_reading", [False, True], ids=["no_shared", "shared"]) + def test_idle_topic( + self: Self, + kikimr: Kikimr, + entity_name: Callable[[str], str], + local_topics: bool, + shared_reading: bool, + ) -> None: + idle_timeout_seconds = 10 + ydb_client = self.get_ydb_client(kikimr, local_topics) + query_name = f"idle_topic_{shared_reading}{local_topics}" + query_name = self._create_query( + kikimr, entity_name, query_name, local_topics, shared_reading, idle_timeout_seconds=idle_timeout_seconds + ) + self._wait_for_shared_reading_start(shared_reading) + + try: + self._write_topic(ydb_client, [self._event(0, "fst-0")], partition_id=0) + self._write_topic(ydb_client, [self._event(0, "snd-0")], partition_id=1) - actual = self.read_stream(1, topic_path=self.output_topic, endpoint=endpoint) - assert actual == [f'["{time_format(now, 0)}"]'] + # Start the idle timeout after both partitions consume the initial events. + self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") - kikimr.ydb_client.query(f'''DROP STREAMING QUERY `{query_name}`;''') + # Let both partitions become idle and trigger state cleanup. + time.sleep(idle_timeout_seconds + 1) + self._write_topic(ydb_client, [self._event(10, "fst-10")], partition_id=0) + self._write_topic(ydb_client, [self._event(10, "snd-10")], partition_id=1) + + # Persist cleanup before sending the next events. + self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") + self._write_topic(ydb_client, [self._event(20, "fst-20")], partition_id=0) + self._write_topic(ydb_client, [self._event(20, "snd-20")], partition_id=1) + + # Persist the final watermark before reading the output topic. + self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") + + expected = ["fst-0", "snd-0", "fst-10", "snd-10"] + self._read_topic_check_rows(ydb_client, expected) + finally: + self._drop_query(kikimr, query_name) + + @link_test_case("#28604") + @pytest.mark.parametrize("local_topics", [True, False]) + @pytest.mark.parametrize("shared_reading", [False, True], ids=["no_shared", "shared"]) + def test_empty_partition( + self: Self, + kikimr: Kikimr, + entity_name: Callable[[str], str], + local_topics: bool, + shared_reading: bool, + ) -> None: + ydb_client = self.get_ydb_client(kikimr, local_topics) + query_name = f"empty_partition_{shared_reading}{local_topics}" + query_name = self._create_query( + kikimr, entity_name, query_name, local_topics, shared_reading, tasks=1, partitions_count=2, + ) + self._wait_for_shared_reading_start(shared_reading) + + try: + self._write_topic(ydb_client, [self._event(0, "fst-0")], partition_id=0) + + # Keep the first partition active while the empty second one becomes idle. + time.sleep(self.idle_timeout_seconds / 2 + 1) + self._write_topic(ydb_client, [self._event(10, "fst-10")], partition_id=0) + + # Let the second partition exceed idle timeout without idling the first. + time.sleep(self.idle_timeout_seconds / 2 + 1) + self._write_topic(ydb_client, [self._event(20, "snd-20")], partition_id=1) + + # Ensure the second-partition event is processed before advancing the first watermark. + self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") + self._write_topic(ydb_client, [self._event(20, "fst-20")], partition_id=0) + self._write_topic(ydb_client, [self._event(30, "fst-30")], partition_id=0) + + # Keep the first partition active until the second becomes idle again. + time.sleep(self.idle_timeout_seconds / 2 + 1) + self._write_topic(ydb_client, [self._event(35, "fst-35")], partition_id=0) + + # Advance the first watermark after the second partition is idle. + time.sleep(self.idle_timeout_seconds / 2 + 1) + self._write_topic(ydb_client, [self._event(40, "fst-40")], partition_id=0) + + expected = ["fst-0", "fst-10", "fst-20", "snd-20"] + self._read_topic_check_rows(ydb_client, expected) + finally: + self._drop_query(kikimr, query_name) @pytest.mark.parametrize("shared_reading", [False, True], ids=["no_shared", "shared"]) @pytest.mark.parametrize("tasks", [1, 2]) @pytest.mark.parametrize("local_topics", [True, False]) - @pytest.mark.parametrize("policy,expected", [ - ("DROP", ['["1970-01-01T00:00:50Z"]']), - ("ADJUST", ['["1970-01-01T00:00:50Z"]', '["1970-01-01T00:00:40Z"]']), - ]) - def test_late_events_policy( - self: StreamingTestBase, + def test_wm_after_parsing( + self: Self, kikimr: Kikimr, entity_name: Callable[[str], str], shared_reading: bool, tasks: int, local_topics: bool, - policy: str, - expected: list[str], ) -> None: - if local_topics and shared_reading: - pytest.skip("Shared reading is not supported for local topics: YQ-5036") + if shared_reading: + pytest.skip("Shared reading is not supported for watermarks after parsing yet") - endpoint = self.get_endpoint(kikimr, local_topics) - query_name = f"wm_late_{policy.lower()}_{shared_reading}{tasks}{local_topics}" - source_name = entity_name(query_name) - self.init_topics(source_name, partitions_count=tasks, endpoint=endpoint) - self.create_source(kikimr, source_name, shared_reading) + ydb_client = self.get_ydb_client(kikimr, local_topics) + query_name = f"wm_after_parsing_{shared_reading}{tasks}{local_topics}" + query_name = self._create_query( + kikimr, entity_name, query_name, local_topics, shared_reading, tasks, input_parsing=True + ) - cluster = f"{source_name}." if not local_topics else "" - idleness_clause = ', WATERMARK_IDLE_TIMEOUT = "PT5S"' if tasks > 1 else '' - sql = f''' - CREATE STREAMING QUERY `{query_name}` WITH ( - WATERMARK_LATE_EVENTS_POLICY = {policy} - ) AS DO BEGIN - PRAGMA ydb.MaxTasksPerStage = '{tasks}'; + try: + self._write_topic( + ydb_client, + [ + f'{self._event(40, "40")}..{self._event(50, "50")}.{self._event(60, "60", filter=True)}', + ], + ) + self._wait_for_idle(shared_reading, tasks) - $input = ( - SELECT - input.*, - CAST(ts AS Timestamp) AS event_time, - FROM - {cluster}{self.input_topic} WITH ( - FORMAT = json_each_row, - SCHEMA (ts String, pass Uint64), - WATERMARK = CAST(ts AS Timestamp) - Interval('PT5S') - {idleness_clause} - ) AS input - ); + expected = ['["40"]', '["50"]'] + self._read_topic_check(ydb_client, expected) + finally: + self._drop_query(kikimr, query_name) - $output = ( - SELECT - CAST(HOP_END() AS String) AS event_time, - AGGREGATE_LIST(ts) AS ts - FROM $input - WHERE - pass > 0 - GROUP BY - HoppingWindow(CAST(event_time AS Timestamp), 'PT1S', 'PT1S') - ); + @pytest.mark.parametrize("local_topics", [True, False]) + @pytest.mark.parametrize("shared_reading", [False, True], ids=["no_shared", "shared"]) + @pytest.mark.parametrize("tasks", [1, 2]) + def test_early_events_policy( + self: Self, + kikimr: Kikimr, + entity_name: Callable[[str], str], + local_topics: bool, + shared_reading: bool, + tasks: int, + ) -> None: + ydb_client = self.get_ydb_client(kikimr, local_topics) + query_name = f"early_events_policy_{shared_reading}{tasks}{local_topics}" + query_name = self._create_query(kikimr, entity_name, query_name, local_topics, shared_reading, tasks) - INSERT INTO {cluster}{self.output_topic} - SELECT ToBytes(Unwrap(Yson::SerializeJson(Yson::From(ts)))) - FROM $output; - END DO; - ''' - kikimr.ydb_client.query(sql) - self.wait_completed_checkpoints(kikimr, f"/Root/{query_name}") + try: + now = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0) - self.write_stream( - data=[ - '{"ts": "1970-01-01T00:00:50Z", "pass": 1}', - '{"ts": "1970-01-01T00:01:00Z", "pass": 0}', - ], - endpoint=endpoint, - partition_key=b'1', - ) - if shared_reading and tasks > 1: - time.sleep(10) # leave a bit more time to fire up idle timeout + self._write_topic( + ydb_client, + [ + self._event(0 * 60, "0", initial_ts=now), + self._event(10 * 60, "600", initial_ts=now), + self._event(4 * 60, "240", initial_ts=now, filter=True), + ], + ) + self._wait_for_idle(shared_reading, tasks) - actual = self.read_stream(1, topic_path=self.output_topic, endpoint=endpoint) + expected = ['["0"]'] + self._read_topic_check(ydb_client, expected) + finally: + self._drop_query(kikimr, query_name) - self.write_stream( - data=[ - '{"ts": "1970-01-01T00:00:40Z", "pass": 1}', - '{"ts": "1970-01-01T00:01:10Z", "pass": 0}', - ], - endpoint=endpoint, - partition_key=b'1', + @pytest.mark.parametrize("shared_reading", [False, True], ids=["no_shared", "shared"]) + @pytest.mark.parametrize("tasks", [1, 2]) + @pytest.mark.parametrize("local_topics", [True, False]) + @pytest.mark.parametrize("policy,expected", [ + ("DROP", ["55", "60"]), + ("ADJUST", ["40", "55", "60"]), + ]) + def test_late_events_policy( + self: Self, + kikimr: Kikimr, + entity_name: Callable[[str], str], + local_topics: bool, + shared_reading: bool, + tasks: int, + policy: str, + expected: list[str], + ) -> None: + ydb_client = self.get_ydb_client(kikimr, local_topics) + query_name = f"late_events_{policy.lower()}_{shared_reading}{tasks}{local_topics}" + query_name = self._create_query( + kikimr, + entity_name, + query_name, + local_topics, + shared_reading, + tasks, + settings={"WATERMARK_LATE_EVENTS_POLICY": policy}, ) - if shared_reading and tasks > 1: - time.sleep(10) # leave a bit more time to fire up idle timeout - if len(expected) > len(actual): - actual += self.read_stream(len(expected) - len(actual), topic_path=self.output_topic, endpoint=endpoint) + try: + self._write_topic( + ydb_client, + [ + self._event(50, "50"), + self._event(60, "60", filter=True), + self._event(55, "55"), + ], + ) + self._wait_for_idle(shared_reading, tasks) - assert sorted(actual) == sorted(expected) + self._read_topic_check_rows(ydb_client, ["50"]) + + self._write_topic( + ydb_client, + [ + self._event(60, "60"), + self._event(40, "40"), + self._event(70, "70", filter=True), + ], + ) + self._wait_for_idle(shared_reading, tasks) - kikimr.ydb_client.query(f'''DROP STREAMING QUERY `{query_name}`;''') + self._read_topic_check_rows(ydb_client, expected) + finally: + self._drop_query(kikimr, query_name) diff --git a/ydb/tests/fq/streaming/ya.make b/ydb/tests/fq/streaming/ya.make index 97c37186f39..3e2b1c6d5d2 100644 --- a/ydb/tests/fq/streaming/ya.make +++ b/ydb/tests/fq/streaming/ya.make @@ -22,6 +22,7 @@ PY_SRCS( ) REQUIREMENTS(cpu:4) +REQUIREMENTS(ram:16) IF (SANITIZER_TYPE) SIZE(LARGE) INCLUDE(${ARCADIA_ROOT}/ydb/tests/large.inc) diff --git a/ydb/tests/fq/streaming_common/common.py b/ydb/tests/fq/streaming_common/common.py index b02339eb7e7..0ffa856006d 100644 --- a/ydb/tests/fq/streaming_common/common.py +++ b/ydb/tests/fq/streaming_common/common.py @@ -1,11 +1,11 @@ import logging import os +import pytest import time +from typing import Self import yatest.common import ydb -import pytest -from typing import Optional from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator from ydb.tests.library.harness.kikimr_runner import KiKiMR from ydb.tests.tools.datastreams_helpers.control_plane import Endpoint @@ -97,32 +97,36 @@ def get_ydb_config(request): class YdbClient: - def __init__(self, endpoint: str, database: str, token: str = "root@builtin", enable_discovery: bool = True): - self.driver_config = ydb.DriverConfig( - endpoint, database, auth_token=token, disable_discovery=not enable_discovery - ) - self.driver = None - self.session_pool = None + WAIT_TIMEOUT: int = 5 + + def __init__(self, driver: ydb.Driver, owns_driver: bool = False): + self.owns_driver = owns_driver + self.driver = driver + if self.owns_driver: + self.driver.wait(self.WAIT_TIMEOUT, fail_fast=True) + + self.session_pool = ydb.QuerySessionPool(self.driver) self.retry_settings = ydb.RetrySettings( - on_ydb_error_callback=lambda e: logger.error(f"Query execution failed and may be retried: {e}") + on_ydb_error_callback=lambda e: logger.error(f"Query execution failed and may be retried: {e}"), ) - self.start() - def start(self): - self.driver = ydb.Driver(self.driver_config) - self.session_pool = ydb.QuerySessionPool(self.driver) + @classmethod + def from_driver_config( + cls, endpoint: str, database: str, token: str = "root@builtin", enable_discovery: bool = True + ) -> Self: + driver_config = ydb.DriverConfig(endpoint, database, auth_token=token, disable_discovery=not enable_discovery) + driver = ydb.Driver(driver_config) + return cls(driver, True) def stop(self): self.session_pool.stop() - self.driver.stop() - - def wait_connection(self, timeout: int = 5): - self.driver.wait(timeout, fail_fast=True) + if self.owns_driver: + self.driver.stop() def query(self, statement: str): return self.session_pool.execute_with_retries(statement, retry_settings=self.retry_settings) - def query_async(self, statement: str, timeout: Optional[float] = None): + def query_async(self, statement: str, timeout: float | None = None): settings = None if timeout is not None: settings = ydb.BaseRequestSettings().with_timeout(timeout) @@ -130,6 +134,86 @@ class YdbClient: statement, settings=settings, retry_settings=self.retry_settings ) + def create_external_data_source( + self, source_name: str, endpoint: str, database: str, shared_reading: bool = False + ) -> None: + self.query(f''' + CREATE EXTERNAL DATA SOURCE `{source_name}` WITH ( + SOURCE_TYPE = 'Ydb', + LOCATION = '{endpoint}', + DATABASE_NAME = '{database}', + {"SHARED_READING = 'TRUE'," if shared_reading else ""} + AUTH_METHOD = 'NONE' + ); + ''') + + def topic_write( + self, + topic: str, + messages: list[str], + timeout: int = plain_or_under_sanitizer_wrapper(120, 150), + *args, + **kwargs, + ) -> None: + writer = self.driver.topic_client.writer(topic, *args, **kwargs) + + try: + writer.write(messages, timeout) + writer.flush() + finally: + writer.close(flush=False) + + def topic_read( + self, + topic: str, + consumer: str, + messages_count: int, + timeout: int = plain_or_under_sanitizer_wrapper(30, 300), + commit: bool = True, + ) -> list[str]: + deadline = time.monotonic() + timeout + + with self.driver.topic_client.reader(topic, consumer=consumer) as reader: + + def _read_single() -> str: + remaining = deadline - time.monotonic() + message = reader.receive_message(timeout=remaining) + + if commit: + reader.commit(message) + + data = message.data + return data.decode() if isinstance(data, bytes) else str(data) + + return [_read_single() for _ in range(messages_count)] + + def topic_read_until( + self, + topic: str, + consumer: str, + messages_count: int, + timeout: int = plain_or_under_sanitizer_wrapper(30, 300), + commit: bool = True, + ) -> list[str]: + deadline = time.monotonic() + timeout + + with self.driver.topic_client.reader(topic, consumer=consumer) as reader: + + def _read_batch() -> list[str]: + remaining = deadline - time.monotonic() + batch = reader.receive_batch(timeout=remaining) + + if commit: + reader.commit(batch) + + datas = [message.data for message in batch.messages] + return [data.decode() if isinstance(data, bytes) else str(data) for data in datas] + + result: list[str] = [] + while len(result) < messages_count: + result.extend(_read_batch()) + return result + class Kikimr: def __init__(self, config: KikimrConfigGenerator, timeout_seconds: int = 240, enable_discovery: bool = True): @@ -141,37 +225,41 @@ class Kikimr: self.first_node = list(self.cluster.nodes.values())[0] self.endpoint = Endpoint(f"{self.first_node.host}:{self.first_node.port}", f"/{config.domain_name}") - self.ydb_client = YdbClient( - database=self.endpoint.database, - endpoint=f"grpc://{self.endpoint.endpoint}", + self.ydb_client = self._setup_ydb_client(self.endpoint, enable_discovery) + + if os.getenv("YDB_ENDPOINT") is None or os.getenv("YDB_DATABASE") is None: + self.external_endpoint = None + self.external_ydb_client = None + else: + self.external_endpoint = Endpoint(os.getenv("YDB_ENDPOINT"), os.getenv("YDB_DATABASE")) + self.external_ydb_client = self._setup_ydb_client(self.external_endpoint, enable_discovery) + + @staticmethod + def _setup_ydb_client(endpoint: Endpoint, enable_discovery: bool) -> YdbClient: + return YdbClient.from_driver_config( + database=endpoint.database, + endpoint=f"grpc://{endpoint.endpoint}", enable_discovery=enable_discovery, ) - self.ydb_client.wait_connection() - def stop(self): + def stop(self) -> None: + if self.external_ydb_client is not None: + self.external_ydb_client.stop() self.ydb_client.stop() self.cluster.stop() class StreamingTestBase(TestYdsBase): - def get_endpoint(self, kikimr, local_topics): - if local_topics: - return kikimr.endpoint - return Endpoint(os.getenv("YDB_ENDPOINT"), os.getenv("YDB_DATABASE")) + def get_endpoint(self, kikimr: Kikimr, local_topics: bool) -> Endpoint: + return kikimr.endpoint if local_topics else kikimr.external_endpoint + + def get_ydb_client(self, kikimr: Kikimr, local_topics: bool) -> YdbClient: + return kikimr.ydb_client if local_topics else kikimr.external_ydb_client - def create_source(self, kikimr: Kikimr, source_name: str, shared: bool = False, endpoint: Endpoint = None): + def create_source(self, kikimr: Kikimr, source_name: str, shared: bool = False, endpoint: Endpoint = None) -> None: if endpoint is None: endpoint = self.get_endpoint(kikimr, local_topics=False) - shared_opt = 'SHARED_READING = "TRUE",\n' if shared else '\n' - kikimr.ydb_client.query(f""" - CREATE EXTERNAL DATA SOURCE `{source_name}` WITH ( - SOURCE_TYPE = "Ydb", - LOCATION = "{endpoint.endpoint}", - DATABASE_NAME = "{endpoint.database}", - {shared_opt} - AUTH_METHOD = "NONE" - ); - """) + kikimr.ydb_client.create_external_data_source(source_name, endpoint.endpoint, endpoint.database, shared) def monitoring_endpoint(self, kikimr: Kikimr, node_id: int) -> str: node = kikimr.cluster.nodes[node_id] diff --git a/ydb/tests/functional/sqs/cloud/test_yandex_cloud_mode.py b/ydb/tests/functional/sqs/cloud/test_yandex_cloud_mode.py index 99e792287ee..9381bcd98a4 100644 --- a/ydb/tests/functional/sqs/cloud/test_yandex_cloud_mode.py +++ b/ydb/tests/functional/sqs/cloud/test_yandex_cloud_mode.py @@ -14,7 +14,7 @@ import pytest import ydb from hamcrest import assert_that, equal_to, not_none, has_item, has_items, is_not, contains_string -from hamcrest import raises, greater_than, not_, less_than +from hamcrest import raises, greater_than from ydb.tests.library.sqs.test_base import IS_FIFO_PARAMS, TABLES_FORMAT_PARAMS from ydb.tests.library.sqs.cloud_test_base import YandexCloudSqsTestBase from ydb.tests.library.sqs.test_base import get_test_with_sqs_tenant_installation @@ -422,10 +422,12 @@ class TestSqsYandexCloudMode(get_test_with_sqs_tenant_installation(YandexCloudSq def get_messages_count(queue_url): attrs = self._sqs_api.get_queue_attributes(queue_url) - msg_count = int(attrs['ApproximateNumberOfMessages']) - infly_msg_count = int(attrs['ApproximateNumberOfMessagesNotVisible']) - assert_that(infly_msg_count, not_(greater_than(msg_count))) - return msg_count + msg_count = int(attrs.get('ApproximateNumberOfMessages', 0)) + infly_msg_count = int(attrs.get('ApproximateNumberOfMessagesNotVisible', 0)) + # Inflight and total can briefly diverge while attributes update asynchronously. + if infly_msg_count <= msg_count: + return msg_count + return None # check that messages are actually moved to dlq and this doesn't break both queues lst = [queue1_url, queue2_url] @@ -437,17 +439,23 @@ class TestSqsYandexCloudMode(get_test_with_sqs_tenant_installation(YandexCloudSq self._send_message_and_assert(q1, msg_body, seq_no=str(seq_no) if is_fifo else None, group_id='group' if is_fifo else None) seq_no += 1 + messages_count_before = None messages_count_metric_update_attempts = 20 while messages_count_metric_update_attempts: messages_count_metric_update_attempts -= 1 messages_count_before = get_messages_count(q1) - assert_that(messages_count_before, not_(less_than(0))) - if messages_count_before == 0: - logging.debug('Wait for proper messages count metric and retry. Attempts left: {}'.format(messages_count_metric_update_attempts)) + if messages_count_before is None or messages_count_before == 0: + logging.debug( + 'Wait for proper messages count metric and retry. Attempts left: {}'.format( + messages_count_metric_update_attempts + ) + ) time.sleep(1) continue + break + assert_that(messages_count_before, not_none()) assert_that(messages_count_before, greater_than(0)) for i in range(max_receive_count): @@ -459,8 +467,7 @@ class TestSqsYandexCloudMode(get_test_with_sqs_tenant_installation(YandexCloudSq if self._is_topic_migration_stage(): msg_from_dlq = self._wait_for_message_body_in_dlq(q2, msg_body) else: - messages_count_after = get_messages_count(q1) - assert_that(messages_count_after, equal_to(messages_count_before - 1)) + self._wait_for_approximate_messages_count(q1, messages_count_before - 1) msg_from_dlq = self._read_single_message_no_wait(q2)[0] assert_that(msg_from_dlq['Body'], equal_to(msg_body)) diff --git a/ydb/tests/functional/sqs/common/test_queue_counters.py b/ydb/tests/functional/sqs/common/test_queue_counters.py index 2dd01a959b3..fb511919332 100644 --- a/ydb/tests/functional/sqs/common/test_queue_counters.py +++ b/ydb/tests/functional/sqs/common/test_queue_counters.py @@ -114,27 +114,30 @@ class TestSqsGettingCounters(KikimrSqsTestBase): self._sqs_api.send_message(queue_url, message_payload) self._read_while_not_empty(queue_url, 1) - sqs_counters = self._get_sqs_counters() + def assert_action_counters(sqs_counters): + successes = self._get_counter_value(sqs_counters, { + 'queue': self.queue_name, + 'sensor': 'ReceiveMessage_Success', + }) + assert successes == 1 - successes = self._get_counter_value(sqs_counters, { - 'queue': self.queue_name, - 'sensor': 'ReceiveMessage_Success', - }) - assert successes == 1 + durations = self._get_counter(sqs_counters, { + 'queue': self.queue_name, + 'sensor': 'ReceiveMessage_Duration', + }) + assert durations is not None + duration_buckets = durations['hist']['buckets'] + assert any(map(lambda x: x > 0, duration_buckets)) - durations = self._get_counter(sqs_counters, { - 'queue': self.queue_name, - 'sensor': 'ReceiveMessage_Duration', - }) - duration_buckets = durations['hist']['buckets'] - assert any(map(lambda x: x > 0, duration_buckets)) + working_durations = self._get_counter(sqs_counters, { + 'queue': self.queue_name, + 'sensor': 'ReceiveMessage_WorkingDuration', + }) + assert working_durations is not None + working_duration_buckets = working_durations['hist']['buckets'] + assert any(map(lambda x: x > 0, working_duration_buckets)) - working_durations = self._get_counter(sqs_counters, { - 'queue': self.queue_name, - 'sensor': 'ReceiveMessage_WorkingDuration', - }) - working_duration_buckets = working_durations['hist']['buckets'] - assert any(map(lambda x: x > 0, working_duration_buckets)) + self._wait_for_sqs_counters(assert_action_counters) def test_receive_message_immediate_duration_counter(self): if self._is_topic_migration_stage(): diff --git a/ydb/tests/functional/sqs/multinode/test_multinode_cluster.py b/ydb/tests/functional/sqs/multinode/test_multinode_cluster.py index 5ee9fc71ca7..3812531243b 100644 --- a/ydb/tests/functional/sqs/multinode/test_multinode_cluster.py +++ b/ydb/tests/functional/sqs/multinode/test_multinode_cluster.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import logging +import os import time import threading @@ -11,7 +12,13 @@ from ydb.tests.library.common.types import Erasure from ydb.tests.library.sqs.matchers import ReadResponseMatcher -from ydb.tests.library.sqs.test_base import KikimrSqsTestBase, STOP_NODE_PARAMS, IS_FIFO_PARAMS, TABLES_FORMAT_PARAMS +from ydb.tests.library.sqs.test_base import ( + KikimrSqsTestBase, + STOP_NODE_PARAMS, + IS_FIFO_PARAMS, + TABLES_FORMAT_PARAMS, + SQS_MIGRATION_STAGES, +) class TestSqsMultinodeCluster(KikimrSqsTestBase): @@ -59,7 +66,8 @@ class TestSqsMultinodeCluster(KikimrSqsTestBase): @pytest.mark.parametrize(**IS_FIFO_PARAMS) @pytest.mark.parametrize(**STOP_NODE_PARAMS) @pytest.mark.skipif( - KikimrSqsTestBase._is_topic_migration_stage(), + # topic_creation also reports MessagesCount via PQ read balancer since #45354 + os.environ.get('YDB_SQS_MIGRATION_STAGE') in SQS_MIGRATION_STAGES, reason='MessagesCount counters use different semantics on topic path', ) def test_has_messages_counters(self, is_fifo, stop_node): diff --git a/ydb/tests/library/stability/run_stress.py b/ydb/tests/library/stability/run_stress.py index 53a881cd2cc..95edf61e6de 100644 --- a/ydb/tests/library/stability/run_stress.py +++ b/ydb/tests/library/stability/run_stress.py @@ -368,10 +368,13 @@ class StressRunExecutor: try: # Disable buffering to ensure output capture - event_prefix = '' + env_exports = ['export YDB_STRESS_EXTENDED_RETRIES=1'] if self.event_process_mode is not None: - event_prefix = f'export YDB_STRESS_UTIL_EVENT_PROCESS_MODE={self.event_process_mode};' - cmd = f"{event_prefix}stdbuf -o0 -e0 {deployed_binary_path} {command_args}" + env_exports.append( + f'export YDB_STRESS_UTIL_EVENT_PROCESS_MODE={self.event_process_mode}' + ) + env_prefix = ''.join(f'{e};' for e in env_exports) + cmd = f"{env_prefix}stdbuf -o0 -e0 {deployed_binary_path} {command_args}" run_config['run_command'] = cmd run_timeout = ( run_config["duration"] + 600 diff --git a/ydb/tests/stress/common/instrumented_client.py b/ydb/tests/stress/common/instrumented_client.py index 428e7f09cd6..91e7b189ea6 100644 --- a/ydb/tests/stress/common/instrumented_client.py +++ b/ydb/tests/stress/common/instrumented_client.py @@ -3,12 +3,20 @@ Instrumented YDB client with automatic metrics collection. This module provides InstrumentedYdbClient class that inherits from YdbClient and automatically collects metrics for all database operations. + +When YDB_STRESS_EXTENDED_RETRIES is enabled, omitted retry_settings use the +extended stress retry policy (see instrumented_pools). """ import inspect import os from typing import Optional from .common import YdbClient +from .instrumented_pools import ( + InstrumentedQuerySessionPool, + InstrumentedSessionPool, + maybe_extended_retry_settings, +) from .publish_metrics import get_metrics_collector @@ -21,6 +29,9 @@ class InstrumentedYdbClient(YdbClient): - Error types - Statistics by operation types + Uses Instrumented* session pools so retry policy hooks apply to both + client.query(...) and direct session_pool.* calls. + Example usage: client = InstrumentedYdbClient(endpoint, database, use_query_service=True) client.query("CREATE TABLE ...", True, operation_name='create_table') @@ -42,6 +53,14 @@ class InstrumentedYdbClient(YdbClient): """ super().__init__(endpoint, database, use_query_service, sessions) + # Replace plain pools with instrumented ones (env-gated retries live there). + self.session_pool.stop() + self.session_pool = ( + InstrumentedQuerySessionPool(self.driver, size=sessions, enable_metrics=enable_metrics) + if use_query_service + else InstrumentedSessionPool(self.driver, size=sessions, enable_metrics=enable_metrics) + ) + self.enable_metrics = enable_metrics self.metrics_collector = get_metrics_collector() if enable_metrics else None @@ -67,6 +86,7 @@ class InstrumentedYdbClient(YdbClient): Returns: Query execution result """ + retry_settings = maybe_extended_retry_settings(retry_settings) if operation_name is None: operation_name = 'ddl' if is_ddl else 'dml' if not self.enable_metrics: diff --git a/ydb/tests/stress/common/instrumented_pools.py b/ydb/tests/stress/common/instrumented_pools.py index 13660d00122..783fba1a103 100644 --- a/ydb/tests/stress/common/instrumented_pools.py +++ b/ydb/tests/stress/common/instrumented_pools.py @@ -3,6 +3,10 @@ Instrumented YDB pool classes with automatic metrics collection through inherita This module provides classes that inherit from ydb.QuerySessionPool and ydb.SessionPool, which automatically collect metrics for all operations. + +When YDB_STRESS_EXTENDED_RETRIES is enabled, omitted retry_settings are replaced with a +more aggressive stress retry policy (more retries + slower backoff). Without the env var, +SDK defaults are used unchanged. """ import inspect @@ -11,6 +15,36 @@ from typing import Optional, Callable import ydb from .publish_metrics import get_metrics_collector +# Enable with: YDB_STRESS_EXTENDED_RETRIES=1 +EXTENDED_RETRIES_ENV = "YDB_STRESS_EXTENDED_RETRIES" + +_EXTENDED_MAX_RETRIES = 50 +_EXTENDED_FAST_BACKOFF = ydb.BackoffSettings(ceiling=6, slot_duration=0.5) +_EXTENDED_SLOW_BACKOFF = ydb.BackoffSettings(ceiling=6, slot_duration=2.0) + + +def extended_retries_enabled() -> bool: + return os.getenv(EXTENDED_RETRIES_ENV, "").lower() in ("1", "true", "yes", "y") + + +def extended_retry_settings(**overrides) -> ydb.RetrySettings: + params = dict( + max_retries=_EXTENDED_MAX_RETRIES, + fast_backoff_settings=_EXTENDED_FAST_BACKOFF, + slow_backoff_settings=_EXTENDED_SLOW_BACKOFF, + ) + params.update(overrides) + return ydb.RetrySettings(**params) + + +def maybe_extended_retry_settings(retry_settings: Optional[ydb.RetrySettings] = None) -> Optional[ydb.RetrySettings]: + """If env is set and retry_settings is omitted, return extended policy; else pass through.""" + if retry_settings is not None: + return retry_settings + if extended_retries_enabled(): + return extended_retry_settings() + return None + class InstrumentedQuerySessionPool(ydb.QuerySessionPool): """ @@ -60,6 +94,7 @@ class InstrumentedQuerySessionPool(ydb.QuerySessionPool): Returns: Query execution result """ + retry_settings = maybe_extended_retry_settings(retry_settings) if operation_name is None: operation_name = 'query_pool_execute' if not self.enable_metrics: @@ -93,6 +128,7 @@ class InstrumentedQuerySessionPool(ydb.QuerySessionPool): Returns: EXPLAIN result """ + retry_settings = maybe_extended_retry_settings(retry_settings) if not self.enable_metrics: return super(InstrumentedQuerySessionPool, self).explain_with_retries(query, retry_settings, *args, @@ -105,6 +141,16 @@ class InstrumentedQuerySessionPool(ydb.QuerySessionPool): operation_name, self.full_name ) + def retry_operation_sync(self, callee: Callable, retry_settings=None, *args, **kwargs): + return super().retry_operation_sync( + callee, maybe_extended_retry_settings(retry_settings), *args, **kwargs + ) + + def retry_tx_sync(self, callee, tx_mode=None, retry_settings=None, *args, **kwargs): + return super().retry_tx_sync( + callee, tx_mode, maybe_extended_retry_settings(retry_settings), *args, **kwargs + ) + def get_metrics_summary(self) -> str: """ Returns summary of collected metrics. @@ -164,6 +210,17 @@ class InstrumentedSessionPool(ydb.SessionPool): if operation_name is None: operation_name = 'session_pool_operation' + # Parent signature: (callee, retry_settings=None, *args, **kwargs). + # Prefer positional retry_settings; only inject a keyword when it was omitted. + if args: + args = (maybe_extended_retry_settings(args[0]),) + args[1:] + elif 'retry_settings' in kwargs: + kwargs['retry_settings'] = maybe_extended_retry_settings(kwargs['retry_settings']) + else: + resolved = maybe_extended_retry_settings(None) + if resolved is not None: + kwargs['retry_settings'] = resolved + if not self.enable_metrics: return super(InstrumentedSessionPool, self).retry_operation_sync(callee, *args, **kwargs) diff --git a/ydb/tests/stress/streaming/__main__.py b/ydb/tests/stress/streaming/__main__.py index 44a650e19a0..860d30b2f4b 100644 --- a/ydb/tests/stress/streaming/__main__.py +++ b/ydb/tests/stress/streaming/__main__.py @@ -14,11 +14,11 @@ if __name__ == '__main__': text = """\033[92mStreaming workload\x1b[0m""" parser = argparse.ArgumentParser(description=text, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('--endpoint', default='localhost:2135', help="An endpoint to be used") - parser.add_argument('--database', default=None, required=True, help='A database to connect') - parser.add_argument('--duration', default=60, type=lambda x: int(x), help='A duration of workload in seconds.') - parser.add_argument('--partitions-count', default=10, type=lambda x: int(x), help='Partitions count.') - parser.add_argument('--prefix', default=None, help='Topic/source name prefix') + parser.add_argument('--endpoint', default='localhost:2135', type=str, help="An endpoint to be used") + parser.add_argument('--database', default=None, type=str, required=True, help='A database to connect') + parser.add_argument('--duration', default=60, type=int, help='A duration of workload in seconds.') + parser.add_argument('--partitions-count', default=10, type=int, help='Partitions count.') + parser.add_argument('--prefix', default=None, type=str, help='Topic/source name prefix') args = parser.parse_args() with Workload(args.endpoint, args.database, args.duration, args.partitions_count, args.prefix) as workload: diff --git a/ydb/tests/stress/streaming/tests/test_workload.py b/ydb/tests/stress/streaming/tests/test_workload.py index e5a2c9aff51..d2a5442edbd 100644 --- a/ydb/tests/stress/streaming/tests/test_workload.py +++ b/ydb/tests/stress/streaming/tests/test_workload.py @@ -27,17 +27,22 @@ class TestYdbWorkload(StressFixture): 'STREAMS_STORAGE_SERVICE': LogLevels.DEBUG, 'FQ_ROW_DISPATCHER': LogLevels.DEBUG, 'KQP_PROXY': LogLevels.DEBUG, - 'KQP_EXECUTER': LogLevels.DEBUG} + 'KQP_EXECUTER': LogLevels.DEBUG, + }, + table_service_config={ + "enable_watermarks": True, + "enable_watermarks_advanced": True, + }, ) def test(self): logger.info("TestYdbWorkload::start test") cmd = [ yatest.common.binary_path(os.getenv("YDB_TEST_PATH")), - "--endpoint", f"localhost:{self.cluster.nodes[1].port}", + "--endpoint", f"localhost:{self.cluster.nodes[1].port}", "--database", self.database, "--duration", self.base_duration, "--partitions-count", "10", - "--prefix", "streaming_stress" + "--prefix", "streaming_stress", ] yatest.common.execute(cmd, wait=True) diff --git a/ydb/tests/stress/streaming/workload/__init__.py b/ydb/tests/stress/streaming/workload/__init__.py index 658854d2997..b2c358fb349 100644 --- a/ydb/tests/stress/streaming/workload/__init__.py +++ b/ydb/tests/stress/streaming/workload/__init__.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) class Workload(): - def __init__(self, endpoint, database, duration, partitions_count, prefix): + def __init__(self, endpoint: str, database: str, duration: int, partitions_count: int, prefix: str): self.database = database self.endpoint = endpoint self.driver = ydb.Driver(ydb.DriverConfig(endpoint, database)) diff --git a/ydb/tests/stress/topic/tests/test_workload_topic.py b/ydb/tests/stress/topic/tests/test_workload_topic.py index 4c79bfa2187..7c6da563993 100644 --- a/ydb/tests/stress/topic/tests/test_workload_topic.py +++ b/ydb/tests/stress/topic/tests/test_workload_topic.py @@ -28,7 +28,7 @@ def create_test_methods(chunk_size): for k in range(count): def make_test_method(index): def test_method(self): - self._run_chunk(k, chunk_size) + self._run_chunk(index, chunk_size) test_method.__name__ = f"test_{index}" return test_method setattr(cls, f"test_{k}", make_test_method(k)) diff --git a/ydb/tests/stress/topic/workload/__init__.py b/ydb/tests/stress/topic/workload/__init__.py index 86e3133d6d6..bf41e15b153 100644 --- a/ydb/tests/stress/topic/workload/__init__.py +++ b/ydb/tests/stress/topic/workload/__init__.py @@ -214,13 +214,14 @@ class YdbTopicWorkload(WorkloadBase): )) def __non_transactional_workload(self): + # Keep wide partition coverage; lower byte_rate only to ease gRPC drain on teardown (#46635). self.run_topic_write_without_tx(TestConfig( partitions=200, partitions_per_tablet=10, producers=20, # producers=int(self.producers), consumers=int(self.consumers), consumer_threads=int(self.consumers), - byte_rate="10M" # byte_rate=self.config.DEFAULT_BYTE_RATE + byte_rate="1M" # byte_rate=self.config.DEFAULT_BYTE_RATE )) @property @@ -243,7 +244,8 @@ class YdbTopicWorkload(WorkloadBase): self._run_workload( self.workload_topic_name, self.duration, - self.config.DEFAULT_BYTE_RATE, + # DEFAULT_BYTE_RATE (100M) overloads a single CI node + self.config.SMALL_BYTE_RATE, self.producers, self.consumers, with_config=True @@ -324,4 +326,12 @@ class YdbTopicWorkload(WorkloadBase): ] if (self.chunk_index is None) or (self.chunk_size is None): return tests - return tests[self.chunk_index * self.chunk_size:(self.chunk_index + 1) * self.chunk_size] + chunk = tests[self.chunk_index * self.chunk_size:(self.chunk_index + 1) * self.chunk_size] + + # One callable so WorkloadBase starts a single worker; run chunk + # scenarios one by one (parallel topic stresses overload CI, #46635). + def run_chunk(): + for f in chunk: + f() + + return [run_chunk] diff --git a/yql/essentials/cfg/tests/gateways-experimental.conf b/yql/essentials/cfg/tests/gateways-experimental.conf index fddec6af53a..83866b60565 100644 --- a/yql/essentials/cfg/tests/gateways-experimental.conf +++ b/yql/essentials/cfg/tests/gateways-experimental.conf @@ -152,6 +152,10 @@ SqlCore { ExtendedTranslationFlags: { Name: "StrictWarningAsError" } + ExtendedTranslationFlags: { + Name: "MaxParseTreeDepth" + Args: ["4096"] + } } YqlCore { diff --git a/yql/essentials/core/common_opt/yql_co_sqlselect.cpp b/yql/essentials/core/common_opt/yql_co_sqlselect.cpp index a47f79f999e..5552d0bbe60 100644 --- a/yql/essentials/core/common_opt/yql_co_sqlselect.cpp +++ b/yql/essentials/core/common_opt/yql_co_sqlselect.cpp @@ -1167,7 +1167,7 @@ void FillInputIndices(const TExprNode::TPtr& from, const TExprNode::TPtr& finalE for (; inputIndex < from->Tail().ChildrenSize(); ++inputIndex) { auto inputAlias = from->Tail().Child(inputIndex)->Child(1)->Content(); const auto& read = from->Tail().Child(inputIndex)->Head(); - const auto& columns = from->Tail().Child(inputIndex)->Tail(); + const auto& columns = *from->Tail().Child(inputIndex)->Child(2); if (x.second.first != Max<ui32>() && x.second.first != inputIndex) { continue; } @@ -1272,7 +1272,7 @@ TExprNode::TListType BuildCleanedColumns(TPositionHandle pos, const TExprNode::T auto inputAlias = from->Tail().Child(i)->Child(1)->Content(); inputAliases.push_back(TString(inputAlias)); if (list->IsCallable("PgResolvedCall")) { - const auto& columns = from->Tail().Child(i)->Tail(); + const auto& columns = *from->Tail().Child(i)->Child(2); if (inputAlias.empty()) { inputAliases.back() = inputAlias = list->Head().Content(); } diff --git a/yql/essentials/core/facade/yql_facade.cpp b/yql/essentials/core/facade/yql_facade.cpp index 60595a52c1a..a81023be262 100644 --- a/yql/essentials/core/facade/yql_facade.cpp +++ b/yql/essentials/core/facade/yql_facade.cpp @@ -890,8 +890,14 @@ bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings) lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); + parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + currentSettings.MaxParseTreeDepth); + parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + currentSettings.MaxParseTreeDepth); NSQLTranslation::TTranslators translators( nullptr, diff --git a/yql/essentials/core/services/mounts/yql_mounts.cpp b/yql/essentials/core/services/mounts/yql_mounts.cpp index e4ae2414a8e..8823c69f9b6 100644 --- a/yql/essentials/core/services/mounts/yql_mounts.cpp +++ b/yql/essentials/core/services/mounts/yql_mounts.cpp @@ -130,12 +130,25 @@ TUserDataTable GetYqlModuleResolverImpl( TUserDataTable mounts; LoadYqlDefaultMounts(mounts); - NSQLTranslationV1::TLexers lexers; - lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); - lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); - NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); + NSQLTranslation::TTranslationSettings settings; + NSQLTranslation::ParseTranslationSettings(sqlFlags, settings); + + NSQLTranslationV1::TLexers lexers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(), + }; + + NSQLTranslationV1::TParsers parsers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), + }; + NSQLTranslation::TTranslators translators( nullptr, NSQLTranslationV1::MakeTranslator(lexers, parsers), diff --git a/yql/essentials/core/type_ann/type_ann_sql.cpp b/yql/essentials/core/type_ann/type_ann_sql.cpp index 21317353432..4e2289973a3 100644 --- a/yql/essentials/core/type_ann/type_ann_sql.cpp +++ b/yql/essentials/core/type_ann/type_ann_sql.cpp @@ -1938,6 +1938,13 @@ bool GatherExtraSortColumns( return false; } +//////////////////////////////////////////////////////////////////////////////// + +bool EnsureSqlSelectWarnings(TExprNode& option, TExprContext& ctx) { + return EnsureTupleSize(option, 2, ctx) && + EnsureTupleOfAtoms(*option.Child(1), ctx); +} + } // namespace //////////////////////////////////////////////////////////////////////////////// @@ -2032,6 +2039,55 @@ TMap<TString, ui32> ExtractExternalColumns(const TExprNode& select) { //////////////////////////////////////////////////////////////////////////////// +IGraphTransformer::TStatus AddSqlSelectWarning( + const TExprNode::TPtr& input, + TExprNode::TPtr& output, + TExprContext& ctx, + TStringBuf name) +{ + YQL_ENSURE(input->IsCallable({"PgSetItem", "YqlSetItem"})); + + TExprNode::TPtr options = input->ChildPtr(0); + TExprNode::TPtr option = GetSetting(*options, "warnings"); + + if (!option) { + // clang-format off + option = ctx.Builder(input->Pos()) + .List() + .Atom(0, "warnings") + .List(1) + .Seal() + .Seal() + .Build(); + // clang-format on + } + + if (!EnsureSqlSelectWarnings(*option, ctx)) { + return IGraphTransformer::TStatus::Error; + } + + TExprNode::TPtr oldTuple = option->ChildPtr(1); + + for (const auto& child : oldTuple->Children()) { + if (child->Content() == name) { + return IGraphTransformer::TStatus::Ok; + } + } + + TExprNode::TListType children = oldTuple->ChildrenList(); + children.reserve(children.size() + 1); + children.emplace_back(ctx.Builder(input->Pos()).Atom(name).Build()); + + TExprNode::TPtr newTuple = ctx.ChangeChildren(*oldTuple, std::move(children)); + + option = ctx.ChangeChild(*option, 1, std::move(newTuple)); + options = ReplaceSetting(*options, option, ctx); + output = ctx.ChangeChild(*input, 0, std::move(options)); + return IGraphTransformer::TStatus::Repeat; +} + +//////////////////////////////////////////////////////////////////////////////// + IGraphTransformer::TStatus SqlSelfWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); if (!EnsureArgsCount(*input, 0, ctx.Expr)) { @@ -2132,7 +2188,10 @@ IGraphTransformer::TStatus SqlColumnRefWrapper(const TExprNode::TPtr& input, TEx IGraphTransformer::TStatus SqlResultItemWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { Y_UNUSED(output); - if (!EnsureArgsCount(*input, 3, ctx.Expr)) { + if (!EnsureMinArgsCount(*input, 3, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (!EnsureMaxArgsCount(*input, 4, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -2173,7 +2232,36 @@ IGraphTransformer::TStatus SqlResultItemWrapper(const TExprNode::TPtr& input, TE } } - auto& lambda = input->ChildRef(2); + if (3 < input->ChildrenSize()) { + auto settings = input->Child(2); + + if (settings->GetTypeAnn() && settings->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Universal) { + input->SetTypeAnn(ctx.Expr.MakeType<TUniversalExprType>()); + return IGraphTransformer::TStatus::Ok; + } + + const auto validator = [&](TStringBuf name, TExprNode& setting, TExprContext& ctx) -> bool { + if (name == "synthetic") { + if (setting.ChildrenSize() != 1) { + ctx.AddError(TIssue( + ctx.GetPosition(setting.Pos()), + TStringBuilder() << "No extra parameters are expected by setting " + << "'" << name << "'")); + return false; + } + + return true; + } + + YQL_ENSURE(false, "unreachable: unexpected setting " << name); + }; + + if (!EnsureValidSettings(*settings, {"synthetic"}, validator, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + } + + auto& lambda = input->TailRef(); bool isUniversal; const auto status = ConvertToLambda(lambda, ctx.Expr, isUniversal, hasType ? 1 : 0); if (status.Level != IGraphTransformer::TStatus::Ok) { @@ -2746,7 +2834,7 @@ IGraphTransformer::TStatus SqlSetItemWrapper(const TExprNode::TPtr& input, TExpr } newLambda = ctx.Expr.NewLambda(column->Pos(), std::move(arguments), std::move(newRoot)); - newColumnChildren[2] = newLambda; + newColumnChildren.back() = newLambda; hasChanges = true; } @@ -2764,7 +2852,7 @@ IGraphTransformer::TStatus SqlSetItemWrapper(const TExprNode::TPtr& input, TExpr newLambda = ctx.Expr.NewLambda(column->Pos(), std::move(arguments), std::move(newRoot)); newColumnChildren[0] = expandedColumns; newColumnChildren[1] = typeNode; - newColumnChildren[2] = newLambda; + newColumnChildren.back() = newLambda; hasChanges = true; } @@ -2932,7 +3020,7 @@ IGraphTransformer::TStatus SqlSetItemWrapper(const TExprNode::TPtr& input, TExpr if (ret != column.TailPtr()) { hasChanges = true; - newResultItems.push_back(ctx.Expr.ChangeChild(column, 2, std::move(ret))); + newResultItems.push_back(ctx.Expr.ChangeChild(column, column.ChildrenSize() - 1, std::move(ret))); } else { newResultItems.push_back(data.ChildPtr(index)); @@ -2963,7 +3051,14 @@ IGraphTransformer::TStatus SqlSetItemWrapper(const TExprNode::TPtr& input, TExpr } for (const auto& p : data.Children()) { - if (!EnsureTupleSize(*p, 3, ctx.Expr)) { + if (!EnsureTupleMinSize(*p, 3, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + + if (!isYql && !EnsureTupleMaxSize(*p, 3, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + if (isYql && !EnsureTupleMaxSize(*p, 4, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -2981,6 +3076,14 @@ IGraphTransformer::TStatus SqlSetItemWrapper(const TExprNode::TPtr& input, TExpr } } + TMaybe<TYqlFromSettings> yqlFromSettings; + if (isYql && 3 < p->ChildrenSize()) { + yqlFromSettings = TYqlFromSettings::Parse(p->ChildPtr(3), ctx); + if (!yqlFromSettings) { + return IGraphTransformer::TStatus::Error; + } + } + auto alias = TString(p->Child(1)->Content()); auto columnOrder = ctx.Types.LookupColumnOrder(p->Head()); const bool isRangeFunction = p->Head().IsCallable("PgResolvedCall"); @@ -3085,20 +3188,40 @@ IGraphTransformer::TStatus SqlSetItemWrapper(const TExprNode::TPtr& input, TExpr return IGraphTransformer::TStatus::Error; } - if (!columnOrder) { - TVector<TString> expected(Reserve(p->Child(2)->ChildrenSize())); - for (const auto& child : p->Child(2)->Children()) { - expected.push_back(TString(child->Content())); - } + TVector<TString> expectedOrder(Reserve(p->Child(2)->ChildrenSize())); + for (const auto& child : p->Child(2)->Children()) { + expectedOrder.push_back(TString(child->Content())); + } + if (!columnOrder) { ctx.Expr.AddError(TIssue( ctx.Expr.GetPosition(option->Head().Pos()), TStringBuilder() << "No column order at source to match pattern " - << "(" << JoinSeq(", ", expected) << ")")); + << "(" << JoinSeq(", ", expectedOrder) << ")")); return IGraphTransformer::TStatus::Error; } + if (yqlFromSettings && + yqlFromSettings->IsCTE && + p->HeadPtr()->GetTypeAnn() && + p->HeadPtr()->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Universal) + { + if (auto actualOrder = InferYqlSimpleColumnOrder(p->HeadPtr())) { + TVector<TPositionHandle> expectedPositions(Reserve(expectedOrder.size())); + for (const auto& child : p->Child(2)->Children()) { + expectedPositions.emplace_back(child->Pos()); + } + + auto status = ValidateYqlExplicitColumnOrders( + input, output, ctx, + p->Pos(), expectedPositions, expectedOrder, *actualOrder); + if (status != IGraphTransformer::TStatus::Ok) { + return status; + } + } + } + TVector<const TItemExprType*> newStructItems; TMaybe<TColumnOrder> newOrder; @@ -4142,6 +4265,11 @@ IGraphTransformer::TStatus SqlSetItemWrapper(const TExprNode::TPtr& input, TExpr return IGraphTransformer::TStatus::Error; } } + else if (optionName == "warnings") { + if (!EnsureSqlSelectWarnings(*option, ctx.Expr)) { + return IGraphTransformer::TStatus::Error; + } + } else { ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(option->Head().Pos()), TStringBuilder() << "Unsupported option: " << optionName)); diff --git a/yql/essentials/core/type_ann/type_ann_sql.h b/yql/essentials/core/type_ann/type_ann_sql.h index a434f3cb3c0..9e62bc12482 100644 --- a/yql/essentials/core/type_ann/type_ann_sql.h +++ b/yql/essentials/core/type_ann/type_ann_sql.h @@ -44,6 +44,14 @@ TMap<TString, ui32> ExtractExternalColumns(const TExprNode& select); //////////////////////////////////////////////////////////////////////////////// +IGraphTransformer::TStatus AddSqlSelectWarning( + const TExprNode::TPtr& input, + TExprNode::TPtr& output, + TExprContext& ctx, + TStringBuf name); + +//////////////////////////////////////////////////////////////////////////////// + IGraphTransformer::TStatus SqlSelfWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); IGraphTransformer::TStatus SqlIterateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx); diff --git a/yql/essentials/core/type_ann/type_ann_yql.cpp b/yql/essentials/core/type_ann/type_ann_yql.cpp index e5712b97d10..19df78f9a24 100644 --- a/yql/essentials/core/type_ann/type_ann_yql.cpp +++ b/yql/essentials/core/type_ann/type_ann_yql.cpp @@ -351,6 +351,33 @@ IGraphTransformer::TStatus TryToUsingEntry( } // namespace +TMaybe<TYqlFromSettings> TYqlFromSettings::Parse(const TExprNode::TPtr& settings, TExtContext& ctx) { + TYqlFromSettings parsed; + + auto validator = [&](TStringBuf name, TExprNode& setting, TExprContext& ctx) -> bool { + if (name == "cte") { + if (setting.ChildrenSize() != 1) { + ctx.AddError(TIssue( + ctx.GetPosition(setting.Pos()), + TStringBuilder() << "No extra parameters are expected by setting " + << "'" << name << "'")); + return false; + } + + parsed.IsCTE = true; + return true; + } + + YQL_ENSURE(false, "unknown setting " << name); + }; + + if (!EnsureValidSettings(*settings, {"cte"}, validator, ctx.Expr)) { + return Nothing(); + } + + return parsed; +} + IGraphTransformer::TStatus PromoteYqlAggOptions( const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { @@ -487,6 +514,95 @@ IGraphTransformer::TStatus InferYqlInferUnionType( return status; } +TMaybe<TVector<std::pair<TString, /*isSynthetic=*/bool>>> +InferYqlSimpleColumnOrder(const TExprNode::TPtr& input) { + if (!input->IsCallable("YqlSelect")) { + return Nothing(); + } + + const auto order = [](const TExprNode::TPtr& item) { + const auto result = GetSetting(item->Head(), "result")->TailPtr(); + + TVector<std::pair<TString, /*isSynthetic=*/bool>> + order(Reserve(result->ChildrenSize())); + for (const auto& item : result->Children()) { + TString name(item->Child(0)->Content()); + bool isSynthetic = (3 < item->ChildrenSize() && + HasSetting(*item->Child(2), "synthetic")); + order.emplace_back(std::move(name), isSynthetic); + } + + return order; + }; + + const auto items = GetSetting(input->Head(), "set_items")->ChildPtr(1); + + auto result = order(items->ChildPtr(0)); + + for (const auto& item : items->Children()) { + auto x = order(item); + if (result != x) { + return Nothing(); + } + } + + return result; +} + +IGraphTransformer::TStatus ValidateYqlExplicitColumnOrders( + const TExprNode::TPtr& input, + TExprNode::TPtr& output, + TExtContext& ctx, + TPositionHandle position, + const TVector<TPositionHandle>& expectedPositions, + const TVector<TString>& expectedOrder, + const TVector<std::pair<TString, /*isSynthetic=*/bool>>& actualOrder) +{ + constexpr size_t Limit = 4; + + TIssue issue( + ctx.Expr.GetPosition(position), + "Column names in SELECT don't match column specification in parenthesis"); + SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_YQL_SOURCE_SELECT_COLUMN_MISMATCH, issue); + + for (size_t i = 0; + (i < Min(actualOrder.size(), expectedOrder.size())) && + (issue.GetSubIssues().size() < Limit); + i += 1) + { + const auto& [alias, isSynthetic] = actualOrder[i]; + if (isSynthetic || alias == expectedOrder[i]) { + continue; + } + + auto subIssue = MakeIntrusive<TIssue>( + ctx.Expr.GetPosition(expectedPositions[i]), + TStringBuilder() + << "At position " << (i + 1) << ' ' + << "actual " << '"' << alias << '"' << ' ' + << "doesn't match " + << "expected " << '"' << expectedOrder[i] << '"'); + SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_YQL_SOURCE_SELECT_COLUMN_MISMATCH, *subIssue); + issue.AddSubIssue(std::move(subIssue)); + } + + if (issue.GetSubIssues().empty()) { + return IGraphTransformer::TStatus::Ok; + } + + if (auto status = AddSqlSelectWarning(input, output, ctx.Expr, "yql_explicit_column_orders"); + status != IGraphTransformer::TStatus::Repeat) + { + return status; + } + + if (!ctx.Expr.AddWarning(issue)) { + return IGraphTransformer::TStatus::Error; + } + + return IGraphTransformer::TStatus::Repeat; +} + IGraphTransformer::TStatus YqlAggFactoryWrapper( const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) { diff --git a/yql/essentials/core/type_ann/type_ann_yql.h b/yql/essentials/core/type_ann/type_ann_yql.h index 8d6b6877b03..34958dd3c03 100644 --- a/yql/essentials/core/type_ann/type_ann_yql.h +++ b/yql/essentials/core/type_ann/type_ann_yql.h @@ -5,6 +5,12 @@ namespace NYql::NTypeAnnImpl { +struct TYqlFromSettings { + bool IsCTE = false; + + static TMaybe<TYqlFromSettings> Parse(const TExprNode::TPtr& settings, TExtContext& ctx); +}; + IGraphTransformer::TStatus PromoteYqlAggOptions( const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); @@ -28,6 +34,19 @@ IGraphTransformer::TStatus InferYqlInferUnionType( bool& areColumnsOrdered, bool& isUniversal); +/// NB: this is a light version only for a simple and sound static analysis. +TMaybe<TVector<std::pair<TString, /*isSynthetic=*/bool>>> +InferYqlSimpleColumnOrder(const TExprNode::TPtr& input); + +IGraphTransformer::TStatus ValidateYqlExplicitColumnOrders( + const TExprNode::TPtr& input, + TExprNode::TPtr& output, + TExtContext& ctx, + TPositionHandle position, + const TVector<TPositionHandle>& expectedPositions, + const TVector<TString>& expectedOrder, + const TVector<std::pair<TString, /*isSynthetic=*/bool>>& actualOrder); + IGraphTransformer::TStatus YqlAggFactoryWrapper( const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx); diff --git a/yql/essentials/core/yql_expr_type_annotation.cpp b/yql/essentials/core/yql_expr_type_annotation.cpp index 75fe4547eaa..54011eca117 100644 --- a/yql/essentials/core/yql_expr_type_annotation.cpp +++ b/yql/essentials/core/yql_expr_type_annotation.cpp @@ -4740,11 +4740,6 @@ IGraphTransformer::TStatus TryConvertToInternal1(TExprNode::TPtr& node, const TT } IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType, - TExprContext& ctx, TConvertFlags flags, bool useTypeDiff) { - return TryConvertToInternal1(node, expectedType, ctx, flags, useTypeDiff, /*typeCtx=*/nullptr); -} - -IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, TConvertFlags flags) { return TryConvertToInternal1(node, expectedType, ctx, flags, typeCtx.UseTypeDiffForConvertToError, &typeCtx); } @@ -4772,11 +4767,6 @@ IGraphTransformer::TStatus TryConvertToInternal2(TExprNode::TPtr& node, const TT } IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, - const TTypeAnnotationNode& expectedType, TExprContext& ctx, TConvertFlags flags, bool useTypeDiff) { - return TryConvertToInternal2(node, sourceType, expectedType, ctx, flags, useTypeDiff, /*typeCtx=*/nullptr); -} - -IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, const TTypeAnnotationNode& expectedType, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, TConvertFlags flags) { return TryConvertToInternal2(node, sourceType, expectedType, ctx, flags, typeCtx.UseTypeDiffForConvertToError, &typeCtx); } @@ -4814,21 +4804,11 @@ IGraphTransformer::TStatus TrySilentConvertToInternal1(TExprNode::TPtr& node, co } IGraphTransformer::TStatus TrySilentConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType, - TExprContext& ctx, TConvertFlags flags) { - return TrySilentConvertToInternal1(node, expectedType, ctx, flags, /*typeCtx=*/nullptr); -} - -IGraphTransformer::TStatus TrySilentConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, TConvertFlags flags) { return TrySilentConvertToInternal1(node, expectedType, ctx, flags, &typeCtx); } IGraphTransformer::TStatus TrySilentConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, - const TTypeAnnotationNode& expectedType, TExprContext& ctx, TConvertFlags flags) { - return TryConvertToImpl(ctx, node, sourceType, expectedType, flags, /*raiseIssues=*/false, /*typeCtx=*/nullptr); -} - -IGraphTransformer::TStatus TrySilentConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, const TTypeAnnotationNode& expectedType, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, TConvertFlags flags) { return TryConvertToImpl(ctx, node, sourceType, expectedType, flags, /*raiseIssues=*/false, &typeCtx); } @@ -5570,15 +5550,6 @@ IGraphTransformer::TStatus SilentInferCommonTypeInternal(TExprNode::TPtr& node1, } IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, TExprNode::TPtr& node2, TExprContext& ctx, - const TTypeAnnotationNode*& commonType, TConvertFlags flags) { - if (!node1->GetTypeAnn() || !node2->GetTypeAnn()) { - return IGraphTransformer::TStatus::Error; - } - - return SilentInferCommonTypeInternal(node1, *node1->GetTypeAnn(), node2, *node2->GetTypeAnn(), ctx, commonType, flags, /*typeCtx=*/nullptr); -} - -IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, TExprNode::TPtr& node2, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, const TTypeAnnotationNode*& commonType, TConvertFlags flags) { if (!node1->GetTypeAnn() || !node2->GetTypeAnn()) { return IGraphTransformer::TStatus::Error; @@ -5588,12 +5559,6 @@ IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, TExprNo } IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const TTypeAnnotationNode& type1, - TExprNode::TPtr& node2, const TTypeAnnotationNode& type2, TExprContext& ctx, - const TTypeAnnotationNode*& commonType, TConvertFlags flags) { - return SilentInferCommonTypeInternal(node1, type1, node2, type2, ctx, commonType, flags, /*typeCtx=*/nullptr); -} - -IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const TTypeAnnotationNode& type1, TExprNode::TPtr& node2, const TTypeAnnotationNode& type2, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, const TTypeAnnotationNode*& commonType, TConvertFlags flags) { return SilentInferCommonTypeInternal(node1, type1, node2, type2, ctx, commonType, flags, &typeCtx); @@ -5620,10 +5585,6 @@ IGraphTransformer::TStatus ConvertChildrenToTypeInternal(const TExprNode::TPtr& return status; } -IGraphTransformer::TStatus ConvertChildrenToType(const TExprNode::TPtr& input, const TTypeAnnotationNode* targetType, TExprContext& ctx, bool useTypeDiff) { - return ConvertChildrenToTypeInternal(input, targetType, ctx, useTypeDiff, /*typeCtx=*/nullptr); -} - IGraphTransformer::TStatus ConvertChildrenToType(const TExprNode::TPtr& input,const TTypeAnnotationNode* targetType, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { return ConvertChildrenToTypeInternal(input, targetType, ctx, typeCtx.UseTypeDiffForConvertToError, &typeCtx); diff --git a/yql/essentials/core/yql_expr_type_annotation.h b/yql/essentials/core/yql_expr_type_annotation.h index 3310fb860db..b989c372c8e 100644 --- a/yql/essentials/core/yql_expr_type_annotation.h +++ b/yql/essentials/core/yql_expr_type_annotation.h @@ -262,51 +262,30 @@ using TConvertFlags = TEnumBitSet<EFlags, DisableTruncation, Last>; using NConvertFlags::TConvertFlags; -//FIXME remove deprecated functions -//#define DEPRECATED [[deprecated]] -#define DEPRECATED - -DEPRECATED -IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, - const TTypeAnnotationNode& expectedType, TExprContext& ctx, TConvertFlags flags = {}, bool useTypeDiff = false); IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, const TTypeAnnotationNode& expectedType, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, TConvertFlags flags = {}); -DEPRECATED -IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType, - TExprContext& ctx, TConvertFlags flags = {}, bool useTypeDiff = false); + IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, TConvertFlags flags = {}); -DEPRECATED -IGraphTransformer::TStatus TrySilentConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType, TExprContext& ctx, - TConvertFlags flags = {}); + IGraphTransformer::TStatus TrySilentConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, TConvertFlags flags = {}); -DEPRECATED -IGraphTransformer::TStatus TrySilentConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, - const TTypeAnnotationNode& expectedType, TExprContext& ctx, TConvertFlags flags = {}); + IGraphTransformer::TStatus TrySilentConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType, const TTypeAnnotationNode& expectedType, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, TConvertFlags flags = {}); + TMaybe<EDataSlot> GetSuperType(EDataSlot dataSlot1, EDataSlot dataSlot2, bool warn = false, TExprContext* ctx = nullptr, TPositionHandle* pos = nullptr); -DEPRECATED -IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, TExprNode::TPtr& node2, TExprContext& ctx, - const TTypeAnnotationNode*& commonType, TConvertFlags flags = {}); + IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, TExprNode::TPtr& node2, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, const TTypeAnnotationNode*& commonType, TConvertFlags flags = {}); -DEPRECATED -IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const TTypeAnnotationNode& type1, - TExprNode::TPtr& node2, const TTypeAnnotationNode& type2, TExprContext& ctx, - const TTypeAnnotationNode*& commonType, TConvertFlags flags = {}); + IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, const TTypeAnnotationNode& type1, TExprNode::TPtr& node2, const TTypeAnnotationNode& type2, TExprContext& ctx, const TTypeAnnotationContext& typeCtx, const TTypeAnnotationNode*& commonType, TConvertFlags flags = {}); -DEPRECATED -IGraphTransformer::TStatus ConvertChildrenToType(const TExprNode::TPtr& input,const TTypeAnnotationNode* targetType, TExprContext& ctx, - bool useTypeDiff = false); + IGraphTransformer::TStatus ConvertChildrenToType(const TExprNode::TPtr& input,const TTypeAnnotationNode* targetType, TExprContext& ctx, const TTypeAnnotationContext& typeCtx); -#undef DEPRECATED - bool IsSqlInCollectionItemsNullable(const NNodes::TCoSqlIn& node); bool IsDataTypeNumeric(EDataSlot dataSlot); diff --git a/yql/essentials/docs/ru/_includes/pg_functions.md b/yql/essentials/docs/ru/_includes/pg_functions.md new file mode 100644 index 00000000000..f53c4239cfd --- /dev/null +++ b/yql/essentials/docs/ru/_includes/pg_functions.md @@ -0,0 +1,4799 @@ +## 9.1. Logical Operators {#logical-operators}
+
+<!-- markdownlint-disable blanks-around-fences blanks-around-headings -->
+
+The usual logical operators are available:
+
+* boolean AND boolean → boolean
+* boolean OR boolean → boolean
+* NOT boolean → boolean
+
+## 9.2. Comparison Functions and Operators {#comparison-functions}
+
+
+The usual comparison operators are available, as shown in Table 9.1.
+
+Table 9.1. Comparison Operators
+
+#|
+|| Operator | Description ||
+|| datatype < datatype → boolean | Less than ||
+|| datatype > datatype → boolean | Greater than ||
+|| datatype <= datatype → boolean | Less than or equal to ||
+|| datatype >= datatype → boolean | Greater than or equal to ||
+|| datatype = datatype → boolean | Equal ||
+|| datatype <> datatype → boolean | Not equal ||
+|| datatype != datatype → boolean | Not equal ||
+|#
+
+There are also some comparison predicates, as shown in Table 9.2. These behave much like operators, but have special syntax mandated by the SQL standard.
+
+#|
+|| Predicate | Description | Example(s) ||
+|| datatype BETWEEN datatype AND datatype → boolean | Between (inclusive of the range endpoints). |
+```sql
+2 BETWEEN 1 AND 3 → true
+2 BETWEEN 3 AND 1 → false
+```||
+|| datatype NOT BETWEEN datatype AND datatype → boolean | Not between (the negation of BETWEEN). |
+```sql
+2 NOT BETWEEN 1 AND 3 → false
+```||
+|| datatype BETWEEN SYMMETRIC datatype AND datatype → boolean | Between, after sorting the two endpoint values.|
+```sql
+2 BETWEEN SYMMETRIC 3 AND 1 → true
+```||
+||datatype NOT BETWEEN SYMMETRIC datatype AND datatype → boolean | Not between, after sorting the two endpoint values. |
+```sql
+2 NOT BETWEEN SYMMETRIC 3 AND 1 → false
+```||
+||datatype IS DISTINCT FROM datatype → boolean | Not equal, treating null as a comparable value. (NOT SUPPORTED)|
+```sql
+#1 IS DISTINCT FROM NULL → true
+#NULL IS DISTINCT FROM NULL → false
+```||
+||datatype IS NOT DISTINCT FROM datatype → boolean | Equal, treating null as a comparable value. (NOT SUPPORTED)|
+```sql
+#1 IS NOT DISTINCT FROM NULL → false
+#NULL IS NOT DISTINCT FROM NULL → true
+```||
+||datatype IS NULL → boolean | Test whether value is null. |
+```sql
+1.5 IS NULL → false
+```||
+||datatype IS NOT NULL → boolean | Test whether value is not null. |
+```sql
+'null' IS NOT NULL → true
+```||
+||datatype ISNULL → boolean | Test whether value is null (nonstandard syntax) |
+```sql
+1.5 ISNULL → false
+```||
+||datatype NOTNULL → boolean | Test whether value is not null (nonstandard syntax) |
+```sql
+1.5 NOTNULL → true
+```||
+||boolean IS TRUE → boolean | Test whether boolean expression yields true. (NOT SUPPORTED)|
+```sql
+true IS TRUE → true
+NULL::boolean IS TRUE → false
+```||
+|#
+
+## 9.3. Mathematical Functions and Operators {#mathematical-functions}
+
+Mathematical operators are provided for many PostgreSQL types. For types without standard mathematical conventions (e.g., date/time types) we describe the actual behavior in subsequent sections.
+
+Table 9.4 shows the mathematical operators that are available for the standard numeric types. Unless otherwise noted, operators shown as accepting numeric_type are available for all the types smallint, integer, bigint, numeric, real, and double precision. Operators shown as accepting integral_type are available for the types smallint, integer, and bigint. Except where noted, each form of an operator returns the same data type as its argument(s). Calls involving multiple argument data types, such as integer + numeric, are resolved by using the type appearing later in these lists.
+
+Table 9.4. Mathematical Operators
+
+#|
+||Operator|Description|Example(s)||
+||numeric_type + numeric_type → numeric_type | Addition |
+```sql
+2 + 3 → 5
+```||
+|| + numeric_type → numeric_type | Unary plus (no operation) |
+```sql
++ 3.5 → 3.5
+```||
+||numeric_type - numeric_type → numeric_type | Subtraction |
+```sql
+2 - 3 → -1
+```||
+|| - numeric_type → numeric_type | Negation |
+```sql
+- (-4) → 4
+```||
+||numeric_type * numeric_type → numeric_type | Multiplication |
+```sql
+2 * 3 → 6
+```||
+||numeric_type / numeric_type → numeric_type|Division (for integral types, division truncates the result towards zero)|
+```sql
+5.0 / 2 → 2.5000000000000000
+5 / 2 → 2
+(-5) / 2 → -2
+```||
+||numeric_type % numeric_type → numeric_type|Modulo (remainder); available for smallint, integer, bigint, and numeric|
+```sql
+5 % 4 → 1
+```||
+||numeric ^ numeric → numeric
+double precision ^ double precision → double precision|
+
+Exponentiation.
+Unlike typical mathematical practice, multiple uses of ^ will associate left to right by default.|
+```sql
+2 ^ 3 → 8
+2 ^ 3 ^ 3 → 512
+2 ^ (3 ^ 3) → 134217728
+```||
+|| \|\/ double precision → double precision|Square root|
+```sql
+|/ 25.0 → 5
+```||
+|| \|\|\/ double precision → double precision|Cube root|
+```sql
+||/ 64.0 → 4
+```||
+||@ numeric_type → numeric_type | Absolute value |
+```sql
+@ -5.0 → 5.0
+```||
+||integral_type & integral_type → integral_type | Bitwise AND |
+```sql
+91 & 15 → 11
+```||
+||integral_type \| integral_type → integral_type | Bitwise OR |
+```sql
+32 | 3 → 35
+```||
+||integral_type # integral_type → integral_type | Bitwise exclusive OR |
+```sql
+17 # 5 → 20
+```||
+||~ integral_type → integral_type | Bitwise NOT |
+```sql
+~1 → -2
+```||
+||integral_type << integer → integral_type | Bitwise shift left |
+```sql
+1 << 4 → 16
+```||
+||integral_type >> integer → integral_type | Bitwise shift right |
+```sql
+8 >> 2 → 2
+```||
+|#
+
+Table 9.5 shows the available mathematical functions. Many of these functions are provided in multiple forms with different argument types. Except where noted, any given form of a function returns the same data type as its argument(s); cross-type cases are resolved in the same way as explained above for operators. The functions working with double precision data are mostly implemented on top of the host system's C library; accuracy and behavior in boundary cases can therefore vary depending on the host system.
+
+Table 9.5. Mathematical Functions
+
+#|
+||Function|Description|Example(s)||
+||abs ( numeric_type ) → numeric_type | Absolute value |
+```sql
+abs(-17.4) → 17.4
+```||
+||cbrt ( double precision ) → double precision | Cube root |
+```sql
+cbrt(64.0) → 4
+```||
+||ceil ( numeric ) → numeric
+ceil ( double precision ) → double precision|Nearest integer greater than or equal to argument|
+```sql
+ceil(42.2) → 43
+ceil(-42.8) → -42
+```||
+||ceiling ( numeric ) → numeric
+ceiling ( double precision ) → double precision|Nearest integer greater than or equal to argument(same as ceil)|
+```sql
+ceiling(95.3) → 96
+```||
+||degrees ( double precision ) → double precision|Converts radians to degrees|
+```sql
+degrees(0.5) → 28.64788975654116
+```||
+||div ( y numeric, x numeric ) → numeric|Integer quotient of y/x (truncates towards zero)|
+```sql
+div(9, 4) → 2
+```||
+||exp ( numeric ) → numeric
+exp ( double precision ) → double precision|Exponential (e raised to the given power)|
+```sql
+exp(1.0) → 2.7182818284590452
+```||
+||factorial ( bigint ) → numeric|Factorial|
+```sql
+factorial(5) → 120
+```||
+||floor ( numeric ) → numeric
+floor ( double precision ) → double precision|Nearest integer less than or equal to argument|
+```sql
+floor(42.8) → 42
+floor(-42.8) → -43
+```||
+||gcd ( numeric_type, numeric_type ) → numeric_type|
+Greatest common divisor (the largest positive number that divides both inputs with no remainder); returns 0 if both inputs are zero; available for integer, bigint, and numeric|
+```sql
+gcd(1071, 462) → 21
+```||
+||lcm ( numeric_type, numeric_type ) → numeric_type|
+Least common multiple (the smallest strictly positive number that is an integral multiple of both inputs); returns 0 if either input is zero; available for integer, bigint, and numeric|
+```sql
+lcm(1071, 462) → 23562
+```||
+||ln ( numeric ) → numeric
+ln ( double precision ) → double precision|Natural logarithm|
+```sql
+ln(2.0) → 0.6931471805599453
+```||
+||log ( numeric ) → numeric
+log ( double precision ) → double precision|Base 10 logarithm|
+```sql
+log(100) → 2
+```||
+||log10 ( numeric ) → numeric
+log10 ( double precision ) → double precision|Base 10 logarithm (same as log)|
+```sql
+log10(1000) → 3
+```||
+||log ( b numeric, x numeric ) → numeric|Logarithm of x to base b|
+```sql
+log(2.0, 64.0) → 6.0000000000000000
+```||
+||min_scale ( numeric ) → integer|Minimum scale (number of fractional decimal digits) needed to represent the supplied value precisely|
+```sql
+min_scale(8.4100) → 2
+```||
+||mod ( y numeric_type, x numeric_type ) → numeric_type|
+Remainder of y/x; available for smallint, integer, bigint, and numeric|
+```sql
+mod(9, 4) → 1
+```||
+||pi ( ) → double precision|Approximate value of π|
+```sql
+pi() → 3.141592653589793
+```||
+||power ( a numeric, b numeric ) → numeric
+power ( a double precision, b double precision ) → double precision|a raised to the power of b|
+```sql
+power(9, 3) → 729
+```||
+||radians ( double precision ) → double precision|Converts degrees to radians|
+```sql
+radians(45.0) → 0.7853981633974483
+```||
+||round ( numeric ) → numeric
+round ( double precision ) → double precision|Rounds to nearest integer. For numeric, ties are broken by rounding away from zero. For double precision, the tie-breaking behavior is platform dependent, but “round to nearest even” is the most common rule.|
+```sql
+round(42.4) → 42
+```||
+||round ( v numeric, s integer ) → numeric|
+Rounds v to s decimal places. Ties are broken by rounding away from zero.|
+```sql
+round(42.4382, 2) → 42.44
+round(1234.56, -1) → 1230
+```||
+||scale ( numeric ) → integer|Scale of the argument (the number of decimal digits in the fractional part)|
+```sql
+scale(8.4100) → 4
+```||
+||sign ( numeric ) → numeric
+sign ( double precision ) → double precision|
+Sign of the argument (-1, 0, or +1)|
+```sql
+sign(-8.4) → -1
+```||
+||sqrt ( numeric ) → numeric
+sqrt ( double precision ) → double precision|
+Square root|
+```sql
+sqrt(2) → 1.4142135623730951
+```||
+||trim_scale ( numeric ) → numeric|
+Reduces the value's scale (number of fractional decimal digits) by removing trailing zeroes|
+```sql
+trim_scale(8.4100) → 8.41
+```||
+||trunc ( numeric ) → numeric
+trunc ( double precision ) → double precision|
+Truncates to integer (towards zero)|
+```sql
+trunc(42.8) → 42
+trunc(-42.8) → -42
+```||
+||trunc ( v numeric, s integer ) → numeric|Truncates v to s decimal places|
+```sql
+trunc(42.4382, 2) → 42.43
+```||
+||width_bucket ( operand numeric, low numeric, high numeric, count integer ) → integer
+width_bucket ( operand double precision, low double precision, high double precision, count integer ) → integer|
+Returns the number of the bucket in which operand falls in a histogram having count equal-width buckets spanning the range low to high. Returns 0 or count+1 for an input outside that range.|
+```sql
+width_bucket(5.35, 0.024, 10.06, 5) → 3
+```||
+||width_bucket ( operand anycompatible, thresholds anycompatiblearray ) → integer (NOT SUPPORTED)|
+Returns the number of the bucket in which operand falls given an array listing the lower bounds of the buckets. Returns 0 for an input less than the first lower bound. operand and the array elements can be of any type having standard comparison operators. The thresholds array must be sorted, smallest first, or unexpected results will be obtained.|
+```sql
+#width_bucket(now(), array['yesterday', 'today', 'tomorrow']::timestamptz[]) → 2
+```||
+|#
+
+Table 9.6. Random Functions
+
+#|
+||Function|Description|Example(s)||
+||random ( ) → double precision|Returns a random value in the range 0.0 <= x < 1.0| ||
+||setseed ( double precision ) → void|
+Sets the seed for subsequent random() calls; argument must be between -1.0 and 1.0, inclusive
+(NOT SUPPORTED)|
+|#
+
+Table 9.7 shows the available trigonometric functions. Each of these functions comes in two variants, one that measures angles in radians and one that measures angles in degrees.
+
+Table 9.7. Trigonometric Functions
+
+#|
+||Function|Description|Example(s)||
+||acos ( double precision ) → double precision|Inverse cosine, result in radians|
+```sql
+acos(1) → 0
+```||
+||acosd ( double precision ) → double precision|Inverse cosine, result in degrees|
+```sql
+acosd(0.5) → 60
+```||
+||asin ( double precision ) → double precision|Inverse sine, result in radians|
+```sql
+asin(1) → 1.5707963267948966
+```||
+||asind ( double precision ) → double precision|Inverse sine, result in degrees|
+```sql
+asind(0.5) → 30
+```||
+||atan ( double precision ) → double precision|Inverse tangent, result in radians|
+```sql
+atan(1) → 0.7853981633974483
+```||
+||atand ( double precision ) → double precision|Inverse tangent, result in degrees|
+```sql
+atand(1) → 45
+```||
+||atan2 ( y double precision, x double precision ) → double precision|Inverse tangent of y/x, result in radians|
+```sql
+atan2(1, 0) → 1.5707963267948966
+```||
+||atan2d ( y double precision, x double precision ) → double precision|Inverse tangent of y/x, result in degrees|
+```sql
+atan2d(1, 0) → 90
+```||
+||cos ( double precision ) → double precision|Cosine, argument in radians|
+```sql
+cos(0) → 1
+```||
+||cosd ( double precision ) → double precision|Cosine, argument in degrees|
+```sql
+cosd(60) → 0.5
+```||
+||cot ( double precision ) → double precision|Cotangent, argument in radians|
+```sql
+cot(0.5) → 1.830487721712452
+```||
+||cotd ( double precision ) → double precision|Cotangent, argument in degrees|
+```sql
+cotd(45) → 1
+```||
+||sin ( double precision ) → double precision|Sine, argument in radians|
+```sql
+sin(1) → 0.8414709848078965
+```||
+||sind ( double precision ) → double precision|Sine, argument in degrees|
+```sql
+sind(30) → 0.5
+```||
+||tan ( double precision ) → double precision|Tangent, argument in radians|
+```sql
+tan(1) → 1.5574077246549023
+```||
+||tand ( double precision ) → double precision|Tangent, argument in degrees|
+```sql
+tand(45) → 1
+```||
+|#
+
+Table 9.8 shows the available hyperbolic functions.
+
+Table 9.8. Hyperbolic Functions
+
+#|
+||Function|Description|Example(s)||
+||sinh ( double precision ) → double precision|Hyperbolic sine|
+```sql
+sinh(1) → 1.1752011936438014
+```||
+||cosh ( double precision ) → double precision|Hyperbolic cosine|
+```sql
+cosh(0) → 1
+```||
+||tanh ( double precision ) → double precision|Hyperbolic tangent|
+```sql
+tanh(1) → 0.7615941559557649
+```||
+||asinh ( double precision ) → double precision|Inverse hyperbolic sine|
+```sql
+asinh(1) → 0.881373587019543
+```||
+||acosh ( double precision ) → double precision|Inverse hyperbolic cosine|
+```sql
+acosh(1) → 0
+```||
+||atanh ( double precision ) → double precision|Inverse hyperbolic tangent|
+```sql
+atanh(0.5) → 0.5493061443340548
+```||
+|#
+
+## 9.4. String Functions and Operators {#string-functions}
+
+This section describes functions and operators for examining and manipulating string values. Strings in this context include values of the types character, character varying, and text. Except where noted, these functions and operators are declared to accept and return type text. They will interchangeably accept character varying arguments. Values of type character will be converted to text before the function or operator is applied, resulting in stripping any trailing spaces in the character value.
+
+SQL defines some string functions that use key words, rather than commas, to separate arguments. Details are in Table 9.9. PostgreSQL also provides versions of these functions that use the regular function invocation syntax (see Table 9.10).
+
+Table 9.9. SQL String Functions and Operators
+
+#|
+||Function/Operator|Description|Example(s)||
+|| text \|\| text → text | Concatenates the two strings |
+```sql
+'Post' || 'greSQL' → PostgreSQL
+```||
+||text \|\| anynonarray → text
+anynonarray \|\| text → text|
+Converts the non-string input to text, then concatenates the two strings. (The non-string input cannot be of an array type, because that would create ambiguity with the array \|\| operators. If you want to concatenate an array's text equivalent, cast it to text explicitly.) (UNSUPPORTED)|
+```sql
+#'Value: ' || 42 → Value: 42
+```||
+||text IS [NOT] [form] NORMALIZED → boolean|
+
+Checks whether the string is in the specified Unicode normalization form. The optional form key word specifies the form: NFC (the default), NFD, NFKC, or NFKD. This expression can only be used when the server encoding is UTF8. Note that checking for normalization using this expression is often faster than normalizing possibly already normalized strings.|
+```sql
+U&'\0061\0308bc' IS NFD NORMALIZED → true
+```||
+||bit_length ( text ) → integer|Returns number of bits in the string (8 times the octet_length)|
+```sql
+bit_length('jose') → 32
+```||
+||char_length ( text ) → integer
+character_length ( text ) → integer|
+Returns number of characters in the string.|
+```sql
+char_length('josé') → 4
+```||
+||lower ( text ) → text|Converts the string to all lower case, according to the rules of the database's locale.|
+```sql
+lower('TOM') → tom
+```||
+||normalize ( text [, form ] ) → text|
+Converts the string to the specified Unicode normalization form. The optional form key word specifies the form: NFC (the default), NFD, NFKC, or NFKD. This function can only be used when the server encoding is UTF8.|
+```sql
+normalize(U&'\0061\0308bc', NFC) → äbc
+```||
+||octet_length ( text ) → integer|
+Returns number of bytes in the string.|
+```sql
+octet_length('josé') → 5
+```||
+||octet_length ( character ) → integer|
+Returns number of bytes in the string. Since this version of the function accepts type character directly, it will not strip trailing spaces.|
+```sql
+octet_length('abc '::character(4)) → 4
+```||
+||overlay ( string text PLACING newsubstring text FROM start integer [ FOR count integer ] ) → text|
+Replaces the substring of string that starts at the start'th character and extends for count characters with newsubstring. If count is omitted, it defaults to the length of newsubstring.|
+```sql
+overlay('Txxxxas' placing 'hom' from 2 for 4) → Thomas
+```||
+||position ( substring text IN string text ) → integer|
+Returns first starting index of the specified substring within string, or zero if it's not present.|
+```sql
+position('om' in 'Thomas') → 3
+```||
+||substring ( string text [ FROM start integer ] [ FOR count integer ] ) → text|
+Extracts the substring of string starting at the start'th character if that is specified, and stopping after count characters if that is specified. Provide at least one of start and count.|
+```sql
+substring('Thomas' from 2 for 3) → hom
+substring('Thomas' from 3) → omas
+substring('Thomas' for 2) → Th
+```||
+||substring ( string text FROM pattern text ) → text|
+Extracts the first substring matching POSIX regular expression; see Section 9.7.3.|
+```sql
+substring('Thomas' from '...$') → mas
+```||
+||substring ( string text SIMILAR pattern text ESCAPE escape text ) → text
+substring ( string text FROM pattern text FOR escape text ) → text|
+Extracts the first substring matching SQL regular expression; see Section 9.7.2. The first form has been specified since SQL:2003; the second form was only in SQL:1999 and should be considered obsolete.|
+```sql
+substring('Thomas' similar '%#"o_a#"_' escape '#') → oma
+```||
+||trim ( [ LEADING \| TRAILING \| BOTH ] [ characters text ] FROM string text ) → text|
+Removes the longest string containing only characters in characters (a space by default) from the start, end, or both ends (BOTH is the default) of string.|
+```sql
+trim(both 'xyz' from 'yxTomxx') → Tom
+```||
+||trim ( [ LEADING \| TRAILING \| BOTH ] [ FROM ] string text [, characters text ] ) → text|
+This is a non-standard syntax for trim().|
+```sql
+trim(both from 'yxTomxx', 'xyz') → Tom
+```||
+||upper ( text ) → text|
+Converts the string to all upper case, according to the rules of the database's locale.|
+```sql
+upper('tom') → TOM
+```||
+|#
+
+Additional string manipulation functions are available and are listed in Table 9.10. Some of them are used internally to implement the SQL-standard string functions listed in Table 9.9.
+
+Table 9.10. Other String Functions
+
+#|
+||Function|Description|Example(s)||
+||ascii ( text ) → integer|
+Returns the numeric code of the first character of the argument. In UTF8 encoding, returns the Unicode code point of the character. In other multibyte encodings, the argument must be an ASCII character.|
+```sql
+ascii('x') → 120
+```||
+||btrim ( string text [, characters text ] ) → text|
+Removes the longest string containing only characters in characters (a space by default) from the start and end of string.|
+```sql
+btrim('xyxtrimyyx', 'xyz') → trim
+```||
+||chr ( integer ) → text|
+Returns the character with the given code. In UTF8 encoding the argument is treated as a Unicode code point. In other multibyte encodings the argument must designate an ASCII character. chr(0) is disallowed because text data types cannot store that character.|
+```sql
+chr(65) → A
+```||
+||concat ( val1 "any" [, val2 "any" [, ...] ] ) → text|
+Concatenates the text representations of all the arguments. NULL arguments are ignored. (NOT SUPPORTED)|
+```sql
+concat('abcde', 2, NULL, 22) → abcde222
+```||
+||concat_ws ( sep text, val1 "any" [, val2 "any" [, ...] ] ) → text|
+Concatenates all but the first argument, with separators. The first argument is used as the separator string, and should not be NULL. Other NULL arguments are ignored. (NOT SUPPORTED)|
+```sql
+concat_ws(',', 'abcde', 2, NULL, 22) → abcde,2,22
+```||
+||format ( formatstr text [, formatarg "any" [, ...] ] ) → text|
+Formats arguments according to a format string; see Section 9.4.1. This function is similar to the C function sprintf. (NOT SUPPORTED)|
+```sql
+format('Hello %s, %1$s', 'World') → Hello World, World
+```||
+||initcap ( text ) → text|
+Converts the first letter of each word to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters.|
+```sql
+initcap('hi THOMAS') → Hi Thomas
+```||
+||left ( string text, n integer ) → text|
+Returns first n characters in the string, or when n is negative, returns all but last \|n\| characters.|
+```sql
+left('abcde', 2) → ab
+```||
+||length ( text ) → integer|
+Returns the number of characters in the string.|
+```sql
+length('jose') → 4
+```||
+||lpad ( string text, length integer [, fill text ] ) → text|
+Extends the string to length length by prepending the characters fill (a space by default). If the string is already longer than length then it is truncated (on the right).|
+```sql
+lpad('hi', 5, 'xy') → xyxhi
+```||
+||ltrim ( string text [, characters text ] ) → text|
+Removes the longest string containing only characters in characters (a space by default) from the start of string.|
+```sql
+ltrim('zzzytest', 'xyz') → test
+```||
+||md5 ( text ) → text|
+Computes the MD5 hash of the argument, with the result written in hexadecimal.|
+```sql
+md5('abc') → 900150983cd24fb0d6963f7d28e17f72
+```||
+||parse_ident ( qualified_identifier text [, strict_mode boolean DEFAULT true ] ) → text[]|
+Splits qualified_identifier into an array of identifiers, removing any quoting of individual identifiers. By default, extra characters after the last identifier are considered an error; but if the second parameter is false, then such extra characters are ignored. (This behavior is useful for parsing names for objects like functions.) Note that this function does not truncate over-length identifiers. If you want truncation you can cast the result to name[].|
+```sql
+parse_ident('"SomeSchema".someTable') → {SomeSchema,someTable}
+```||
+||pg_client_encoding ( ) → name|
+Returns current client encoding name.|
+```sql
+pg_client_encoding() → UTF8
+```||
+||quote_ident ( text ) → text|
+Returns the given string suitably quoted to be used as an identifier in an SQL statement string. Quotes are added only if necessary (i.e., if the string contains non-identifier characters or would be case-folded). Embedded quotes are properly doubled. See also Example 43.1.|
+```sql
+quote_ident('Foo bar') → "Foo bar"
+```||
+||quote_literal ( text ) → text|
+Returns the given string suitably quoted to be used as a string literal in an SQL statement string. Embedded single-quotes and backslashes are properly doubled. Note that quote_literal returns null on null input; if the argument might be null, quote_nullable is often more suitable. See also Example 43.1.|
+```sql
+quote_literal(E'O\'Reilly') → ''O''Reilly''
+```||
+||quote_literal ( anyelement ) → text|
+Converts the given value to text and then quotes it as a literal. Embedded single-quotes and backslashes are properly doubled. (NOT SUPPORTED)|
+```sql
+#quote_literal(42.5) → '42.5'
+```||
+||quote_nullable ( text ) → text|
+Returns the given string suitably quoted to be used as a string literal in an SQL statement string; or, if the argument is null, returns NULL. Embedded single-quotes and backslashes are properly doubled. See also Example 43.1.|
+```sql
+quote_nullable(NULL) → NULL
+```||
+||quote_nullable ( anyelement ) → text|
+Converts the given value to text and then quotes it as a literal; or, if the argument is null, returns NULL. Embedded single-quotes and backslashes are properly doubled. (NOT SUPPORTED)|
+```sql
+#quote_nullable(42.5) → '42.5'
+```||
+||regexp_match ( string text, pattern text [, flags text ] ) → text[]|
+Returns captured substrings resulting from the first match of a POSIX regular expression to the string; see Section 9.7.3.|
+```sql
+regexp_match('foobarbequebaz', '(bar)(beque)') → {bar,beque}
+```||
+||regexp_matches ( string text, pattern text [, flags text ] ) → setof text[]|
+Returns captured substrings resulting from the first match of a POSIX regular expression to the string, or multiple matches if the g flag is used; see Section 9.7.3. (NOT SUPPORTED)|
+```sql
+#regexp_matches('foobarbequebaz', 'ba.', 'g') → {bar},{baz}
+```||
+||regexp_replace ( string text, pattern text, replacement text [, flags text ] ) → text|
+Replaces substrings resulting from the first match of a POSIX regular expression, or multiple substring matches if the g flag is used; see Section 9.7.3.|
+```sql
+regexp_replace('Thomas', '.[mN]a.', 'M') → ThM
+```||
+||regexp_split_to_array ( string text, pattern text [, flags text ] ) → text[]|
+Splits string using a POSIX regular expression as the delimiter, producing an array of results; see Section 9.7.3.|
+```sql
+regexp_split_to_array('hello world', '\s+') → {hello,world}
+```||
+||regexp_split_to_table ( string text, pattern text [, flags text ] ) → setof text|
+Splits string using a POSIX regular expression as the delimiter, producing a set of results; see Section 9.7.3. (NOT SUPPORTED)|
+```sql
+#regexp_split_to_table('hello world', '\s+') → hello,world
+```||
+||repeat ( string text, number integer ) → text|
+Repeats string the specified number of times.|
+```sql
+repeat('Pg', 4) → PgPgPgPg
+```||
+||replace ( string text, from text, to text ) → text|
+Replaces all occurrences in string of substring from with substring to.|
+```sql
+replace('abcdefabcdef', 'cd', 'XX') → abXXefabXXef
+```||
+||reverse ( text ) → text|
+Reverses the order of the characters in the string.|
+```sql
+reverse('abcde') → edcba
+```||
+||right ( string text, n integer ) → text|
+Returns last n characters in the string, or when n is negative, returns all but first \|n\| characters.|
+```sql
+right('abcde', 2) → de
+```||
+||rpad ( string text, length integer [, fill text ] ) → text|
+Extends the string to length length by appending the characters fill (a space by default). If the string is already longer than length then it is truncated.|
+```sql
+rpad('hi', 5, 'xy') → hixyx
+```||
+||rtrim ( string text [, characters text ] ) → text|
+Removes the longest string containing only characters in characters (a space by default) from the end of string.|
+```sql
+rtrim('testxxzx', 'xyz') → test
+```||
+||split_part ( string text, delimiter text, n integer ) → text|
+Splits string at occurrences of delimiter and returns the n'th field (counting from one), or when n is negative, returns the \|n\|'th-from-last field.|
+```sql
+split_part('abc~@~def~@~ghi', '~@~', 2) → def
+split_part('abc,def,ghi,jkl', ',', -2) → ghi
+```||
+||strpos ( string text, substring text ) → integer|
+Returns first starting index of the specified substring within string, or zero if it's not present. (Same as position(substring in string), but note the reversed argument order.)|
+```sql
+strpos('high', 'ig') → 2
+```||
+||substr ( string text, start integer [, count integer ] ) → text|
+Extracts the substring of string starting at the start'th character, and extending for count characters if that is specified. (Same as substring(string from start for count).)|
+```sql
+substr('alphabet', 3) → phabet
+substr('alphabet', 3, 2) → ph
+```||
+||starts_with ( string text, prefix text ) → boolean|
+Returns true if string starts with prefix.|
+```sql
+starts_with('alphabet', 'alph') → true
+```||
+||string_to_array ( string text, delimiter text [, null_string text ] ) → text[]|
+Splits the string at occurrences of delimiter and forms the resulting fields into a text array. If delimiter is NULL, each character in the string will become a separate element in the array. If delimiter is an empty string, then the string is treated as a single field. If null_string is supplied and is not NULL, fields matching that string are replaced by NULL.|
+```sql
+string_to_array('xx~~yy~~zz', '~~', 'yy') → {xx,NULL,zz}
+```||
+||string_to_table ( string text, delimiter text [, null_string text ] ) → setof text|
+Splits the string at occurrences of delimiter and returns the resulting fields as a set of text rows. If delimiter is NULL, each character in the string will become a separate row of the result. If delimiter is an empty string, then the string is treated as a single field. If null_string is supplied and is not NULL, fields matching that string are replaced by NULL. (NOT SUPPORTED)|
+```sql
+#string_to_table('xx~^~yy~^~zz', '~^~', 'yy') → [xx,NULL,zz]
+```||
+||to_ascii ( string text ) → text
+to_ascii ( string text, encoding name ) → text
+to_ascii ( string text, encoding integer ) → text|
+Converts string to ASCII from another encoding, which may be identified by name or number. If encoding is omitted the database encoding is assumed (which in practice is the only useful case). The conversion consists primarily of dropping accents. Conversion is only supported from LATIN1, LATIN2, LATIN9, and WIN1250 encodings. (See the unaccent module for another, more flexible solution.) (NOT SUPPORTED)|
+```sql
+#to_ascii('Karél') → Karel
+```||
+||to_hex ( integer ) → text
+to_hex ( bigint ) → text|
+Converts the number to its equivalent hexadecimal representation.|
+```sql
+to_hex(2147483647) → 7fffffff
+```||
+||translate ( string text, from text, to text ) → text|
+Replaces each character in string that matches a character in the from set with the corresponding character in the to set. If from is longer than to, occurrences of the extra characters in from are deleted.|
+```sql
+translate('12345', '143', 'ax') → a2x5
+```||
+||unistr ( text ) → text|
+Evaluate escaped Unicode characters in the argument. Unicode characters can be specified as \XXXX (4 hexadecimal digits), \+XXXXXX (6 hexadecimal digits), \uXXXX (4 hexadecimal digits), or \UXXXXXXXX (8 hexadecimal digits). To specify a backslash, write two backslashes. All other characters are taken literally.
+
+If the server encoding is not UTF-8, the Unicode code point identified by one of these escape sequences is converted to the actual server encoding; an error is reported if that's not possible.
+
+This function provides a (non-standard) alternative to string constants with Unicode escapes (see Section 4.1.2.3).|
+```sql
+unistr('d\0061t\+000061') → data
+unistr('d\u0061t\U00000061') → data
+```||
+|#
+
+## 9.5. Binary String Functions and Operators {#binary-string-functions}
+This section describes functions and operators for examining and manipulating binary strings, that is values of type bytea. Many of these are equivalent, in purpose and syntax, to the text-string functions described in the previous section.
+
+SQL defines some string functions that use key words, rather than commas, to separate arguments. Details are in Table 9.11. PostgreSQL also provides versions of these functions that use the regular function invocation syntax (see Table 9.12).
+
+Table 9.11. SQL Binary String Functions and Operators
+
+#|
+||Function/Operator|Description|Example(s)||
+||bytea \|\| bytea → bytea|
+Concatenates the two binary strings.|
+```sql
+'\x123456'::bytea || '\x789a00bcde'::bytea → \x123456789a00bcde
+```||
+||bit_length ( bytea ) → integer|
+Returns number of bits in the binary string (8 times the octet_length).|
+```sql
+bit_length('\x123456'::bytea) → 24
+```||
+||octet_length ( bytea ) → integer|
+Returns number of bytes in the binary string.|
+```sql
+octet_length('\x123456'::bytea) → 3
+```||
+||overlay ( bytes bytea PLACING newsubstring bytea FROM start integer [ FOR count integer ] ) → bytea|
+Replaces the substring of bytes that starts at the start'th byte and extends for count bytes with newsubstring. If count is omitted, it defaults to the length of newsubstring.|
+```sql
+overlay('\x1234567890'::bytea placing '\002\003'::bytea from 2 for 3) → \x12020390
+```||
+||position ( substring bytea IN bytes bytea ) → integer|
+Returns first starting index of the specified substring within bytes, or zero if it's not present.|
+```sql
+position('\x5678'::bytea in '\x1234567890'::bytea) → 3
+```||
+||substring ( bytes bytea [ FROM start integer ] [ FOR count integer ] ) → bytea|
+Extracts the substring of bytes starting at the start'th byte if that is specified, and stopping after count bytes if that is specified. Provide at least one of start and count.|
+```sql
+substring('\x1234567890'::bytea from 3 for 2) → \x5678
+```||
+||trim ( [ LEADING \| TRAILING \| BOTH ] bytesremoved bytea FROM bytes bytea ) → bytea|
+Removes the longest string containing only bytes appearing in bytesremoved from the start, end, or both ends (BOTH is the default) of bytes.|
+```sql
+trim('\x9012'::bytea from '\x1234567890'::bytea) → \x345678
+```||
+||trim ( [ LEADING \| TRAILING \| BOTH ] [ FROM ] bytes bytea, bytesremoved bytea ) → bytea|
+This is a non-standard syntax for trim().|
+```sql
+trim(both from '\x1234567890'::bytea, '\x9012'::bytea) → \x345678
+```||
+|#
+
+Additional binary string manipulation functions are available and are listed in Table 9.12. Some of them are used internally to implement the SQL-standard string functions listed in Table 9.11.
+
+Table 9.12. Other Binary String Functions
+
+#|
+||Function|Description|Example(s)||
+||bit_count ( bytes bytea ) → bigint|
+Returns the number of bits set in the binary string (also known as “popcount”).|
+```sql
+bit_count('\x1234567890'::bytea) → 15
+```||
+||btrim ( bytes bytea, bytesremoved bytea ) → bytea|
+Removes the longest string containing only bytes appearing in bytesremoved from the start and end of bytes.|
+```sql
+btrim('\x1234567890'::bytea, '\x9012'::bytea) → \x345678
+```||
+||get_bit ( bytes bytea, n bigint ) → integer|
+Extracts n'th bit from binary string.|
+```sql
+get_bit('\x1234567890'::bytea, 30) → 1
+```||
+||get_byte ( bytes bytea, n integer ) → integer|
+Extracts n'th byte from binary string.|
+```sql
+get_byte('\x1234567890'::bytea, 4) → 144
+```||
+||length ( bytea ) → integer|
+Returns the number of bytes in the binary string.|
+```sql
+length('\x1234567890'::bytea) → 5
+```||
+||length ( bytes bytea, encoding name ) → integer|
+Returns the number of characters in the binary string, assuming that it is text in the given encoding.|
+```sql
+length('jose'::bytea, 'UTF8') → 4
+```||
+||ltrim ( bytes bytea, bytesremoved bytea ) → bytea|
+Removes the longest string containing only bytes appearing in bytesremoved from the start of bytes.|
+```sql
+ltrim('\x1234567890'::bytea, '\x9012'::bytea) → \x34567890
+```||
+||md5 ( bytea ) → text|
+Computes the MD5 hash of the binary string, with the result written in hexadecimal.|
+```sql
+md5('Th\000omas'::bytea) → 8ab2d3c9689aaf18b4958c334c82d8b1
+```||
+||rtrim ( bytes bytea, bytesremoved bytea ) → bytea|
+Removes the longest string containing only bytes appearing in bytesremoved from the end of bytes.|
+```sql
+rtrim('\x1234567890'::bytea, '\x9012'::bytea) → \x12345678
+```||
+||set_bit ( bytes bytea, n bigint, newvalue integer ) → bytea|
+Sets n'th bit in binary string to newvalue.|
+```sql
+set_bit('\x1234567890'::bytea, 30, 0) → \x1234563890
+```||
+||set_byte ( bytes bytea, n integer, newvalue integer ) → bytea|
+Sets n'th byte in binary string to newvalue.|
+```sql
+set_byte('\x1234567890'::bytea, 4, 64) → \x1234567840
+```||
+||sha224 ( bytea ) → bytea|
+Computes the SHA-224 hash of the binary string.|
+```sql
+sha224('abc'::bytea) → \x23097d223405d8228642a477bda255b32aadbce4bda0b3f7e36c9da7
+```||
+||sha256 ( bytea ) → bytea|
+Computes the SHA-256 hash of the binary string.|
+```sql
+sha256('abc'::bytea) → \xba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad
+```||
+||sha384 ( bytea ) → bytea|
+Computes the SHA-384 hash of the binary string.|
+```sql
+sha384('abc'::bytea) → \xcb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7
+```||
+||sha512 ( bytea ) → bytea|
+Computes the SHA-512 hash of the binary string.|
+```sql
+sha512('abc'::bytea) → \xddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f
+```||
+||substr ( bytes bytea, start integer [, count integer ] ) → bytea|
+Extracts the substring of bytes starting at the start'th byte, and extending for count bytes if that is specified. (Same as substring(bytes from start for count).)|
+```sql
+substr('\x1234567890'::bytea, 3, 2) → \x5678
+```||
+|#
+
+Functions get_byte and set_byte number the first byte of a binary string as byte 0. Functions get_bit and set_bit number bits from the right within each byte; for example bit 0 is the least significant bit of the first byte, and bit 15 is the most significant bit of the second byte.
+
+For historical reasons, the function md5 returns a hex-encoded value of type text whereas the SHA-2 functions return type bytea. Use the functions encode and decode to convert between the two. For example write encode(sha256('abc'), 'hex') to get a hex-encoded text representation, or decode(md5('abc'), 'hex') to get a bytea value.
+
+Functions for converting strings between different character sets (encodings), and for representing arbitrary binary data in textual form, are shown in Table 9.13. For these functions, an argument or result of type text is expressed in the database's default encoding, while arguments or results of type bytea are in an encoding named by another argument.
+
+Table 9.13. Text/Binary String Conversion Functions
+
+#|
+||Function|Description|Example(s)||
+||convert ( bytes bytea, src_encoding name, dest_encoding name ) → bytea|
+Converts a binary string representing text in encoding src_encoding to a binary string in encoding dest_encoding (see Section 24.3.4 for available conversions). (NOT SUPPORTED)|
+```sql
+#convert('text_in_utf8', 'UTF8', 'LATIN1') → \x746578745f696e5f75746638
+```||
+||convert_from ( bytes bytea, src_encoding name ) → text|
+Converts a binary string representing text in encoding src_encoding to text in the database encoding (see Section 24.3.4 for available conversions).|
+```sql
+convert_from('text_in_utf8', 'UTF8') → text_in_utf8
+```||
+||convert_to ( string text, dest_encoding name ) → bytea|
+Converts a text string (in the database encoding) to a binary string encoded in encoding dest_encoding (see Section 24.3.4 for available conversions).|
+```sql
+convert_to('some_text', 'UTF8') → \x736f6d655f74657874
+```||
+||encode ( bytes bytea, format text ) → text|
+Encodes binary data into a textual representation; supported format values are: base64, escape, hex.|
+```sql
+encode('123\000\001', 'base64') → MTIzAAE=
+```||
+||decode ( string text, format text ) → bytea|
+Decodes binary data from a textual representation; supported format values are the same as for encode.|
+```sql
+decode('MTIzAAE=', 'base64') → \x3132330001
+```||
+|#
+
+## 9.6. Bit String Functions and Operators {#bit-string-functions}
+This section describes functions and operators for examining and manipulating bit strings, that is values of the types bit and bit varying. (While only type bit is mentioned in these tables, values of type bit varying can be used interchangeably.) Bit strings support the usual comparison operators shown in Table 9.1, as well as the operators shown in Table 9.14.
+
+Table 9.14. Bit String Operators
+
+#|
+||Operator|Description|Example(s)||
+bit \|\| bit → bit |
+Concatenation |
+```sql
+B'10001' || B'011' → 10001011
+```||
+||bit & bit → bit|
+Bitwise AND (inputs must be of equal length)|
+```sql
+B'10001' & B'01101' → 00001
+```||
+||bit \| bit → bit|
+Bitwise OR (inputs must be of equal length)|
+```sql
+B'10001' | B'01101' → 11101
+```||
+||bit # bit → bit|
+Bitwise exclusive OR (inputs must be of equal length)|
+```sql
+B'10001' # B'01101' → 11100
+```||
+||~ bit → bit|
+Bitwise NOT|
+```sql
+~ B'10001' → 01110
+```||
+||bit << integer → bit|
+Bitwise shift left (string length is preserved)|
+```sql
+B'10001' << 3 → 01000
+```||
+||bit >> integer → bit|
+Bitwise shift right (string length is preserved)|
+```sql
+B'10001' >> 2 → 00100
+```||
+|#
+
+Some of the functions available for binary strings are also available for bit strings, as shown in Table 9.15.
+
+Table 9.15. Bit String Functions
+
+#|
+||Function|Description|Example(s)||
+||bit_count ( bit ) → bigint|
+Returns the number of bits set in the bit string (also known as “popcount”).|
+```sql
+bit_count(B'10111') → 4
+```||
+||bit_length ( bit ) → integer|
+Returns number of bits in the bit string.|
+```sql
+bit_length(B'10111') → 5
+```||
+||length ( bit ) → integer|
+Returns number of bits in the bit string.|
+```sql
+length(B'10111') → 5
+```||
+||octet_length ( bit ) → integer|
+Returns number of bytes in the bit string.|
+```sql
+octet_length(B'1011111011') → 2
+```||
+||overlay ( bits bit PLACING newsubstring bit FROM start integer [ FOR count integer ] ) → bit|
+Replaces the substring of bits that starts at the start'th bit and extends for count bits with newsubstring. If count is omitted, it defaults to the length of newsubstring.|
+```sql
+overlay(B'01010101010101010' placing B'11111' from 2 for 3) → 0111110101010101010
+```||
+||position ( substring bit IN bits bit ) → integer|
+Returns first starting index of the specified substring within bits, or zero if it's not present.|
+```sql
+position(B'010' in B'000001101011') → 8
+```||
+||substring ( bits bit [ FROM start integer ] [ FOR count integer ] ) → bit|
+Extracts the substring of bits starting at the start'th bit if that is specified, and stopping after count bits if that is specified. Provide at least one of start and count.|
+```sql
+substring(B'110010111111' from 3 for 2) → 00
+```||
+||get_bit ( bits bit, n integer ) → integer|
+Extracts n'th bit from bit string; the first (leftmost) bit is bit 0.|
+```sql
+get_bit(B'101010101010101010', 6) → 1
+```||
+||set_bit ( bits bit, n integer, newvalue integer ) → bit|
+Sets n'th bit in bit string to newvalue; the first (leftmost) bit is bit 0.|
+```sql
+set_bit(B'101010101010101010', 6, 0) → 101010001010101010
+```||
+|#
+
+In addition, it is possible to cast integral values to and from type bit. Casting an integer to bit(n) copies the rightmost n bits. Casting an integer to a bit string width wider than the integer itself will sign-extend on the left. Some examples:
+
+```sql
+44::bit(10) → 0000101100
+44::bit(3) → 100
+cast(-44 as bit(12)) → 111111010100
+'1110'::bit(4)::integer → 14
+```
+Note that casting to just “bit” means casting to bit(1), and so will deliver only the least significant bit of the integer.
+
+## 9.7. Pattern Matching {#pattern-matching}
+
+9.7.1. LIKE
+
+```sql
+string LIKE pattern [ESCAPE escape-character]
+string NOT LIKE pattern [ESCAPE escape-character]
+```
+The LIKE expression returns true if the string matches the supplied pattern. (As expected, the NOT LIKE expression returns false if LIKE returns true, and vice versa. An equivalent expression is NOT (string LIKE pattern).)
+
+If pattern does not contain percent signs or underscores, then the pattern only represents the string itself; in that case LIKE acts like the equals operator. An underscore (_) in pattern stands for (matches) any single character; a percent sign (%) matches any sequence of zero or more characters.
+
+Some examples:
+
+```sql
+'abc' LIKE 'abc' → true
+'abc' LIKE 'a%' → true
+'abc' LIKE '_b_' → true
+'abc' LIKE 'c' → false
+```
+
+LIKE pattern matching always covers the entire string. Therefore, if it's desired to match a sequence anywhere within a string, the pattern must start and end with a percent sign.
+
+To match a literal underscore or percent sign without matching other characters, the respective character in pattern must be preceded by the escape character. The default escape character is the backslash but a different one can be selected by using the ESCAPE clause. To match the escape character itself, write two escape characters.
+
+Note
+If you have standard_conforming_strings turned off, any backslashes you write in literal string constants will need to be doubled. See Section 4.1.2.1 for more information.
+
+It's also possible to select no escape character by writing ESCAPE ''. This effectively disables the escape mechanism, which makes it impossible to turn off the special meaning of underscore and percent signs in the pattern.
+
+According to the SQL standard, omitting ESCAPE means there is no escape character (rather than defaulting to a backslash), and a zero-length ESCAPE value is disallowed. PostgreSQL's behavior in this regard is therefore slightly nonstandard.
+
+The key word ILIKE can be used instead of LIKE to make the match case-insensitive according to the active locale. This is not in the SQL standard but is a PostgreSQL extension.
+
+The operator `~~` is equivalent to LIKE, and `~~*` corresponds to ILIKE. There are also `!~~` and `!~~*` operators that represent NOT LIKE and NOT ILIKE, respectively. All of these operators are PostgreSQL-specific. You may see these operator names in EXPLAIN output and similar places, since the parser actually translates LIKE et al. to these operators.
+
+The phrases LIKE, ILIKE, NOT LIKE, and NOT ILIKE are generally treated as operators in PostgreSQL syntax; for example they can be used in expression operator ANY (subquery) constructs, although an ESCAPE clause cannot be included there. In some obscure cases it may be necessary to use the underlying operator names instead.
+
+Also see the prefix operator ^@ and corresponding starts_with function, which are useful in cases where simply matching the beginning of a string is needed.
+
+9.7.2. SIMILAR TO Regular Expressions
+
+```sql
+string SIMILAR TO pattern [ESCAPE escape-character] (NOT SUPPORTED)
+string NOT SIMILAR TO pattern [ESCAPE escape-character] (NOT SUPPORTED)
+```
+
+The SIMILAR TO operator returns true or false depending on whether its pattern matches the given string. It is similar to LIKE, except that it interprets the pattern using the SQL standard's definition of a regular expression. SQL regular expressions are a curious cross between LIKE notation and common (POSIX) regular expression notation.
+
+Like LIKE, the SIMILAR TO operator succeeds only if its pattern matches the entire string; this is unlike common regular expression behavior where the pattern can match any part of the string. Also like LIKE, SIMILAR TO uses _ and % as wildcard characters denoting any single character and any string, respectively (these are comparable to . and .* in POSIX regular expressions).
+
+In addition to these facilities borrowed from LIKE, SIMILAR TO supports these pattern-matching metacharacters borrowed from POSIX regular expressions:
+
+| denotes alternation (either of two alternatives).
+
+* denotes repetition of the previous item zero or more times.
+
++ denotes repetition of the previous item one or more times.
+
+? denotes repetition of the previous item zero or one time.
+
+{m} denotes repetition of the previous item exactly m times.
+
+{m,} denotes repetition of the previous item m or more times.
+
+{m,n} denotes repetition of the previous item at least m and not more than n times.
+
+Parentheses () can be used to group items into a single logical item.
+
+A bracket expression [...] specifies a character class, just as in POSIX regular expressions.
+
+Notice that the period (.) is not a metacharacter for SIMILAR TO.
+
+As with LIKE, a backslash disables the special meaning of any of these metacharacters. A different escape character can be specified with ESCAPE, or the escape capability can be disabled by writing ESCAPE ''.
+
+According to the SQL standard, omitting ESCAPE means there is no escape character (rather than defaulting to a backslash), and a zero-length ESCAPE value is disallowed. PostgreSQL's behavior in this regard is therefore slightly nonstandard.
+
+Another nonstandard extension is that following the escape character with a letter or digit provides access to the escape sequences defined for POSIX regular expressions; see Table 9.20, Table 9.21, and Table 9.22 below.
+
+Some examples:
+
+```sql
+#'abc' SIMILAR TO 'abc' → true
+#'abc' SIMILAR TO 'a' → false
+#'abc' SIMILAR TO '%(b|d)%' → true
+#'abc' SIMILAR TO '(b|c)%' → false
+#'-abc-' SIMILAR TO '%\mabc\M%' → true
+#'xabcy' SIMILAR TO '%\mabc\M%' → false
+```
+
+The substring function with three parameters provides extraction of a substring that matches an SQL regular expression pattern. The function can be written according to standard SQL syntax:
+
+```sql
+substring(string similar pattern escape escape-character)
+```
+or using the now obsolete SQL:1999 syntax:
+```sql
+substring(string from pattern for escape-character)
+```
+
+or as a plain three-argument function:
+```sql
+substring(string, pattern, escape-character)
+```
+
+As with SIMILAR TO, the specified pattern must match the entire data string, or else the function fails and returns null. To indicate the part of the pattern for which the matching data sub-string is of interest, the pattern should contain two occurrences of the escape character followed by a double quote ("). The text matching the portion of the pattern between these separators is returned when the match is successful.
+
+The escape-double-quote separators actually divide substring's pattern into three independent regular expressions; for example, a vertical bar (|) in any of the three sections affects only that section. Also, the first and third of these regular expressions are defined to match the smallest possible amount of text, not the largest, when there is any ambiguity about how much of the data string matches which pattern. (In POSIX parlance, the first and third regular expressions are forced to be non-greedy.)
+
+As an extension to the SQL standard, PostgreSQL allows there to be just one escape-double-quote separator, in which case the third regular expression is taken as empty; or no separators, in which case the first and third regular expressions are taken as empty.
+
+Some examples, with #" delimiting the return string:
+
+```sql
+substring('foobar' similar '%#"o_b#"%' escape '#') → oob
+substring('foobar' similar '#"o_b#"%' escape '#') → NULL
+```
+
+9.7.3. POSIX Regular Expressions
+Table 9.16 lists the available operators for pattern matching using POSIX regular expressions.
+
+Table 9.16. Regular Expression Match Operators
+
+#|
+||Operator|Description|Example(s)||
+||ext ~ text → boolean|
+String matches regular expression, case sensitively|
+```sql
+'thomas' ~ 't.*ma' → true
+```||
+||text ~* text → boolean|
+String matches regular expression, case insensitively|
+```sql
+'thomas' ~* 'T.*ma' → true
+```||
+||text !~ text → boolean|
+String does not match regular expression, case sensitively|
+```sql
+'thomas' !~ 't.*max' → true
+```||
+||text !~* text → boolean|
+String does not match regular expression, case insensitively|
+```sql
+'thomas' !~* 'T.*ma' → false
+```||
+|#
+
+POSIX regular expressions provide a more powerful means for pattern matching than the LIKE and SIMILAR TO operators. Many Unix tools such as egrep, sed, or awk use a pattern matching language that is similar to the one described here.
+
+A regular expression is a character sequence that is an abbreviated definition of a set of strings (a regular set). A string is said to match a regular expression if it is a member of the regular set described by the regular expression. As with LIKE, pattern characters match string characters exactly unless they are special characters in the regular expression language — but regular expressions use different special characters than LIKE does. Unlike LIKE patterns, a regular expression is allowed to match anywhere within a string, unless the regular expression is explicitly anchored to the beginning or end of the string.
+
+Some examples:
+
+```sql
+'abcd' ~ 'bc' → true
+'abcd' ~ 'a.c' → true /* dot matches any character */
+'abcd' ~ 'a.*d' → true /* * repeats the preceding pattern item */
+'abcd' ~ '(b|x)' → true /* | means OR, parentheses group */
+'abcd' ~ '^a' → true /* ^ anchors to start of string */
+'abcd' ~ '^(b|c)' → false /* would match except for anchoring */
+```
+The POSIX pattern language is described in much greater detail below.
+
+The substring function with two parameters, substring(string from pattern), provides extraction of a substring that matches a POSIX regular expression pattern. It returns null if there is no match, otherwise the first portion of the text that matched the pattern. But if the pattern contains any parentheses, the portion of the text that matched the first parenthesized subexpression (the one whose left parenthesis comes first) is returned. You can put parentheses around the whole expression if you want to use parentheses within it without triggering this exception. If you need parentheses in the pattern before the subexpression you want to extract, see the non-capturing parentheses described below.
+
+Some examples:
+
+```sql
+substring('foobar' from 'o.b') → oob
+substring('foobar' from 'o(.)b') → o
+```
+
+The regexp_replace function provides substitution of new text for substrings that match POSIX regular expression patterns. It has the syntax regexp_replace(source, pattern, replacement [, flags ]). The source string is returned unchanged if there is no match to the pattern. If there is a match, the source string is returned with the replacement string substituted for the matching substring. The replacement string can contain \n, where n is 1 through 9, to indicate that the source substring matching the n'th parenthesized subexpression of the pattern should be inserted, and it can contain \& to indicate that the substring matching the entire pattern should be inserted. Write \\ if you need to put a literal backslash in the replacement text. The flags parameter is an optional text string containing zero or more single-letter flags that change the function's behavior. Flag i specifies case-insensitive matching, while flag g specifies replacement of each matching substring rather than only the first one. Supported flags (though not g) are described in Table 9.24.
+
+Some examples:
+
+```sql
+regexp_replace('foobarbaz', 'b..', 'X') → fooXbaz
+regexp_replace('foobarbaz', 'b..', 'X', 'g') → fooXX
+regexp_replace('foobarbaz', 'b(..)', 'X\1Y', 'g') → fooXarYXazY
+```
+
+The regexp_match function returns a text array of captured substring(s) resulting from the first match of a POSIX regular expression pattern to a string. It has the syntax regexp_match(string, pattern [, flags ]). If there is no match, the result is NULL. If a match is found, and the pattern contains no parenthesized subexpressions, then the result is a single-element text array containing the substring matching the whole pattern. If a match is found, and the pattern contains parenthesized subexpressions, then the result is a text array whose n'th element is the substring matching the n'th parenthesized subexpression of the pattern (not counting “non-capturing” parentheses; see below for details). The flags parameter is an optional text string containing zero or more single-letter flags that change the function's behavior. Supported flags are described in Table 9.24.
+
+Some examples:
+
+```sql
+regexp_match('foobarbequebaz', 'bar.*que') → {barbeque}
+regexp_match('foobarbequebaz', '(bar)(beque)') → {bar,beque}
+```
+
+In the common case where you just want the whole matching substring or NULL for no match, write something like
+
+```sql
+#(regexp_match('foobarbequebaz', 'bar.*que'))[1] → barbeque
+```
+
+The regexp_matches function returns a set of text arrays of captured substring(s) resulting from matching a POSIX regular expression pattern to a string. It has the same syntax as regexp_match. This function returns no rows if there is no match, one row if there is a match and the g flag is not given, or N rows if there are N matches and the g flag is given. Each returned row is a text array containing the whole matched substring or the substrings matching parenthesized subexpressions of the pattern, just as described above for regexp_match. regexp_matches accepts all the flags shown in Table 9.24, plus the g flag which commands it to return all matches, not just the first one.
+
+Some examples:
+
+```sql
+SELECT * FROM regexp_matches('foo', 'not there') a → [
+]
+```
+
+```sql
+SELECT * FROM regexp_matches('foobarbequebazilbarfbonk', '(b[^b]+)(b[^b]+)', 'g') a → [
+{bar,beque}
+{bazil,barf}
+]
+```
+
+Tip
+In most cases regexp_matches() should be used with the g flag, since if you only want the first match, it's easier and more efficient to use regexp_match(). However, regexp_match() only exists in PostgreSQL version 10 and up. When working in older versions, a common trick is to place a regexp_matches() call in a sub-select, for example:
+
+```sql
+SELECT col1, (SELECT regexp_matches(col2, '(bar)(beque)')) FROM tab;
+```
+
+This produces a text array if there's a match, or NULL if not, the same as regexp_match() would do. Without the sub-select, this query would produce no output at all for table rows without a match, which is typically not the desired behavior.
+
+The regexp_split_to_table function splits a string using a POSIX regular expression pattern as a delimiter. It has the syntax regexp_split_to_table(string, pattern [, flags ]). If there is no match to the pattern, the function returns the string. If there is at least one match, for each match it returns the text from the end of the last match (or the beginning of the string) to the beginning of the match. When there are no more matches, it returns the text from the end of the last match to the end of the string. The flags parameter is an optional text string containing zero or more single-letter flags that change the function's behavior. regexp_split_to_table supports the flags described in Table 9.24.
+
+The regexp_split_to_array function behaves the same as regexp_split_to_table, except that regexp_split_to_array returns its result as an array of text. It has the syntax regexp_split_to_array(string, pattern [, flags ]). The parameters are the same as for regexp_split_to_table.
+
+Some examples:
+
+```sql
+SELECT foo FROM regexp_split_to_table('the quick brown fox jumps over the lazy dog', '\s+') AS foo → [
+the
+quick
+brown
+fox
+jumps
+over
+the
+lazy
+dog
+]
+```
+
+```sql
+SELECT regexp_split_to_array('the quick brown fox jumps over the lazy dog', '\s+') → {the,quick,brown,fox,jumps,over,the,lazy,dog}
+```
+
+```sql
+SELECT foo FROM regexp_split_to_table('the quick brown fox', '\s*') AS foo → [
+t
+h
+e
+q
+u
+i
+c
+k
+b
+r
+o
+w
+n
+f
+o
+x
+]
+```
+
+As the last example demonstrates, the regexp split functions ignore zero-length matches that occur at the start or end of the string or immediately after a previous match. This is contrary to the strict definition of regexp matching that is implemented by regexp_match and regexp_matches, but is usually the most convenient behavior in practice. Other software systems such as Perl use similar definitions.
+
+9.7.3.5. Regular Expression Matching Rules
+In the event that an RE could match more than one substring of a given string, the RE matches the one starting earliest in the string. If the RE could match more than one substring starting at that point, either the longest possible match or the shortest possible match will be taken, depending on whether the RE is greedy or non-greedy.
+
+Whether an RE is greedy or not is determined by the following rules:
+
+Most atoms, and all constraints, have no greediness attribute (because they cannot match variable amounts of text anyway).
+
+Adding parentheses around an RE does not change its greediness.
+
+A quantified atom with a fixed-repetition quantifier ({m} or {m}?) has the same greediness (possibly none) as the atom itself.
+
+A quantified atom with other normal quantifiers (including {m,n} with m equal to n) is greedy (prefers longest match).
+
+A quantified atom with a non-greedy quantifier (including {m,n}? with m equal to n) is non-greedy (prefers shortest match).
+
+A branch — that is, an RE that has no top-level | operator — has the same greediness as the first quantified atom in it that has a greediness attribute.
+
+An RE consisting of two or more branches connected by the | operator is always greedy.
+
+The above rules associate greediness attributes not only with individual quantified atoms, but with branches and entire REs that contain quantified atoms. What that means is that the matching is done in such a way that the branch, or whole RE, matches the longest or shortest possible substring as a whole. Once the length of the entire match is determined, the part of it that matches any particular subexpression is determined on the basis of the greediness attribute of that subexpression, with subexpressions starting earlier in the RE taking priority over ones starting later.
+
+An example of what this means:
+
+```sql
+SUBSTRING('XY1234Z', 'Y*([0-9]{1,3})') → 123
+SUBSTRING('XY1234Z', 'Y*?([0-9]{1,3})') → 1
+```
+
+In the first case, the RE as a whole is greedy because `Y*` is greedy. It can match beginning at the Y, and it matches the longest possible string starting there, i.e., Y123. The output is the parenthesized part of that, or 123. In the second case, the RE as a whole is non-greedy because `Y*?` is non-greedy. It can match beginning at the Y, and it matches the shortest possible string starting there, i.e., Y1. The subexpression [0-9]{1,3} is greedy but it cannot change the decision as to the overall match length; so it is forced to match just 1.
+
+In short, when an RE contains both greedy and non-greedy subexpressions, the total match length is either as long as possible or as short as possible, according to the attribute assigned to the whole RE. The attributes assigned to the subexpressions only affect how much of that match they are allowed to “eat” relative to each other.
+
+The quantifiers {1,1} and {1,1}? can be used to force greediness or non-greediness, respectively, on a subexpression or a whole RE. This is useful when you need the whole RE to have a greediness attribute different from what's deduced from its elements. As an example, suppose that we are trying to separate a string containing some digits into the digits and the parts before and after them. We might try to do that like this:
+
+```sql
+regexp_match('abc01234xyz', '(.*)(\d+)(.*)') → {abc0123,4,xyz}
+```
+
+That didn't work: the first .* is greedy so it “eats” as much as it can, leaving the \d+ to match at the last possible place, the last digit. We might try to fix that by making it non-greedy:
+
+```sql
+regexp_match('abc01234xyz', '(.*?)(\d+)(.*)') → {abc,0,""}
+```
+
+That didn't work either, because now the RE as a whole is non-greedy and so it ends the overall match as soon as possible. We can get what we want by forcing the RE as a whole to be greedy:
+
+```sql
+regexp_match('abc01234xyz', '(?:(.*?)(\d+)(.*)){1,1}') → {abc,01234,xyz}
+```
+
+Controlling the RE's overall greediness separately from its components' greediness allows great flexibility in handling variable-length patterns.
+
+When deciding what is a longer or shorter match, match lengths are measured in characters, not collating elements. An empty string is considered longer than no match at all. For example: `bb*` matches the three middle characters of abbbc; (week|wee)(night|knights) matches all ten characters of weeknights; when `(.*).*` is matched against abc the parenthesized subexpression matches all three characters; and when (a*)* is matched against bc both the whole RE and the parenthesized subexpression match an empty string.
+
+If case-independent matching is specified, the effect is much as if all case distinctions had vanished from the alphabet. When an alphabetic that exists in multiple cases appears as an ordinary character outside a bracket expression, it is effectively transformed into a bracket expression containing both cases, e.g., x becomes [xX]. When it appears inside a bracket expression, all case counterparts of it are added to the bracket expression, e.g., [x] becomes [xX] and [^x] becomes [^xX].
+
+If newline-sensitive matching is specified, . and bracket expressions using ^ will never match the newline character (so that matches will not cross lines unless the RE explicitly includes a newline) and ^ and $ will match the empty string after and before a newline respectively, in addition to matching at beginning and end of string respectively. But the ARE escapes \A and \Z continue to match beginning or end of string only. Also, the character class shorthands \D and \W will match a newline regardless of this mode. (Before PostgreSQL 14, they did not match newlines when in newline-sensitive mode. Write [^[:digit:]] or [^[:word:]] to get the old behavior.)
+
+If partial newline-sensitive matching is specified, this affects . and bracket expressions as with newline-sensitive matching, but not ^ and $.
+
+If inverse partial newline-sensitive matching is specified, this affects ^ and $ as with newline-sensitive matching, but not . and bracket expressions. This isn't very useful but is provided for symmetry
+
+## 9.8. Data Type Formatting Functions {#data-type-formatting-functions}
+
+The PostgreSQL formatting functions provide a powerful set of tools for converting various data types (date/time, integer, floating point, numeric) to formatted strings and for converting from formatted strings to specific data types. Table 9.25 lists them. These functions all follow a common calling convention: the first argument is the value to be formatted and the second argument is a template that defines the output or input format.
+
+Table 9.25. Formatting Functions
+
+#|
+||Function|Description|Example(s)||
+||to_char ( timestamp with time zone, text ) → text|
+Converts time stamp to string according to the given format.|
+```sql
+to_char(timestamp '2002-04-20 17:31:12.66', 'HH12:MI:SS') → 05:31:12
+```||
+||to_char ( interval, text ) → text|
+Converts interval to string according to the given format.|
+```sql
+to_char(interval '15h 2m 12s', 'HH24:MI:SS') → 15:02:12
+```||
+||to_char ( numeric_type, text ) → text|
+Converts number to string according to the given format; available for integer, bigint, numeric, real, double precision.|
+```sql
+to_char(125, '999') → ' 125'
+to_char(125.8::real, '999D9') → ' 125.8'
+to_char(-125.8, '999D99S') → '125.80-'
+```||
+||to_date ( text, text ) → date|
+Converts string to date according to the given format.|
+```sql
+to_date('05 Dec 2000', 'DD Mon YYYY') → '2000-12-05'
+```||
+||to_number ( text, text ) → numeric|
+Converts string to numeric according to the given format.|
+```sql
+to_number('12,454.8-', '99G999D9S') → '-12454.8'
+```||
+||to_timestamp ( text, text ) → timestamp with time zone|
+Converts string to time stamp according to the given format. (See also to_timestamp(double precision) in Table 9.32.)|
+```sql
+cast(to_timestamp('05 Dec 2000', 'DD Mon YYYY') as timestamp) → '2000-12-05 00:00:00'
+```||
+|#
+
+Table 9.30 shows some examples of the use of the to_char function.
+
+Table 9.30. to_char Examples
+
+```sql
+to_char('2000-06-06 05:39:18'::timestamp, 'Day, DD HH12:MI:SS') → 'Tuesday , 06 05:39:18'
+to_char('2000-06-06 05:39:18'::timestamp, 'FMDay, FMDD HH12:MI:SS') → 'Tuesday, 6 05:39:18'
+to_char(-0.1, '99.99') → ' -.10'
+to_char(-0.1, 'FM9.99') → '-.1'
+to_char(-0.1, 'FM90.99') → '-0.1'
+to_char(0.1, '0.9') → ' 0.1'
+to_char(12, '9990999.9') → ' 0012.0'
+to_char(12, 'FM9990999.9') → '0012.'
+to_char(485, '999') → ' 485'
+to_char(-485, '999') → '-485'
+to_char(485, '9 9 9') → ' 4 8 5'
+to_char(1485, '9,999') → ' 1,485'
+to_char(1485, '9G999') → ' 1,485'
+to_char(148.5, '999.999') → ' 148.500'
+to_char(148.5, 'FM999.999') → '148.5'
+to_char(148.5, 'FM999.990') → '148.500'
+to_char(148.5, '999D999') → ' 148.500'
+to_char(3148.5, '9G999D999') → ' 3,148.500'
+to_char(-485, '999S') → '485-'
+to_char(-485, '999MI') → '485-'
+to_char(485, '999MI') → '485 '
+to_char(485, 'FM999MI') → '485'
+to_char(485, 'PL999') → '+ 485'
+to_char(485, 'SG999') → '+485'
+to_char(-485, 'SG999') → '-485'
+to_char(-485, '9SG99') → '4-85'
+to_char(-485, '999PR') → '<485>'
+to_char(485, 'L999') → ' 485'
+to_char(485, 'RN') → ' CDLXXXV'
+to_char(485, 'FMRN') → 'CDLXXXV'
+to_char(5.2, 'FMRN') → 'V'
+to_char(482, '999th') → ' 482nd'
+to_char(485, '"Good number:"999') → 'Good number: 485'
+to_char(485.8, '"Pre:"999" Post:" .999') → 'Pre: 485 Post: .800'
+to_char(12, '99V999') → ' 12000'
+to_char(12.4, '99V999') → ' 12400'
+to_char(12.45, '99V9') → ' 125'
+to_char(0.0004859, '9.99EEEE') → ' 4.86e-04'
+```
+
+## 9.9. Date/Time Functions and Operators {#date-time-functions}
+
+Table 9.32 shows the available functions for date/time value processing, with details appearing in the following subsections. Table 9.31 illustrates the behaviors of the basic arithmetic operators (+, *, etc.). For formatting functions, refer to Section 9.8. You should be familiar with the background information on date/time data types from Section 8.5.
+
+In addition, the usual comparison operators shown in Table 9.1 are available for the date/time types. Dates and timestamps (with or without time zone) are all comparable, while times (with or without time zone) and intervals can only be compared to other values of the same data type. When comparing a timestamp without time zone to a timestamp with time zone, the former value is assumed to be given in the time zone specified by the TimeZone configuration parameter, and is rotated to UTC for comparison to the latter value (which is already in UTC internally). Similarly, a date value is assumed to represent midnight in the TimeZone zone when comparing it to a timestamp.
+
+All the functions and operators described below that take time or timestamp inputs actually come in two variants: one that takes time with time zone or timestamp with time zone, and one that takes time without time zone or timestamp without time zone. For brevity, these variants are not shown separately. Also, the + and * operators come in commutative pairs (for example both date + integer and integer + date); we show only one of each such pair.
+
+Table 9.31. Date/Time Operators
+
+#|
+||Operator|Description|Example(s)||
+||date + integer → date|
+Add a number of days to a date|
+```sql
+date '2001-09-28' + 7 → 2001-10-05
+```||
+||date + interval → timestamp|
+Add an interval to a date|
+```sql
+date '2001-09-28' + interval '1 hour' → 2001-09-28 01:00:00
+```||
+||date + time → timestamp|
+Add a time-of-day to a date|
+```sql
+date '2001-09-28' + time '03:00' → 2001-09-28 03:00:00
+```||
+||interval + interval → interval|
+Add intervals|
+```sql
+interval '1 day' + interval '1 hour' → 1 day 01:00:00
+```||
+||timestamp + interval → timestamp|
+Add an interval to a timestamp|
+```sql
+timestamp '2001-09-28 01:00' + interval '23 hours' → 2001-09-29 00:00:00
+```||
+||time + interval → time|
+Add an interval to a time|
+```sql
+time '01:00' + interval '3 hours' → 04:00:00
+```||
+||- interval → interval|
+Negate an interval|
+```sql
+- interval '23 hours' → -23:00:00
+```||
+||date - date → integer|
+Subtract dates, producing the number of days elapsed|
+```sql
+date '2001-10-01' - date '2001-09-28' → 3
+```||
+||date - integer → date|
+Subtract a number of days from a date|
+```sql
+date '2001-10-01' - 7 → 2001-09-24
+```||
+||date - interval → timestamp|
+Subtract an interval from a date|
+```sql
+date '2001-09-28' - interval '1 hour' → 2001-09-27 23:00:00
+```||
+||time - time → interval|
+Subtract times|
+```sql
+time '05:00' - time '03:00' → 02:00:00
+```||
+||time - interval → time|
+Subtract an interval from a time|
+```sql
+time '05:00' - interval '2 hours' → 03:00:00
+```||
+||timestamp - interval → timestamp|
+Subtract an interval from a timestamp|
+```sql
+timestamp '2001-09-28 23:00' - interval '23 hours' → 2001-09-28 00:00:00
+```||
+||interval - interval → interval|
+Subtract intervals|
+```sql
+interval '1 day' - interval '1 hour' → 1 day -01:00:00
+```||
+||timestamp - timestamp → interval|
+Subtract timestamps (converting 24-hour intervals into days, similarly to justify_hours())|
+```sql
+timestamp '2001-09-29 03:00' - timestamp '2001-07-27 12:00' → 63 days 15:00:00
+```||
+||interval * double precision → interval|
+Multiply an interval by a scalar|
+```sql
+interval '1 second' * 900 → 00:15:00
+interval '1 day' * 21 → 21 days
+interval '1 hour' * 3.5 → 03:30:00
+```||
+||interval / double precision → interval|
+Divide an interval by a scalar|
+```sql
+interval '1 hour' / 1.5 → 00:40:00
+```||
+|#
+
+Table 9.32. Date/Time Functions
+
+#|
+||Function|Description|Example(s)||
+||age ( timestamp, timestamp ) → interval|
+Subtract arguments, producing a “symbolic” result that uses years and months, rather than just days|
+```sql
+age(timestamp '2001-04-10', timestamp '1957-06-13') → 43 years 9 mons 27 days
+```||
+||age ( timestamp ) → interval|
+Subtract argument from current_date (at midnight)|
+```sql
+age(timestamp '1957-06-13') ~→ 62 years 6 mons 10 days
+```||
+||clock_timestamp ( ) → timestamp with time zone|
+Current date and time (changes during statement execution); see Section 9.9.5|
+```sql
+clock_timestamp() ~→ 2019-12-23 14:39:53.662522-05
+```||
+||current_date → date|
+Current date; see Section 9.9.5|
+```sql
+current_date ~→ 2019-12-23
+```||
+||current_time → time with time zone|
+Current time of day; see Section 9.9.5|
+```sql
+current_time ~→ 14:39:53.662522-05
+```||
+||current_time ( integer ) → time with time zone|
+Current time of day, with limited precision; see Section 9.9.5|
+```sql
+current_time(2) ~→ 14:39:53.66-05
+```||
+||current_timestamp → timestamp with time zone|
+Current date and time (start of current transaction); see Section 9.9.5|
+```sql
+current_timestamp ~→ 2019-12-23 14:39:53.662522-05
+```||
+||current_timestamp ( integer ) → timestamp with time zone|
+Current date and time (start of current transaction), with limited precision; see Section 9.9.5|
+```sql
+current_timestamp(0) ~→ 2019-12-23 14:39:53-05
+```||
+||date_bin ( interval, timestamp, timestamp ) → timestamp|
+Bin input into specified interval aligned with specified origin; see Section 9.9.3|
+```sql
+date_bin('15 minutes', timestamp '2001-02-16 20:38:40', timestamp '2001-02-16 20:05:00') → 2001-02-16 20:35:00
+```||
+||date_part ( text, timestamp ) → double precision|
+Get timestamp subfield (equivalent to extract); see Section 9.9.1|
+```sql
+date_part('hour', timestamp '2001-02-16 20:38:40') → 20
+```||
+||date_part ( text, interval ) → double precision|
+Get interval subfield (equivalent to extract); see Section 9.9.1|
+```sql
+date_part('month', interval '2 years 3 months') → 3
+```||
+||date_trunc ( text, timestamp ) → timestamp|
+Truncate to specified precision; see Section 9.9.2|
+```sql
+date_trunc('hour', timestamp '2001-02-16 20:38:40') → 2001-02-16 20:00:00
+```||
+||date_trunc ( text, timestamp with time zone, text ) → timestamp with time zone|
+Truncate to specified precision in the specified time zone; see Section 9.9.2 (NOT SUPPORTED)|
+```sql
+#date_trunc('day', timestamptz '2001-02-16 20:38:40+00', 'Australia/Sydney') → 2001-02-16 13:00:00+00
+```||
+||date_trunc ( text, interval ) → interval|
+Truncate to specified precision; see Section 9.9.2|
+```sql
+date_trunc('hour', interval '2 days 3 hours 40 minutes') → 2 days 03:00:00
+```||
+||extract ( field from timestamp ) → numeric|
+Get timestamp subfield; see Section 9.9.1|
+```sql
+extract(hour from timestamp '2001-02-16 20:38:40') → 20
+```||
+||extract ( field from interval ) → numeric|
+Get interval subfield; see Section 9.9.1|
+```sql
+extract(month from interval '2 years 3 months') → 3
+```||
+||isfinite ( date ) → boolean|
+Test for finite date (not +/-infinity)|
+```sql
+isfinite(date '2001-02-16') → true
+```||
+||isfinite ( timestamp ) → boolean|
+Test for finite timestamp (not +/-infinity)|
+```sql
+isfinite(timestamp 'infinity') → false
+```||
+||isfinite ( interval ) → boolean|
+Test for finite interval (currently always true)|
+```sql
+isfinite(interval '4 hours') → true
+```||
+||justify_days ( interval ) → interval|
+Adjust interval so 30-day time periods are represented as months|
+```sql
+justify_days(interval '35 days') → 1 mon 5 days
+```||
+||justify_hours ( interval ) → interval|
+Adjust interval so 24-hour time periods are represented as days|
+```sql
+justify_hours(interval '27 hours') → 1 day 03:00:00
+```||
+||justify_interval ( interval ) → interval|
+Adjust interval using justify_days and justify_hours, with additional sign adjustments|
+```sql
+justify_interval(interval '1 mon -1 hour') → 29 days 23:00:00
+```||
+||localtime → time|
+Current time of day; see Section 9.9.5 (NOT SUPPORTED)|
+```sql
+#localtime ~→ 14:39:53.662522
+```||
+||localtime ( integer ) → time|
+Current time of day, with limited precision; see Section 9.9.5 (NOT SUPPORTED)|
+```sql
+#localtime(0) ~→ 14:39:53
+```||
+||localtimestamp → timestamp|
+Current date and time (start of current transaction); see Section 9.9.5 (NOT SUPPORTED)|
+```sql
+#localtimestamp ~→ 2019-12-23 14:39:53.662522
+```||
+||localtimestamp ( integer ) → timestamp|
+Current date and time (start of current transaction), with limited precision; see Section 9.9.5 (NOT SUPPORTED)|
+```sql
+#localtimestamp(2) ~→ 2019-12-23 14:39:53.66
+```||
+||make_date ( year int, month int, day int ) → date|
+Create date from year, month and day fields (negative years signify BC)|
+```sql
+make_date(2013, 7, 15) → 2013-07-15
+```||
+||make_interval ( [ years int [, months int [, weeks int [, days int [, hours int [, mins int [, secs double precision ]]]]]]] ) → interval|
+Create interval from years, months, weeks, days, hours, minutes and seconds fields, each of which can default to zero (NOT SUPPORTED)|
+```sql
+#make_interval(days => 10) → 10 days
+```||
+||make_time ( hour int, min int, sec double precision ) → time|
+Create time from hour, minute and seconds fields|
+```sql
+make_time(8, 15, 23.5) → 08:15:23.5
+```||
+||make_timestamp ( year int, month int, day int, hour int, min int, sec double precision ) → timestamp|
+Create timestamp from year, month, day, hour, minute and seconds fields (negative years signify BC)|
+```sql
+make_timestamp(2013, 7, 15, 8, 15, 23.5) → 2013-07-15 08:15:23.5
+```||
+||make_timestamptz ( year int, month int, day int, hour int, min int, sec double precision [, timezone text ] ) → timestamp with time zone|
+Create timestamp with time zone from year, month, day, hour, minute and seconds fields (negative years signify BC). If timezone is not specified, the current time zone is used; the examples assume the session time zone is Europe/London|
+```sql
+make_timestamptz(2013, 7, 15, 8, 15, 23.5) ~→ 2013-07-15 08:15:23.5+01
+#make_timestamptz(2013, 7, 15, 8, 15, 23.5, 'America/New_York') ~→ 2013-07-15 13:15:23.5+01
+```||
+||now ( ) → timestamp with time zone|
+Current date and time (start of current transaction); see Section 9.9.5|
+```sql
+now() ~→ 2019-12-23 14:39:53.662522-05
+```||
+||statement_timestamp ( ) → timestamp with time zone|
+Current date and time (start of current statement); see Section 9.9.5|
+```sql
+statement_timestamp() ~→ 2019-12-23 14:39:53.662522-05
+```||
+||timeofday ( ) → text|
+Current date and time (like clock_timestamp, but as a text string); see Section 9.9.5|
+```sql
+timeofday() ~→ Mon Dec 23 14:39:53.662522 2019 EST
+```||
+||transaction_timestamp ( ) → timestamp with time zone|
+Current date and time (start of current transaction); see Section 9.9.5|
+```sql
+transaction_timestamp() ~→ 2019-12-23 14:39:53.662522-05
+```||
+||to_timestamp ( double precision ) → timestamp with time zone|
+Convert Unix epoch (seconds since 1970-01-01 00:00:00+00) to timestamp with time zone|
+```sql
+to_timestamp(1284352323) → 2010-09-13 04:32:03+00
+```||
+|#
+
+In addition to these functions, the SQL OVERLAPS operator is supported: (NOT SUPPORTED)
+
+```sql
+(start1, end1) OVERLAPS (start2, end2)
+(start1, length1) OVERLAPS (start2, length2)
+```
+
+This expression yields true when two time periods (defined by their endpoints) overlap, false when they do not overlap. The endpoints can be specified as pairs of dates, times, or time stamps; or as a date, time, or time stamp followed by an interval. When a pair of values is provided, either the start or the end can be written first; OVERLAPS automatically takes the earlier value of the pair as the start. Each time period is considered to represent the half-open interval start <= time < end, unless start and end are equal in which case it represents that single time instant. This means for instance that two time periods with only an endpoint in common do not overlap.
+
+```sql
+(DATE '2001-02-16', DATE '2001-12-21') OVERLAPS (DATE '2001-10-30', DATE '2002-10-30') → true
+(DATE '2001-02-16', INTERVAL '100 days') OVERLAPS (DATE '2001-10-30', DATE '2002-10-30') → false
+(DATE '2001-10-29', DATE '2001-10-30') OVERLAPS (DATE '2001-10-30', DATE '2001-10-31') → false
+(DATE '2001-10-30', DATE '2001-10-30') OVERLAPS (DATE '2001-10-30', DATE '2001-10-31') → true
+```
+
+When adding an interval value to (or subtracting an interval value from) a timestamp with time zone value, the days component advances or decrements the date of the timestamp with time zone by the indicated number of days, keeping the time of day the same. Across daylight saving time changes (when the session time zone is set to a time zone that recognizes DST), this means interval '1 day' does not necessarily equal interval '24 hours'. For example, with the session time zone set to America/Denver:
+
+```sql
+timestamp with time zone '2005-04-02 12:00:00-07' + interval '1 day' ~→ 2005-04-03 12:00:00-06
+timestamp with time zone '2005-04-02 12:00:00-07' + interval '24 hours' ~→ 2005-04-03 13:00:00-06
+```
+
+This happens because an hour was skipped due to a change in daylight saving time at 2005-04-03 02:00:00 in time zone America/Denver.
+
+Note there can be ambiguity in the months field returned by age because different months have different numbers of days. PostgreSQL's approach uses the month from the earlier of the two dates when calculating partial months. For example, age('2004-06-01', '2004-04-30') uses April to yield 1 mon 1 day, while using May would yield 1 mon 2 days because May has 31 days, while April has only 30.
+
+Subtraction of dates and timestamps can also be complex. One conceptually simple way to perform subtraction is to convert each value to a number of seconds using EXTRACT(EPOCH FROM ...), then subtract the results; this produces the number of seconds between the two values. This will adjust for the number of days in each month, timezone changes, and daylight saving time adjustments. Subtraction of date or timestamp values with the “-” operator returns the number of days (24-hours) and hours/minutes/seconds between the values, making the same adjustments. The age function returns years, months, days, and hours/minutes/seconds, performing field-by-field subtraction and then adjusting for negative field values. The following queries illustrate the differences in these approaches. The sample results were produced with timezone = 'US/Eastern'; there is a daylight saving time change between the two dates used:
+
+```sql
+EXTRACT(EPOCH FROM timestamptz '2013-07-01 12:00:00') - EXTRACT(EPOCH FROM timestamptz '2013-03-01 12:00:00') ~→ 10537200.000000
+(EXTRACT(EPOCH FROM timestamptz '2013-07-01 12:00:00') - EXTRACT(EPOCH FROM timestamptz '2013-03-01 12:00:00')) / 60 / 60 / 24 ~→ 121.9583333333333333
+timestamptz '2013-07-01 12:00:00' - timestamptz '2013-03-01 12:00:00' ~→ 121 days 23:00:00
+age(timestamptz '2013-07-01 12:00:00', timestamptz '2013-03-01 12:00:00') → 4 mons
+```
+
+9.9.1. EXTRACT, date_part
+
+```sql
+EXTRACT(field FROM source)
+```
+
+The extract function retrieves subfields such as year or hour from date/time values. source must be a value expression of type timestamp, time, or interval. (Expressions of type date are cast to timestamp and can therefore be used as well.) field is an identifier or string that selects what field to extract from the source value. The extract function returns values of type numeric. The following are valid field names:
+
+century
+The century
+
+```sql
+EXTRACT(CENTURY FROM TIMESTAMP '2000-12-16 12:21:13') → 20
+EXTRACT(CENTURY FROM TIMESTAMP '2001-02-16 20:38:40') → 21
+```
+
+The first century starts at 0001-01-01 00:00:00 AD, although they did not know it at the time. This definition applies to all Gregorian calendar countries. There is no century number 0, you go from -1 century to 1 century. If you disagree with this, please write your complaint to: Pope, Cathedral Saint-Peter of Roma, Vatican.
+
+day
+For timestamp values, the day (of the month) field (1–31) ; for interval values, the number of days
+
+```sql
+EXTRACT(DAY FROM TIMESTAMP '2001-02-16 20:38:40') → 16
+ EXTRACT(DAY FROM INTERVAL '40 days 1 minute') → 40
+```
+
+decade
+The year field divided by 10
+
+```sql
+EXTRACT(DECADE FROM TIMESTAMP '2001-02-16 20:38:40') → 200
+```
+
+dow
+The day of the week as Sunday (0) to Saturday (6)
+
+```sql
+EXTRACT(DOW FROM TIMESTAMP '2001-02-16 20:38:40') → 5
+```
+
+Note that extract's day of the week numbering differs from that of the to_char(..., 'D') function.
+
+doy
+The day of the year (1–365/366)
+
+```sql
+EXTRACT(DOY FROM TIMESTAMP '2001-02-16 20:38:40') → 47
+```
+
+epoch
+For timestamp with time zone values, the number of seconds since 1970-01-01 00:00:00 UTC (negative for timestamps before that); for date and timestamp values, the nominal number of seconds since 1970-01-01 00:00:00, without regard to timezone or daylight-savings rules; for interval values, the total number of seconds in the interval
+
+```sql
+EXTRACT(EPOCH FROM TIMESTAMP WITH TIME ZONE '2001-02-16 20:38:40.12-08') → 982384720.120000
+EXTRACT(EPOCH FROM TIMESTAMP '2001-02-16 20:38:40.12') → 982355920.120000
+EXTRACT(EPOCH FROM INTERVAL '5 days 3 hours') → 442800.000000
+```
+
+You can convert an epoch value back to a timestamp with time zone with to_timestamp:
+
+```sql
+to_timestamp(982384720.12) → 2001-02-17 04:38:40.12+00
+```
+Beware that applying to_timestamp to an epoch extracted from a date or timestamp value could produce a misleading result: the result will effectively assume that the original value had been given in UTC, which might not be the case.
+
+hour
+The hour field (0–23)
+
+```sql
+EXTRACT(HOUR FROM TIMESTAMP '2001-02-16 20:38:40') → 20
+```
+
+isodow
+The day of the week as Monday (1) to Sunday (7)
+
+```sql
+EXTRACT(ISODOW FROM TIMESTAMP '2001-02-18 20:38:40') → 7
+```
+
+This is identical to dow except for Sunday. This matches the ISO 8601 day of the week numbering.
+
+isoyear
+The ISO 8601 week-numbering year that the date falls in (not applicable to intervals)
+
+```sql
+EXTRACT(ISOYEAR FROM DATE '2006-01-01') → 2005
+EXTRACT(ISOYEAR FROM DATE '2006-01-02') → 2006
+```
+
+Each ISO 8601 week-numbering year begins with the Monday of the week containing the 4th of January, so in early January or late December the ISO year may be different from the Gregorian year. See the week field for more information.
+
+This field is not available in PostgreSQL releases prior to 8.3.
+
+julian
+The Julian Date corresponding to the date or timestamp (not applicable to intervals). Timestamps that are not local midnight result in a fractional value. See Section B.7 for more information.
+
+```sql
+EXTRACT(JULIAN FROM DATE '2006-01-01') → 2453737
+EXTRACT(JULIAN FROM TIMESTAMP '2006-01-01 12:00') → 2453737.50000000000000000000
+```
+
+microseconds
+The seconds field, including fractional parts, multiplied by 1 000 000; note that this includes full seconds
+
+```sql
+EXTRACT(MICROSECONDS FROM TIME '17:12:28.5') → 28500000
+```
+
+millennium
+The millennium
+
+```sql
+EXTRACT(MILLENNIUM FROM TIMESTAMP '2001-02-16 20:38:40') → 3
+```
+Years in the 1900s are in the second millennium. The third millennium started January 1, 2001.
+
+milliseconds
+The seconds field, including fractional parts, multiplied by 1000. Note that this includes full seconds.
+
+```sql
+EXTRACT(MILLISECONDS FROM TIME '17:12:28.5') → 28500.000
+```
+
+minute
+The minutes field (0–59)
+
+```sql
+EXTRACT(MINUTE FROM TIMESTAMP '2001-02-16 20:38:40') → 38
+```
+
+month
+For timestamp values, the number of the month within the year (1–12) ; for interval values, the number of months, modulo 12 (0–11)
+
+```sql
+SELECT EXTRACT(MONTH FROM TIMESTAMP '2001-02-16 20:38:40') → 2
+SELECT EXTRACT(MONTH FROM INTERVAL '2 years 3 months') → 3
+SELECT EXTRACT(MONTH FROM INTERVAL '2 years 13 months') → 1
+```
+
+quarter
+The quarter of the year (1–4) that the date is in
+
+```sql
+SELECT EXTRACT(QUARTER FROM TIMESTAMP '2001-02-16 20:38:40') → 1
+```
+
+second
+The seconds field, including any fractional seconds
+
+```sql
+SELECT EXTRACT(SECOND FROM TIMESTAMP '2001-02-16 20:38:40') → 40.000000
+SELECT EXTRACT(SECOND FROM TIME '17:12:28.5') → 28.500000
+```
+
+timezone
+The time zone offset from UTC, measured in seconds. Positive values correspond to time zones east of UTC, negative values to zones west of UTC. (Technically, PostgreSQL does not use UTC because leap seconds are not handled.)
+
+timezone_hour
+The hour component of the time zone offset
+
+timezone_minute
+The minute component of the time zone offset
+
+week
+The number of the ISO 8601 week-numbering week of the year. By definition, ISO weeks start on Mondays and the first week of a year contains January 4 of that year. In other words, the first Thursday of a year is in week 1 of that year.
+
+In the ISO week-numbering system, it is possible for early-January dates to be part of the 52nd or 53rd week of the previous year, and for late-December dates to be part of the first week of the next year. For example, 2005-01-01 is part of the 53rd week of year 2004, and 2006-01-01 is part of the 52nd week of year 2005, while 2012-12-31 is part of the first week of 2013. It's recommended to use the isoyear field together with week to get consistent results.
+
+```sql
+EXTRACT(WEEK FROM TIMESTAMP '2001-02-16 20:38:40') → 7
+```
+
+year
+The year field. Keep in mind there is no 0 AD, so subtracting BC years from AD years should be done with care.
+
+```sql
+EXTRACT(YEAR FROM TIMESTAMP '2001-02-16 20:38:40') → 2001
+```
+
+Note
+When the input value is +/-Infinity, extract returns +/-Infinity for monotonically-increasing fields (epoch, julian, year, isoyear, decade, century, and millennium). For other fields, NULL is returned. PostgreSQL versions before 9.6 returned zero for all cases of infinite input.
+
+The extract function is primarily intended for computational processing. For formatting date/time values for display, see Section 9.8.
+
+The date_part function is modeled on the traditional Ingres equivalent to the SQL-standard function extract:
+
+```sql
+date_part('field', source)
+```
+
+Note that here the field parameter needs to be a string value, not a name. The valid field names for date_part are the same as for extract. For historical reasons, the date_part function returns values of type double precision. This can result in a loss of precision in certain uses. Using extract is recommended instead.
+
+```sql
+date_part('day', TIMESTAMP '2001-02-16 20:38:40') → 16
+date_part('hour', INTERVAL '4 hours 3 minutes') → 4
+```
+
+9.9.2. date_trunc
+
+The function date_trunc is conceptually similar to the trunc function for numbers.
+
+```sql
+date_trunc(field, source [, time_zone ])
+```
+
+source is a value expression of type timestamp, timestamp with time zone, or interval. (Values of type date and time are cast automatically to timestamp or interval, respectively.) field selects to which precision to truncate the input value. The return value is likewise of type timestamp, timestamp with time zone, or interval, and it has all fields that are less significant than the selected one set to zero (or one, for day and month).
+
+Valid values for field are:
+
+microseconds
+milliseconds
+second
+minute
+hour
+day
+week
+month
+quarter
+year
+decade
+century
+millennium
+
+When the input value is of type timestamp with time zone, the truncation is performed with respect to a particular time zone; for example, truncation to day produces a value that is midnight in that zone. By default, truncation is done with respect to the current TimeZone setting, but the optional time_zone argument can be provided to specify a different time zone. The time zone name can be specified in any of the ways described in Section 8.5.3.
+
+A time zone cannot be specified when processing timestamp without time zone or interval inputs. These are always taken at face value.
+
+Examples (assuming the local time zone is America/New_York):
+
+```sql
+date_trunc('hour', TIMESTAMP '2001-02-16 20:38:40') ~→ 2001-02-16 20:00:00
+date_trunc('year', TIMESTAMP '2001-02-16 20:38:40') ~→ 2001-01-01 00:00:00
+date_trunc('day', TIMESTAMP WITH TIME ZONE '2001-02-16 20:38:40+00') ~→ 2001-02-16 00:00:00-05
+#date_trunc('day', TIMESTAMP WITH TIME ZONE '2001-02-16 20:38:40+00', 'Australia/Sydney') ~→ 2001-02-16 08:00:00-05
+date_trunc('hour', INTERVAL '3 days 02:47:33') → 3 days 02:00:00
+```
+
+9.9.3. date_bin
+
+The function date_bin “bins” the input timestamp into the specified interval (the stride) aligned with a specified origin.
+
+```sql
+date_bin(stride, source, origin)
+```
+
+source is a value expression of type timestamp or timestamp with time zone. (Values of type date are cast automatically to timestamp.) stride is a value expression of type interval. The return value is likewise of type timestamp or timestamp with time zone, and it marks the beginning of the bin into which the source is placed.
+
+Examples:
+
+```sql
+date_bin('15 minutes', TIMESTAMP '2020-02-11 15:44:17', TIMESTAMP '2001-01-01') → 2020-02-11 15:30:00
+date_bin('15 minutes', TIMESTAMP '2020-02-11 15:44:17', TIMESTAMP '2001-01-01 00:02:30') → 2020-02-11 15:32:30
+```
+
+In the case of full units (1 minute, 1 hour, etc.), it gives the same result as the analogous date_trunc call, but the difference is that date_bin can truncate to an arbitrary interval.
+
+The stride interval must be greater than zero and cannot contain units of month or larger.
+
+9.9.4. AT TIME ZONE
+
+The AT TIME ZONE operator converts time stamp without time zone to/from time stamp with time zone, and time with time zone values to different time zones. Table 9.33 shows its variants. (NOT SUPPORTED)
+
+Table 9.33. AT TIME ZONE Variants
+
+#|
+||Operator|Description|Example(s)|
+||timestamp without time zone AT TIME ZONE zone → timestamp with time zone|
+Converts given time stamp without time zone to time stamp with time zone, assuming the given value is in the named time zone.|
+```sql
+#timestamp '2001-02-16 20:38:40' at time zone 'America/Denver' → 2001-02-17 03:38:40+00
+```||
+||timestamp with time zone AT TIME ZONE zone → timestamp without time zone|
+Converts given time stamp with time zone to time stamp without time zone, as the time would appear in that zone.|
+```sql
+#timestamp with time zone '2001-02-16 20:38:40-05' at time zone 'America/Denver' → 2001-02-16 18:38:40
+```||
+||time with time zone AT TIME ZONE zone → time with time zone|
+Converts given time with time zone to a new time zone. Since no date is supplied, this uses the currently active UTC offset for the named destination zone.|
+```sql
+#time with time zone '05:34:17-05' at time zone 'UTC' → 10:34:17+00
+```||
+|#
+
+In these expressions, the desired time zone zone can be specified either as a text value (e.g., 'America/Los_Angeles') or as an interval (e.g., INTERVAL '-08:00'). In the text case, a time zone name can be specified in any of the ways described in Section 8.5.3. The interval case is only useful for zones that have fixed offsets from UTC, so it is not very common in practice.
+
+Examples (assuming the current TimeZone setting is America/Los_Angeles):
+
+```sql
+#TIMESTAMP '2001-02-16 20:38:40' AT TIME ZONE 'America/Denver' ~→ 2001-02-16 19:38:40-08
+#TIMESTAMP WITH TIME ZONE '2001-02-16 20:38:40-05' AT TIME ZONE 'America/Denver' ~→ 2001-02-16 18:38:40
+#TIMESTAMP '2001-02-16 20:38:40' AT TIME ZONE 'Asia/Tokyo' AT TIME ZONE 'America/Chicago' ~→ 2001-02-16 05:38:40
+```
+
+The first example adds a time zone to a value that lacks it, and displays the value using the current TimeZone setting. The second example shifts the time stamp with time zone value to the specified time zone, and returns the value without a time zone. This allows storage and display of values different from the current TimeZone setting. The third example converts Tokyo time to Chicago time.
+
+The function timezone(zone, timestamp) is equivalent to the SQL-conforming construct timestamp AT TIME ZONE zone.
+
+9.9.5. Current Date/Time
+
+PostgreSQL provides a number of functions that return values related to the current date and time. These SQL-standard functions all return values based on the start time of the current transaction:
+
+```sql
+CURRENT_DATE
+CURRENT_TIME
+CURRENT_TIMESTAMP
+CURRENT_TIME(precision)
+CURRENT_TIMESTAMP(precision)
+LOCALTIME
+LOCALTIMESTAMP
+LOCALTIME(precision)
+LOCALTIMESTAMP(precision)
+```
+
+CURRENT_TIME and CURRENT_TIMESTAMP deliver values with time zone; LOCALTIME and LOCALTIMESTAMP deliver values without time zone.
+
+CURRENT_TIME, CURRENT_TIMESTAMP, LOCALTIME, and LOCALTIMESTAMP can optionally take a precision parameter, which causes the result to be rounded to that many fractional digits in the seconds field. Without a precision parameter, the result is given to the full available precision.
+
+Some examples:
+
+```sql
+CURRENT_TIME ~→ 14:39:53.662522-05
+CURRENT_DATE ~→ 2019-12-23
+CURRENT_TIMESTAMP ~→ 2019-12-23 14:39:53.662522-05
+CURRENT_TIMESTAMP(2) ~→ 2019-12-23 14:39:53.66-05
+#LOCALTIMESTAMP ~→ 2019-12-23 14:39:53.662522
+```
+
+Since these functions return the start time of the current transaction, their values do not change during the transaction. This is considered a feature: the intent is to allow a single transaction to have a consistent notion of the “current” time, so that multiple modifications within the same transaction bear the same time stamp.
+
+Note
+Other database systems might advance these values more frequently.
+
+PostgreSQL also provides functions that return the start time of the current statement, as well as the actual current time at the instant the function is called. The complete list of non-SQL-standard time functions is:
+
+```sql
+transaction_timestamp()
+statement_timestamp()
+clock_timestamp()
+timeofday()
+now()
+```
+
+transaction_timestamp() is equivalent to CURRENT_TIMESTAMP, but is named to clearly reflect what it returns. statement_timestamp() returns the start time of the current statement (more specifically, the time of receipt of the latest command message from the client). statement_timestamp() and transaction_timestamp() return the same value during the first command of a transaction, but might differ during subsequent commands. clock_timestamp() returns the actual current time, and therefore its value changes even within a single SQL command. timeofday() is a historical PostgreSQL function. Like clock_timestamp(), it returns the actual current time, but as a formatted text string rather than a timestamp with time zone value. now() is a traditional PostgreSQL equivalent to transaction_timestamp().
+
+All the date/time data types also accept the special literal value now to specify the current date and time (again, interpreted as the transaction start time). Thus, the following three all return the same result:
+
+```sql
+SELECT CURRENT_TIMESTAMP;
+SELECT now();
+SELECT TIMESTAMP 'now'; /* but see tip below */
+```
+
+Tip
+Do not use the third form when specifying a value to be evaluated later, for example in a DEFAULT clause for a table column. The system will convert now to a timestamp as soon as the constant is parsed, so that when the default value is needed, the time of the table creation would be used! The first two forms will not be evaluated until the default value is used, because they are function calls. Thus they will give the desired behavior of defaulting to the time of row insertion. (See also Section 8.5.1.4.)
+
+9.9.6. Delaying Execution
+
+The following functions are available to delay execution of the server process:
+
+```sql
+pg_sleep ( double precision )
+pg_sleep_for ( interval )
+pg_sleep_until ( timestamp with time zone )
+```
+
+pg_sleep makes the current session's process sleep until the given number of seconds have elapsed. Fractional-second delays can be specified. pg_sleep_for is a convenience function to allow the sleep time to be specified as an interval. pg_sleep_until is a convenience function for when a specific wake-up time is desired. For example:
+
+```sql
+SELECT pg_sleep(1.5);
+SELECT pg_sleep_for('5 minutes');
+SELECT pg_sleep_until('tomorrow 03:00');
+```
+
+Note
+The effective resolution of the sleep interval is platform-specific; 0.01 seconds is a common value. The sleep delay will be at least as long as specified. It might be longer depending on factors such as server load. In particular, pg_sleep_until is not guaranteed to wake up exactly at the specified time, but it will not wake up any earlier.
+
+Warning
+Make sure that your session does not hold more locks than necessary when calling pg_sleep or its variants. Otherwise other sessions might have to wait for your sleeping process, slowing down the entire system.
+
+## 9.10. Enum Support Functions (NOT SUPPORTED) {#enum-support-functions}
+
+## 9.11. Geometric Functions and Operators {#geometric-functions}
+The geometric types point, box, lseg, line, path, polygon, and circle have a large set of native support functions and operators, shown in Table 9.35, Table 9.36, and Table 9.37.
+
+Table 9.35. Geometric Operators
+
+#|
+||Operator|Description|Example(s)||
+||geometric_type + point → geometric_type|
+Adds the coordinates of the second point to those of each point of the first argument, thus performing translation. Available for point, box, path, circle.|
+```sql
+box '(1,1),(0,0)' + point '(2,0)' → (3,1),(2,0)
+```||
+||path + path → path|
+Concatenates two open paths (returns NULL if either path is closed).|
+```sql
+path '[(0,0),(1,1)]' + path '[(2,2),(3,3),(4,4)]' → [(0,0),(1,1),(2,2),(3,3),(4,4)]
+```||
+||geometric_type - point → geometric_type|
+Subtracts the coordinates of the second point from those of each point of the first argument, thus performing translation. Available for point, box, path, circle.|
+```sql
+box '(1,1),(0,0)' - point '(2,0)' → (-1,1),(-2,0)
+```||
+||geometric_type * point → geometric_type|
+Multiplies each point of the first argument by the second point (treating a point as being a complex number represented by real and imaginary parts, and performing standard complex multiplication). If one interprets the second point as a vector, this is equivalent to scaling the object's size and distance from the origin by the length of the vector, and rotating it counterclockwise around the origin by the vector's angle from the x axis. Available for point, box,[a] path, circle.|
+```sql
+path '((0,0),(1,0),(1,1))' * point '(3.0,0)' → ((0,0),(3,0),(3,3))
+path '((0,0),(1,0),(1,1))' * point(cosd(45), sind(45)) → ((0,0),(0.7071067811865475,0.7071067811865475),(0,1.414213562373095))
+```||
+||geometric_type / point → geometric_type|
+Divides each point of the first argument by the second point (treating a point as being a complex number represented by real and imaginary parts, and performing standard complex division). If one interprets the second point as a vector, this is equivalent to scaling the object's size and distance from the origin down by the length of the vector, and rotating it clockwise around the origin by the vector's angle from the x axis. Available for point, box,[a] path, circle.|
+```sql
+path '((0,0),(1,0),(1,1))' / point '(2.0,0)' → ((0,0),(0.5,0),(0.5,0.5))
+path '((0,0),(1,0),(1,1))' / point(cosd(45), sind(45)) → ((0,0),(0.7071067811865476,-0.7071067811865476),(1.4142135623730951,0))
+```||
+||@-@ geometric_type → double precision|
+Computes the total length. Available for lseg, path.|
+```sql
+@-@ path '[(0,0),(1,0),(1,1)]' → 2
+```||
+||@@ geometric_type → point|
+Computes the center point. Available for box, lseg, polygon, circle.|
+```sql
+@@ box '(2,2),(0,0)' → (1,1)
+```||
+||\# geometric_type → integer|
+Returns the number of points. Available for path, polygon.|
+```sql
+# path '((1,0),(0,1),(-1,0))' → 3
+```||
+||geometric_type # geometric_type → point|
+Computes the point of intersection, or NULL if there is none. Available for lseg, line.|
+```sql
+lseg '[(0,0),(1,1)]' # lseg '[(1,0),(0,1)]' → (0.5,0.5)
+```||
+||box # box → box|
+Computes the intersection of two boxes, or NULL if there is none.|
+```sql
+box '(2,2),(-1,-1)' # box '(1,1),(-2,-2)' → (1,1),(-1,-1)
+```||
+||geometric_type ## geometric_type → point|
+Computes the closest point to the first object on the second object. Available for these pairs of types: (point, box), (point, lseg), (point, line), (lseg, box), (lseg, lseg), (line, lseg).|
+```sql
+point '(0,0)' ## lseg '[(2,0),(0,2)]' → (1,1)
+```||
+||geometric_type <-> geometric_type → double precision|
+Computes the distance between the objects. Available for all geometric types except polygon, for all combinations of point with another geometric type, and for these additional pairs of types: (box, lseg), (lseg, line), (polygon, circle) (and the commutator cases).|
+```sql
+circle '<(0,0),1>' <-> circle '<(5,0),1>' → 3
+```||
+||geometric_type @> geometric_type → boolean|
+Does first object contain second? Available for these pairs of types: (box, point), (box, box), (path, point), (polygon, point), (polygon, polygon), (circle, point), (circle, circle).|
+```sql
+circle '<(0,0),2>' @> point '(1,1)' → true
+```||
+||geometric_type <@ geometric_type → boolean|
+Is first object contained in or on second? Available for these pairs of types: (point, box), (point, lseg), (point, line), (point, path), (point, polygon), (point, circle), (box, box), (lseg, box), (lseg, line), (polygon, polygon), (circle, circle).|
+```sql
+point '(1,1)' <@ circle '<(0,0),2>' → true
+```||
+||geometric_type && geometric_type → boolean|
+Do these objects overlap? (One point in common makes this true.) Available for box, polygon, circle.|
+```sql
+box '(1,1),(0,0)' && box '(2,2),(0,0)' → true
+```||
+||geometric_type << geometric_type → boolean|
+Is first object strictly left of second? Available for point, box, polygon, circle.|
+```sql
+circle '<(0,0),1>' << circle '<(5,0),1>' → true
+```||
+||geometric_type >> geometric_type → boolean|
+Is first object strictly right of second? Available for point, box, polygon, circle.|
+```sql
+circle '<(5,0),1>' >> circle '<(0,0),1>' → true
+```||
+||geometric_type &< geometric_type → boolean|
+Does first object not extend to the right of second? Available for box, polygon, circle.|
+```sql
+box '(1,1),(0,0)' &< box '(2,2),(0,0)' → true
+```||
+||geometric_type &> geometric_type → boolean|
+Does first object not extend to the left of second? Available for box, polygon, circle.|
+```sql
+box '(3,3),(0,0)' &> box '(2,2),(0,0)' → true
+```||
+||geometric_type <<\| geometric_type → boolean|
+Is first object strictly below second? Available for point, box, polygon, circle.|
+```sql
+box '(3,3),(0,0)' <<| box '(5,5),(3,4)' → true
+```||
+||geometric_type \|>> geometric_type → boolean|
+Is first object strictly above second? Available for point, box, polygon, circle.|
+```sql
+box '(5,5),(3,4)' |>> box '(3,3),(0,0)' → true
+```||
+||geometric_type &<\| geometric_type → boolean|
+Does first object not extend above second? Available for box, polygon, circle.|
+```sql
+box '(1,1),(0,0)' &<| box '(2,2),(0,0)' → true
+```||
+||geometric_type \|&> geometric_type → boolean|
+Does first object not extend below second? Available for box, polygon, circle.|
+```sql
+box '(3,3),(0,0)' |&> box '(2,2),(0,0)' → true
+```||
+||box <^ box → boolean|
+Is first object below second (allows edges to touch)?|
+```sql
+box '((1,1),(0,0))' <^ box '((2,2),(1,1))' → true
+```||
+||box >^ box → boolean|
+Is first object above second (allows edges to touch)?|
+```sql
+box '((2,2),(1,1))' >^ box '((1,1),(0,0))' → true
+```||
+||geometric_type ?# geometric_type → boolean|
+Do these objects intersect? Available for these pairs of types: (box, box), (lseg, box), (lseg, lseg), (lseg, line), (line, box), (line, line), (path, path).|
+```sql
+lseg '[(-1,0),(1,0)]' ?# box '(2,2),(-2,-2)' → true
+```||
+||?- line → boolean
+?- lseg → boolean|
+Is line horizontal?|
+```sql
+?- lseg '[(-1,0),(1,0)]' → true
+```||
+||point ?- point → boolean|
+Are points horizontally aligned (that is, have same y coordinate)?|
+```sql
+point '(1,0)' ?- point '(0,0)' → true
+```||
+||?\| line → boolean
+?\| lseg → boolean|
+Is line vertical?|
+```sql
+?| lseg '[(-1,0),(1,0)]' → false
+```||
+||point ?\| point → boolean|
+Are points vertically aligned (that is, have same x coordinate)?|
+```sql
+point '(0,1)' ?| point '(0,0)' → true
+```||
+||line ?-\| line → boolean
+lseg ?-\| lseg → boolean|
+Are lines perpendicular?|
+```sql
+lseg '[(0,0),(0,1)]' ?-| lseg '[(0,0),(1,0)]' → true
+```||
+||line ?\|\| line → boolean
+lseg ?\|\| lseg → boolean|
+Are lines parallel?|
+```sql
+lseg '[(-1,0),(1,0)]' ?|| lseg '[(-1,2),(1,2)]' → true
+```||
+||geometric_type ~= geometric_type → boolean|
+Are these objects the same? Available for point, box, polygon, circle.|
+```sql
+polygon '((0,0),(1,1))' ~= polygon '((1,1),(0,0))' → true
+```||
+|#
+
+[a] “Rotating” a box with these operators only moves its corner points: the box is still considered to have sides parallel to the axes. Hence the box's size is not preserved, as a true rotation would do.
+
+Caution
+Note that the “same as” operator, ~=, represents the usual notion of equality for the point, box, polygon, and circle types. Some of the geometric types also have an = operator, but = compares for equal areas only. The other scalar comparison operators (<= and so on), where available for these types, likewise compare areas.
+
+Note
+Before PostgreSQL 14, the point is strictly below/above comparison operators point <<\| point and point \|>> point were respectively called <^ and >^. These names are still available, but are deprecated and will eventually be removed.
+
+Table 9.36. Geometric Functions
+
+#|
+||Function|Description|Example(s)||
+||area ( geometric_type ) → double precision|
+Computes area. Available for box, path, circle. A path input must be closed, else NULL is returned. Also, if the path is self-intersecting, the result may be meaningless.|
+```sql
+area(box '(2,2),(0,0)') → 4
+```||
+||center ( geometric_type ) → point|
+Computes center point. Available for box, circle.|
+```sql
+center(box '(1,2),(0,0)') → (0.5,1)
+```||
+||diagonal ( box ) → lseg|
+Extracts box's diagonal as a line segment (same as lseg(box)).|
+```sql
+diagonal(box '(1,2),(0,0)') → [(1,2),(0,0)]
+```||
+||diameter ( circle ) → double precision|
+Computes diameter of circle.|
+```sql
+diameter(circle '<(0,0),2>') → 4
+```||
+||height ( box ) → double precision|
+Computes vertical size of box.|
+```sql
+height(box '(1,2),(0,0)') → 2
+```||
+||isclosed ( path ) → boolean|
+Is path closed?|
+```sql
+isclosed(path '((0,0),(1,1),(2,0))') → true
+```||
+||isopen ( path ) → boolean|
+Is path open?|
+```sql
+isopen(path '[(0,0),(1,1),(2,0)]') → true
+```||
+||length ( geometric_type ) → double precision|
+Computes the total length. Available for lseg, path.|
+```sql
+length(path '((-1,0),(1,0))') → 4
+```||
+||npoints ( geometric_type ) → integer|
+Returns the number of points. Available for path, polygon.|
+```sql
+npoints(path '[(0,0),(1,1),(2,0)]') → 3
+```||
+||pclose ( path ) → path|
+Converts path to closed form.|
+```sql
+pclose(path '[(0,0),(1,1),(2,0)]') → ((0,0),(1,1),(2,0))
+```||
+||popen ( path ) → path|
+Converts path to open form.|
+```sql
+popen(path '((0,0),(1,1),(2,0))') → [(0,0),(1,1),(2,0)]
+```||
+||radius ( circle ) → double precision|
+Computes radius of circle.|
+```sql
+radius(circle '<(0,0),2>') → 2
+```||
+||slope ( point, point ) → double precision|
+Computes slope of a line drawn through the two points.|
+```sql
+slope(point '(0,0)', point '(2,1)') → 0.5
+```||
+||width ( box ) → double precision|
+Computes horizontal size of box.|
+```sql
+width(box '(1,2),(0,0)') → 1
+```||
+|#
+
+Table 9.37. Geometric Type Conversion Functions
+
+#|
+||Function|Description|Example(s)||
+||box ( circle ) → box|
+Computes box inscribed within the circle.|
+```sql
+box(circle '<(0,0),2>') → (1.414213562373095,1.414213562373095),(-1.414213562373095,-1.414213562373095)
+```||
+||box ( point ) → box|
+Converts point to empty box.|
+```sql
+box(point '(1,0)') → (1,0),(1,0)
+```||
+||box ( point, point ) → box|
+Converts any two corner points to box.|
+```sql
+box(point '(0,1)', point '(1,0)') → (1,1),(0,0)
+```||
+||box ( polygon ) → box|
+Computes bounding box of polygon.|
+```sql
+box(polygon '((0,0),(1,1),(2,0))') → (2,1),(0,0)
+```||
+||bound_box ( box, box ) → box|
+Computes bounding box of two boxes.|
+```sql
+bound_box(box '(1,1),(0,0)', box '(4,4),(3,3)') → (4,4),(0,0)
+```||
+||circle ( box ) → circle|
+Computes smallest circle enclosing box.|
+```sql
+circle(box '(1,1),(0,0)') → <(0.5,0.5),0.7071067811865476>
+```||
+||circle ( point, double precision ) → circle|
+Constructs circle from center and radius.|
+```sql
+circle(point '(0,0)', 2.0) → <(0,0),2>
+```||
+||circle ( polygon ) → circle|
+Converts polygon to circle. The circle's center is the mean of the positions of the polygon's points, and the radius is the average distance of the polygon's points from that center.|
+```sql
+circle(polygon '((0,0),(1,3),(2,0))') → <(1,1),1.6094757082487299>
+```||
+||line ( point, point ) → line|
+Converts two points to the line through them.|
+```sql
+line(point '(-1,0)', point '(1,0)') → {0,-1,0}
+```||
+||lseg ( box ) → lseg|
+Extracts box's diagonal as a line segment.|
+```sql
+lseg(box '(1,0),(-1,0)') → [(1,0),(-1,0)]
+```||
+||lseg ( point, point ) → lseg|
+Constructs line segment from two endpoints.|
+```sql
+lseg(point '(-1,0)', point '(1,0)') → [(-1,0),(1,0)]
+```||
+||path ( polygon ) → path|
+Converts polygon to a closed path with the same list of points.|
+```sql
+path(polygon '((0,0),(1,1),(2,0))') → ((0,0),(1,1),(2,0))
+```||
+||point ( double precision, double precision ) → point|
+Constructs point from its coordinates.|
+```sql
+point(23.4, -44.5) → (23.4,-44.5)
+```||
+||point ( box ) → point|
+Computes center of box.|
+```sql
+point(box '(1,0),(-1,0)') → (0,0)
+```||
+||point ( circle ) → point|
+Computes center of circle.|
+```sql
+point(circle '<(0,0),2>') → (0,0)
+```||
+||point ( lseg ) → point|
+Computes center of line segment.|
+```sql
+point(lseg '[(-1,0),(1,0)]') → (0,0)
+```||
+||point ( polygon ) → point|
+Computes center of polygon (the mean of the positions of the polygon's points).|
+```sql
+point(polygon '((0,0),(1,1),(2,0))') → (1,0.3333333333333333)
+```||
+||polygon ( box ) → polygon|
+Converts box to a 4-point polygon.|
+```sql
+polygon(box '(1,1),(0,0)') → ((0,0),(0,1),(1,1),(1,0))
+```||
+||polygon ( circle ) → polygon|
+Converts circle to a 12-point polygon.|
+```sql
+polygon(circle '<(0,0),2>') → ((-2,0),(-1.7320508075688774,0.9999999999999999),(-1.0000000000000002,1.7320508075688772),(-1.2246467991473532e-16,2),(0.9999999999999996,1.7320508075688774),(1.732050807568877,1.0000000000000007),(2,2.4492935982947064e-16),(1.7320508075688776,-0.9999999999999994),(1.0000000000000009,-1.7320508075688767),(3.6739403974420594e-16,-2),(-0.9999999999999987,-1.732050807568878),(-1.7320508075688767,-1.0000000000000009))
+```||
+||polygon ( path ) → polygon|
+Converts closed path to a polygon with the same list of points.|
+```sql
+polygon(path '((0,0),(1,1),(2,0))') → ((0,0),(1,1),(2,0))
+```||
+|#
+
+## 9.12. Network Address Functions and Operators {#network-address-functions}
+The IP network address types, cidr and inet, support the usual comparison operators shown in Table 9.1 as well as the specialized operators and functions shown in Table 9.38 and Table 9.39.
+
+Any cidr value can be cast to inet implicitly; therefore, the operators and functions shown below as operating on inet also work on cidr values. (Where there are separate functions for inet and cidr, it is because the behavior should be different for the two cases.) Also, it is permitted to cast an inet value to cidr. When this is done, any bits to the right of the netmask are silently zeroed to create a valid cidr value.
+
+Table 9.38. IP Address Operators
+
+#|
+||Operator|Description|Example(s)||
+||inet << inet → boolean|
+Is subnet strictly contained by subnet? This operator, and the next four, test for subnet inclusion. They consider only the network parts of the two addresses (ignoring any bits to the right of the netmasks) and determine whether one network is identical to or a subnet of the other.|
+```sql
+inet '192.168.1.5' << inet '192.168.1/24' → true
+inet '192.168.0.5' << inet '192.168.1/24' → false
+inet '192.168.1/24' << inet '192.168.1/24' → false
+```||
+||inet <<= inet → boolean|
+Is subnet contained by or equal to subnet?|
+```sql
+inet '192.168.1/24' <<= inet '192.168.1/24' → true
+```||
+||inet >> inet → boolean|
+Does subnet strictly contain subnet?|
+```sql
+inet '192.168.1/24' >> inet '192.168.1.5' → true
+```||
+||inet >>= inet → boolean|
+Does subnet contain or equal subnet?|
+```sql
+inet '192.168.1/24' >>= inet '192.168.1/24' → true
+```||
+||inet && inet → boolean|
+Does either subnet contain or equal the other?|
+```sql
+inet '192.168.1/24' && inet '192.168.1.80/28' → true
+inet '192.168.1/24' && inet '192.168.2.0/28' → false
+```||
+||~ inet → inet|
+Computes bitwise NOT.|
+```sql
+~ inet '192.168.1.6' → 63.87.254.249
+```||
+||inet & inet → inet|
+Computes bitwise AND.|
+```sql
+inet '192.168.1.6' & inet '0.0.0.255' → 0.0.0.6
+```||
+||inet \| inet → inet|
+Computes bitwise OR.|
+```sql
+inet '192.168.1.6' | inet '0.0.0.255' → 192.168.1.255
+```||
+||inet + bigint → inet|
+Adds an offset to an address.|
+```sql
+inet '192.168.1.6' + 25 → 192.168.1.31
+```||
+||bigint + inet → inet|
+Adds an offset to an address. (NOT SUPPORTED)|
+```sql
+#200 + inet '::ffff:fff0:1' → ::ffff:255.240.0.201
+```||
+||inet - bigint → inet|
+Subtracts an offset from an address.|
+```sql
+inet '192.168.1.43' - 36 → 192.168.1.7
+```||
+||inet - inet → bigint|
+Computes the difference of two addresses.|
+```sql
+inet '192.168.1.43' - inet '192.168.1.19' → 24
+inet '::1' - inet '::ffff:1' → -4294901760
+```||
+|#
+
+Table 9.39. IP Address Functions
+
+#|
+||Function|Description|Example(s)||
+||abbrev ( inet ) → text|
+Creates an abbreviated display format as text. (The result is the same as the inet output function produces; it is “abbreviated” only in comparison to the result of an explicit cast to text, which for historical reasons will never suppress the netmask part.)|
+```sql
+abbrev(inet '10.1.0.0/32') → 10.1.0.0
+```||
+||abbrev ( cidr ) → text|
+Creates an abbreviated display format as text. (The abbreviation consists of dropping all-zero octets to the right of the netmask; more examples are in Table 8.22.)|
+```sql
+abbrev(cidr '10.1.0.0/16') → 10.1/16
+```||
+||broadcast ( inet ) → inet|
+Computes the broadcast address for the address's network.|
+```sql
+broadcast(inet '192.168.1.5/24') → 192.168.1.255/24
+```||
+||family ( inet ) → integer|
+Returns the address's family: 4 for IPv4, 6 for IPv6.|
+```sql
+family(inet '::1') → 6
+```||
+||host ( inet ) → text|
+Returns the IP address as text, ignoring the netmask.|
+```sql
+host(inet '192.168.1.0/24') → 192.168.1.0
+```||
+||hostmask ( inet ) → inet|
+Computes the host mask for the address's network.|
+```sql
+hostmask(inet '192.168.23.20/30') → 0.0.0.3
+```||
+||inet_merge ( inet, inet ) → cidr|
+Computes the smallest network that includes both of the given networks.|
+```sql
+inet_merge(inet '192.168.1.5/24', inet '192.168.2.5/24') → 192.168.0.0/22
+```||
+||inet_same_family ( inet, inet ) → boolean|
+Tests whether the addresses belong to the same IP family.|
+```sql
+inet_same_family(inet '192.168.1.5/24', inet '::1') → false
+```||
+||masklen ( inet ) → integer|
+Returns the netmask length in bits.|
+```sql
+masklen(inet '192.168.1.5/24') → 24
+```||
+||netmask ( inet ) → inet|
+Computes the network mask for the address's network.|
+```sql
+netmask(inet '192.168.1.5/24') → 255.255.255.0
+```||
+||network ( inet ) → cidr|
+Returns the network part of the address, zeroing out whatever is to the right of the netmask. (This is equivalent to casting the value to cidr.)|
+```sql
+network(inet '192.168.1.5/24') → 192.168.1.0/24
+```||
+||set_masklen ( inet, integer ) → inet|
+Sets the netmask length for an inet value. The address part does not change.|
+```sql
+set_masklen(inet '192.168.1.5/24', 16) → 192.168.1.5/16
+```||
+||set_masklen ( cidr, integer ) → cidr|
+Sets the netmask length for a cidr value. Address bits to the right of the new netmask are set to zero.|
+```sql
+set_masklen(cidr '192.168.1.0/24', 16) → 192.168.0.0/16
+```||
+||text ( inet ) → text|
+Returns the unabbreviated IP address and netmask length as text. (This has the same result as an explicit cast to text.)|
+```sql
+text(inet '192.168.1.5') → 192.168.1.5/32
+```||
+|#
+
+Tip
+The abbrev, host, and text functions are primarily intended to offer alternative display formats for IP addresses.
+
+The MAC address types, macaddr and macaddr8, support the usual comparison operators shown in Table 9.1 as well as the specialized functions shown in Table 9.40. In addition, they support the bitwise logical operators ~, & and | (NOT, AND and OR), just as shown above for IP addresses.
+
+Table 9.40. MAC Address Functions
+
+#|
+||Function|Description|Example(s)||
+||trunc ( macaddr ) → macaddr|
+Sets the last 3 bytes of the address to zero. The remaining prefix can be associated with a particular manufacturer (using data not included in PostgreSQL).|
+```sql
+trunc(macaddr '12:34:56:78:90:ab') → 12:34:56:00:00:00
+```||
+||trunc ( macaddr8 ) → macaddr8|
+Sets the last 5 bytes of the address to zero. The remaining prefix can be associated with a particular manufacturer (using data not included in PostgreSQL).|
+```sql
+trunc(macaddr8 '12:34:56:78:90:ab:cd:ef') → 12:34:56:00:00:00:00:00
+```||
+||macaddr8_set7bit ( macaddr8 ) → macaddr8|
+Sets the 7th bit of the address to one, creating what is known as modified EUI-64, for inclusion in an IPv6 address.|
+```sql
+macaddr8_set7bit(macaddr8 '00:34:56:ab:cd:ef') → 02:34:56:ff:fe:ab:cd:ef
+```||
+|#
+
+
+## 9.13. Text Search Functions and Operators (NOT SUPPORTED) {#text-search-functions}
+
+## 9.14. UUID Functions {#uuid-functions}
+
+PostgreSQL includes one function to generate a UUID:
+
+gen_random_uuid () → uuid
+
+This function returns a version 4 (random) UUID. This is the most commonly used type of UUID and is appropriate for most applications.
+
+## 9.15. XML Functions {#xml-functions}
+
+The functions and function-like expressions described in this section operate on values of type xml. See Section 8.13 for information about the xml type. The function-like expressions xmlparse and xmlserialize for converting to and from type xml are documented there, not in this section.
+
+Use of most of these functions requires PostgreSQL to have been built with configure --with-libxml.
+
+9.15.1. Producing XML Content
+
+A set of functions and function-like expressions is available for producing XML content from SQL data. As such, they are particularly suitable for formatting query results into XML documents for processing in client applications.
+
+9.15.1.1. Xmlcomment
+
+xmlcomment ( text ) → xml
+
+The function xmlcomment creates an XML value containing an XML comment with the specified text as content. The text cannot contain “--” or end with a “-”, otherwise the resulting construct would not be a valid XML comment. If the argument is null, the result is null.
+
+Example:
+```sql
+xmlcomment('hello') → <!--hello-->
+```
+
+9.15.1.2. Xmlconcat
+
+xmlconcat ( xml [, ...] ) → xml (NOT SUPPORTED)
+
+The function xmlconcat concatenates a list of individual XML values to create a single value containing an XML content fragment. Null values are omitted; the result is only null if there are no nonnull arguments.
+
+Example:
+```sql
+#xmlconcat('<abc/>', '<bar>foo</bar>') → <abc/><bar>foo</bar>
+```
+
+XML declarations, if present, are combined as follows. If all argument values have the same XML version declaration, that version is used in the result, else no version is used. If all argument values have the standalone declaration value “yes”, then that value is used in the result. If all argument values have a standalone declaration value and at least one is “no”, then that is used in the result. Else the result will have no standalone declaration. If the result is determined to require a standalone declaration but no version declaration, a version declaration with version 1.0 will be used because XML requires an XML declaration to contain a version declaration. Encoding declarations are ignored and removed in all cases.
+
+Example:
+```sql
+#xmlconcat('<?xml version="1.1"?><foo/>', '<?xml version="1.1" standalone="no"?><bar/>') → <?xml version="1.1"?><foo/><bar/>
+```
+
+9.15.1.3. Xmlelement
+
+xmlelement ( NAME name [, XMLATTRIBUTES ( attvalue [ AS attname ] [, ...] ) ] [, content [, ...]] ) → xml (NOT SUPPORTED)
+
+```sql
+#xmlelement(name foo) → <foo/>
+#xmlelement(name foo, xmlattributes('xyz' as bar)) → <foo bar="xyz"/>
+#xmlelement(name foo, xmlattributes(current_date as bar), 'cont', 'ent') ~→ <foo bar="2007-01-26">content</foo>
+```
+
+Element and attribute names that are not valid XML names are escaped by replacing the offending characters by the sequence _xHHHH_, where HHHH is the character's Unicode codepoint in hexadecimal notation. For example:
+
+```sql
+#xmlelement(name "foo$bar", xmlattributes('xyz' as "a&b")) → <foo_x0024_bar a_x0026_b="xyz"/>
+```
+
+An explicit attribute name need not be specified if the attribute value is a column reference, in which case the column's name will be used as the attribute name by default. In other cases, the attribute must be given an explicit name. So this example is valid:
+
+Element content, if specified, will be formatted according to its data type. If the content is itself of type xml, complex XML documents can be constructed. For example:
+
+```sql
+#xmlelement(name foo, xmlattributes('xyz' as bar), xmlelement(name abc), xmlcomment('test'), xmlelement(name xyz)) → <foo bar="xyz"><abc/><!--test--><xyz/></foo>
+```
+
+Content of other types will be formatted into valid XML character data. This means in particular that the characters <, >, and & will be converted to entities. Binary data (data type bytea) will be represented in base64 or hex encoding, depending on the setting of the configuration parameter xmlbinary. The particular behavior for individual data types is expected to evolve in order to align the PostgreSQL mappings with those specified in SQL:2006 and later, as discussed in Section D.3.1.3.
+
+9.15.1.4. Xmlforest
+
+xmlforest ( content [ AS name ] [, ...] ) → xml (NOT SUPPORTED)
+
+9.15.1.5. Xmlpi
+
+xmlpi ( NAME name [, content ] ) → xml (NOT SUPPORTED)
+
+9.15.1.6. Xmlroot
+
+xmlroot ( xml, VERSION {text|NO VALUE} [, STANDALONE {YES|NO|NO VALUE} ] ) → xml (NOT SUPPORTED)
+
+9.15.1.7. Xmlagg
+
+xmlagg ( xml ) → xml
+
+The function xmlagg is, unlike the other functions described here, an aggregate function. It concatenates the input values to the aggregate function call, much like xmlconcat does, except that concatenation occurs across rows rather than across expressions in a single row. See Section 9.21 for additional information about aggregate functions.
+
+Example:
+```sql
+SELECT xmlagg(x::xml) from (values ('<a/>'),('<b/>')) a(x) → <a/><b/>
+```
+
+9.15.2. XML Predicates
+The expressions described in this section check properties of xml values.
+
+9.15.2.1. IS DOCUMENT
+
+xml IS DOCUMENT → boolean (NOT SUPPORTED)
+
+The expression IS DOCUMENT returns true if the argument XML value is a proper XML document, false if it is not (that is, it is a content fragment), or null if the argument is null. See Section 8.13 about the difference between documents and content fragments.
+
+9.15.2.2. IS NOT DOCUMENT
+
+xml IS NOT DOCUMENT → boolean (NOT SUPPORTED)
+
+The expression IS NOT DOCUMENT returns false if the argument XML value is a proper XML document, true if it is not (that is, it is a content fragment), or null if the argument is null.
+
+9.15.2.3. XMLEXISTS
+
+XMLEXISTS ( text PASSING [BY {REF|VALUE}] xml [BY {REF|VALUE}] ) → boolean
+
+The function xmlexists evaluates an XPath 1.0 expression (the first argument), with the passed XML value as its context item. The function returns false if the result of that evaluation yields an empty node-set, true if it yields any other value. The function returns null if any argument is null. A nonnull value passed as the context item must be an XML document, not a content fragment or any non-XML value.
+
+Example:
+```sql
+xmlexists('//town[text() = ''Toronto'']' PASSING BY VALUE '<towns><town>Toronto</town><town>Ottawa</town></towns>') → true
+```
+
+9.15.2.4. Xml_is_well_formed (NOT SUPPORTED)
+
+xml_is_well_formed ( text ) → boolean
+xml_is_well_formed_document ( text ) → boolean
+xml_is_well_formed_content ( text ) → boolean
+
+9.15.3. Processing XML
+To process values of data type xml, PostgreSQL offers the functions xpath and xpath_exists, which evaluate XPath 1.0 expressions, and the XMLTABLE table function.
+
+9.15.3.1. Xpath
+
+xpath ( xpath text, xml xml [, nsarray text[] ] ) → xml[] (NOT SUPPORTED)
+
+
+9.15.3.2. Xpath_exists
+
+xpath_exists ( xpath text, xml xml [, nsarray text[] ] ) → boolean
+
+The function xpath_exists is a specialized form of the xpath function. Instead of returning the individual XML values that satisfy the XPath 1.0 expression, this function returns a Boolean indicating whether the query was satisfied or not (specifically, whether it produced any value other than an empty node-set). This function is equivalent to the XMLEXISTS predicate, except that it also offers support for a namespace mapping argument.
+
+Example:
+```sql
+xpath_exists('/my:a/text()', '<my:a xmlns:my="http://example.com">test</my:a>',ARRAY[ARRAY['my', 'http://example.com']]) → true
+```
+
+9.15.3.3. Xmltable
+
+XMLTABLE (
+ [ XMLNAMESPACES ( namespace_uri AS namespace_name [, ...] ), ]
+ row_expression PASSING [BY {REF|VALUE}] document_expression [BY {REF|VALUE}]
+ COLUMNS name { type [PATH column_expression] [DEFAULT default_expression] [NOT NULL | NULL]
+ | FOR ORDINALITY }
+ [, ...]
+) → setof record (NOT SUPPORTED)
+
+9.15.4. Mapping Tables to XML (NOT SUPPORTED)
+
+The following functions map the contents of relational tables to XML values. They can be thought of as XML export functionality:
+
+table_to_xml ( table regclass, nulls boolean,
+ tableforest boolean, targetns text ) → xml
+query_to_xml ( query text, nulls boolean,
+ tableforest boolean, targetns text ) → xml
+cursor_to_xml ( cursor refcursor, count integer, nulls boolean,
+ tableforest boolean, targetns text ) → xml
+
+## 9.16. JSON Functions and Operators {#json-functions}
+
+This section describes:
+
+functions and operators for processing and creating JSON data
+
+the SQL/JSON path language
+
+To learn more about the SQL/JSON standard, see [sqltr-19075-6]. For details on JSON types supported in PostgreSQL, see Section 8.14.
+
+9.16.1. Processing and Creating JSON Data
+
+Table 9.44 shows the operators that are available for use with JSON data types (see Section 8.14). In addition, the usual comparison operators shown in Table 9.1 are available for jsonb, though not for json. The comparison operators follow the ordering rules for B-tree operations outlined in Section 8.14.4. See also Section 9.21 for the aggregate function json_agg which aggregates record values as JSON, the aggregate function json_object_agg which aggregates pairs of values into a JSON object, and their jsonb equivalents, jsonb_agg and jsonb_object_agg.
+
+Table 9.44. json and jsonb Operators
+
+#|
+||Operator|Description|Example(s)||
+||json -> integer → json
+jsonb -> integer → jsonb|
+Extracts n'th element of JSON array (array elements are indexed from zero, but negative integers count from the end).|
+```sql
+'[{"a":"foo"},{"b":"bar"},{"c":"baz"}]'::json -> 2 → {"c":"baz"}
+'[{"a":"foo"},{"b":"bar"},{"c":"baz"}]'::json -> -3 → {"a":"foo"}
+```||
+||json -> text → json
+jsonb -> text → jsonb|
+Extracts JSON object field with the given key.|
+```sql
+'{"a": {"b":"foo"}}'::json -> 'a' → {"b":"foo"}
+```||
+||json ->> integer → text
+jsonb ->> integer → text|
+Extracts n'th element of JSON array, as text.|
+```sql
+'[1,2,3]'::json ->> 2 → 3
+```||
+||json ->> text → text
+jsonb ->> text → text|
+Extracts JSON object field with the given key, as text.|
+```sql
+'{"a":1,"b":2}'::json ->> 'b' → 2
+```||
+||json #> text[] → json
+jsonb #> text[] → jsonb|
+Extracts JSON sub-object at the specified path, where path elements can be either field keys or array indexes. (NOT SUPPORTED)|
+```sql
+#'{"a": {"b": ["foo","bar"]}}'::json #> '{a,b,1}' → "bar"
+```||
+||json #>> text[] → text
+jsonb #>> text[] → text|
+Extracts JSON sub-object at the specified path as text. (NOT SUPPORTED)|
+```sql
+#'{"a": {"b": ["foo","bar"]}}'::json #>> '{a,b,1}' → bar
+```||
+|#
+
+Note
+The field/element/path extraction operators return NULL, rather than failing, if the JSON input does not have the right structure to match the request; for example if no such key or array element exists.
+
+Some further operators exist only for jsonb, as shown in Table 9.45. Section 8.14.4 describes how these operators can be used to effectively search indexed jsonb data.
+
+Table 9.45. Additional jsonb Operators
+
+#|
+||Operator|Description|Example(s)||
+||jsonb @> jsonb → boolean|
+Does the first JSON value contain the second? (See Section 8.14.3 for details about containment.)|
+```sql
+'{"a":1, "b":2}'::jsonb @> '{"b":2}'::jsonb → true
+```||
+||jsonb <@ jsonb → boolean|
+Is the first JSON value contained in the second?|
+```sql
+'{"b":2}'::jsonb <@ '{"a":1, "b":2}'::jsonb → true
+```||
+||jsonb ? text → boolean|
+Does the text string exist as a top-level key or array element within the JSON value?|
+```sql
+'{"a":1, "b":2}'::jsonb ? 'b' → true
+'["a", "b", "c"]'::jsonb ? 'b' → true
+```||
+||jsonb ?\| text[] → boolean|
+Do any of the strings in the text array exist as top-level keys or array elements?|
+```sql
+'{"a":1, "b":2, "c":3}'::jsonb ?| array['b', 'd'] → true
+```||
+||jsonb ?& text[] → boolean|
+Do all of the strings in the text array exist as top-level keys or array elements?|
+```sql
+'["a", "b", "c"]'::jsonb ?& array['a', 'b'] → true
+```||
+||jsonb \|\| jsonb → jsonb|
+Concatenates two jsonb values. Concatenating two arrays generates an array containing all the elements of each input. Concatenating two objects generates an object containing the union of their keys, taking the second object's value when there are duplicate keys. All other cases are treated by converting a non-array input into a single-element array, and then proceeding as for two arrays. Does not operate recursively: only the top-level array or object structure is merged.|
+```sql
+'["a", "b"]'::jsonb || '["a", "d"]'::jsonb → ["a", "b", "a", "d"]
+'{"a": "b"}'::jsonb || '{"c": "d"}'::jsonb → {"a": "b", "c": "d"}
+'[1, 2]'::jsonb || '3'::jsonb → [1, 2, 3]
+'{"a": "b"}'::jsonb || '42'::jsonb → [{"a": "b"}, 42]
+```
+
+To append an array to another array as a single entry, wrap it in an additional layer of array, for example:
+
+```sql
+'[1, 2]'::jsonb || jsonb_build_array('[3, 4]'::jsonb) → [1, 2, [3, 4]]
+```||
+||jsonb - text → jsonb|
+Deletes a key (and its value) from a JSON object, or matching string value(s) from a JSON array.|
+```sql
+'{"a": "b", "c": "d"}'::jsonb - 'a' → {"c": "d"}
+'["a", "b", "c", "b"]'::jsonb - 'b' → ["a", "c"]
+```||
+||jsonb - text[] → jsonb|
+Deletes all matching keys or array elements from the left operand. (NOT SUPPORTED)|
+```sql
+#'{"a": "b", "c": "d"}'::jsonb - '{a,c}'::text[] → {}
+```||
+||jsonb - integer → jsonb|
+Deletes the array element with specified index (negative integers count from the end). Throws an error if JSON value is not an array.|
+```sql
+'["a", "b"]'::jsonb - 1 → ["a"]
+```||
+||jsonb #- text[] → jsonb|
+Deletes the field or array element at the specified path, where path elements can be either field keys or array indexes.|
+```sql
+'["a", {"b":1}]'::jsonb #- '{1,b}' → ["a", {}]
+```||
+||jsonb @? jsonpath → boolean|
+Does JSON path return any item for the specified JSON value?|
+```sql
+'{"a":[1,2,3,4,5]}'::jsonb @? '$.a[*] ? (@ > 2)' → true
+```||
+||jsonb @@ jsonpath → boolean|
+Returns the result of a JSON path predicate check for the specified JSON value. Only the first item of the result is taken into account. If the result is not Boolean, then NULL is returned.|
+```sql
+'{"a":[1,2,3,4,5]}'::jsonb @@ '$.a[*] > 2' → true
+```||
+|#
+
+Note
+The jsonpath operators @? and @@ suppress the following errors: missing object field or array element, unexpected JSON item type, datetime and numeric errors. The jsonpath-related functions described below can also be told to suppress these types of errors. This behavior might be helpful when searching JSON document collections of varying structure.
+
+Table 9.46 shows the functions that are available for constructing json and jsonb values.
+
+Table 9.46. JSON Creation Functions
+
+#|
+||Function|Description|Example(s)||
+||to_json ( anyelement ) → json
+to_jsonb ( anyelement ) → jsonb|
+Converts any SQL value to json or jsonb. Arrays and composites are converted recursively to arrays and objects (multidimensional arrays become arrays of arrays in JSON). Otherwise, if there is a cast from the SQL data type to json, the cast function will be used to perform the conversion;[a] otherwise, a scalar JSON value is produced. For any scalar other than a number, a Boolean, or a null value, the text representation will be used, with escaping as necessary to make it a valid JSON string value. (NOT SUPPORTED)|
+```sql
+#to_json('Fred said "Hi."'::text) → "Fred said \"Hi.\""
+#to_jsonb(row(42, 'Fred said "Hi."'::text)) → {"f1": 42, "f2": "Fred said \"Hi.\""}
+```||
+||array_to_json ( anyarray [, boolean ] ) → json|
+Converts an SQL array to a JSON array. The behavior is the same as to_json except that line feeds will be added between top-level array elements if the optional boolean parameter is true.|
+```sql
+array_to_json('{{1,5},{99,100}}'::int[]) → [[1,5],[99,100]]
+```||
+||row_to_json ( record [, boolean ] ) → json|
+Converts an SQL composite value to a JSON object. The behavior is the same as to_json except that line feeds will be added between top-level elements if the optional boolean parameter is true. (NOT SUPPORTED)|
+```sql
+#row_to_json(row(1,'foo')) → {"f1":1,"f2":"foo"}
+```||
+||json_build_array ( VARIADIC "any" ) → json
+jsonb_build_array ( VARIADIC "any" ) → jsonb|
+Builds a possibly-heterogeneously-typed JSON array out of a variadic argument list. Each argument is converted as per to_json or to_jsonb. (NOT SUPPORTED)|
+```sql
+json_build_array(1, 2, 'foo', 4, 5) → [1, 2, "foo", 4, 5]
+```||
+||json_build_object ( VARIADIC "any" ) → json
+jsonb_build_object ( VARIADIC "any" ) → jsonb|
+Builds a JSON object out of a variadic argument list. By convention, the argument list consists of alternating keys and values. Key arguments are coerced to text; value arguments are converted as per to_json or to_jsonb. (NOT SUPPORTED)|
+```sql
+#json_build_object('foo', 1, 2, row(3,'bar')) → {"foo" : 1, "2" : {"f1":3,"f2":"bar"}}
+```||
+||json_object ( text[] ) → json
+jsonb_object ( text[] ) → jsonb|
+Builds a JSON object out of a text array. The array must have either exactly one dimension with an even number of members, in which case they are taken as alternating key/value pairs, or two dimensions such that each inner array has exactly two elements, which are taken as a key/value pair. All values are converted to JSON strings.|
+```sql
+json_object('{a, 1, b, "def", c, 3.5}') → {"a" : "1", "b" : "def", "c" : "3.5"}
+json_object('{{a, 1}, {b, "def"}, {c, 3.5}}') → {"a" : "1", "b" : "def", "c" : "3.5"}
+```||
+||json_object ( keys text[], values text[] ) → json
+jsonb_object ( keys text[], values text[] ) → jsonb|
+This form of json_object takes keys and values pairwise from separate text arrays. Otherwise it is identical to the one-argument form.|
+```sql
+json_object('{a,b}', '{1,2}') → {"a" : "1", "b" : "2"}
+```||
+|#
+
+Table 9.47 shows the functions that are available for processing json and jsonb values.
+
+Table 9.47. JSON Processing Functions
+
+#|
+||Function|Description|Example(s)||
+||json_array_elements ( json ) → setof json
+jsonb_array_elements ( jsonb ) → setof jsonb|
+Expands the top-level JSON array into a set of JSON values.|
+```sql
+SELECT * FROM json_array_elements('[1,true, [2,false]]') as a → [
+1
+true
+[2,false]
+]
+```||
+||json_array_elements_text ( json ) → setof text
+jsonb_array_elements_text ( jsonb ) → setof text|
+Expands the top-level JSON array into a set of text values.|
+```sql
+SELECT * FROM json_array_elements_text('["foo", "bar"]') as a → [
+foo
+bar
+]
+```||
+||json_array_length ( json ) → integer
+jsonb_array_length ( jsonb ) → integer|
+Returns the number of elements in the top-level JSON array.|
+```sql
+json_array_length('[1,2,3,{"f1":1,"f2":[5,6]},4]') → 5
+jsonb_array_length('[]') → 0
+```||
+||json_each ( json ) → setof record ( key text, value json )
+jsonb_each ( jsonb ) → setof record ( key text, value jsonb )|
+Expands the top-level JSON object into a set of key/value pairs.|
+
+```sql
+SELECT * FROM json_each('{"a":"foo", "b":"bar"}') as a → [
+a,"foo"
+b,"bar"
+]
+```||
+||json_each_text ( json ) → setof record ( key text, value text )
+jsonb_each_text ( jsonb ) → setof record ( key text, value text )|
+Expands the top-level JSON object into a set of key/value pairs. The returned values will be of type text.|
+```sql
+SELECT * FROM json_each_text('{"a":"foo", "b":"bar"}') as a → [
+a,foo
+b,bar
+]
+```||
+||json_extract_path ( from_json json, VARIADIC path_elems text[] ) → json
+jsonb_extract_path ( from_json jsonb, VARIADIC path_elems text[] ) → jsonb|
+Extracts JSON sub-object at the specified path. (This is functionally equivalent to the #> operator, but writing the path out as a variadic list can be more convenient in some cases.) (NOT SUPPORTED)|
+```sql
+json_extract_path('{"f2":{"f3":1},"f4":{"f5":99,"f6":"foo"}}', 'f4', 'f6') → "foo"
+```||
+||json_extract_path_text ( from_json json, VARIADIC path_elems text[] ) → text
+jsonb_extract_path_text ( from_json jsonb, VARIADIC path_elems text[] ) → text|
+Extracts JSON sub-object at the specified path as text. (This is functionally equivalent to the #>> operator.) (NOT SUPPORTED)|
+```sql
+json_extract_path_text('{"f2":{"f3":1},"f4":{"f5":99,"f6":"foo"}}', 'f4', 'f6') → foo
+```||
+||json_object_keys ( json ) → setof text
+jsonb_object_keys ( jsonb ) → setof text|
+Returns the set of keys in the top-level JSON object.|
+```sql
+SELECT * FROM json_object_keys('{"f1":"abc","f2":{"f3":"a", "f4":"b"}}') as a → [
+f1
+f2
+]
+```||
+||json_populate_record ( base anyelement, from_json json ) → anyelement
+jsonb_populate_record ( base anyelement, from_json jsonb ) → anyelement|
+Expands the top-level JSON object to a row having the composite type of the base argument. The JSON object is scanned for fields whose names match column names of the output row type, and their values are inserted into those columns of the output. (Fields that do not correspond to any output column name are ignored.) In typical use, the value of base is just NULL, which means that any output columns that do not match any object field will be filled with nulls. However, if base isn't NULL then the values it contains will be used for unmatched columns. (NOT SUPPORTED)
+
+To convert a JSON value to the SQL type of an output column, the following rules are applied in sequence:
+
+A JSON null value is converted to an SQL null in all cases.
+
+If the output column is of type json or jsonb, the JSON value is just reproduced exactly.
+
+If the output column is a composite (row) type, and the JSON value is a JSON object, the fields of the object are converted to columns of the output row type by recursive application of these rules.
+
+Likewise, if the output column is an array type and the JSON value is a JSON array, the elements of the JSON array are converted to elements of the output array by recursive application of these rules.
+
+Otherwise, if the JSON value is a string, the contents of the string are fed to the input conversion function for the column's data type.
+
+Otherwise, the ordinary text representation of the JSON value is fed to the input conversion function for the column's data type.
+
+While the example below uses a constant JSON value, typical use would be to reference a json or jsonb column laterally from another table in the query's FROM clause. Writing json_populate_record in the FROM clause is good practice, since all of the extracted columns are available for use without duplicate function calls.
+
+```sql
+#CREATE TYPE subrowtype as (d int, e text);
+#CREATE type myrowtype as (a int, b text[], c subrowtype);
+#SELECT * FROM json_populate_record(null::myrowtype, '{"a": 1, "b": ["2", "a b"], "c": {"d": 4, "e": "a b c"}, "x": "foo"}') → 1,{2,"a b"},(4,"a b c")
+```||
+||json_populate_recordset ( base anyelement, from_json json ) → setof anyelement
+jsonb_populate_recordset ( base anyelement, from_json jsonb ) → setof anyelement|
+Expands the top-level JSON array of objects to a set of rows having the composite type of the base argument. Each element of the JSON array is processed as described above for json[b]_populate_record. (NOT SUPPORTED)|
+```sql
+#CREATE TYPE twoints as (a int, b int);
+#SELECT * FROM json_populate_recordset(null::twoints, '[{"a":1,"b":2}, {"a":3,"b":4}]') → [
+1,2
+3,4
+]
+```||
+||json_to_record ( json ) → record
+jsonb_to_record ( jsonb ) → record|
+Expands the top-level JSON object to a row having the composite type defined by an AS clause. (As with all functions returning record, the calling query must explicitly define the structure of the record with an AS clause.) The output record is filled from fields of the JSON object, in the same way as described above for json[b]_populate_record. Since there is no input record value, unmatched columns are always filled with nulls. (NOT SUPPORTED)|
+```sql
+#CREATE TYPE myrowtype as (a int, b text);
+#SELECT * FROM json_to_record('{"a":1,"b":[1,2,3],"c":[1,2,3],"e":"bar","r": {"a": 123, "b": "a b c"}}') as x(a int, b text, c int[], d text, r myrowtype) → 1,[1,2,3],{1,2,3},(123,"a b c")
+```||
+||json_to_recordset ( json ) → setof record
+jsonb_to_recordset ( jsonb ) → setof record|
+Expands the top-level JSON array of objects to a set of rows having the composite type defined by an AS clause. (As with all functions returning record, the calling query must explicitly define the structure of the record with an AS clause.) Each element of the JSON array is processed as described above for json[b]_populate_record. (NOT SUPPORTED)|
+```SQL
+#SELECT * from json_to_recordset('[{"a":1,"b":"foo"}, {"a":"2","c":"bar"}]') as x(a int, b text) → [
+1,foo
+2,
+]
+```||
+||jsonb_set ( target jsonb, path text[], new_value jsonb [, create_if_missing boolean ] ) → jsonb|
+Returns target with the item designated by path replaced by new_value, or with new_value added if create_if_missing is true (which is the default) and the item designated by path does not exist. All earlier steps in the path must exist, or the target is returned unchanged. As with the path oriented operators, negative integers that appear in the path count from the end of JSON arrays. If the last path step is an array index that is out of range, and create_if_missing is true, the new value is added at the beginning of the array if the index is negative, or at the end of the array if it is positive.|
+```sql
+jsonb_set('[{"f1":1,"f2":null},2,null,3]', '{0,f1}', '[2,3,4]', false) → [{"f1": [2, 3, 4], "f2": null}, 2, null, 3]
+jsonb_set('[{"f1":1,"f2":null},2]', '{0,f3}', '[2,3,4]') → [{"f1": 1, "f2": null, "f3": [2, 3, 4]}, 2]
+```||
+||jsonb_set_lax ( target jsonb, path text[], new_value jsonb [, create_if_missing boolean [, null_value_treatment text ]] ) → jsonb|
+If new_value is not NULL, behaves identically to jsonb_set. Otherwise behaves according to the value of null_value_treatment which must be one of 'raise_exception', 'use_json_null', 'delete_key', or 'return_target'. The default is 'use_json_null'.|
+```sql
+jsonb_set_lax('[{"f1":1,"f2":null},2,null,3]', '{0,f1}', null) → [{"f1": null, "f2": null}, 2, null, 3]
+jsonb_set_lax('[{"f1":99,"f2":null},2]', '{0,f3}', null, true, 'return_target') → [{"f1": 99, "f2": null}, 2]
+```||
+||jsonb_insert ( target jsonb, path text[], new_value jsonb [, insert_after boolean ] ) → jsonb|
+Returns target with new_value inserted. If the item designated by the path is an array element, new_value will be inserted before that item if insert_after is false (which is the default), or after it if insert_after is true. If the item designated by the path is an object field, new_value will be inserted only if the object does not already contain that key. All earlier steps in the path must exist, or the target is returned unchanged. As with the path oriented operators, negative integers that appear in the path count from the end of JSON arrays. If the last path step is an array index that is out of range, the new value is added at the beginning of the array if the index is negative, or at the end of the array if it is positive.|
+```sql
+jsonb_insert('{"a": [0,1,2]}', '{a, 1}', '"new_value"') → {"a": [0, "new_value", 1, 2]}
+jsonb_insert('{"a": [0,1,2]}', '{a, 1}', '"new_value"', true) → {"a": [0, 1, "new_value", 2]}
+```||
+||json_strip_nulls ( json ) → json
+jsonb_strip_nulls ( jsonb ) → jsonb|
+Deletes all object fields that have null values from the given JSON value, recursively. Null values that are not object fields are untouched.|
+```sql
+json_strip_nulls('[{"f1":1, "f2":null}, 2, null, 3]') → [{"f1":1},2,null,3]
+```||
+||jsonb_path_exists ( target jsonb, path jsonpath [, vars jsonb [, silent boolean ]] ) → boolean|
+Checks whether the JSON path returns any item for the specified JSON value. If the vars argument is specified, it must be a JSON object, and its fields provide named values to be substituted into the jsonpath expression. If the silent argument is specified and is true, the function suppresses the same errors as the @? and @@ operators do.|
+```sql
+jsonb_path_exists('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2, "max":4}', false) → true
+```||
+||jsonb_path_match ( target jsonb, path jsonpath [, vars jsonb [, silent boolean ]] ) → boolean|
+Returns the result of a JSON path predicate check for the specified JSON value. Only the first item of the result is taken into account. If the result is not Boolean, then NULL is returned. The optional vars and silent arguments act the same as for jsonb_path_exists.|
+```sql
+jsonb_path_match('{"a":[1,2,3,4,5]}', 'exists($.a[*] ? (@ >= $min && @ <= $max))', '{"min":2, "max":4}', false) → true
+```||
+||jsonb_path_query ( target jsonb, path jsonpath [, vars jsonb [, silent boolean ]] ) → setof jsonb|
+Returns all JSON items returned by the JSON path for the specified JSON value. The optional vars and silent arguments act the same as for jsonb_path_exists.|
+```sql
+SELECT * FROM jsonb_path_query('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2, "max":4}', false) as a → [
+2
+3
+4
+]
+```||
+||jsonb_path_query_array ( target jsonb, path jsonpath [, vars jsonb [, silent boolean ]] ) → jsonb|
+Returns all JSON items returned by the JSON path for the specified JSON value, as a JSON array. The optional vars and silent arguments act the same as for jsonb_path_exists.|
+```sql
+jsonb_path_query_array('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2, "max":4}', false) → [2, 3, 4]
+```||
+||jsonb_path_query_first ( target jsonb, path jsonpath [, vars jsonb [, silent boolean ]] ) → jsonb|
+Returns the first JSON item returned by the JSON path for the specified JSON value. Returns NULL if there are no results. The optional vars and silent arguments act the same as for jsonb_path_exists.|
+```sql
+jsonb_path_query_first('{"a":[1,2,3,4,5]}', '$.a[*] ? (@ >= $min && @ <= $max)', '{"min":2, "max":4}', false) → 2
+```||
+||jsonb_path_exists_tz ( target jsonb, path jsonpath [, vars jsonb [, silent boolean ]] ) → boolean
+jsonb_path_match_tz ( target jsonb, path jsonpath [, vars jsonb [, silent boolean ]] ) → boolean
+jsonb_path_query_tz ( target jsonb, path jsonpath [, vars jsonb [, silent boolean ]] ) → setof jsonb
+jsonb_path_query_array_tz ( target jsonb, path jsonpath [, vars jsonb [, silent boolean ]] ) → jsonb
+jsonb_path_query_first_tz ( target jsonb, path jsonpath [, vars jsonb [, silent boolean ]] ) → jsonb|
+These functions act like their counterparts described above without the _tz suffix, except that these functions support comparisons of date/time values that require timezone-aware conversions. The example below requires interpretation of the date-only value 2015-08-02 as a timestamp with time zone, so the result depends on the current TimeZone setting. Due to this dependency, these functions are marked as stable, which means these functions cannot be used in indexes. Their counterparts are immutable, and so can be used in indexes; but they will throw errors if asked to make such comparisons. (NOT SUPPORTED)|
+```sql
+jsonb_path_exists_tz('["2015-08-01 12:00:00-05"]', '$[*] ? (@.datetime() < "2015-08-02".datetime())', '{}', false) → true
+```||
+||jsonb_pretty ( jsonb ) → text|
+Converts the given JSON value to pretty-printed, indented text.|
+```sql
+jsonb_pretty('[{"f1":1,"f2":null}, 2]') → """[
+ {
+ "f1": 1,
+ "f2": null
+ },
+ 2
+]"""
+```||
+||json_typeof ( json ) → text
+jsonb_typeof ( jsonb ) → text|
+Returns the type of the top-level JSON value as a text string. Possible types are object, array, string, number, boolean, and null. (The null result should not be confused with an SQL NULL; see the examples.)|
+```sql
+json_typeof('-123.4') → number
+json_typeof('null'::json) → null
+json_typeof(NULL::json) IS NULL → true
+```||
+|#
+
+9.16.2. The SQL/JSON Path Language
+
+SQL/JSON path expressions specify the items to be retrieved from the JSON data, similar to XPath expressions used for SQL access to XML. In PostgreSQL, path expressions are implemented as the jsonpath data type and can use any elements described in Section 8.14.7.
+
+JSON query functions and operators pass the provided path expression to the path engine for evaluation. If the expression matches the queried JSON data, the corresponding JSON item, or set of items, is returned. Path expressions are written in the SQL/JSON path language and can include arithmetic expressions and functions.
+
+A path expression consists of a sequence of elements allowed by the jsonpath data type. The path expression is normally evaluated from left to right, but you can use parentheses to change the order of operations. If the evaluation is successful, a sequence of JSON items is produced, and the evaluation result is returned to the JSON query function that completes the specified computation.
+
+To refer to the JSON value being queried (the context item), use the $ variable in the path expression. It can be followed by one or more accessor operators, which go down the JSON structure level by level to retrieve sub-items of the context item. Each operator that follows deals with the result of the previous evaluation step.
+
+For example, suppose you have some JSON data from a GPS tracker that you would like to parse, such as:
+
+{
+ "track": {
+ "segments": [
+ {
+ "location": [ 47.763, 13.4034 ],
+ "start time": "2018-10-14 10:05:14",
+ "HR": 73
+ },
+ {
+ "location": [ 47.706, 13.2635 ],
+ "start time": "2018-10-14 10:39:21",
+ "HR": 135
+ }
+ ]
+ }
+}
+To retrieve the available track segments, you need to use the .key accessor operator to descend through surrounding JSON objects:
+
+$.track.segments
+To retrieve the contents of an array, you typically use the [*] operator. For example, the following path will return the location coordinates for all the available track segments:
+
+$.track.segments[*].location
+To return the coordinates of the first segment only, you can specify the corresponding subscript in the [] accessor operator. Recall that JSON array indexes are 0-relative:
+
+$.track.segments[0].location
+The result of each path evaluation step can be processed by one or more jsonpath operators and methods listed in Section 9.16.2.2. Each method name must be preceded by a dot. For example, you can get the size of an array:
+
+$.track.segments.size()
+More examples of using jsonpath operators and methods within path expressions appear below in Section 9.16.2.2.
+
+When defining a path, you can also use one or more filter expressions that work similarly to the WHERE clause in SQL. A filter expression begins with a question mark and provides a condition in parentheses:
+
+? (condition)
+Filter expressions must be written just after the path evaluation step to which they should apply. The result of that step is filtered to include only those items that satisfy the provided condition. SQL/JSON defines three-valued logic, so the condition can be true, false, or unknown. The unknown value plays the same role as SQL NULL and can be tested for with the is unknown predicate. Further path evaluation steps use only those items for which the filter expression returned true.
+
+The functions and operators that can be used in filter expressions are listed in Table 9.49. Within a filter expression, the @ variable denotes the value being filtered (i.e., one result of the preceding path step). You can write accessor operators after @ to retrieve component items.
+
+For example, suppose you would like to retrieve all heart rate values higher than 130. You can achieve this using the following expression:
+
+$.track.segments[*].HR ? (@ > 130)
+To get the start times of segments with such values, you have to filter out irrelevant segments before returning the start times, so the filter expression is applied to the previous step, and the path used in the condition is different:
+
+$.track.segments[*] ? (@.HR > 130)."start time"
+You can use several filter expressions in sequence, if required. For example, the following expression selects start times of all segments that contain locations with relevant coordinates and high heart rate values:
+
+$.track.segments[*] ? (@.location[1] < 13.4) ? (@.HR > 130)."start time"
+Using filter expressions at different nesting levels is also allowed. The following example first filters all segments by location, and then returns high heart rate values for these segments, if available:
+
+$.track.segments[*] ? (@.location[1] < 13.4).HR ? (@ > 130)
+You can also nest filter expressions within each other:
+
+$.track ? (exists(@.segments[*] ? (@.HR > 130))).segments.size()
+This expression returns the size of the track if it contains any segments with high heart rate values, or an empty sequence otherwise.
+
+PostgreSQL's implementation of the SQL/JSON path language has the following deviations from the SQL/JSON standard:
+
+A path expression can be a Boolean predicate, although the SQL/JSON standard allows predicates only in filters. This is necessary for implementation of the @@ operator. For example, the following jsonpath expression is valid in PostgreSQL:
+
+$.track.segments[*].HR < 70
+There are minor differences in the interpretation of regular expression patterns used in like_regex filters, as described in Section 9.16.2.3.
+
+9.16.2.1. Strict And Lax Modes
+
+When you query JSON data, the path expression may not match the actual JSON data structure. An attempt to access a non-existent member of an object or element of an array results in a structural error. SQL/JSON path expressions have two modes of handling structural errors:
+
+lax (default) — the path engine implicitly adapts the queried data to the specified path. Any remaining structural errors are suppressed and converted to empty SQL/JSON sequences.
+
+strict — if a structural error occurs, an error is raised.
+
+The lax mode facilitates matching of a JSON document structure and path expression if the JSON data does not conform to the expected schema. If an operand does not match the requirements of a particular operation, it can be automatically wrapped as an SQL/JSON array or unwrapped by converting its elements into an SQL/JSON sequence before performing this operation. Besides, comparison operators automatically unwrap their operands in the lax mode, so you can compare SQL/JSON arrays out-of-the-box. An array of size 1 is considered equal to its sole element. Automatic unwrapping is not performed only when:
+
+The path expression contains type() or size() methods that return the type and the number of elements in the array, respectively.
+
+The queried JSON data contain nested arrays. In this case, only the outermost array is unwrapped, while all the inner arrays remain unchanged. Thus, implicit unwrapping can only go one level down within each path evaluation step.
+
+For example, when querying the GPS data listed above, you can abstract from the fact that it stores an array of segments when using the lax mode:
+
+lax $.track.segments.location
+In the strict mode, the specified path must exactly match the structure of the queried JSON document to return an SQL/JSON item, so using this path expression will cause an error. To get the same result as in the lax mode, you have to explicitly unwrap the segments array:
+
+strict $.track.segments[*].location
+The .\*\* accessor can lead to surprising results when using the lax mode. For instance, the following query selects every HR value twice:
+
+lax $.\*\*.HR
+This happens because the .\*\* accessor selects both the segments array and each of its elements, while the .HR accessor automatically unwraps arrays when using the lax mode. To avoid surprising results, we recommend using the .\*\* accessor only in the strict mode. The following query selects each HR value just once:
+
+strict $.\*\*.HR
+
+9.16.2.2. SQL/JSON Path Operators And Methods
+
+Table 9.48 shows the operators and methods available in jsonpath. Note that while the unary operators and methods can be applied to multiple values resulting from a preceding path step, the binary operators (addition etc.) can only be applied to single values.
+
+Table 9.48. jsonpath Operators and Methods
+
+#|
+||Operator/Method|Description|Example(s)||
+||number + number → number|
+Addition|
+```sql
+jsonb_path_query_array('[2]', '$[0] + 3', '{}', false) → [5]
+```||
+||\+ number → number|
+Unary plus (no operation); unlike addition, this can iterate over multiple values|
+```sql
+jsonb_path_query_array('{"x": [2,3,4]}', '+ $.x', '{}', false) → [2, 3, 4]
+```||
+||number - number → number|
+Subtraction|
+```sql
+jsonb_path_query_array('[2]', '7 - $[0]', '{}', false) → [5]
+```||
+||\- number → number|
+Negation; unlike subtraction, this can iterate over multiple values|
+```sql
+jsonb_path_query_array('{"x": [2,3,4]}', '- $.x', '{}', false) → [-2, -3, -4]
+```||
+||number * number → number|
+Multiplication|
+```sql
+jsonb_path_query_array('[4]', '2 * $[0]', '{}', false) → [8]
+```||
+||number / number → number|
+Division|
+```sql
+jsonb_path_query_array('[8.5]', '$[0] / 2', '{}', false) → [4.2500000000000000]
+```||
+||number % number → number|
+Modulo (remainder)|
+```sql
+jsonb_path_query_array('[32]', '$[0] % 10', '{}', false) → [2]
+```||
+||value . type() → string|
+Type of the JSON item (see json_typeof)|
+```sql
+jsonb_path_query_array('[1, "2", {}]', '$[*].type()', '{}', false) → ["number", "string", "object"]
+```||
+||value . size() → number|
+Size of the JSON item (number of array elements, or 1 if not an array)|
+```sql
+jsonb_path_query_array('{"m": [11, 15]}', '$.m.size()', '{}', false) → [2]
+```||
+||value . double() → number|
+Approximate floating-point number converted from a JSON number or string|
+```sql
+jsonb_path_query_array('{"len": "1.9"}', '$.len.double() * 2', '{}', false) → [3.8]
+```||
+||number . ceiling() → number|
+Nearest integer greater than or equal to the given number|
+```sql
+jsonb_path_query_array('{"h": 1.3}', '$.h.ceiling()', '{}', false) → [2]
+```||
+||number . floor() → number|
+Nearest integer less than or equal to the given number|
+```sql
+jsonb_path_query_array('{"h": 1.7}', '$.h.floor()', '{}', false) → [1]
+```||
+||number . abs() → number|
+Absolute value of the given number|
+```sql
+jsonb_path_query_array('{"z": -0.3}', '$.z.abs()', '{}', false) → [0.3]
+```||
+||string . datetime() → datetime_type (see note)|
+Date/time value converted from a string (NOT SUPPORTED)|
+```sql
+jsonb_path_query_array('["2015-8-1", "2015-08-12"]', '$[*] ? (@.datetime() < "2015-08-2".datetime())', '{}', false) → ["2015-8-1"]
+```||
+||string . datetime(template) → datetime_type (see note)|
+Date/time value converted from a string using the specified to_timestamp template (NOT SUPPORTED)|
+```sql
+jsonb_path_query_array('["12:30", "18:40"]', '$[*].datetime("HH24:MI")', '{}', false) → ["12:30:00", "18:40:00"]
+```||
+||object . keyvalue() → array|
+The object's key-value pairs, represented as an array of objects containing three fields: "key", "value", and "id"; "id" is a unique identifier of the object the key-value pair belongs to|
+```sql
+jsonb_path_query_array('{"x": "20", "y": 32}', '$.keyvalue()', '{}', false) → [{"id": 0, "key": "x", "value": "20"}, {"id": 0, "key": "y", "value": 32}]
+```||
+|#
+
+Note
+The result type of the datetime() and datetime(template) methods can be date, timetz, time, timestamptz, or timestamp. Both methods determine their result type dynamically.
+
+The datetime() method sequentially tries to match its input string to the ISO formats for date, timetz, time, timestamptz, and timestamp. It stops on the first matching format and emits the corresponding data type.
+
+The datetime(template) method determines the result type according to the fields used in the provided template string.
+
+The datetime() and datetime(template) methods use the same parsing rules as the to_timestamp SQL function does (see Section 9.8), with three exceptions. First, these methods don't allow unmatched template patterns. Second, only the following separators are allowed in the template string: minus sign, period, solidus (slash), comma, apostrophe, semicolon, colon and space. Third, separators in the template string must exactly match the input string.
+
+If different date/time types need to be compared, an implicit cast is applied. A date value can be cast to timestamp or timestamptz, timestamp can be cast to timestamptz, and time to timetz. However, all but the first of these conversions depend on the current TimeZone setting, and thus can only be performed within timezone-aware jsonpath functions.
+
+Table 9.49 shows the available filter expression elements.
+
+Table 9.49. jsonpath Filter Expression Elements
+
+#|
+||Predicate/Value|Description|Example(s)||
+||value == value → boolean|
+Equality comparison (this, and the other comparison operators, work on all JSON scalar values)|
+```sql
+jsonb_path_query_array('[1, "a", 1, 3]', '$[*] ? (@ == 1)', '{}', false) → [1, 1]
+jsonb_path_query_array('[1, "a", 1, 3]', '$[*] ? (@ == "a")', '{}', false) → ["a"]
+```||
+||value != value → boolean
+value <> value → boolean|
+Non-equality comparison|
+```sql
+jsonb_path_query_array('[1, 2, 1, 3]', '$[*] ? (@ != 1)', '{}', false) → [2, 3]
+jsonb_path_query_array('["a", "b", "c"]', '$[*] ? (@ <> "b")', '{}', false) → ["a", "c"]
+```||
+||value < value → boolean|
+Less-than comparison|
+```sql
+jsonb_path_query_array('[1, 2, 3]', '$[*] ? (@ < 2)', '{}', false) → [1]
+```||
+||value <= value → boolean|
+Less-than-or-equal-to comparison|
+```sql
+jsonb_path_query_array('["a", "b", "c"]', '$[*] ? (@ <= "b")', '{}', false) → ["a", "b"]
+```||
+||value > value → boolean|
+Greater-than comparison|
+```sql
+jsonb_path_query_array('[1, 2, 3]', '$[*] ? (@ > 2)', '{}', false) → [3]
+```||
+||value >= value → boolean|
+Greater-than-or-equal-to comparison|
+```sql
+jsonb_path_query_array('[1, 2, 3]', '$[*] ? (@ >= 2)', '{}', false) → [2, 3]
+```||
+||true → boolean|
+JSON constant true|
+```sql
+jsonb_path_query_array('[{"name": "John", "parent": false}, {"name": "Chris", "parent": true}]', '$[*] ? (@.parent == true)', '{}', false) → [{"name": "Chris", "parent": true}]
+```||
+||false → boolean|
+JSON constant false|
+```sql
+jsonb_path_query_array('[{"name": "John", "parent": false}, {"name": "Chris", "parent": true}]', '$[*] ? (@.parent == false)', '{}', false) → [{"name": "John", "parent": false}]
+```||
+||null → value|
+JSON constant null (note that, unlike in SQL, comparison to null works normally)|
+```sql
+jsonb_path_query_array('[{"name": "Mary", "job": null}, {"name": "Michael", "job": "driver"}]', '$[*] ? (@.job == null) .name', '{}', false) → ["Mary"]
+```||
+||boolean && boolean → boolean|
+Boolean AND|
+```sql
+jsonb_path_query_array('[1, 3, 7]', '$[*] ? (@ > 1 && @ < 5)', '{}', false) → [3]
+```||
+||boolean \|\| boolean → boolean|
+Boolean OR|
+```sql
+jsonb_path_query_array('[1, 3, 7]', '$[*] ? (@ < 1 || @ > 5)', '{}', false) → [7]
+```||
+||! boolean → boolean|
+Boolean NOT|
+```sql
+jsonb_path_query_array('[1, 3, 7]', '$[*] ? (!(@ < 5))', '{}', false) → [7]
+```||
+||boolean is unknown → boolean|
+Tests whether a Boolean condition is unknown.|
+```sql
+jsonb_path_query_array('[-1, 2, 7, "foo"]', '$[*] ? ((@ > 0) is unknown)', '{}', false) → ["foo"]
+```||
+||string like_regex string [ flag string ] → boolean|
+Tests whether the first operand matches the regular expression given by the second operand, optionally with modifications described by a string of flag characters (see Section 9.16.2.3).|
+```sql
+jsonb_path_query_array('["abc", "abd", "aBdC", "abdacb", "babc"]', '$[*] ? (@ like_regex "^ab.*c")', '{}', false) → ["abc", "abdacb"]
+jsonb_path_query_array('["abc", "abd", "aBdC", "abdacb", "babc"]', '$[*] ? (@ like_regex "^ab.*c" flag "i")', '{}', false) → ["abc", "aBdC", "abdacb"]
+```||
+||string starts with string → boolean|
+Tests whether the second operand is an initial substring of the first operand.|
+```sql
+jsonb_path_query_array('["John Smith", "Mary Stone", "Bob Johnson"]', '$[*] ? (@ starts with "John")', '{}', false) → ["John Smith"]
+```||
+||exists ( path_expression ) → boolean|
+Tests whether a path expression matches at least one SQL/JSON item. Returns unknown if the path expression would result in an error; the second example uses this to avoid a no-such-key error in strict mode.|
+```sql
+jsonb_path_query_array('{"x": [1, 2], "y": [2, 4]}', 'strict $.* ? (exists (@ ? (@[*] > 2)))', '{}', false) → [[2, 4]]
+jsonb_path_query_array('{"value": 41}', 'strict $ ? (exists (@.name)) .name', '{}', false) → []
+```||
+|#
+
+9.16.2.3. SQL/JSON Regular Expressions
+SQL/JSON path expressions allow matching text to a regular expression with the like_regex filter. For example, the following SQL/JSON path query would case-insensitively match all strings in an array that start with an English vowel:
+
+$[*] ? (@ like_regex "^[aeiou]" flag "i")
+
+The optional flag string may include one or more of the characters i for case-insensitive match, m to allow ^ and $ to match at newlines, s to allow . to match a newline, and q to quote the whole pattern (reducing the behavior to a simple substring match).
+
+The SQL/JSON standard borrows its definition for regular expressions from the LIKE_REGEX operator, which in turn uses the XQuery standard. PostgreSQL does not currently support the LIKE_REGEX operator. Therefore, the like_regex filter is implemented using the POSIX regular expression engine described in Section 9.7.3. This leads to various minor discrepancies from standard SQL/JSON behavior, which are cataloged in Section 9.7.3.8. Note, however, that the flag-letter incompatibilities described there do not apply to SQL/JSON, as it translates the XQuery flag letters to match what the POSIX engine expects.
+
+Keep in mind that the pattern argument of like_regex is a JSON path string literal, written according to the rules given in Section 8.14.7. This means in particular that any backslashes you want to use in the regular expression must be doubled. For example, to match string values of the root document that contain only digits:
+
+`$.* ? (@ like_regex "^\\d+$")`
+
+## 9.17. Sequence Manipulation Functions (NOT SUPPORTED) {#sequence-manipulation-functions}
+
+## 9.18. Conditional Expressions {#conditional-expressions}
+
+9.18.1. CASE
+The SQL CASE expression is a generic conditional expression, similar to if/else statements in other programming languages:
+
+CASE WHEN condition THEN result
+ [WHEN ...]
+ [ELSE result]
+END
+CASE clauses can be used wherever an expression is valid. Each condition is an expression that returns a boolean result. If the condition's result is true, the value of the CASE expression is the result that follows the condition, and the remainder of the CASE expression is not processed. If the condition's result is not true, any subsequent WHEN clauses are examined in the same manner. If no WHEN condition yields true, the value of the CASE expression is the result of the ELSE clause. If the ELSE clause is omitted and no condition is true, the result is null.
+
+An example:
+
+```sql
+SELECT a,
+ CASE WHEN a=1 THEN 'one'
+ WHEN a=2 THEN 'two'
+ ELSE 'other'
+ END as case
+ FROM (VALUES (1),(2),(3)) as x(a)
+
+ a | case
+---+-------
+ 1 | one
+ 2 | two
+ 3 | other
+```
+
+The data types of all the result expressions must be convertible to a single output type. See Section 10.5 for more details.
+
+There is a “simple” form of CASE expression that is a variant of the general form above:
+
+CASE expression
+ WHEN value THEN result
+ [WHEN ...]
+ [ELSE result]
+END
+
+The first expression is computed, then compared to each of the value expressions in the WHEN clauses until one is found that is equal to it. If no match is found, the result of the ELSE clause (or a null value) is returned. This is similar to the switch statement in C.
+
+The example above can be written using the simple CASE syntax:
+
+```sql
+SELECT a,
+ CASE a WHEN 1 THEN 'one'
+ WHEN 2 THEN 'two'
+ ELSE 'other'
+ END as case
+ FROM (VALUES (1),(2),(3)) as x(a)
+
+ a | case
+---+-------
+ 1 | one
+ 2 | two
+ 3 | other
+```
+
+A CASE expression does not evaluate any subexpressions that are not needed to determine the result. For example, this is a possible way of avoiding a division-by-zero failure:
+
+```sql
+SELECT ... WHERE CASE WHEN x <> 0 THEN y/x > 1.5 ELSE false END;
+```
+
+Note
+As described in Section 4.2.14, there are various situations in which subexpressions of an expression are evaluated at different times, so that the principle that “CASE evaluates only necessary subexpressions” is not ironclad. For example a constant 1/0 subexpression will usually result in a division-by-zero failure at planning time, even if it's within a CASE arm that would never be entered at run time.
+
+9.18.2. COALESCE
+
+COALESCE(value [, ...])
+The COALESCE function returns the first of its arguments that is not null. Null is returned only if all arguments are null. It is often used to substitute a default value for null values when data is retrieved for display, for example:
+
+```sql
+SELECT COALESCE(description, short_description, '(none)') ...
+```
+
+This returns description if it is not null, otherwise short_description if it is not null, otherwise (none).
+
+The arguments must all be convertible to a common data type, which will be the type of the result (see Section 10.5 for details).
+
+Like a CASE expression, COALESCE only evaluates the arguments that are needed to determine the result; that is, arguments to the right of the first non-null argument are not evaluated. This SQL-standard function provides capabilities similar to NVL and IFNULL, which are used in some other database systems.
+
+9.18.3. NULLIF
+
+NULLIF(value1, value2) (NOT SUPPORTED)
+The NULLIF function returns a null value if value1 equals value2; otherwise it returns value1. This can be used to perform the inverse operation of the COALESCE example given above:
+
+```sql
+SELECT NULLIF(value, '(none)') ...
+```
+
+In this example, if value is (none), null is returned, otherwise the value of value is returned.
+
+The two arguments must be of comparable types. To be specific, they are compared exactly as if you had written value1 = value2, so there must be a suitable = operator available.
+
+The result has the same type as the first argument — but there is a subtlety. What is actually returned is the first argument of the implied = operator, and in some cases that will have been promoted to match the second argument's type. For example, NULLIF(1, 2.2) yields numeric, because there is no integer = numeric operator, only numeric = numeric.
+
+9.18.4. GREATEST and LEAST
+
+GREATEST(value [, ...])
+LEAST(value [, ...])
+
+The GREATEST and LEAST functions select the largest or smallest value from a list of any number of expressions. The expressions must all be convertible to a common data type, which will be the type of the result (see Section 10.5 for details). NULL values in the list are ignored. The result will be NULL only if all the expressions evaluate to NULL. (NOT SUPPORTED)
+
+Note that GREATEST and LEAST are not in the SQL standard, but are a common extension. Some other databases make them return NULL if any argument is NULL, rather than only when all are NULL.
+
+## 9.19. Array Functions and Operators {#array-functions}
+
+Table 9.51 shows the specialized operators available for array types. In addition to those, the usual comparison operators shown in Table 9.1 are available for arrays. The comparison operators compare the array contents element-by-element, using the default B-tree comparison function for the element data type, and sort based on the first difference. In multidimensional arrays the elements are visited in row-major order (last subscript varies most rapidly). If the contents of two arrays are equal but the dimensionality is different, the first difference in the dimensionality information determines the sort order.
+
+Table 9.51. Array Operators
+
+#|
+||Operator|Description|Example(s)||
+||anyarray @> anyarray → boolean|
+Does the first array contain the second, that is, does each element appearing in the second array equal some element of the first array? (Duplicates are not treated specially, thus ARRAY[1] and ARRAY[1,1] are each considered to contain the other.)|
+```sql
+ARRAY[1,4,3] @> ARRAY[3,1,3] → true
+```||
+||anyarray <@ anyarray → boolean|
+Is the first array contained by the second?|
+```sql
+ARRAY[2,2,7] <@ ARRAY[1,7,4,2,6] → true
+```||
+||anyarray && anyarray → boolean|
+Do the arrays overlap, that is, have any elements in common?|
+```sql
+ARRAY[1,4,3] && ARRAY[2,1] → true
+```||
+||anycompatiblearray \|\| anycompatiblearray → anycompatiblearray|
+Concatenates the two arrays. Concatenating a null or empty array is a no-op; otherwise the arrays must have the same number of dimensions (as illustrated by the first example) or differ in number of dimensions by one (as illustrated by the second). If the arrays are not of identical element types, they will be coerced to a common type (see Section 10.5). (NOT SUPPORTED)|
+```sql
+#ARRAY[1,2,3] || ARRAY[4,5,6,7] → {1,2,3,4,5,6,7}
+#ARRAY[1,2,3] || ARRAY[[4,5,6],[7,8,9.9]] → {{1,2,3},{4,5,6},{7,8,9.9}}
+```||
+||anycompatible \|\| anycompatiblearray → anycompatiblearray|
+Concatenates an element onto the front of an array (which must be empty or one-dimensional). (NOT SUPPORTED)|
+```sql
+#3 || ARRAY[4,5,6] → {3,4,5,6}
+```||
+||anycompatiblearray \|\| anycompatible → anycompatiblearray|
+Concatenates an element onto the end of an array (which must be empty or one-dimensional). (NOT SUPPORTED)|
+```sql
+#ARRAY[4,5,6] || 7 → {4,5,6,7}
+```||
+|#
+
+See Section 8.15 for more details about array operator behavior. See Section 11.2 for more details about which operators support indexed operations.
+
+Table 9.52 shows the functions available for use with array types. See Section 8.15 for more information and examples of the use of these functions.
+
+Table 9.52. Array Functions
+
+#|
+||Function|Description|Example(s)||
+||array_append ( anycompatiblearray, anycompatible ) → anycompatiblearray|
+Appends an element to the end of an array (same as the anycompatiblearray \|\| anycompatible operator). (NOT SUPPORTED)|
+```sql
+#array_append(ARRAY[1,2], 3) → {1,2,3}
+```||
+||array_cat ( anycompatiblearray, anycompatiblearray ) → anycompatiblearray|
+Concatenates two arrays (same as the anycompatiblearray \|\| anycompatiblearray operator). (NOT SUPPORTED)|
+```sql
+#array_cat(ARRAY[1,2,3], ARRAY[4,5]) → {1,2,3,4,5}
+```||
+||array_dims ( anyarray ) → text|
+Returns a text representation of the array's dimensions.|
+```sql
+array_dims(ARRAY[[1,2,3], [4,5,6]]) → [1:2][1:3]
+```||
+||array_fill ( anyelement, integer[] [, integer[] ] ) → anyarray|
+Returns an array filled with copies of the given value, having dimensions of the lengths specified by the second argument. The optional third argument supplies lower-bound values for each dimension (which default to all 1). (NOT SUPPORTED)|
+```sql
+#array_fill(11, ARRAY[2,3]) → {{11,11,11},{11,11,11}}
+#array_fill(7, ARRAY[3], ARRAY[2]) → [2:4]={7,7,7}
+```||
+||array_length ( anyarray, integer ) → integer|
+Returns the length of the requested array dimension. (Produces NULL instead of 0 for empty or missing array dimensions.)|
+```sql
+array_length(array[1,2,3], 1) → 3
+#array_length(array[]::int[], 1) → NULL
+array_length(array['text'], 2) → NULL
+```||
+||array_lower ( anyarray, integer ) → integer|
+Returns the lower bound of the requested array dimension.|
+```sql
+array_lower('[0:2]={1,2,3}'::integer[], 1) → 0
+```||
+||array_ndims ( anyarray ) → integer|
+Returns the number of dimensions of the array.|
+```sql
+array_ndims(ARRAY[[1,2,3], [4,5,6]]) → 2
+```||
+||array_position ( anycompatiblearray, anycompatible [, integer ] ) → integer|
+Returns the subscript of the first occurrence of the second argument in the array, or NULL if it's not present. If the third argument is given, the search begins at that subscript. The array must be one-dimensional. Comparisons are done using IS NOT DISTINCT FROM semantics, so it is possible to search for NULL. (NOT SUPPORTED)|
+```sql
+#array_position(ARRAY['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat'], 'mon') → 2
+```||
+||array_positions ( anycompatiblearray, anycompatible ) → integer[]|
+Returns an array of the subscripts of all occurrences of the second argument in the array given as first argument. The array must be one-dimensional. Comparisons are done using IS NOT DISTINCT FROM semantics, so it is possible to search for NULL. NULL is returned only if the array is NULL; if the value is not found in the array, an empty array is returned. (NOT SUPPORTED)|
+```sql
+#array_positions(ARRAY['A','A','B','A'], 'A') → {1,2,4}
+```||
+||array_prepend ( anycompatible, anycompatiblearray ) → anycompatiblearray|
+Prepends an element to the beginning of an array (same as the anycompatible \|\| anycompatiblearray operator). (NOT SUPPORTED)|
+```sql
+#array_prepend(1, ARRAY[2,3]) → {1,2,3}
+```||
+||array_remove ( anycompatiblearray, anycompatible ) → anycompatiblearray|
+Removes all elements equal to the given value from the array. The array must be one-dimensional. Comparisons are done using IS NOT DISTINCT FROM semantics, so it is possible to remove NULLs. (NOT SUPPORTED)|
+```sql
+#array_remove(ARRAY[1,2,3,2], 2) → {1,3}
+```||
+||array_replace ( anycompatiblearray, anycompatible, anycompatible ) → anycompatiblearray|
+Replaces each array element equal to the second argument with the third argument. (NOT SUPPORTED)|
+```sql
+#array_replace(ARRAY[1,2,5,4], 5, 3) → {1,2,3,4}
+```||
+||array_to_string ( array anyarray, delimiter text [, null_string text ] ) → text|
+Converts each array element to its text representation, and concatenates those separated by the delimiter string. If null_string is given and is not NULL, then NULL array entries are represented by that string; otherwise, they are omitted.|
+```sql
+array_to_string(ARRAY[1, 2, 3, NULL, 5], ',', '*') → 1,2,3,*,5
+```||
+||array_upper ( anyarray, integer ) → integer|
+Returns the upper bound of the requested array dimension.|
+```sql
+array_upper(ARRAY[1,8,3,7], 1) → 4
+```||
+||cardinality ( anyarray ) → integer|
+Returns the total number of elements in the array, or 0 if the array is empty.|
+```sql
+cardinality(ARRAY[[1,2],[3,4]]) → 4
+```||
+||trim_array ( array anyarray, n integer ) → anyarray|
+Trims an array by removing the last n elements. If the array is multidimensional, only the first dimension is trimmed.|
+```sql
+trim_array(ARRAY[1,2,3,4,5,6], 2) → {1,2,3,4}
+```||
+||unnest ( anyarray ) → setof anyelement|
+Expands an array into a set of rows. The array's elements are read out in storage order. (NOT SUPPORTED)|
+
+```sql
+SELECT * FROM unnest(ARRAY[1,2]) as a → [
+1
+2
+]
+
+SELECT * FROM unnest(ARRAY[['foo','bar'],['baz','quux']]) as a → [
+foo
+bar
+baz
+quux
+]
+```||
+||unnest ( anyarray, anyarray [, ... ] ) → setof anyelement, anyelement [, ... ]|
+Expands multiple arrays (possibly of different data types) into a set of rows. If the arrays are not all the same length then the shorter ones are padded with NULLs. This form is only allowed in a query's FROM clause; see Section 7.2.1.4. (NOT SUPPORTED)|
+```sql
+#SELECT * FROM unnest(ARRAY[1,2], ARRAY['foo','bar','baz']) as x(a,b) → [
+1,foo
+2,bar
+,baz
+]
+```||
+|#
+
+Note
+There are two differences in the behavior of string_to_array from pre-9.1 versions of PostgreSQL. First, it will return an empty (zero-element) array rather than NULL when the input string is of zero length. Second, if the delimiter string is NULL, the function splits the input into individual characters, rather than returning NULL as before.
+
+See also Section 9.21 about the aggregate function array_agg for use with arrays.
+
+## 9.20. Range/Multirange Functions and Operators (NOT SUPPORTED) {#range-multirange-functions}
+
+## 9.21. Aggregate Functions {#aggregate-functions}
+
+Aggregate functions compute a single result from a set of input values. The built-in general-purpose aggregate functions are listed in Table 9.57 while statistical aggregates are in Table 9.58. The built-in within-group ordered-set aggregate functions are listed in Table 9.59 while the built-in within-group hypothetical-set ones are in Table 9.60. Grouping operations, which are closely related to aggregate functions, are listed in Table 9.61. The special syntax considerations for aggregate functions are explained in Section 4.2.7. Consult Section 2.7 for additional introductory information.
+
+Aggregate functions that support Partial Mode are eligible to participate in various optimizations, such as parallel aggregation.
+
+Table 9.57. General-Purpose Aggregate Functions
+
+#|
+||Function|Description|Partial Mode|Example||
+||array_agg ( anynonarray ) → anyarray|
+Collects all the input values, including nulls, into an array.|
+No|
+```sql
+SELECT array_agg(x) FROM (VALUES (1),(2)) a(x) → {1,2}
+```||
+||array_agg ( anyarray ) → anyarray|
+Concatenates all the input arrays into an array of one higher dimension. (The inputs must all have the same dimensionality, and cannot be empty or null.)|
+No|
+```sql
+SELECT array_agg(x) FROM (VALUES (Array[1,2]),(Array[3,4])) a(x) → {{1,2},{3,4}}
+```||
+||avg ( smallint ) → numeric
+avg ( integer ) → numeric
+avg ( bigint ) → numeric
+avg ( numeric ) → numeric
+avg ( real ) → double precision
+avg ( double precision ) → double precision
+avg ( interval ) → interval|
+Computes the average (arithmetic mean) of all the non-null input values.|
+Yes|
+```sql
+SELECT avg(x::smallint) FROM (VALUES (1),(2),(3)) a(x) → 2.0000000000000000
+SELECT avg(x::integer) FROM (VALUES (1),(2),(3)) a(x) → 2.0000000000000000
+SELECT avg(x::bigint) FROM (VALUES (1),(2),(3)) a(x) → 2.0000000000000000
+SELECT avg(x::numeric) FROM (VALUES (1),(2),(3)) a(x) → 2.0000000000000000
+SELECT avg(x::real) FROM (VALUES (1),(2),(3)) a(x) → 2
+SELECT avg(x::double precision) FROM (VALUES (1),(2),(3)) a(x) → 2
+SELECT avg(cast(x as interval day)) FROM (VALUES ('1'),('2'),('3')) a(x) → 2 days
+```||
+||bit_and ( smallint ) → smallint
+bit_and ( integer ) → integer
+bit_and ( bigint ) → bigint
+bit_and ( bit ) → bit|
+Computes the bitwise AND of all non-null input values.|
+Yes|
+```sql
+SELECT bit_and(x::smallint) FROM (VALUES (5),(6),(7)) a(x) → 4
+SELECT bit_and(x::integer) FROM (VALUES (5),(6),(7)) a(x) → 4
+SELECT bit_and(x::bigint) FROM (VALUES (5),(6),(7)) a(x) → 4
+SELECT bit_and(x::bit(3)) FROM (VALUES ('101'),('110'),('111')) a(x) → 100
+```||
+||bit_or ( smallint ) → smallint
+bit_or ( integer ) → integer
+bit_or ( bigint ) → bigint
+bit_or ( bit ) → bit|
+Computes the bitwise OR of all non-null input values.|
+Yes|
+```sql
+SELECT bit_or(x::smallint) FROM (VALUES (4),(5),(6)) a(x) → 7
+SELECT bit_or(x::integer) FROM (VALUES (4),(5),(6)) a(x) → 7
+SELECT bit_or(x::bigint) FROM (VALUES (4),(5),(6)) a(x) → 7
+SELECT bit_or(x::bit(3)) FROM (VALUES ('100'),('101'),('110')) a(x) → 111
+```||
+||bit_xor ( smallint ) → smallint
+bit_xor ( integer ) → integer
+bit_xor ( bigint ) → bigint
+bit_xor ( bit ) → bit|
+Computes the bitwise exclusive OR of all non-null input values. Can be useful as a checksum for an unordered set of values.|
+Yes|
+```sql
+SELECT bit_xor(x::smallint) FROM (VALUES (5),(6),(6)) a(x) → 5
+SELECT bit_xor(x::integer) FROM (VALUES (5),(6),(6)) a(x) → 5
+SELECT bit_xor(x::bigint) FROM (VALUES (5),(6),(6)) a(x) → 5
+SELECT bit_xor(x::bit(3)) FROM (VALUES ('101'),('110'),('110')) a(x) → 101
+```||
+||bool_and ( boolean ) → boolean|
+Returns true if all non-null input values are true, otherwise false.|
+Yes|
+```sql
+SELECT bool_and(x) FROM (VALUES (null),(false),(true)) a(x) → false
+SELECT bool_and(x) FROM (VALUES (null),(true),(true)) a(x) → true
+SELECT bool_and(x) FROM (VALUES (null::bool),(null::bool),(null::bool)) a(x) → NULL
+```||
+||bool_or ( boolean ) → boolean|
+Returns true if any non-null input value is true, otherwise false.|
+Yes|
+```sql
+SELECT bool_or(x) FROM (VALUES (null),(false),(false)) a(x) → false
+SELECT bool_or(x) FROM (VALUES (null),(false),(true)) a(x) → true
+SELECT bool_or(x) FROM (VALUES (null::bool),(null::bool),(null::bool)) a(x) → NULL
+```||
+||count ( * ) → bigint|
+Computes the number of input rows.|
+Yes|
+```sql
+SELECT count(*) FROM (VALUES (4),(5),(6)) a(x) → 3
+```||
+||count ( any ) → bigint|
+Computes the number of input rows in which the input value is not null.|
+Yes|
+```sql
+SELECT count(x) FROM (VALUES (4),(null),(6)) a(x) → 2
+```||
+||every ( boolean ) → boolean|
+This is the SQL standard's equivalent to bool_and|
+Yes|
+```sql
+SELECT every(x) FROM (VALUES (null),(false),(true)) a(x) → false
+SELECT every(x) FROM (VALUES (null),(true),(true)) a(x) → true
+SELECT every(x) FROM (VALUES (null::bool),(null::bool),(null::bool)) a(x) → NULL
+```||
+||json_agg ( anyelement ) → json
+jsonb_agg ( anyelement ) → jsonb|
+Collects all the input values, including nulls, into a JSON array. Values are converted to JSON as per to_json or to_jsonb. (NOT SUPPORTED)|
+No|
+```sql
+#SELECT json_agg(x) FROM (VALUES (1),(2),(3)) a(x) → [1,2,3]
+#SELECT jsonb_agg(x) FROM (VALUES ('a'),('b'),('c')) a(x) → ["a","b","c"]
+```||
+||json_object_agg ( key any, value any ) → json
+jsonb_object_agg ( key any, value any ) → jsonb|
+Collects all the key/value pairs into a JSON object. Key arguments are coerced to text; value arguments are converted as per to_json or to_jsonb. Values can be null, but not keys.|
+No|
+```sql
+SELECT json_object_agg(x,y) FROM (VALUES ('a',1),('b',2),('c',3)) a(x,y) → [
+{ "a" : 1, "b" : 2, "c" : 3 }
+]
+
+SELECT jsonb_object_agg(x,y) FROM (VALUES ('x','a'),('y','b'),('z','c')) a(x,y) → [
+{"x": "a", "y": "b", "z": "c"}
+]
+
+```||
+||max ( see text ) → same as input type|
+Computes the maximum of the non-null input values. Available for any numeric, string, date/time, or enum type, as well as inet, interval, money, oid, pg_lsn, tid, and arrays of any of these types. (Arrays aren't supported)|
+Yes|
+```sql
+SELECT max(x::smallint) FROM (VALUES (1),(2),(3)) a(x) → 3
+SELECT max(x::integer) FROM (VALUES (1),(2),(3)) a(x) → 3
+SELECT max(x::bigint) FROM (VALUES (1),(2),(3)) a(x) → 3
+SELECT max(x::real) FROM (VALUES (1),(2),(3)) a(x) → 3
+SELECT max(x::double precision) FROM (VALUES (1),(2),(3)) a(x) → 3
+SELECT max(x::numeric) FROM (VALUES (1),(2),(3)) a(x) → 3
+SELECT max(x) FROM (VALUES ('a'),('b'),('c')) a(x) → 'c'
+SELECT max(x::date) FROM (VALUES ('2001-01-01'),('2001-02-03'),('2002-01-01')) a(x) → 2002-01-01
+SELECT max(x::timestamp) FROM (VALUES ('2001-01-01 23:05:04'),('2001-01-01 23:06:03'),('2001-01-01 23:59:00')) a(x) → 2001-01-01 23:59:00
+SELECT max(x::time) FROM (VALUES ('10:00:05'),('11:00:01'),('12:50:00')) a(x) → 12:50:00
+SELECT max(x) FROM (VALUES (interval '1' day),(interval '2' day),(interval '3' day)) a(x) → 3 days
+
+SELECT max(array[x,x]::smallint[]) FROM (VALUES (1),(2),(3)) a(x) → {3,3}
+SELECT max(array[x,x]::integer[]) FROM (VALUES (1),(2),(3)) a(x) → {3,3}
+SELECT max(array[x,x]::bigint[]) FROM (VALUES (1),(2),(3)) a(x) → {3,3}
+SELECT max(array[x,x]::real[]) FROM (VALUES (1),(2),(3)) a(x) → {3,3}
+SELECT max(array[x,x]::double precision[]) FROM (VALUES (1),(2),(3)) a(x) → {3,3}
+SELECT max(array[x,x]::numeric[]) FROM (VALUES (1),(2),(3)) a(x) → {3,3}
+SELECT max(array[x,x]) FROM (VALUES ('a'),('b'),('c')) a(x) → {c,c}
+SELECT max(array[x,x]::date[]) FROM (VALUES ('2001-01-01'),('2001-02-03'),('2002-01-01')) a(x) → {2002-01-01,2002-01-01}
+SELECT max(array[x,x]::timestamp[]) FROM (VALUES ('2001-01-01 23:05:04'),('2001-01-01 23:06:03'),('2001-01-01 23:59:00')) a(x) → {"2001-01-01 23:59:00","2001-01-01 23:59:00"}
+SELECT max(array[x,x]::time[]) FROM (VALUES ('10:00:05'),('11:00:01'),('12:50:00')) a(x) → {12:50:00,12:50:00}
+SELECT max(array[x,x]) FROM (VALUES (interval '1' day),(interval '2' day),(interval '3' day)) a(x) → {"3 days","3 days"}
+```||
+||min ( see text ) → same as input type|
+Computes the minimum of the non-null input values. Available for any numeric, string, date/time, or enum type, as well as inet, interval, money, oid, pg_lsn, tid, and arrays of any of these types. (Arrays aren't supported)|
+Yes|
+```sql
+SELECT min(x::smallint) FROM (VALUES (1),(2),(3)) a(x) → 1
+SELECT min(x::integer) FROM (VALUES (1),(2),(3)) a(x) → 1
+SELECT min(x::bigint) FROM (VALUES (1),(2),(3)) a(x) → 1
+SELECT min(x::real) FROM (VALUES (1),(2),(3)) a(x) → 1
+SELECT min(x::double precision) FROM (VALUES (1),(2),(3)) a(x) → 1
+SELECT min(x::numeric) FROM (VALUES (1),(2),(3)) a(x) → 1
+SELECT min(x) FROM (VALUES ('a'),('b'),('c')) a(x) → 'a'
+SELECT min(x::date) FROM (VALUES ('2001-01-01'),('2001-02-03'),('2002-01-01')) a(x) → 2001-01-01
+SELECT min(x::timestamp) FROM (VALUES ('2001-01-01 23:05:04'),('2001-01-01 23:06:03'),('2001-01-01 23:59:00')) a(x) → 2001-01-01 23:05:04
+SELECT min(x::time) FROM (VALUES ('10:00:05'),('11:00:01'),('12:50:00')) a(x) → 10:00:05
+SELECT min(x) FROM (VALUES (interval '1' day),(interval '2' day),(interval '3' day)) a(x) → 1 day
+
+SELECT min(array[x,x]::smallint[]) FROM (VALUES (1),(2),(3)) a(x) → {1,1}
+SELECT min(array[x,x]::integer[]) FROM (VALUES (1),(2),(3)) a(x) → {1,1}
+SELECT min(array[x,x]::bigint[]) FROM (VALUES (1),(2),(3)) a(x) → {1,1}
+SELECT min(array[x,x]::real[]) FROM (VALUES (1),(2),(3)) a(x) → {1,1}
+SELECT min(array[x,x]::double precision[]) FROM (VALUES (1),(2),(3)) a(x) → {1,1}
+SELECT min(array[x,x]::numeric[]) FROM (VALUES (1),(2),(3)) a(x) → {1,1}
+SELECT min(array[x,x]) FROM (VALUES ('a'),('b'),('c')) a(x) → {a,a}
+SELECT min(array[x,x]::date[]) FROM (VALUES ('2001-01-01'),('2001-02-03'),('2002-01-01')) a(x) → {2001-01-01,2001-01-01}
+SELECT min(array[x,x]::timestamp[]) FROM (VALUES ('2001-01-01 23:05:04'),('2001-01-01 23:06:03'),('2001-01-01 23:59:00')) a(x) → {"2001-01-01 23:05:04","2001-01-01 23:05:04"}
+SELECT min(array[x,x]::time[]) FROM (VALUES ('10:00:05'),('11:00:01'),('12:50:00')) a(x) → {10:00:05,10:00:05}
+SELECT min(array[x,x]) FROM (VALUES (interval '1' day),(interval '2' day),(interval '3' day)) a(x) → {"1 day","1 day"}
+```||
+||range_agg ( value anyrange ) → anymultirange|
+Computes the union of the non-null input values. (NOT SUPPORTED)|
+No|
+||
+||range_intersect_agg ( value anyrange ) → anyrange
+range_intersect_agg ( value anymultirange ) → anymultirange|
+Computes the intersection of the non-null input values. (NOT SUPPORTED)|
+No|
+||
+||string_agg ( value text, delimiter text ) → text
+string_agg ( value bytea, delimiter bytea ) → bytea|
+Concatenates the non-null input values into a string. Each value after the first is preceded by the corresponding delimiter (if it's not null).|
+No|
+```sql
+SELECT string_agg(x,'') FROM (VALUES ('a'),('b'),('c')) a(x) → abc
+SELECT string_agg(x::bytea,','::bytea) FROM (VALUES ('a'),('b'),('c')) a(x) → a,b,c
+```||
+||sum ( smallint ) → bigint
+sum ( integer ) → bigint
+sum ( bigint ) → numeric
+sum ( numeric ) → numeric
+sum ( real ) → real
+sum ( double precision ) → double precision
+sum ( interval ) → interval
+sum ( money ) → money|
+Computes the sum of the non-null input values.|
+Yes|
+```sql
+SELECT sum(x::smallint) FROM (VALUES (1),(2),(3)) a(x) → 6
+SELECT sum(x::integer) FROM (VALUES (1),(2),(3)) a(x) → 6
+SELECT sum(x::bigint) FROM (VALUES (1),(2),(3)) a(x) → 6
+SELECT sum(x::real) FROM (VALUES (1),(2),(3)) a(x) → 6
+SELECT sum(x::double precision) FROM (VALUES (1),(2),(3)) a(x) → 6
+SELECT sum(x::numeric) FROM (VALUES (1),(2),(3)) a(x) → 6
+SELECT sum(x) FROM (VALUES (interval '1' day),(interval '2' day),(interval '3' day)) a(x) → 6 days
+```||
+||xmlagg ( xml ) → xml|
+Concatenates the non-null XML input values (see Section 9.15.1.7).|
+No|
+||
+|#
+
+It should be noted that except for count, these functions return a null value when no rows are selected. In particular, sum of no rows returns null, not zero as one might expect, and array_agg returns null rather than an empty array when there are no input rows. The coalesce function can be used to substitute zero or an empty array for null when necessary.
+
+The aggregate functions array_agg, json_agg, jsonb_agg, json_object_agg, jsonb_object_agg, string_agg, and xmlagg, as well as similar user-defined aggregate functions, produce meaningfully different result values depending on the order of the input values. This ordering is unspecified by default, but can be controlled by writing an ORDER BY clause within the aggregate call, as shown in Section 4.2.7. Alternatively, supplying the input values from a sorted subquery will usually work. For example:
+
+```sql
+SELECT xmlagg(x) FROM (SELECT x FROM test ORDER BY y DESC) AS tab;
+```
+
+Beware that this approach can fail if the outer query level contains additional processing, such as a join, because that might cause the subquery's output to be reordered before the aggregate is computed.
+
+Note
+The boolean aggregates bool_and and bool_or correspond to the standard SQL aggregates every and any or some. PostgreSQL supports every, but not any or some, because there is an ambiguity built into the standard syntax:
+
+```sql
+SELECT b1 = ANY((SELECT b2 FROM t2 ...)) FROM t1 ...;
+```
+
+Here ANY can be considered either as introducing a subquery, or as being an aggregate function, if the subquery returns one row with a Boolean value. Thus the standard name cannot be given to these aggregates.
+
+Note
+Users accustomed to working with other SQL database management systems might be disappointed by the performance of the count aggregate when it is applied to the entire table. A query like:
+
+SELECT count(*) FROM sometable;
+will require effort proportional to the size of the table: PostgreSQL will need to scan either the entire table or the entirety of an index that includes all rows in the table.
+
+Table 9.58 shows aggregate functions typically used in statistical analysis. (These are separated out merely to avoid cluttering the listing of more-commonly-used aggregates.) Functions shown as accepting numeric_type are available for all the types smallint, integer, bigint, numeric, real, and double precision. Where the description mentions N, it means the number of input rows for which all the input expressions are non-null. In all cases, null is returned if the computation is meaningless, for example when N is zero.
+
+Table 9.58. Aggregate Functions for Statistics
+
+#|
+||Function|Description|Partial Mode|Examples||
+||corr ( Y double precision, X double precision ) → double precision|
+Computes the correlation coefficient.|
+Yes|
+```sql
+SELECT corr(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → -0.5
+```||
+||covar_pop ( Y double precision, X double precision ) → double precision|
+Computes the population covariance.|
+Yes|
+```sql
+SELECT covar_pop(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → -0.3333333333333333
+```||
+||covar_samp ( Y double precision, X double precision ) → double precision|
+Computes the sample covariance.|
+Yes|
+```sql
+SELECT covar_samp(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → -0.5
+```||
+||regr_avgx ( Y double precision, X double precision ) → double precision|
+Computes the average of the independent variable, sum(X)/N.|
+Yes|
+```sql
+SELECT regr_avgx(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → 2
+```||
+||regr_avgy ( Y double precision, X double precision ) → double precision|
+Computes the average of the dependent variable, sum(Y)/N.|
+Yes|
+```sql
+SELECT regr_avgy(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → 2
+```||
+||regr_count ( Y double precision, X double precision ) → bigint|
+Computes the number of rows in which both inputs are non-null.|
+Yes|
+```sql
+SELECT regr_count(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → 3
+```||
+||regr_intercept ( Y double precision, X double precision ) → double precision|
+Computes the y-intercept of the least-squares-fit linear equation determined by the (X, Y) pairs.|
+Yes|
+```sql
+SELECT regr_intercept(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → 3
+```||
+||regr_r2 ( Y double precision, X double precision ) → double precision|
+Computes the square of the correlation coefficient.|
+Yes|
+```sql
+SELECT regr_r2(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → 0.25
+```||
+||regr_slope ( Y double precision, X double precision ) → double precision|
+Computes the slope of the least-squares-fit linear equation determined by the (X, Y) pairs.|
+Yes|
+```sql
+SELECT regr_slope(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → -0.5
+```||
+||regr_sxx ( Y double precision, X double precision ) → double precision|
+Computes the “sum of squares” of the independent variable, `sum(X^2) - sum(X)^2/N`.|
+Yes|
+```sql
+SELECT regr_sxx(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → 2
+```||
+||regr_sxy ( Y double precision, X double precision ) → double precision|
+Computes the “sum of products” of independent times dependent variables, `sum(X*Y) - sum(X) * sum(Y)/N`.|
+Yes|
+```sql
+SELECT regr_sxy(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → -1
+```||
+||regr_syy ( Y double precision, X double precision ) → double precision|
+Computes the “sum of squares” of the dependent variable, `sum(Y^2) - sum(Y)^2/N`.|
+Yes|
+```sql
+SELECT regr_syy(x,y) FROM (VALUES (1,2),(2,3),(3,1)) a(x,y) → 2
+```||
+||stddev ( numeric_type ) → double precision for real or double precision, otherwise numeric|
+This is a historical alias for stddev_samp.|
+Yes|
+```sql
+SELECT stddev(x) FROM (VALUES (1),(2),(3)) a(x) → 1.00000000000000000000
+```||
+||stddev_pop ( numeric_type ) → double precision for real or double precision, otherwise numeric|
+Computes the population standard deviation of the input values.|
+Yes|
+```sql
+SELECT stddev_pop(x) FROM (VALUES (1),(2),(3)) a(x) → 0.81649658092772603273
+```||
+||stddev_samp ( numeric_type ) → double precision for real or double precision, otherwise numeric|
+Computes the sample standard deviation of the input values.|
+Yes|
+```sql
+SELECT stddev_samp(x) FROM (VALUES (1),(2),(3)) a(x) → 1.00000000000000000000
+```||
+||variance ( numeric_type ) → double precision for real or double precision, otherwise numeric|
+This is a historical alias for var_samp.|
+Yes|
+```sql
+SELECT variance(x) FROM (VALUES (1),(2),(3)) a(x) → 1.00000000000000000000
+```||
+||var_pop ( numeric_type ) → double precision for real or double precision, otherwise numeric|
+Computes the population variance of the input values (square of the population standard deviation).|
+Yes|
+```sql
+SELECT var_pop(x) FROM (VALUES (1),(2),(3)) a(x) → 0.66666666666666666667
+```||
+||var_samp ( numeric_type ) → double precision for real or double precision, otherwise numeric|
+Computes the sample variance of the input values (square of the sample standard deviation).|
+Yes|
+```sql
+SELECT var_samp(x) FROM (VALUES (1),(2),(3)) a(x) → 1.00000000000000000000
+```||
+|#
+
+Table 9.59 shows some aggregate functions that use the ordered-set aggregate syntax. These functions are sometimes referred to as “inverse distribution” functions. Their aggregated input is introduced by ORDER BY, and they may also take a direct argument that is not aggregated, but is computed only once. All these functions ignore null values in their aggregated input. For those that take a fraction parameter, the fraction value must be between 0 and 1; an error is thrown if not. However, a null fraction value simply produces a null result.
+
+Table 9.59. Ordered-Set Aggregate Functions (NOT SUPPORTED)
+
+#|
+||Function|Description|Partial Mode||
+||mode () WITHIN GROUP ( ORDER BY anyelement ) → anyelement|
+Computes the mode, the most frequent value of the aggregated argument (arbitrarily choosing the first one if there are multiple equally-frequent values). The aggregated argument must be of a sortable type.|
+No||
+||percentile_cont ( fraction double precision ) WITHIN GROUP ( ORDER BY double precision ) → double precision
+percentile_cont ( fraction double precision ) WITHIN GROUP ( ORDER BY interval ) → interval|
+Computes the continuous percentile, a value corresponding to the specified fraction within the ordered set of aggregated argument values. This will interpolate between adjacent input items if needed.|
+No||
+||percentile_cont ( fractions double precision[] ) WITHIN GROUP ( ORDER BY double precision ) → double precision[]
+percentile_cont ( fractions double precision[] ) WITHIN GROUP ( ORDER BY interval ) → interval[]|
+Computes multiple continuous percentiles. The result is an array of the same dimensions as the fractions parameter, with each non-null element replaced by the (possibly interpolated) value corresponding to that percentile.|
+No||
+||percentile_disc ( fraction double precision ) WITHIN GROUP ( ORDER BY anyelement ) → anyelement|
+Computes the discrete percentile, the first value within the ordered set of aggregated argument values whose position in the ordering equals or exceeds the specified fraction. The aggregated argument must be of a sortable type.|
+No||
+||percentile_disc ( fractions double precision[] ) WITHIN GROUP ( ORDER BY anyelement ) → anyarray|
+Computes multiple discrete percentiles. The result is an array of the same dimensions as the fractions parameter, with each non-null element replaced by the input value corresponding to that percentile. The aggregated argument must be of a sortable type.|
+No||
+|#
+
+Each of the “hypothetical-set” aggregates listed in Table 9.60 is associated with a window function of the same name defined in Section 9.22. In each case, the aggregate's result is the value that the associated window function would have returned for the “hypothetical” row constructed from args, if such a row had been added to the sorted group of rows represented by the sorted_args. For each of these functions, the list of direct arguments given in args must match the number and types of the aggregated arguments given in sorted_args. Unlike most built-in aggregates, these aggregates are not strict, that is they do not drop input rows containing nulls. Null values sort according to the rule specified in the ORDER BY clause.
+
+Table 9.60. Hypothetical-Set Aggregate Functions (NOT SUPPORTED)
+
+#|
+||Function|Description|Partial Mode||
+||rank ( args ) WITHIN GROUP ( ORDER BY sorted_args ) → bigint|
+Computes the rank of the hypothetical row, with gaps; that is, the row number of the first row in its peer group.|
+No||
+||dense_rank ( args ) WITHIN GROUP ( ORDER BY sorted_args ) → bigint|
+Computes the rank of the hypothetical row, without gaps; this function effectively counts peer groups.|
+No||
+||percent_rank ( args ) WITHIN GROUP ( ORDER BY sorted_args ) → double precision|
+Computes the relative rank of the hypothetical row, that is (rank - 1) / (total rows - 1). The value thus ranges from 0 to 1 inclusive.|
+No||
+||cume_dist ( args ) WITHIN GROUP ( ORDER BY sorted_args ) → double precision|
+Computes the cumulative distribution, that is (number of rows preceding or peers with hypothetical row) / (total rows). The value thus ranges from 1/N to 1.|
+No||
+|#
+
+Table 9.61. Grouping Operations
+
+#|
+||Function|Description||
+||GROUPING ( group_by_expression(s) ) → integer|
+Returns a bit mask indicating which GROUP BY expressions are not included in the current grouping set. Bits are assigned with the rightmost argument corresponding to the least-significant bit; each bit is 0 if the corresponding expression is included in the grouping criteria of the grouping set generating the current result row, and 1 if it is not included. (NOT SUPPORTED)||
+|#
+
+The grouping operations shown in Table 9.61 are used in conjunction with grouping sets (see Section 7.2.4) to distinguish result rows. The arguments to the GROUPING function are not actually evaluated, but they must exactly match expressions given in the GROUP BY clause of the associated query level. For example:
+
+```sql
+=> SELECT * FROM items_sold;
+ make | model | sales
+-------+-------+-------
+ Foo | GT | 10
+ Foo | Tour | 20
+ Bar | City | 15
+ Bar | Sport | 5
+(4 rows)
+
+=> SELECT make, model, GROUPING(make,model), sum(sales) FROM items_sold GROUP BY ROLLUP(make,model);
+ make | model | grouping | sum
+-------+-------+----------+-----
+ Foo | GT | 0 | 10
+ Foo | Tour | 0 | 20
+ Bar | City | 0 | 15
+ Bar | Sport | 0 | 5
+ Foo | | 1 | 30
+ Bar | | 1 | 20
+ | | 3 | 50
+(7 rows)
+```
+
+Here, the grouping value 0 in the first four rows shows that those have been grouped normally, over both the grouping columns. The value 1 indicates that model was not grouped by in the next-to-last two rows, and the value 3 indicates that neither make nor model was grouped by in the last row (which therefore is an aggregate over all the input rows).
+
+## 9.22. Window Functions {#window-functions}
+
+Window functions provide the ability to perform calculations across sets of rows that are related to the current query row. See Section 3.5 for an introduction to this feature, and Section 4.2.8 for syntax details.
+
+The built-in window functions are listed in Table 9.62. Note that these functions must be invoked using window function syntax, i.e., an OVER clause is required.
+
+In addition to these functions, any built-in or user-defined ordinary aggregate (i.e., not ordered-set or hypothetical-set aggregates) can be used as a window function; see Section 9.21 for a list of the built-in aggregates. Aggregate functions act as window functions only when an OVER clause follows the call; otherwise they act as plain aggregates and return a single row for the entire set.
+
+Table 9.62. General-Purpose Window Functions
+
+#|
+||Function|Description|Examples||
+||row_number () → bigint|
+Returns the number of the current row within its partition, counting from 1.|
+```sql
+SELECT row_number() OVER (ORDER BY x) FROM (VALUES (4),(5),(6)) a(x) → [
+1
+2
+3
+]
+```||
+||rank () → bigint|
+Returns the rank of the current row, with gaps; that is, the row_number of the first row in its peer group.|
+```sql
+SELECT rank() OVER (ORDER BY x) FROM (VALUES (4),(5),(5),(6)) a(x) → [
+1
+2
+2
+4
+]
+```||
+||dense_rank () → bigint|
+Returns the rank of the current row, without gaps; this function effectively counts peer groups.|
+```sql
+SELECT dense_rank() OVER (ORDER BY x) FROM (VALUES (4),(5),(5),(6)) a(x) → [
+1
+2
+2
+3
+]
+```||
+||percent_rank () → double precision|
+Returns the relative rank of the current row, that is (rank - 1) / (total partition rows - 1). The value thus ranges from 0 to 1 inclusive.|
+```sql
+SELECT percent_rank() OVER (ORDER BY x) FROM (VALUES (4),(5),(5),(6)) a(x) → [
+0
+0.3333333333333333
+0.3333333333333333
+1
+]
+```||
+||cume_dist () → double precision|
+Returns the cumulative distribution, that is (number of partition rows preceding or peers with current row) / (total partition rows). The value thus ranges from 1/N to 1.|
+```sql
+SELECT cume_dist() OVER (ORDER BY x) FROM (VALUES (4),(5),(5),(6)) a(x) → [
+0.25
+0.75
+0.75
+1
+]
+```||
+||ntile ( num_buckets integer ) → integer|
+Returns an integer ranging from 1 to the argument value, dividing the partition as equally as possible.|
+```sql
+SELECT ntile(2) OVER (ORDER BY x) FROM (VALUES (4),(5),(5),(6)) a(x) → [
+1
+1
+2
+2
+]
+```||
+||lag ( value anycompatible [, offset integer [, default anycompatible ]] ) → anycompatible|
+Returns value evaluated at the row that is offset rows before the current row within the partition; if there is no such row, instead returns default (which must be of a type compatible with value). Both offset and default are evaluated with respect to the current row. If omitted, offset defaults to 1 and default to NULL.|
+```sql
+SELECT lag(x) OVER (ORDER BY x) FROM (VALUES (4),(5),(5),(6)) a(x) → [
+NULL
+4
+5
+5
+]
+```||
+||lead ( value anycompatible [, offset integer [, default anycompatible ]] ) → anycompatible|
+Returns value evaluated at the row that is offset rows after the current row within the partition; if there is no such row, instead returns default (which must be of a type compatible with value). Both offset and default are evaluated with respect to the current row. If omitted, offset defaults to 1 and default to NULL.|
+```sql
+SELECT lead(x) OVER (ORDER BY x) FROM (VALUES (4),(5),(5),(6)) a(x) → [
+5
+5
+6
+NULL
+]
+```||
+||first_value ( value anyelement ) → anyelement|
+Returns value evaluated at the row that is the first row of the window frame.|
+```sql
+SELECT first_value(x) OVER (ORDER BY x) FROM (VALUES (4),(5),(5),(6)) a(x) → [
+4
+4
+4
+4
+]
+```||
+||last_value ( value anyelement ) → anyelement|
+Returns value evaluated at the row that is the last row of the window frame.|
+```sql
+SELECT last_value(x) OVER (ORDER BY x) FROM (VALUES (4),(5),(5),(6)) a(x) → [
+4
+5
+5
+6
+]
+```||
+||nth_value ( value anyelement, n integer ) → anyelement|
+Returns value evaluated at the row that is the n'th row of the window frame (counting from 1); returns NULL if there is no such row.|
+```sql
+SELECT nth_value(x,2) OVER (ORDER BY x) FROM (VALUES (4),(5),(5),(6)) a(x) → [
+NULL
+5
+5
+5
+]
+```||
+|#
+
+All of the functions listed in Table 9.62 depend on the sort ordering specified by the ORDER BY clause of the associated window definition. Rows that are not distinct when considering only the ORDER BY columns are said to be peers. The four ranking functions (including cume_dist) are defined so that they give the same answer for all rows of a peer group.
+
+Note that first_value, last_value, and nth_value consider only the rows within the “window frame”, which by default contains the rows from the start of the partition through the last peer of the current row. This is likely to give unhelpful results for last_value and sometimes also nth_value. You can redefine the frame by adding a suitable frame specification (RANGE, ROWS or GROUPS) to the OVER clause. See Section 4.2.8 for more information about frame specifications.
+
+When an aggregate function is used as a window function, it aggregates over the rows within the current row's window frame. An aggregate used with ORDER BY and the default window frame definition produces a “running sum” type of behavior, which may or may not be what's wanted. To obtain aggregation over the whole partition, omit ORDER BY or use ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING. Other frame specifications can be used to obtain other effects.
+
+Note
+The SQL standard defines a RESPECT NULLS or IGNORE NULLS option for lead, lag, first_value, last_value, and nth_value. This is not implemented in PostgreSQL: the behavior is always the same as the standard's default, namely RESPECT NULLS. Likewise, the standard's FROM FIRST or FROM LAST option for nth_value is not implemented: only the default FROM FIRST behavior is supported. (You can achieve the result of FROM LAST by reversing the ORDER BY ordering.)
+
+## 9.23. Subquery Expressions {#subquery-expressions}
+
+This section describes the SQL-compliant subquery expressions available in PostgreSQL. All of the expression forms documented in this section return Boolean (true/false) results.
+
+9.23.1. EXISTS
+
+EXISTS (subquery)
+The argument of EXISTS is an arbitrary SELECT statement, or subquery. The subquery is evaluated to determine whether it returns any rows. If it returns at least one row, the result of EXISTS is “true”; if the subquery returns no rows, the result of EXISTS is “false”.
+
+The subquery can refer to variables from the surrounding query, which will act as constants during any one evaluation of the subquery.
+
+The subquery will generally only be executed long enough to determine whether at least one row is returned, not all the way to completion. It is unwise to write a subquery that has side effects (such as calling sequence functions); whether the side effects occur might be unpredictable.
+
+Since the result depends only on whether any rows are returned, and not on the contents of those rows, the output list of the subquery is normally unimportant. A common coding convention is to write all EXISTS tests in the form EXISTS(SELECT 1 WHERE ...). There are exceptions to this rule however, such as subqueries that use INTERSECT.
+
+This simple example is like an inner join on col2, but it produces at most one output row for each tab1 row, even if there are several matching tab2 rows:
+
+```sql
+SELECT col1
+FROM tab1
+WHERE EXISTS (SELECT 1 FROM tab2 WHERE col2 = tab1.col2);
+```
+
+Example
+```sql
+SELECT x FROM (VALUES (1),(2),(3)) a(x) WHERE EXISTS (SELECT 1 FROM (VALUES (3),(4),(5)) b(y) WHERE x=y) → [
+3
+]
+```
+
+9.23.2. IN
+
+expression IN (subquery)
+
+The right-hand side is a parenthesized subquery, which must return exactly one column. The left-hand expression is evaluated and compared to each row of the subquery result. The result of IN is “true” if any equal subquery row is found. The result is “false” if no equal row is found (including the case where the subquery returns no rows).
+
+Note that if the left-hand expression yields null, or if there are no equal right-hand values and at least one right-hand row yields null, the result of the IN construct will be null, not false. This is in accordance with SQL's normal rules for Boolean combinations of null values.
+
+As with EXISTS, it's unwise to assume that the subquery will be evaluated completely.
+
+row_constructor IN (subquery) (NOT SUPPORTED)
+
+The left-hand side of this form of IN is a row constructor, as described in Section 4.2.13. The right-hand side is a parenthesized subquery, which must return exactly as many columns as there are expressions in the left-hand row. The left-hand expressions are evaluated and compared row-wise to each row of the subquery result. The result of IN is “true” if any equal subquery row is found. The result is “false” if no equal row is found (including the case where the subquery returns no rows).
+
+As usual, null values in the rows are combined per the normal rules of SQL Boolean expressions. Two rows are considered equal if all their corresponding members are non-null and equal; the rows are unequal if any corresponding members are non-null and unequal; otherwise the result of that row comparison is unknown (null). If all the per-row results are either unequal or null, with at least one null, then the result of IN is null.
+
+Example
+```sql
+SELECT x FROM (VALUES (1),(2),(3)) a(x) WHERE x IN (SELECT y FROM (VALUES (3),(4),(5)) b(y)) → [
+3
+]
+```
+
+9.23.3. NOT IN
+
+expression NOT IN (subquery)
+
+The right-hand side is a parenthesized subquery, which must return exactly one column. The left-hand expression is evaluated and compared to each row of the subquery result. The result of NOT IN is “true” if only unequal subquery rows are found (including the case where the subquery returns no rows). The result is “false” if any equal row is found.
+
+Note that if the left-hand expression yields null, or if there are no equal right-hand values and at least one right-hand row yields null, the result of the NOT IN construct will be null, not true. This is in accordance with SQL's normal rules for Boolean combinations of null values.
+
+As with EXISTS, it's unwise to assume that the subquery will be evaluated completely.
+
+row_constructor NOT IN (subquery) (NOT SUPPORTED)
+
+The left-hand side of this form of NOT IN is a row constructor, as described in Section 4.2.13. The right-hand side is a parenthesized subquery, which must return exactly as many columns as there are expressions in the left-hand row. The left-hand expressions are evaluated and compared row-wise to each row of the subquery result. The result of NOT IN is “true” if only unequal subquery rows are found (including the case where the subquery returns no rows). The result is “false” if any equal row is found.
+
+As usual, null values in the rows are combined per the normal rules of SQL Boolean expressions. Two rows are considered equal if all their corresponding members are non-null and equal; the rows are unequal if any corresponding members are non-null and unequal; otherwise the result of that row comparison is unknown (null). If all the per-row results are either unequal or null, with at least one null, then the result of NOT IN is null.
+
+Example
+```sql
+SELECT x FROM (VALUES (1),(2),(3)) a(x) WHERE x NOT IN (SELECT y FROM (VALUES (3),(4),(5)) b(y)) ORDER BY x → [
+1
+2
+]
+```
+
+9.23.4. ANY/SOME
+
+expression operator ANY (subquery)
+expression operator SOME (subquery)
+
+The right-hand side is a parenthesized subquery, which must return exactly one column. The left-hand expression is evaluated and compared to each row of the subquery result using the given operator, which must yield a Boolean result. The result of ANY is “true” if any true result is obtained. The result is “false” if no true result is found (including the case where the subquery returns no rows).
+
+SOME is a synonym for ANY. IN is equivalent to = ANY.
+
+Note that if there are no successes and at least one right-hand row yields null for the operator's result, the result of the ANY construct will be null, not false. This is in accordance with SQL's normal rules for Boolean combinations of null values.
+
+As with EXISTS, it's unwise to assume that the subquery will be evaluated completely.
+
+row_constructor operator ANY (subquery) (NOT SUPPORTED)
+row_constructor operator SOME (subquery) (NOT SUPPORTED)
+
+The left-hand side of this form of ANY is a row constructor, as described in Section 4.2.13. The right-hand side is a parenthesized subquery, which must return exactly as many columns as there are expressions in the left-hand row. The left-hand expressions are evaluated and compared row-wise to each row of the subquery result, using the given operator. The result of ANY is “true” if the comparison returns true for any subquery row. The result is “false” if the comparison returns false for every subquery row (including the case where the subquery returns no rows). The result is NULL if no comparison with a subquery row returns true, and at least one comparison returns NULL.
+
+See Section 9.24.5 for details about the meaning of a row constructor comparison.
+
+Example
+```sql
+SELECT x FROM (VALUES (1),(2),(3)) a(x) WHERE x = ANY (SELECT y FROM (VALUES (3),(4),(5)) b(y)) → [
+3
+]
+```
+
+9.23.5. ALL
+
+expression operator ALL (subquery)
+
+The right-hand side is a parenthesized subquery, which must return exactly one column. The left-hand expression is evaluated and compared to each row of the subquery result using the given operator, which must yield a Boolean result. The result of ALL is “true” if all rows yield true (including the case where the subquery returns no rows). The result is “false” if any false result is found. The result is NULL if no comparison with a subquery row returns false, and at least one comparison returns NULL.
+
+NOT IN is equivalent to <> ALL.
+
+As with EXISTS, it's unwise to assume that the subquery will be evaluated completely.
+
+row_constructor operator ALL (subquery) (NOT SUPPORTED)
+
+The left-hand side of this form of ALL is a row constructor, as described in Section 4.2.13. The right-hand side is a parenthesized subquery, which must return exactly as many columns as there are expressions in the left-hand row. The left-hand expressions are evaluated and compared row-wise to each row of the subquery result, using the given operator. The result of ALL is “true” if the comparison returns true for all subquery rows (including the case where the subquery returns no rows). The result is “false” if the comparison returns false for any subquery row. The result is NULL if no comparison with a subquery row returns false, and at least one comparison returns NULL.
+
+See Section 9.24.5 for details about the meaning of a row constructor comparison.
+
+Example
+```sql
+SELECT x FROM (VALUES (1),(2),(3)) a(x) WHERE x <> ALL (SELECT y FROM (VALUES (3),(4),(5)) b(y)) ORDER BY x → [
+1
+2
+]
+```
+
+9.23.6. Single-Row Comparison
+
+row_constructor operator (subquery) (NOT SUPPORTED)
+
+The left-hand side is a row constructor, as described in Section 4.2.13. The right-hand side is a parenthesized subquery, which must return exactly as many columns as there are expressions in the left-hand row. Furthermore, the subquery cannot return more than one row. (If it returns zero rows, the result is taken to be null.) The left-hand side is evaluated and compared row-wise to the single subquery result row.
+
+See Section 9.24.5 for details about the meaning of a row constructor comparison.
+
+## 9.24. Row and Array Comparisons {#row-and-array-comparisons}
+
+This section describes several specialized constructs for making multiple comparisons between groups of values. These forms are syntactically related to the subquery forms of the previous section, but do not involve subqueries. The forms involving array subexpressions are PostgreSQL extensions; the rest are SQL-compliant. All of the expression forms documented in this section return Boolean (true/false) results.
+
+9.24.1. IN
+
+expression IN (value [, ...])
+The right-hand side is a parenthesized list of expressions. The result is “true” if the left-hand expression's result is equal to any of the right-hand expressions. This is a shorthand notation for
+
+expression = value1
+OR
+expression = value2
+OR
+...
+Note that if the left-hand expression yields null, or if there are no equal right-hand values and at least one right-hand expression yields null, the result of the IN construct will be null, not false. This is in accordance with SQL's normal rules for Boolean combinations of null values.
+
+Example
+```sql
+SELECT x FROM (VALUES (1),(2),(3)) a(x) WHERE x IN (1,2) ORDER BY x → [
+1
+2
+]
+
+SELECT x IN (y, z) FROM (VALUES (1,1,2),(2,3,null),(3,4,5),(4,null,null)) a(x,y,z) ORDER BY x → [
+true
+NULL
+false
+NULL
+]
+```
+
+9.24.2. NOT IN
+
+expression NOT IN (value [, ...])
+The right-hand side is a parenthesized list of expressions. The result is “true” if the left-hand expression's result is unequal to all of the right-hand expressions. This is a shorthand notation for
+
+expression <> value1
+AND
+expression <> value2
+AND
+...
+Note that if the left-hand expression yields null, or if there are no equal right-hand values and at least one right-hand expression yields null, the result of the NOT IN construct will be null, not true as one might naively expect. This is in accordance with SQL's normal rules for Boolean combinations of null values.
+
+Tip
+x NOT IN y is equivalent to NOT (x IN y) in all cases. However, null values are much more likely to trip up the novice when working with NOT IN than when working with IN. It is best to express your condition positively if possible.
+
+Example
+```sql
+SELECT x FROM (VALUES (1),(2),(3)) a(x) WHERE x NOT IN (1,2) ORDER BY x → [
+3
+]
+
+SELECT x NOT IN (y, z) FROM (VALUES (1,1,2),(2,3,null),(3,4,5),(4,null,null)) a(x,y,z) ORDER BY x → [
+false
+NULL
+true
+NULL
+]
+```
+
+9.24.3. ANY/SOME (array)
+
+expression operator ANY (array expression)
+expression operator SOME (array expression)
+
+The right-hand side is a parenthesized expression, which must yield an array value. The left-hand expression is evaluated and compared to each element of the array using the given operator, which must yield a Boolean result. The result of ANY is “true” if any true result is obtained. The result is “false” if no true result is found (including the case where the array has zero elements).
+
+If the array expression yields a null array, the result of ANY will be null. If the left-hand expression yields null, the result of ANY is ordinarily null (though a non-strict comparison operator could possibly yield a different result). Also, if the right-hand array contains any null elements and no true comparison result is obtained, the result of ANY will be null, not false (again, assuming a strict comparison operator). This is in accordance with SQL's normal rules for Boolean combinations of null values.
+
+SOME is a synonym for ANY.
+
+Example
+```sql
+SELECT x FROM (VALUES (1),(2),(3)) a(x) WHERE x = ANY (array[1,2]) ORDER BY x → [
+1
+2
+]
+]
+```
+
+9.24.4. ALL (array)
+
+expression operator ALL (array expression)
+
+The right-hand side is a parenthesized expression, which must yield an array value. The left-hand expression is evaluated and compared to each element of the array using the given operator, which must yield a Boolean result. The result of ALL is “true” if all comparisons yield true (including the case where the array has zero elements). The result is “false” if any false result is found.
+
+If the array expression yields a null array, the result of ALL will be null. If the left-hand expression yields null, the result of ALL is ordinarily null (though a non-strict comparison operator could possibly yield a different result). Also, if the right-hand array contains any null elements and no false comparison result is obtained, the result of ALL will be null, not true (again, assuming a strict comparison operator). This is in accordance with SQL's normal rules for Boolean combinations of null values.
+
+Example
+```sql
+SELECT x FROM (VALUES (1),(2),(3)) a(x) WHERE x <> ALL (array[1,2]) ORDER BY x → [
+3
+]
+]
+```
+
+9.24.5. Row Constructor Comparison (NOT SUPPORTED)
+
+row_constructor operator row_constructor
+
+Each side is a row constructor, as described in Section 4.2.13. The two row constructors must have the same number of fields. The given operator is applied to each pair of corresponding fields. (Since the fields could be of different types, this means that a different specific operator could be selected for each pair.) All the selected operators must be members of some B-tree operator class, or be the negator of an = member of a B-tree operator class, meaning that row constructor comparison is only possible when the operator is =, <>, <, <=, >, or >=, or has semantics similar to one of these.
+
+The = and <> cases work slightly differently from the others. Two rows are considered equal if all their corresponding members are non-null and equal; the rows are unequal if any corresponding members are non-null and unequal; otherwise the result of the row comparison is unknown (null).
+
+For the <, <=, > and >= cases, the row elements are compared left-to-right, stopping as soon as an unequal or null pair of elements is found. If either of this pair of elements is null, the result of the row comparison is unknown (null); otherwise comparison of this pair of elements determines the result. For example, ROW(1,2,NULL) < ROW(1,3,0) yields true, not null, because the third pair of elements are not considered.
+
+Note
+Prior to PostgreSQL 8.2, the <, <=, > and >= cases were not handled per SQL specification. A comparison like ROW(a,b) < ROW(c,d) was implemented as a < c AND b < d whereas the correct behavior is equivalent to a < c OR (a = c AND b < d).
+
+row_constructor IS DISTINCT FROM row_constructor
+
+This construct is similar to a <> row comparison, but it does not yield null for null inputs. Instead, any null value is considered unequal to (distinct from) any non-null value, and any two nulls are considered equal (not distinct). Thus the result will either be true or false, never null.
+
+row_constructor IS NOT DISTINCT FROM row_constructor
+
+This construct is similar to a = row comparison, but it does not yield null for null inputs. Instead, any null value is considered unequal to (distinct from) any non-null value, and any two nulls are considered equal (not distinct). Thus the result will always be either true or false, never null.
+
+9.24.6. Composite Type Comparison (NOT SUPPORTED)
+
+record operator record
+
+The SQL specification requires row-wise comparison to return NULL if the result depends on comparing two NULL values or a NULL and a non-NULL. PostgreSQL does this only when comparing the results of two row constructors (as in Section 9.24.5) or comparing a row constructor to the output of a subquery (as in Section 9.23). In other contexts where two composite-type values are compared, two NULL field values are considered equal, and a NULL is considered larger than a non-NULL. This is necessary in order to have consistent sorting and indexing behavior for composite types.
+
+Each side is evaluated and they are compared row-wise. Composite type comparisons are allowed when the operator is =, <>, <, <=, > or >=, or has semantics similar to one of these. (To be specific, an operator can be a row comparison operator if it is a member of a B-tree operator class, or is the negator of the = member of a B-tree operator class.) The default behavior of the above operators is the same as for IS [ NOT ] DISTINCT FROM for row constructors (see Section 9.24.5).
+
+To support matching of rows which include elements without a default B-tree operator class, the following operators are defined for composite type comparison: `*=`, `*<>`, `*<`, `*<=`, `*>`, and `*>=`. These operators compare the internal binary representation of the two rows. Two rows might have a different binary representation even though comparisons of the two rows with the equality operator is true. The ordering of rows under these comparison operators is deterministic but not otherwise meaningful. These operators are used internally for materialized views and might be useful for other specialized purposes such as replication and B-Tree deduplication (see Section 64.4.3). They are not intended to be generally useful for writing queries, though.
+
+## 9.25. Set Returning Functions {#set-returning-functions}
+
+This section describes functions that possibly return more than one row. The most widely used functions in this class are series generating functions, as detailed in Table 9.63 and Table 9.64. Other, more specialized set-returning functions are described elsewhere in this manual. See Section 7.2.1.4 for ways to combine multiple set-returning functions.
+
+Table 9.63. Series Generating Functions
+
+#|
+||Function|Description||
+||generate_series ( start integer, stop integer [, step integer ] ) → setof integer
+generate_series ( start bigint, stop bigint [, step bigint ] ) → setof bigint
+generate_series ( start numeric, stop numeric [, step numeric ] ) → setof numeric|
+Generates a series of values from start to stop, with a step size of step. step defaults to 1.||
+||generate_series ( start timestamp, stop timestamp, step interval ) → setof timestamp
+generate_series ( start timestamp with time zone, stop timestamp with time zone, step interval ) → setof timestamp with time zone|
+Generates a series of values from start to stop, with a step size of step.||
+|#
+
+When step is positive, zero rows are returned if start is greater than stop. Conversely, when step is negative, zero rows are returned if start is less than stop. Zero rows are also returned if any input is NULL. It is an error for step to be zero. Some examples follow:
+
+```sql
+SELECT * FROM generate_series(2,4) a → [
+2
+3
+4
+]
+
+SELECT * FROM generate_series(5,1,-2) a → [
+5
+3
+1
+]
+
+SELECT * FROM generate_series(4,3) a → [
+]
+
+SELECT * FROM generate_series(1.1, 4, 1.3) a → [
+1.1
+2.4
+3.7
+]
+
+-- this example relies on the date-plus-integer operator:
+SELECT date '2004-02-05' + s.a AS dates FROM generate_series(0,14,7) AS s(a) → [
+2004-02-05
+2004-02-12
+2004-02-19
+]
+
+SELECT * FROM generate_series('2008-03-01 00:00'::timestamp, '2008-03-04 12:00', '10 hours') a → [
+2008-03-01 00:00:00
+2008-03-01 10:00:00
+2008-03-01 20:00:00
+2008-03-02 06:00:00
+2008-03-02 16:00:00
+2008-03-03 02:00:00
+2008-03-03 12:00:00
+2008-03-03 22:00:00
+2008-03-04 08:00:00
+]
+```
+
+Table 9.64. Subscript Generating Functions (NOT SUPPORTED)
+
+#|
+||Function|Description||
+||generate_subscripts ( array anyarray, dim integer ) → setof integer|
+Generates a series comprising the valid subscripts of the dim'th dimension of the given array.||
+||generate_subscripts ( array anyarray, dim integer, reverse boolean ) → setof integer|
+Generates a series comprising the valid subscripts of the dim'th dimension of the given array. When reverse is true, returns the series in reverse order.||
+|#
+
+```sql
+-- basic usage:
+#SELECT generate_subscripts('{NULL,1,NULL,2}'::int[], 1) AS s → [
+1
+2
+3
+4
+]
+
+SELECT a AS array, s AS subscript, a[s] AS value FROM (SELECT generate_subscripts(a, 1) AS s, a FROM (VALUES (array[-1,-2]),(array[100,200,300])) s(a)) foo;
+```
+
+## 9.26. System Information Functions and Operators (NOT SUPPORTED) {#system-information-functions}
+
+## 9.27. System Administration Functions (NOT SUPPORTED) {#system-administration-functions}
+
+## 9.28. Trigger Functions (NOT SUPPORTED) {#trigger-functions}
+
+## 9.29. Event Trigger Functions (NOT SUPPORTED) {#event-trigger-functions}
+
+## 9.30. Statistics Information Functions (NOT SUPPORTED) {#statistics-information-functions}
diff --git a/yql/essentials/docs/ru/udf/list/postgres.md b/yql/essentials/docs/ru/udf/list/postgres.md index d138307411e..6bc7cbe8044 100644 --- a/yql/essentials/docs/ru/udf/list/postgres.md +++ b/yql/essentials/docs/ru/udf/list/postgres.md @@ -298,3 +298,6 @@ SELECT PgNot(PgBool(true)), -- PgBool(false) ; ``` +## Функции {#functions} + +{% include [pg_functions](../../_includes/pg_functions.md) %}
\ No newline at end of file diff --git a/yql/essentials/parser/antlr_ast/ya.make b/yql/essentials/parser/antlr_ast/ya.make index d3b33a2dfa4..432e143fb6c 100644 --- a/yql/essentials/parser/antlr_ast/ya.make +++ b/yql/essentials/parser/antlr_ast/ya.make @@ -1,4 +1,3 @@ RECURSE( gen ) - diff --git a/yql/essentials/parser/common/antlr4/depth_limiting_listener.cpp b/yql/essentials/parser/common/antlr4/depth_limiting_listener.cpp new file mode 100644 index 00000000000..41873352412 --- /dev/null +++ b/yql/essentials/parser/common/antlr4/depth_limiting_listener.cpp @@ -0,0 +1,42 @@ +#include "depth_limiting_listener.h" + +#include <util/string/cast.h> +#include <util/system/compiler.h> + +namespace NAntlrAST { + +TDepthLimitingListener::TDepthLimitingListener(size_t maxDepth) + : MaxDepth_(maxDepth) +{ +} + +void TDepthLimitingListener::Reset() { + CurrentDepth_ = 0; +} + +void TDepthLimitingListener::enterEveryRule(antlr4::ParserRuleContext* ctx) { + Y_UNUSED(ctx); + + ++CurrentDepth_; + if (MaxDepth_ < CurrentDepth_) { + throw antlr4::ParseCancellationException( + "Maximum parse tree depth exceeded: " + ToString(MaxDepth_)); + } +} + +void TDepthLimitingListener::exitEveryRule(antlr4::ParserRuleContext* ctx) { + Y_UNUSED(ctx); + + Y_ENSURE(0 < CurrentDepth_); + --CurrentDepth_; +} + +void TDepthLimitingListener::visitTerminal(antlr4::tree::TerminalNode* node) { + Y_UNUSED(node); +} + +void TDepthLimitingListener::visitErrorNode(antlr4::tree::ErrorNode* node) { + Y_UNUSED(node); +} + +} // namespace NAntlrAST diff --git a/yql/essentials/parser/common/antlr4/depth_limiting_listener.h b/yql/essentials/parser/common/antlr4/depth_limiting_listener.h new file mode 100644 index 00000000000..eb824318564 --- /dev/null +++ b/yql/essentials/parser/common/antlr4/depth_limiting_listener.h @@ -0,0 +1,28 @@ +#pragma once + +#ifdef ERROR + #undef ERROR +#endif +#include <contrib/libs/antlr4_cpp_runtime/src/antlr4-runtime.h> + +#include <util/generic/maybe.h> + +namespace NAntlrAST { + +class TDepthLimitingListener: public antlr4::tree::ParseTreeListener { +public: + explicit TDepthLimitingListener(size_t maxDepth); + + void Reset(); + + void enterEveryRule(antlr4::ParserRuleContext* ctx) override; + void exitEveryRule(antlr4::ParserRuleContext* ctx) override; + void visitTerminal(antlr4::tree::TerminalNode* node) override; + void visitErrorNode(antlr4::tree::ErrorNode* node) override; + +private: + const size_t MaxDepth_; + size_t CurrentDepth_ = 0; +}; + +} // namespace NAntlrAST diff --git a/yql/essentials/parser/common/antlr4/ya.make b/yql/essentials/parser/common/antlr4/ya.make index d74107fe119..af34c19c1a4 100644 --- a/yql/essentials/parser/common/antlr4/ya.make +++ b/yql/essentials/parser/common/antlr4/ya.make @@ -6,6 +6,7 @@ PEERDIR( ) SRCS( + depth_limiting_listener.cpp error_listener.cpp ) diff --git a/yql/essentials/parser/pg_wrapper/test/test_doc.py b/yql/essentials/parser/pg_wrapper/test/test_doc.py index 52214190e4b..b959375b2df 100644 --- a/yql/essentials/parser/pg_wrapper/test/test_doc.py +++ b/yql/essentials/parser/pg_wrapper/test/test_doc.py @@ -62,7 +62,7 @@ def test_doc(): print("WILL SKIP TESTS BEFORE: ", skip_before) if stop_at is not None: print("WILL STOP AT: ", stop_at) - doc_src = yatest.common.source_path("contrib/ydb/docs/ru/core/postgresql/_includes/functions.md") + doc_src = yatest.common.source_path("yql/essentials/docs/ru/_includes/pg_functions.md") with open(doc_src) as f: doc_data = f.readlines() in_code = False diff --git a/yql/essentials/parser/pg_wrapper/test/ya.make b/yql/essentials/parser/pg_wrapper/test/ya.make index a7af659b96c..53edfb9539d 100644 --- a/yql/essentials/parser/pg_wrapper/test/ya.make +++ b/yql/essentials/parser/pg_wrapper/test/ya.make @@ -22,7 +22,7 @@ REQUIREMENTS( ) DATA( - arcadia/contrib/ydb/docs/ru/core/postgresql/_includes/functions.md + arcadia/yql/essentials/docs/ru/_includes/pg_functions.md arcadia/yql/essentials/cfg/udf_test arcadia/yql/essentials/mount ) diff --git a/yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h b/yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h index 93cee5b55f4..eafcf156ea2 100644 --- a/yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h +++ b/yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h @@ -1,6 +1,7 @@ #pragma once #include <yql/essentials/parser/common/error.h> +#include <yql/essentials/parser/common/antlr4/depth_limiting_listener.h> #include <yql/essentials/parser/common/antlr4/error_listener.h> #include <yql/essentials/parser/common/antlr4/lexer_tokens_collector.h> @@ -11,7 +12,8 @@ #endif #include <contrib/libs/antlr4_cpp_runtime/src/antlr4-runtime.h> -#include <utility> +#include <util/generic/maybe.h> +#include <util/generic/scope.h> namespace NProtoAST { using namespace NAST; @@ -41,9 +43,11 @@ public: TString queryName = "query", google::protobuf::Arena* arena = nullptr, bool isAmbiguityError = false, - bool isAmbiguityDebugging = false) + bool isAmbiguityDebugging = false, + TMaybe<size_t> maxParseTreeDepth = Nothing()) : QueryName_(std::move(queryName)) , IsAmbiguityError_(isAmbiguityError) + , MaxParseTreeDepth_(maxParseTreeDepth) , InputStream_(data) , Lexer_(&InputStream_) , TokenStream_(&Lexer_) @@ -57,25 +61,38 @@ public: } google::protobuf::Message* BuildAST(IErrorCollector& errors) { - // TODO: find a better way to break on lexer errors - typename antlr4::YqlErrorListener listener(&errors, &Parser_.error, IsAmbiguityError_); Parser_.removeErrorListeners(); - Parser_.addErrorListener(&listener); + + // TODO: find a better way to break on lexer errors + typename antlr4::YqlErrorListener errorListener(&errors, &Parser_.error, IsAmbiguityError_); + Parser_.addErrorListener(&errorListener); + Y_DEFER { + Parser_.removeErrorListener(&errorListener); + }; + + TMaybe<NAntlrAST::TDepthLimitingListener> depthLimiter; + if (MaxParseTreeDepth_) { + depthLimiter.ConstructInPlace(*MaxParseTreeDepth_); + Parser_.addParseListener(depthLimiter.Get()); + } + Y_DEFER { + if (depthLimiter) { + Parser_.removeParseListener(depthLimiter.Get()); + } + }; + try { auto result = Parser_.Parse(&errors); - Parser_.removeErrorListener(&listener); if (Parser_.error) { result = nullptr; } Parser_.error = false; return result; } catch (const TTooManyErrors&) { - Parser_.removeErrorListener(&listener); Parser_.error = false; return nullptr; } catch (...) { errors.Error(0, 0, CurrentExceptionMessage()); - Parser_.removeErrorListener(&listener); Parser_.error = false; return nullptr; } @@ -84,6 +101,7 @@ public: private: TString QueryName_; bool IsAmbiguityError_; + TMaybe<size_t> MaxParseTreeDepth_; antlr4::ANTLRInputStream InputStream_; TLexer Lexer_; diff --git a/yql/essentials/parser/proto_ast/antlr4/ya.make b/yql/essentials/parser/proto_ast/antlr4/ya.make index 18c68a249bc..386e91324fb 100644 --- a/yql/essentials/parser/proto_ast/antlr4/ya.make +++ b/yql/essentials/parser/proto_ast/antlr4/ya.make @@ -11,5 +11,3 @@ SRCS( ) END() - - diff --git a/yql/essentials/public/fastcheck/check_state.cpp b/yql/essentials/public/fastcheck/check_state.cpp index c893d7f1d13..297f55aca1f 100644 --- a/yql/essentials/public/fastcheck/check_state.cpp +++ b/yql/essentials/public/fastcheck/check_state.cpp @@ -97,9 +97,16 @@ google::protobuf::Message* TCheckState::ParseSql(TIssues& issues) { return nullptr; } - NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); + NSQLTranslationV1::TParsers parsers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), + }; result.Msg = NSQLTranslationV1::SqlAST( parsers, @@ -148,12 +155,21 @@ const TAstParseResult* TCheckState::TranslateSql(TIssues& issues) { return &TranslateCache_->Result; } - NSQLTranslationV1::TLexers lexers; - lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); - lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); - NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); + NSQLTranslationV1::TLexers lexers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(), + }; + + NSQLTranslationV1::TParsers parsers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), + }; result.Result = NSQLTranslationV1::SqlASTToYql( lexers, diff --git a/yql/essentials/public/fastcheck/fastcheck.cpp b/yql/essentials/public/fastcheck/fastcheck.cpp index 174547e7eaf..6819f168296 100644 --- a/yql/essentials/public/fastcheck/fastcheck.cpp +++ b/yql/essentials/public/fastcheck/fastcheck.cpp @@ -42,12 +42,19 @@ bool CheckProgram(const TString& program, const TOptions& options, TIssues& erro return false; } - NSQLTranslationV1::TLexers lexers; - lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); - lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); - NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); + NSQLTranslationV1::TLexers lexers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(), + }; + + NSQLTranslationV1::TParsers parsers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false), + }; NSQLTranslation::TTranslators translators( nullptr, diff --git a/yql/essentials/public/fastcheck/format.cpp b/yql/essentials/public/fastcheck/format.cpp index b70f1848065..a83d59e049b 100644 --- a/yql/essentials/public/fastcheck/format.cpp +++ b/yql/essentials/public/fastcheck/format.cpp @@ -90,12 +90,22 @@ private: settings.AnsiLexer = request.IsAnsiLexer; settings.LangVer = request.LangVer; - NSQLTranslationV1::TLexers lexers; - lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); - lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); - NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); + NSQLTranslationV1::TLexers lexers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(), + }; + + NSQLTranslationV1::TParsers parsers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + /*maxParseTreeDepth=*/settings.MaxParseTreeDepth), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + /*maxParseTreeDepth=*/settings.MaxParseTreeDepth), + }; + auto formatter = NSQLFormat::MakeSqlFormatter(lexers, parsers, settings); TString formattedQuery; res.Success = formatter->Format(request.Program, formattedQuery, res.Issues); diff --git a/yql/essentials/public/purecalc/common/worker_factory.cpp b/yql/essentials/public/purecalc/common/worker_factory.cpp index 4a0791e25a7..da57ef98af7 100644 --- a/yql/essentials/public/purecalc/common/worker_factory.cpp +++ b/yql/essentials/public/purecalc/common/worker_factory.cpp @@ -272,8 +272,14 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile( lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); + parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth); + parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth); NSQLTranslation::TTranslators translators( nullptr, diff --git a/yql/essentials/public/sql_format/sql_format.cpp b/yql/essentials/public/sql_format/sql_format.cpp index 230beae42a1..9b892a8375c 100644 --- a/yql/essentials/public/sql_format/sql_format.cpp +++ b/yql/essentials/public/sql_format/sql_format.cpp @@ -9,12 +9,20 @@ namespace NSQLFormat { bool SqlFormatSimple(const TString& query, TString& formattedQuery, TString& error) { - NSQLTranslationV1::TLexers lexers; - lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); - lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); - NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); + NSQLTranslationV1::TLexers lexers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(), + }; + + NSQLTranslationV1::TParsers parsers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false), + }; + return SqlFormatSimple(lexers, parsers, query, formattedQuery, error); } diff --git a/yql/essentials/sql/settings/translation_settings.cpp b/yql/essentials/sql/settings/translation_settings.cpp index 81cf2785150..02fd18501b2 100644 --- a/yql/essentials/sql/settings/translation_settings.cpp +++ b/yql/essentials/sql/settings/translation_settings.cpp @@ -6,6 +6,7 @@ #include <library/cpp/deprecated/split/split_iterator.h> +#include <util/string/cast.h> #include <util/string/split.h> #include <util/string/join.h> #include <util/system/env.h> @@ -33,7 +34,7 @@ public: }; [[noreturn]] TString ThrowBad(TStringBuf flag, const TVector<TString>& args) { - YQL_ENSURE(false, "Bad " << flag << "args [" << JoinSeq(", ", args) << "]"); + YQL_ENSURE(false, "Bad " << flag << " args [" << JoinSeq(", ", args) << "]"); } } // namespace @@ -188,6 +189,21 @@ void ParseTranslationSettings(const TExtendedSqlFlags& flags, TTranslationSettin } }, }, + { + "MaxParseTreeDepth", + [](const TVector<TString>& args, TTranslationSettings& s) { + if (args.size() != 1) { + ThrowBad("MaxParseTreeDepth", args); + } + + size_t value = 0; + if (!TryFromString(args[0], value)) { + ThrowBad("MaxParseTreeDepth", args); + } + + s.MaxParseTreeDepth = value; + }, + }, }; for (const auto& [flag, args] : flags) { diff --git a/yql/essentials/sql/settings/translation_settings.h b/yql/essentials/sql/settings/translation_settings.h index 9252d84caf6..a5598aa6eb8 100644 --- a/yql/essentials/sql/settings/translation_settings.h +++ b/yql/essentials/sql/settings/translation_settings.h @@ -21,7 +21,9 @@ class IAutoParamBuilderFactory; } // namespace NYql namespace NSQLTranslation { + constexpr const size_t SQL_MAX_PARSER_ERRORS = 100; +constexpr const size_t SQL_MAX_PARSE_TREE_DEPTH = 4096; enum class ESqlMode { QUERY = 0, @@ -121,6 +123,7 @@ struct TTranslationSettings { bool PGDisable; bool WarnOnV0; bool TestAntlr4; // TODO(YQL-19017): remove. + TMaybe<size_t> MaxParseTreeDepth; ISqlFeaturePolicy::TPtr V0WarnAsError; ISqlFeaturePolicy::TPtr DqDefaultAuto; ISqlFeaturePolicy::TPtr BlockDefaultAuto; diff --git a/yql/essentials/sql/v1/format/check/check_format.cpp b/yql/essentials/sql/v1/format/check/check_format.cpp index 41ca6dfedf2..a1e891df807 100644 --- a/yql/essentials/sql/v1/format/check/check_format.cpp +++ b/yql/essentials/sql/v1/format/check/check_format.cpp @@ -133,8 +133,14 @@ TMaybe<TString> CheckedFormat( }; NSQLTranslationV1::TParsers parsers = { - .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(), - .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(), + .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), }; auto formatter = NSQLFormat::MakeSqlFormatter(lexers, parsers, settings); diff --git a/yql/essentials/sql/v1/ide/completion/check/check_complete.cpp b/yql/essentials/sql/v1/ide/completion/check/check_complete.cpp index a0d7b448df0..bb8ad16d72d 100644 --- a/yql/essentials/sql/v1/ide/completion/check/check_complete.cpp +++ b/yql/essentials/sql/v1/ide/completion/check/check_complete.cpp @@ -12,6 +12,8 @@ #include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h> #include <yql/essentials/sql/v1/lexer/antlr4_pure_ansi/lexer.h> +#include <contrib/libs/antlr4_cpp_runtime/src/Exceptions.h> + #include <util/charset/utf8.h> #include <util/random/random.h> @@ -93,15 +95,21 @@ bool CheckComplete(TStringBuf query, TYqlContext ctx) { bool CheckComplete(TStringBuf query, NYql::TExprNode::TPtr root, NYql::TExprContext& ctx, NYql::TIssues& issues) try { return CheckComplete(query, MakeYqlAnalysis()->Analyze(root, ctx)); +} catch (const antlr4::ParseCancellationException& e) { + Y_UNUSED(e); + return true; } catch (...) { - issues.AddIssue(FormatCurrentException()); + issues.AddIssue(CurrentExceptionMessage()); return false; } bool CheckComplete(TStringBuf query, NYql::TAstNode& root, NYql::TIssues& issues) try { return CheckComplete(query, MakeYqlAnalysis()->Analyze(root, issues).GetOrElse({})); +} catch (const antlr4::ParseCancellationException& e) { + Y_UNUSED(e); + return true; } catch (...) { - issues.AddIssue(FormatCurrentException()); + issues.AddIssue(CurrentExceptionMessage()); return false; } diff --git a/yql/essentials/sql/v1/ide/completion/sql_complete_ut.cpp b/yql/essentials/sql/v1/ide/completion/sql_complete_ut.cpp index 092f7803a5e..7f2e1511e3d 100644 --- a/yql/essentials/sql/v1/ide/completion/sql_complete_ut.cpp +++ b/yql/essentials/sql/v1/ide/completion/sql_complete_ut.cpp @@ -2401,4 +2401,23 @@ Y_UNIT_TEST(CachedSchema) { } } +Y_UNIT_TEST(NoStackOverflowOnDeeplyNestedSubquery) { + constexpr size_t Depth = 4 * 1024; + + auto engine = MakeSqlCompletionEngineUT(); + + TStringBuilder query; + query << '#'; + for (size_t i = 0; i < Depth; ++i) { + query << "SELECT * FROM ("; + } + query << "SELECT 1"; + for (size_t i = 0; i < Depth; ++i) { + query << ")"; + } + + UNIT_ASSERT_EXCEPTION_CONTAINS( + Complete(engine, query), std::exception, "Maximum parse tree depth exceeded"); +} + } // Y_UNIT_TEST_SUITE(SqlCompleteTests) diff --git a/yql/essentials/sql/v1/ide/pure_ast/parser.cpp b/yql/essentials/sql/v1/ide/pure_ast/parser.cpp index 4b9fd1e8d59..0b0e6d0805a 100644 --- a/yql/essentials/sql/v1/ide/pure_ast/parser.cpp +++ b/yql/essentials/sql/v1/ide/pure_ast/parser.cpp @@ -2,11 +2,13 @@ #include "parse_tree.h" +#include <yql/essentials/parser/common/antlr4/depth_limiting_listener.h> #include <yql/essentials/parser/antlr_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h> #include <yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h> #include <util/system/yassert.h> #include <util/charset/utf8.h> +#include <util/generic/maybe.h> #include <util/string/builder.h> namespace NSQLPureAST { @@ -22,6 +24,8 @@ public: template <bool IsAnsiLexer> class TParser: public IParser { + static constexpr size_t MaxParseTreeDepth = 4096; + public: using TLexer = std::conditional_t< IsAnsiLexer, @@ -32,11 +36,13 @@ public: : Chars_() , Lexer_(&Chars_) , Tokens_(&Lexer_) + , DepthLimiter_(/*maxDepth=*/MaxParseTreeDepth) , Parser_(&Tokens_) { Lexer_.removeErrorListeners(); Parser_.removeErrorListeners(); Parser_.setErrorHandler(std::make_shared<TErrorStrategy>()); + Parser_.addParseListener(&DepthLimiter_); } TParseTree Parse(TStringBuf text) override { @@ -61,6 +67,7 @@ private: Chars_.load(text.Data(), text.Size(), /* lenient = */ false); Lexer_.reset(); Tokens_.setTokenSource(&Lexer_); + DepthLimiter_.Reset(); Parser_.reset(); return Parser_.sql_query(); } @@ -89,6 +96,7 @@ private: antlr4::ANTLRInputStream Chars_; TLexer Lexer_; antlr4::CommonTokenStream Tokens_; + NAntlrAST::TDepthLimitingListener DepthLimiter_; SQLv1 Parser_; }; diff --git a/yql/essentials/sql/v1/ide/pure_ast/ya.make b/yql/essentials/sql/v1/ide/pure_ast/ya.make index ed83b11e704..c0eab67f51d 100644 --- a/yql/essentials/sql/v1/ide/pure_ast/ya.make +++ b/yql/essentials/sql/v1/ide/pure_ast/ya.make @@ -1,6 +1,7 @@ LIBRARY() PEERDIR( + yql/essentials/parser/common/antlr4 yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4 yql/essentials/parser/antlr_ast/gen/v1_antlr4 ) diff --git a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp index 2d29d0a6ffd..71aa6aba9ef 100644 --- a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp +++ b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp @@ -10,9 +10,10 @@ namespace { class TParser: public NSQLTranslation::IParser { public: - explicit TParser(bool isAmbuguityError, bool isAmbiguityDebugging) - : IsAmbiguityError_(isAmbuguityError) + TParser(bool isAmbiguityError, bool isAmbiguityDebugging, TMaybe<size_t> maxParseTreeDepth) + : IsAmbiguityError_(isAmbiguityError) , IsAmbiguityDebugging_(isAmbiguityDebugging) + , MaxParseTreeDepth_(maxParseTreeDepth) { } @@ -23,39 +24,43 @@ public: NProtoAST::TProtoASTBuilder4< NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> - builder(query, queryName, arena, IsAmbiguityError_, IsAmbiguityDebugging_); + builder(query, queryName, arena, IsAmbiguityError_, IsAmbiguityDebugging_, MaxParseTreeDepth_); return builder.BuildAST(err); } private: - bool IsAmbiguityError_; - bool IsAmbiguityDebugging_; + const bool IsAmbiguityError_; + const bool IsAmbiguityDebugging_; + const TMaybe<size_t> MaxParseTreeDepth_; }; class TFactory: public NSQLTranslation::IParserFactory { public: - explicit TFactory(bool isAmbuguityError, bool isAmbiguityDebugging) - : IsAmbiguityError_(isAmbuguityError) + TFactory(bool isAmbiguityError, bool isAmbiguityDebugging, TMaybe<size_t> maxParseTreeDepth) + : IsAmbiguityError_(isAmbiguityError) , IsAmbiguityDebugging_(isAmbiguityDebugging) + , MaxParseTreeDepth_(maxParseTreeDepth) { } std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final { - return std::make_unique<TParser>(IsAmbiguityError_, IsAmbiguityDebugging_); + return std::make_unique<TParser>(IsAmbiguityError_, IsAmbiguityDebugging_, MaxParseTreeDepth_); } private: - bool IsAmbiguityError_; - bool IsAmbiguityDebugging_; + const bool IsAmbiguityError_; + const bool IsAmbiguityDebugging_; + const TMaybe<size_t> MaxParseTreeDepth_; }; } // namespace NSQLTranslation::TParserFactoryPtr MakeAntlr4ParserFactory( bool isAmbiguityError, - bool isAmbiguityDebugging) + bool isAmbiguityDebugging, + TMaybe<size_t> maxParseTreeDepth) { - return MakeIntrusive<TFactory>(isAmbiguityError, isAmbiguityDebugging); + return MakeIntrusive<TFactory>(isAmbiguityError, isAmbiguityDebugging, maxParseTreeDepth); } } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h index 146dcfff7aa..aeb83d7212f 100644 --- a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h +++ b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h @@ -1,10 +1,13 @@ #pragma once #include <yql/essentials/parser/proto_ast/common.h> +#include <util/generic/maybe.h> + namespace NSQLTranslationV1 { NSQLTranslation::TParserFactoryPtr MakeAntlr4ParserFactory( bool isAmbiguityError = false, - bool isAmbiguityDebugging = false); + bool isAmbiguityDebugging = false, + TMaybe<size_t> maxParseTreeDepth = Nothing()); } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp index 89eb7c5aa1a..d9aed46eae9 100644 --- a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp +++ b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp @@ -10,9 +10,10 @@ namespace { class TParser: public NSQLTranslation::IParser { public: - explicit TParser(bool isAmbuguityError, bool isAmbiguityDebugging) - : IsAmbiguityError_(isAmbuguityError) + TParser(bool isAmbiguityError, bool isAmbiguityDebugging, TMaybe<size_t> maxParseTreeDepth) + : IsAmbiguityError_(isAmbiguityError) , IsAmbiguityDebugging_(isAmbiguityDebugging) + , MaxParseTreeDepth_(maxParseTreeDepth) { } @@ -23,39 +24,43 @@ public: NProtoAST::TProtoASTBuilder4< NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> - builder(query, queryName, arena, IsAmbiguityError_, IsAmbiguityDebugging_); + builder(query, queryName, arena, IsAmbiguityError_, IsAmbiguityDebugging_, MaxParseTreeDepth_); return builder.BuildAST(err); } private: - bool IsAmbiguityError_; - bool IsAmbiguityDebugging_; + const bool IsAmbiguityError_; + const bool IsAmbiguityDebugging_; + const TMaybe<size_t> MaxParseTreeDepth_; }; class TFactory: public NSQLTranslation::IParserFactory { public: - explicit TFactory(bool isAmbuguityError, bool isAmbiguityDebugging) - : IsAmbiguityError_(isAmbuguityError) + TFactory(bool isAmbiguityError, bool isAmbiguityDebugging, TMaybe<size_t> maxParseTreeDepth) + : IsAmbiguityError_(isAmbiguityError) , IsAmbiguityDebugging_(isAmbiguityDebugging) + , MaxParseTreeDepth_(maxParseTreeDepth) { } std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final { - return std::make_unique<TParser>(IsAmbiguityError_, IsAmbiguityDebugging_); + return std::make_unique<TParser>(IsAmbiguityError_, IsAmbiguityDebugging_, MaxParseTreeDepth_); } private: - bool IsAmbiguityError_; - bool IsAmbiguityDebugging_; + const bool IsAmbiguityError_; + const bool IsAmbiguityDebugging_; + const TMaybe<size_t> MaxParseTreeDepth_; }; } // namespace NSQLTranslation::TParserFactoryPtr MakeAntlr4AnsiParserFactory( bool isAmbiguityError, - bool isAmbiguityDebugging) + bool isAmbiguityDebugging, + TMaybe<size_t> maxParseTreeDepth) { - return MakeIntrusive<TFactory>(isAmbiguityError, isAmbiguityDebugging); + return MakeIntrusive<TFactory>(isAmbiguityError, isAmbiguityDebugging, maxParseTreeDepth); } } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h index 39ff5d6eba2..05a716c1c75 100644 --- a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h +++ b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h @@ -1,10 +1,13 @@ #pragma once #include <yql/essentials/parser/proto_ast/common.h> +#include <util/generic/maybe.h> + namespace NSQLTranslationV1 { NSQLTranslation::TParserFactoryPtr MakeAntlr4AnsiParserFactory( bool isAmbiguityError = false, - bool isAmbiguityDebugging = false); + bool isAmbiguityDebugging = false, + TMaybe<size_t> maxParseTreeDepth = Nothing()); } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/proto_parser/proto_parser_ut.cpp b/yql/essentials/sql/v1/proto_parser/proto_parser_ut.cpp new file mode 100644 index 00000000000..c45189d976a --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/proto_parser_ut.cpp @@ -0,0 +1,89 @@ +#include "proto_parser.h" + +#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h> +#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h> +#include <yql/essentials/sql/v1/lexer/lexer.h> +#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h> +#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h> + +#include <library/cpp/testing/unittest/registar.h> + +Y_UNIT_TEST_SUITE(ProtoParserTests) { + +google::protobuf::Message* SqlAST( + const TString& query, + NSQLTranslation::TTranslationSettings settings, + NYql::TIssues& issues) +{ + NSQLTranslationV1::TLexers lexers; + NSQLTranslationV1::TParsers parsers; + + if (!ParseTranslationSettings(query, settings, issues)) { + return nullptr; + } + + lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); + lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); + auto lexer = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer); + auto onNextToken = [&](NSQLTranslation::TParsedToken&& token) { + Y_UNUSED(token); + }; + + if (!lexer->Tokenize(query, "", onNextToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) { + return nullptr; + } + + parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth); + + parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth); + + return NSQLTranslationV1::SqlAST( + parsers, + query, + /* queryName = */ "", + issues, + NSQLTranslation::SQL_MAX_PARSER_ERRORS, + settings.AnsiLexer, + settings.Arena); +} + +TString GenerateQuery(size_t depth = 8 * 1024) { + const TString prefix = "SELECT 1 FROM ("; + const TString core = "SELECT 1"; + const TString suffix = ")"; + + TString query; + query.reserve(prefix.size() * depth + core.size() + suffix.size() * depth); + for (size_t i = 0; i < depth; ++i) { + query += prefix; + } + query += core; + for (size_t i = 0; i < depth; ++i) { + query += suffix; + } + + return query; +} + +Y_UNIT_TEST(StackOverflowSubquery) { + const auto query = GenerateQuery(); + + google::protobuf::Arena arena; + NSQLTranslation::TTranslationSettings settings; + settings.Arena = &arena; + settings.MaxParseTreeDepth = 4 * 1024; + + NYql::TIssues issues; + const auto* m = SqlAST(query, settings, issues); + + UNIT_ASSERT(!m); + UNIT_ASSERT_STRING_CONTAINS(issues.ToOneLineString(), "Maximum parse tree depth exceeded"); +} + +} // Y_UNIT_TEST_SUITE(ProtoParserTests) diff --git a/yql/essentials/sql/v1/proto_parser/ut/ya.make b/yql/essentials/sql/v1/proto_parser/ut/ya.make new file mode 100644 index 00000000000..f52f74321bf --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/ut/ya.make @@ -0,0 +1,15 @@ +UNITTEST_FOR(yql/essentials/sql/v1/proto_parser) + +SRCS( + proto_parser_ut.cpp +) + +PEERDIR( + yql/essentials/sql/v1/lexer/antlr4 + yql/essentials/sql/v1/lexer/antlr4_ansi + yql/essentials/sql/v1/lexer + yql/essentials/sql/v1/proto_parser/antlr4 + yql/essentials/sql/v1/proto_parser/antlr4_ansi +) + +END() diff --git a/yql/essentials/sql/v1/proto_parser/ya.make b/yql/essentials/sql/v1/proto_parser/ya.make index 91047f7a32e..a1587d6aed5 100644 --- a/yql/essentials/sql/v1/proto_parser/ya.make +++ b/yql/essentials/sql/v1/proto_parser/ya.make @@ -20,3 +20,5 @@ RECURSE( antlr4 antlr4_ansi ) + +RECURSE_FOR_TESTS(ut) diff --git a/yql/essentials/sql/v1/select_yql.cpp b/yql/essentials/sql/v1/select_yql.cpp index 47e0b6e028b..c9b89286923 100644 --- a/yql/essentials/sql/v1/select_yql.cpp +++ b/yql/essentials/sql/v1/select_yql.cpp @@ -123,7 +123,7 @@ public: return new TYqlValuesNode(*this); } - bool SetColumns(TVector<TString> columns, TContext& ctx) { + bool SetColumns(TVector<TYqlColumnRef> columns, TContext& ctx) { if (columns.empty()) { return true; } @@ -142,14 +142,17 @@ private: TNodePtr BuildColumnList() const { TNodePtr columns = Y(); for (size_t i = 0; i < Width_; ++i) { + TPosition position = Pos_; TString name; if (!Columns_ || Columns_->size() <= i) { name = TStringBuilder() << "column" << i; } else { - name = Columns_->at(i); + const auto& c = Columns_->at(i); + position = c.Position; + name = c.Name; } - columns->Add(BuildQuotedAtom(Pos_, name)); + columns->Add(BuildQuotedAtom(std::move(position), name)); } return columns; } @@ -184,7 +187,7 @@ private: TNodePtr Values_; size_t Width_ = 0; - TMaybe<TVector<TString>> Columns_; + TMaybe<TVector<TYqlColumnRef>> Columns_; }; class TYqlSelectLikeNode: public INode { @@ -224,6 +227,11 @@ protected: }; class TYqlSetItemNode final: public TYqlSelectLikeNode, private TYqlSetItemArgs { + struct TProjectionItem { + TNodePtr Term; + bool IsAliasSynthetic = false; + }; + public: explicit TYqlSetItemNode(TYqlSetItemArgs&& args) : TYqlSelectLikeNode(args.Position) @@ -232,7 +240,9 @@ public: } bool DoInit(TContext& ctx, ISource* src) override { - if (!InitProjection(ctx, src) || + auto projection = InitProjection(ctx, src); + + if (!projection || !InitSource(ctx, src) || (Where && !Where->GetRef().Init(ctx, src)) || (GroupBy && !Init(ctx, src, *GroupBy)) || @@ -240,13 +250,14 @@ public: !Init(ctx, src, Windows) || !TYqlSelectLikeNode::Init(ctx, src, OrderBy) || (Limit && !Limit->GetRef().Init(ctx, src)) || - (Offset && !Offset->GetRef().Init(ctx, src))) { + (Offset && !Offset->GetRef().Init(ctx, src))) + { return false; } TNodePtr item = Y(); { - TNodePtr items = BuildYqlResultItems(Projection); + TNodePtr items = BuildYqlResultItems(*projection); if (!items) { return false; } @@ -354,31 +365,43 @@ public: } private: - bool InitProjection(TContext& ctx, ISource* src) const { + TMaybe<TVector<TProjectionItem>> InitProjection(TContext& ctx, ISource* src) const { return std::visit( TOverloaded{ [&](const TVector<TNodePtr>& terms) { + YQL_ENSURE(!terms.empty()); return InitTerms(ctx, src, terms); }, - [](const TPlainAsterisk&) { - return true; + [&](const TPlainAsterisk&) -> TMaybe<TVector<TProjectionItem>> { + return TVector<TProjectionItem>{}; }, }, Projection); } - bool InitTerms(TContext& ctx, ISource* src, const TVector<TNodePtr>& terms) const { - THashSet<TString> used = UsedLables(terms); + TMaybe<TVector<TProjectionItem>> + InitTerms(TContext& ctx, ISource* src, const TVector<TNodePtr>& terms) const { + THashSet<TString> used = UsedLabels(terms); + TVector<TProjectionItem> items(Reserve(terms.size())); for (size_t i = 0; i < terms.size(); ++i) { const TNodePtr& term = terms[i]; - TString label = TermAlias(term, i, used); + auto [label, isSyntheticA] = TermAlias(term, i, used); used.emplace(label); term->SetLabel(label); + + items.push_back({ + .Term = term, + .IsAliasSynthetic = isSyntheticA, + }); } - return ::NSQLTranslationV1::Init(ctx, src, terms); + if (!::NSQLTranslationV1::Init(ctx, src, terms)) { + return Nothing(); + } + + return items; } bool InitSource(TContext& ctx, ISource* src) const { @@ -457,7 +480,7 @@ private: return true; } - THashSet<TString> UsedLables(const TVector<TNodePtr>& terms) const { + THashSet<TString> UsedLabels(const TVector<TNodePtr>& terms) const { THashSet<TString> used(terms.size()); for (const TNodePtr& term : terms) { used.emplace(term->GetLabel()); @@ -465,46 +488,51 @@ private: return used; } - TString TermAlias(const TNodePtr& term, size_t i, const THashSet<TString>& used) const { + std::pair<TString, /*isSynthetic=*/bool> + TermAlias(const TNodePtr& term, size_t i, const THashSet<TString>& used) const { if (const TString& label = term->GetLabel(); !label.empty()) { - return label; + return {label, false}; } if (TMaybe<TString> alias = ColumnAlias(term)) { - return std::move(*alias); + return {std::move(*alias), false}; } for (;; ++i) { TString alias = TStringBuilder() << "column" << i; if (!used.contains(alias)) { - return alias; + return {alias, true}; } } } - TNodePtr BuildYqlResultItems(const TProjection& projection) const { - return std::visit( - TOverloaded{ - [&](const TVector<TNodePtr>& terms) { return BuildYqlResultItems(terms); }, - [&](const TPlainAsterisk& terms) { return BuildYqlResultItems(terms); }, - }, projection); - } + TNodePtr BuildYqlResultItems(const TVector<TProjectionItem>& projection) const { + if (projection.empty()) { + return BuildYqlResultItems(TPlainAsterisk()); + } - TNodePtr BuildYqlResultItems(const TVector<TNodePtr>& terms) const { TNodePtr items = Y(); - for (const TNodePtr& term : terms) { - items->Add(BuildYqlResultItem(term->GetLabel(), term)); + for (const auto& [term, isSynthetic] : projection) { + items->Add(BuildYqlResultItem(term->GetLabel(), isSynthetic, term)); } return items; } TNodePtr BuildYqlResultItems(const TPlainAsterisk&) const { - return Y(BuildYqlResultItem("", Y("YqlStar"))); + return Y(BuildYqlResultItem(/*name=*/"", /*isSynthetic=*/false, Y("YqlStar"))); } - TNodePtr BuildYqlResultItem(TString name, TNodePtr term) const { + TNodePtr BuildYqlResultItem(TString name, bool isSynthetic, TNodePtr term) const { TNodePtr nameAtom = BuildQuotedAtom(Pos_, name); - return Y("YqlResultItem", std::move(nameAtom), Y("Void"), Y("lambda", Q(Y()), std::move(term))); + + TNodePtr item = Y("YqlResultItem"); + item = L(std::move(item), std::move(nameAtom)); + item = L(std::move(item), Y("Void")); + if (isSynthetic) { + item = L(std::move(item), Q(Y(Q(Y(Q("synthetic")))))); + } + item = L(std::move(item), Y("lambda", Q(Y()), std::move(term))); + return item; } TMaybe<TString> ColumnAlias(const TNodePtr& term) const { @@ -524,16 +552,28 @@ private: } TMaybe<TNodePtr> BuildFromElement(TContext& ctx, const TYqlSource& source) const { - const auto build = [this](TNodePtr node, TString name, const TVector<TString>& columns) { + const auto build = [this](TNodePtr node, + TString name, + const TVector<TYqlColumnRef>& columns, + bool isCTE = false) + { YQL_ENSURE(!name.empty(), "An empty source name is unsupported"); + TNodePtr nameAtom = BuildQuotedAtom(Pos_, name); + TNodePtr columnList = Y(); - for (const TString& column : columns) { - columnList = L(std::move(columnList), BuildQuotedAtom(Pos_, column)); + for (const TYqlColumnRef& c : columns) { + columnList = L(std::move(columnList), BuildQuotedAtom(c.Position, c.Name)); } - TNodePtr nameAtom = BuildQuotedAtom(Pos_, name); - return Q(Y(std::move(node), std::move(nameAtom), Q(std::move(columnList)))); + TNodePtr x = Y(); + x = L(std::move(x), std::move(node)); + x = L(std::move(x), std::move(nameAtom)); + x = L(std::move(x), Q(std::move(columnList))); + if (isCTE) { + x = L(std::move(x), Q(Y(Q(Y(Q("cte")))))); + } + return Q(std::move(x)); }; if (!source.Alias) { @@ -550,7 +590,7 @@ private: } if (source.Alias->Kind == TYqlSourceAlias::EKind::CTE) { - return build(source.Node, source.Alias->Name, columns); + return build(source.Node, source.Alias->Name, columns, /*isCTE=*/true); } ctx.Error() << "Qualified by column names source alias " diff --git a/yql/essentials/sql/v1/select_yql.h b/yql/essentials/sql/v1/select_yql.h index 51debe6352f..1f9e58fb534 100644 --- a/yql/essentials/sql/v1/select_yql.h +++ b/yql/essentials/sql/v1/select_yql.h @@ -4,6 +4,11 @@ namespace NSQLTranslationV1 { +struct TYqlColumnRef { + TPosition Position; + TString Name; +}; + struct TYqlSourceAlias { enum class EKind { Subquery, @@ -12,7 +17,7 @@ struct TYqlSourceAlias { TPosition Position; TString Name; - TVector<TString> Columns; + TVector<TYqlColumnRef> Columns; EKind Kind = EKind::Subquery; }; diff --git a/yql/essentials/sql/v1/sql_select_yql.cpp b/yql/essentials/sql/v1/sql_select_yql.cpp index 0fc2d3f327c..369e4f715f9 100644 --- a/yql/essentials/sql/v1/sql_select_yql.cpp +++ b/yql/essentials/sql/v1/sql_select_yql.cpp @@ -143,7 +143,7 @@ private: return std::unexpected(ESQLError::Basic); } - TVector<TString> columns; + TVector<TYqlColumnRef> columns; if (rule.HasBlock2()) { columns = TableColumns(rule.GetBlock2().GetRule_pure_column_list1()); } @@ -568,6 +568,7 @@ private: } Token(rule.GetToken2()); + setItem.Position = Ctx_.Pos(); if (Mode_ != NSQLTranslation::ESqlMode::QUERY) { return Unsupported("ESqlMode != QUERY"); @@ -1547,15 +1548,23 @@ private: } } - TVector<TString> TableColumns(const TRule_pure_column_list& rule) { - TVector<TString> columns; - columns.emplace_back(Id(rule.GetRule_an_id2(), *this)); + TVector<TYqlColumnRef> TableColumns(const TRule_pure_column_list& rule) { + TVector<TYqlColumnRef> columns(Reserve(1 + rule.GetBlock3().size())); + columns.emplace_back(TableColumn(rule.GetRule_an_id2())); for (const auto& id : rule.GetBlock3()) { - columns.emplace_back(Id(id.GetRule_an_id2(), *this)); + columns.emplace_back(TableColumn(id.GetRule_an_id2())); } return columns; } + TYqlColumnRef TableColumn(const TRule_an_id& rule) { + TString id = Id(rule, *this); + return { + .Position = Ctx_.Pos(), + .Name = std::move(id), + }; + } + EYqlSetOp ToOp(const TRule_union_op& node) { const TString token = ToLowerUTF8(node.GetToken1().GetValue()); diff --git a/yql/essentials/tests/common/test_framework/yql_utils.py b/yql/essentials/tests/common/test_framework/yql_utils.py index 529ba1b8324..3ddeb7e1d16 100644 --- a/yql/essentials/tests/common/test_framework/yql_utils.py +++ b/yql/essentials/tests/common/test_framework/yql_utils.py @@ -1215,4 +1215,3 @@ class LoggingDowngrade(object): for name, level in self.loggers: log = logging.getLogger(name) log.setLevel(level) - return True diff --git a/yql/essentials/tests/sql/minirun/part11/canondata/result.json b/yql/essentials/tests/sql/minirun/part11/canondata/result.json index 075c74984af..bd06a4e9655 100644 --- a/yql/essentials/tests/sql/minirun/part11/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part11/canondata/result.json @@ -1283,7 +1283,7 @@ { "checksum": "d1a1ecd0a254a6d78f041fb9f2898440", "size": 2158, - "uri": "https://{canondata_backend}/1899731/450d16e5954d0f0d78bafa54b230a5d645089cc9/resource.tar.gz#test.test_select_yql-with_cte_graph-default.txt-Debug_/opt.yql" + "uri": "https://{canondata_backend}/1130705/450d16e5954d0f0d78bafa54b230a5d645089cc9/resource.tar.gz#test.test_select_yql-with_cte_graph-default.txt-Debug_/opt.yql" } ], "test.test[select_yql-with_cte_graph-default.txt-Results]": [ diff --git a/yql/essentials/tests/sql/minirun/part14/canondata/test.test_select_yql-group_by_a_avg_sum_count_b-default.txt-Results_/extracted b/yql/essentials/tests/sql/minirun/part14/canondata/test.test_select_yql-group_by_a_avg_sum_count_b-default.txt-Results_/extracted index ca766826fc4..74104a2558e 100644 --- a/yql/essentials/tests/sql/minirun/part14/canondata/test.test_select_yql-group_by_a_avg_sum_count_b-default.txt-Results_/extracted +++ b/yql/essentials/tests/sql/minirun/part14/canondata/test.test_select_yql-group_by_a_avg_sum_count_b-default.txt-Results_/extracted @@ -1,10 +1,16 @@ <tmp_path>/program.sql:<main>: Warning: Type annotation - <tmp_path>/program.sql:<main>: Warning: At function: RemovePrefixMembers, At function: Unordered, At function: YqlSelect, At tuple, At tuple, At tuple, At function: YqlSetItem, At tuple, At tuple, At tuple, At function: YqlResultItem, At lambda - - <tmp_path>/program.sql:<main>:6:12: Warning: At function: / - Sum(b) / Count(b) AS avg2 + <tmp_path>/program.sql:<main>:3:1: Warning: At function: RemovePrefixMembers, At function: Unordered, At function: YqlSelect, At tuple, At tuple, At tuple, At function: YqlSetItem, At tuple + SELECT + ^ + <tmp_path>/program.sql:<main>: Warning: At tuple + + <tmp_path>/program.sql:<main>:3:1: Warning: At tuple, At function: YqlResultItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:6:12: Warning: At function: / + Sum(b) / Count(b) AS avg2 ^ - <tmp_path>/program.sql:<main>:6:12: Warning: Integral type implicit bitcast: Int64 and Uint64 - Sum(b) / Count(b) AS avg2 + <tmp_path>/program.sql:<main>:6:12: Warning: Integral type implicit bitcast: Int64 and Uint64 + Sum(b) / Count(b) AS avg2 ^
\ No newline at end of file diff --git a/yql/essentials/tests/sql/minirun/part15/canondata/result.json b/yql/essentials/tests/sql/minirun/part15/canondata/result.json index daaf80a7a77..800862f3e6d 100644 --- a/yql/essentials/tests/sql/minirun/part15/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part15/canondata/result.json @@ -1123,16 +1123,16 @@ ], "test.test[select_yql-with_cte_column_order-default.txt-Debug]": [ { - "checksum": "f8aad33ffd7164186ecc7ce4c75ab1fc", - "size": 1027, - "uri": "https://{canondata_backend}/1784826/24d0ba8d8815179c2043e869e443713cc813220d/resource.tar.gz#test.test_select_yql-with_cte_column_order-default.txt-Debug_/opt.yql" + "checksum": "bef80e3bd8c9ead7160adae4f95e9719", + "size": 966, + "uri": "https://{canondata_backend}/1900335/b619bd3e2003c8b1549523b7199881c565c90c81/resource.tar.gz#test.test_select_yql-with_cte_column_order-default.txt-Debug_/opt.yql" } ], "test.test[select_yql-with_cte_column_order-default.txt-Results]": [ { - "checksum": "e28d345d542fcad785426cdf347f9dc3", - "size": 6747, - "uri": "https://{canondata_backend}/1784826/24d0ba8d8815179c2043e869e443713cc813220d/resource.tar.gz#test.test_select_yql-with_cte_column_order-default.txt-Results_/results.txt" + "checksum": "0dc3d1d129aaec2986fa6fe3b7991c90", + "size": 5772, + "uri": "https://{canondata_backend}/1900335/b619bd3e2003c8b1549523b7199881c565c90c81/resource.tar.gz#test.test_select_yql-with_cte_column_order-default.txt-Results_/results.txt" } ], "test.test[select_yql-with_cte_shadow-default.txt-Debug]": [ diff --git a/yql/essentials/tests/sql/minirun/part5/canondata/result.json b/yql/essentials/tests/sql/minirun/part5/canondata/result.json index 4784cba1848..5a99db535ad 100644 --- a/yql/essentials/tests/sql/minirun/part5/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part5/canondata/result.json @@ -1219,6 +1219,23 @@ "uri": "https://{canondata_backend}/1847551/58f37c8e1b8c74f66657e94c231b8acc11b82bf5/resource.tar.gz#test.test_select_yql-window_fn_lag_1-default.txt-Results_/results.txt" } ], + "test.test[select_yql-with_cte_column_order_warning_1-default.txt-Debug]": [ + { + "checksum": "6e8842d5e64c78f1dd590a5077a3919a", + "size": 363, + "uri": "https://{canondata_backend}/1942100/aab35e85ff469f67e894dd1870fc858f4bf47a51/resource.tar.gz#test.test_select_yql-with_cte_column_order_warning_1-default.txt-Debug_/opt.yql" + } + ], + "test.test[select_yql-with_cte_column_order_warning_1-default.txt-Results]": [ + { + "checksum": "bdea5726342a2fbb8acd35387e11e129", + "size": 1388, + "uri": "https://{canondata_backend}/1942100/aab35e85ff469f67e894dd1870fc858f4bf47a51/resource.tar.gz#test.test_select_yql-with_cte_column_order_warning_1-default.txt-Results_/results.txt" + }, + { + "uri": "file://test.test_select_yql-with_cte_column_order_warning_1-default.txt-Results_/extracted" + } + ], "test.test[simple_pg-now-default.txt-Debug]": [ { "checksum": "2d3c63e18948b1dd8cab2769647acdd0", diff --git a/yql/essentials/tests/sql/minirun/part5/canondata/test.test_select_yql-with_cte_column_order_warning_1-default.txt-Results_/extracted b/yql/essentials/tests/sql/minirun/part5/canondata/test.test_select_yql-with_cte_column_order_warning_1-default.txt-Results_/extracted new file mode 100644 index 00000000000..07ee6c4cd85 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part5/canondata/test.test_select_yql-with_cte_column_order_warning_1-default.txt-Results_/extracted @@ -0,0 +1,20 @@ +<tmp_path>/program.sql:<main>: Warning: Type annotation + + <tmp_path>/program.sql:<main>:4:1: Warning: At function: RemovePrefixMembers, At function: Unordered, At function: YqlSelect, At tuple, At tuple, At tuple, At function: YqlSetItem, Column names in SELECT don't match column specification in parenthesis + SELECT 1 FROM x; + ^ + <tmp_path>/program.sql:<main>:3:9: Warning: At position 1 actual "b" doesn't match expected "a" + WITH x (a, b) AS (SELECT 1 AS b, '2' AS a) + ^ + <tmp_path>/program.sql:<main>:3:12: Warning: At position 2 actual "a" doesn't match expected "b" + WITH x (a, b) AS (SELECT 1 AS b, '2' AS a) + ^ + <tmp_path>/program.sql:<main>:7:1: Warning: At function: RemovePrefixMembers, At function: Unordered, At function: YqlSelect, At tuple, At tuple, At tuple, At function: YqlSetItem, Column names in SELECT don't match column specification in parenthesis + SELECT 1 FROM x; + ^ + <tmp_path>/program.sql:<main>:6:9: Warning: At position 1 actual "c" doesn't match expected "a" + WITH x (a, b) AS (SELECT 1 AS c, '2' AS d) + ^ + <tmp_path>/program.sql:<main>:6:12: Warning: At position 2 actual "d" doesn't match expected "b" + WITH x (a, b) AS (SELECT 1 AS c, '2' AS d) + ^
\ No newline at end of file diff --git a/yql/essentials/tests/sql/minirun/part6/canondata/result.json b/yql/essentials/tests/sql/minirun/part6/canondata/result.json index dabd5d061d9..7348d93d8aa 100644 --- a/yql/essentials/tests/sql/minirun/part6/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part6/canondata/result.json @@ -1591,6 +1591,20 @@ "uri": "https://{canondata_backend}/1917492/53531106e44e755ea77008ec27c01bc41d81b31c/resource.tar.gz#test.test_select_yql-window_fn_rownumber_0-default.txt-Results_/results.txt" } ], + "test.test[select_yql-with_cte_column_order_warning_3-default.txt-Debug]": [ + { + "checksum": "10170f139ee9936913349710c58a7f21", + "size": 1178, + "uri": "https://{canondata_backend}/1130705/6a59cef7793dee5c0be5997fa74d991bd4c1fa65/resource.tar.gz#test.test_select_yql-with_cte_column_order_warning_3-default.txt-Debug_/opt.yql" + } + ], + "test.test[select_yql-with_cte_column_order_warning_3-default.txt-Results]": [ + { + "checksum": "3432be66794780aaeaad6d43b71e08e5", + "size": 695, + "uri": "https://{canondata_backend}/1130705/6a59cef7793dee5c0be5997fa74d991bd4c1fa65/resource.tar.gz#test.test_select_yql-with_cte_column_order_warning_3-default.txt-Results_/results.txt" + } + ], "test.test[select_yql_tpch-q12-default.txt-Debug]": [ { "checksum": "016a78e3b03b83737abd950a8b11c5b3", diff --git a/yql/essentials/tests/sql/minirun/part7/canondata/result.json b/yql/essentials/tests/sql/minirun/part7/canondata/result.json index 7b32e632916..dfbe4abe05c 100644 --- a/yql/essentials/tests/sql/minirun/part7/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part7/canondata/result.json @@ -1154,6 +1154,23 @@ "uri": "https://{canondata_backend}/1903280/44f4f1690f38bce1fadfbce889480224fbb8b527/resource.tar.gz#test.test_select_yql-window_fn_rank_0_2-default.txt-Results_/results.txt" } ], + "test.test[select_yql-with_cte_column_order_warning_2-default.txt-Debug]": [ + { + "checksum": "faaf016532aede5fdfd1ec1209ca6002", + "size": 422, + "uri": "https://{canondata_backend}/1889210/395b3905dee71983895b616ba65cc3a4c3be2eba/resource.tar.gz#test.test_select_yql-with_cte_column_order_warning_2-default.txt-Debug_/opt.yql" + } + ], + "test.test[select_yql-with_cte_column_order_warning_2-default.txt-Results]": [ + { + "checksum": "4024f25cdc45effaeb85c61883503ad7", + "size": 2081, + "uri": "https://{canondata_backend}/1889210/395b3905dee71983895b616ba65cc3a4c3be2eba/resource.tar.gz#test.test_select_yql-with_cte_column_order_warning_2-default.txt-Results_/results.txt" + }, + { + "uri": "file://test.test_select_yql-with_cte_column_order_warning_2-default.txt-Results_/extracted" + } + ], "test.test[subselect-partition_by-default.txt-Debug]": [ { "checksum": "f4ef6bca9cf8ae3b87242f8510e75833", diff --git a/yql/essentials/tests/sql/minirun/part7/canondata/test.test_select_yql-with_cte_column_order_warning_2-default.txt-Results_/extracted b/yql/essentials/tests/sql/minirun/part7/canondata/test.test_select_yql-with_cte_column_order_warning_2-default.txt-Results_/extracted new file mode 100644 index 00000000000..e8ac776f526 --- /dev/null +++ b/yql/essentials/tests/sql/minirun/part7/canondata/test.test_select_yql-with_cte_column_order_warning_2-default.txt-Results_/extracted @@ -0,0 +1,8 @@ +<tmp_path>/program.sql:<main>: Warning: Type annotation + + <tmp_path>/program.sql:<main>:10:1: Warning: At function: RemovePrefixMembers, At function: Unordered, At function: YqlSelect, At tuple, At tuple, At tuple, At function: YqlSetItem, Column names in SELECT don't match column specification in parenthesis + SELECT 1 FROM x; + ^ + <tmp_path>/program.sql:<main>:9:12: Warning: At position 2 actual "c" doesn't match expected "b" + WITH x (a, b) AS (SELECT 1, '2' AS c) + ^
\ No newline at end of file diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json index f7b3d04f180..31cdc16f860 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/result.json +++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json @@ -9423,23 +9423,23 @@ ], "test_sql2yql.test[select_yql-alias]": [ { - "checksum": "1bfc06cd9cfb45243dd3e1077f6224ce", - "size": 2200, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-alias_/sql.yql" + "checksum": "19c58044aa34742ef7db3ce4bcb8b92b", + "size": 2268, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-alias_/sql.yql" } ], "test_sql2yql.test[select_yql-correlated_asterisk_subquery_scalar_with_join]": [ { - "checksum": "e276b1495ae66e2bebde1fc566948093", - "size": 1648, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-correlated_asterisk_subquery_scalar_with_join_/sql.yql" + "checksum": "a65af77b740884fc6ff81c02ff96088f", + "size": 1665, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-correlated_asterisk_subquery_scalar_with_join_/sql.yql" } ], "test_sql2yql.test[select_yql-correlated_asterisk_subquery_scalar_with_source]": [ { - "checksum": "7c3fd2d9f8758f70a9c12b41475b075e", - "size": 1383, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-correlated_asterisk_subquery_scalar_with_source_/sql.yql" + "checksum": "ffcb4cb1d1cb2f871568f8f2925ca628", + "size": 1400, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-correlated_asterisk_subquery_scalar_with_source_/sql.yql" } ], "test_sql2yql.test[select_yql-correlated_order_by_trivial]": [ @@ -9458,30 +9458,30 @@ ], "test_sql2yql.test[select_yql-correlated_projection_group_by]": [ { - "checksum": "a58fe73633380c018c4110853c8075c3", - "size": 1886, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-correlated_projection_group_by_/sql.yql" + "checksum": "b11b34035e7717d03c832d863e9f87cc", + "size": 1903, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-correlated_projection_group_by_/sql.yql" } ], "test_sql2yql.test[select_yql-correlated_projection_in]": [ { - "checksum": "54eef2d7627901c6dbbae680ecc0d293", - "size": 1903, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-correlated_projection_in_/sql.yql" + "checksum": "a17329e9580d33c72f1c5ca975c27eea", + "size": 1937, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-correlated_projection_in_/sql.yql" } ], "test_sql2yql.test[select_yql-correlated_projection_trivial]": [ { - "checksum": "b7cdee087fd32d126d606567c1e7a48f", - "size": 1157, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-correlated_projection_trivial_/sql.yql" + "checksum": "91ee84fe327282e0e5034246a357b98a", + "size": 1174, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-correlated_projection_trivial_/sql.yql" } ], "test_sql2yql.test[select_yql-correlated_projection_unique]": [ { - "checksum": "e745b78c41036be21c343526f6f935d1", - "size": 1584, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-correlated_projection_unique_/sql.yql" + "checksum": "a5d6945dfbd4e6721a8067d28a4492e6", + "size": 1618, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-correlated_projection_unique_/sql.yql" } ], "test_sql2yql.test[select_yql-correlated_where_exists]": [ @@ -9493,23 +9493,23 @@ ], "test_sql2yql.test[select_yql-correlated_where_group_by]": [ { - "checksum": "119671ab9f34f341e1e317f1e21e1ee4", - "size": 2613, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-correlated_where_group_by_/sql.yql" + "checksum": "a267cb75d0ea9eab5b9cabe8cee4136a", + "size": 2630, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-correlated_where_group_by_/sql.yql" } ], "test_sql2yql.test[select_yql-correlated_where_in]": [ { - "checksum": "c1542d1ec275558ca06dce6510568f14", - "size": 1886, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-correlated_where_in_/sql.yql" + "checksum": "0d04c984516c860d7a080206520a08bf", + "size": 1903, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-correlated_where_in_/sql.yql" } ], "test_sql2yql.test[select_yql-correlated_where_in_group_by]": [ { - "checksum": "806dfe9f49dc2f3160fbec650f23e4f6", - "size": 2113, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-correlated_where_in_group_by_/sql.yql" + "checksum": "8ab7dadeceeb92f2aa3810b478f80dd7", + "size": 2130, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-correlated_where_in_group_by_/sql.yql" } ], "test_sql2yql.test[select_yql-correlated_where_trivial]": [ @@ -9521,9 +9521,9 @@ ], "test_sql2yql.test[select_yql-correlated_where_unique]": [ { - "checksum": "dafae1ca2ba45a63df8077c493bc78c2", - "size": 1626, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-correlated_where_unique_/sql.yql" + "checksum": "9793680eea1b27c1877ac32d0d763d03", + "size": 1643, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-correlated_where_unique_/sql.yql" } ], "test_sql2yql.test[select_yql-expr]": [ @@ -9535,9 +9535,9 @@ ], "test_sql2yql.test[select_yql-from_anon_values]": [ { - "checksum": "70c02275534c7c8f719b1d304f388166", - "size": 915, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-from_anon_values_/sql.yql" + "checksum": "925712f4209f27a1efae9a436eca1d4f", + "size": 932, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-from_anon_values_/sql.yql" } ], "test_sql2yql.test[select_yql-from_select]": [ @@ -9626,9 +9626,9 @@ ], "test_sql2yql.test[select_yql-group_by_a_avg_b]": [ { - "checksum": "20ae69bfad5b568593be4afbefefc191", - "size": 1271, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_avg_b_/sql.yql" + "checksum": "be6f51acf9504e9797fe36f37ef5b33e", + "size": 1288, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_avg_b_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_avg_sum_count_b]": [ @@ -9640,254 +9640,254 @@ ], "test_sql2yql.test[select_yql-group_by_a_b_count_c]": [ { - "checksum": "8ea79a7280ad6d476017d5e8793dcf00", - "size": 1466, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_b_count_c_/sql.yql" + "checksum": "7c8a938082f18ea8f579994aaf7292f5", + "size": 1483, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_b_count_c_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_count_all]": [ { - "checksum": "121a7656908c124990537999fd2ba3df", - "size": 1254, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_all_/sql.yql" + "checksum": "93cf433a50b2ae2721d8b9e6ba119c6c", + "size": 1271, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_all_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_count_all_null]": [ { - "checksum": "7e7e7a53d37522c82f78fa29b0116a4d", - "size": 1233, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_all_null_/sql.yql" + "checksum": "0f403e5418eb8dbf1bd12df489086b7d", + "size": 1250, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_all_null_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_count_all_nulls]": [ { - "checksum": "554b56a6b12359b5d20c68b1f4f33f0d", - "size": 1615, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_all_nulls_/sql.yql" + "checksum": "329011f7d2fb1e5b770e3a6aaa1d6ab7", + "size": 1632, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_all_nulls_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_count_b]": [ { - "checksum": "6efe41b6463941efcecbf183b9e2cce6", - "size": 1267, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_b_/sql.yql" + "checksum": "10a9f392f2cac4082b6d898274afa2d4", + "size": 1284, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_b_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_count_b_null]": [ { - "checksum": "942201519e6b9c11a33c8b54eca63240", - "size": 1246, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_b_null_/sql.yql" + "checksum": "c78bc5f6f7202d73f5d8694d2e52d367", + "size": 1263, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_b_null_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_count_b_nulls]": [ { - "checksum": "edf5a2556066184c0294640a5ee505fe", - "size": 1628, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_b_nulls_/sql.yql" + "checksum": "9b1b403e92537fc9cffccf7e7b6d4d85", + "size": 1645, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_count_b_nulls_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_having_b]": [ { - "checksum": "5f6ffcb46186637655c4b7aeaad1aa1f", - "size": 1607, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_having_b_/sql.yql" + "checksum": "20c9dcebe562242b3698d10e3a4760bd", + "size": 1624, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_having_b_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_having_b_where_b]": [ { - "checksum": "bc8e224d20fdb568b66fa70e99b47bd6", - "size": 1689, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_having_b_where_b_/sql.yql" + "checksum": "41ffd410ccb454f6f3581eb78d55efca", + "size": 1706, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_having_b_where_b_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_max_b]": [ { - "checksum": "eb22802587d37709420ad09ba351ae72", - "size": 1265, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_max_b_/sql.yql" + "checksum": "272543fc49bec2e72e85315b7d9fec73", + "size": 1282, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_max_b_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_min_b]": [ { - "checksum": "98d349778d26d3761e8f94bfb4eac289", - "size": 1265, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_min_b_/sql.yql" + "checksum": "8064688629fb1c3f09b7826c3bee3f0e", + "size": 1282, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_min_b_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_sum_b]": [ { - "checksum": "0b10a0c1e90e0938364e92c3ff1fc3d2", - "size": 1263, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_sum_b_/sql.yql" + "checksum": "3e63b5d88d079aac11594f8dd1d33f2e", + "size": 1280, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_sum_b_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_sum_distinct_b]": [ { - "checksum": "d74d66de5ee071ab8649946bade27f4e", - "size": 1457, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_sum_distinct_b_/sql.yql" + "checksum": "8bae4cf157f12bdd74625276d057d8e5", + "size": 1474, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_sum_distinct_b_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_a_where_count_b]": [ { - "checksum": "5440c52ba2edcdd080d980d3d05d752f", - "size": 1475, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_where_count_b_/sql.yql" + "checksum": "1aa01378c24209d4f78ac0353f1520ad", + "size": 1492, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_a_where_count_b_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_apb_z]": [ { - "checksum": "40df2ad8a77df53c909831ada094a2b0", - "size": 1294, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_apb_z_/sql.yql" + "checksum": "94652be102e3c36276ec37e4b2064150", + "size": 1328, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_apb_z_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_count_b]": [ { - "checksum": "77fb23e8cc5af367f53cf648daf488b2", - "size": 997, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_count_b_/sql.yql" + "checksum": "1924d6eca93036d971ea3158ec947432", + "size": 1014, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_count_b_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_having_b]": [ { - "checksum": "6ee5c2813789c1c81d936c7c585e4fdd", - "size": 1137, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_having_b_/sql.yql" + "checksum": "34ed17d4b8464715f95126b63ba6e635", + "size": 1154, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_having_b_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_min_max]": [ { - "checksum": "ee7847aa4a394ecdb45e84a3dd6d3408", - "size": 1151, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_min_max_/sql.yql" + "checksum": "53e3bc77932052261112ea573a83d380", + "size": 1168, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_min_max_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_nokey_promotion]": [ { - "checksum": "a0d0122047ec9a8ba12a870b2c3051c3", - "size": 3658, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_nokey_promotion_/sql.yql" + "checksum": "9cfeabcd660d2297612fceccfa5b0256", + "size": 3760, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_nokey_promotion_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_optionality]": [ { - "checksum": "ab7ee13fdc472baf8b693e1508c873de", - "size": 11004, - "uri": "https://{canondata_backend}/1871102/eb25b1ebe2b0e4a641cfac3e711cbc8b7d5fc148/resource.tar.gz#test_sql2yql.test_select_yql-group_by_optionality_/sql.yql" + "checksum": "2e854fe48b6fedc776ffb0e90be363f3", + "size": 11157, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_optionality_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_optionality_null]": [ { - "checksum": "366a9f9cd77cf52aa3bc49420113da5a", - "size": 10781, - "uri": "https://{canondata_backend}/1903280/3231d7a47b92cae05de9e903f30e0b0e9e69f173/resource.tar.gz#test_sql2yql.test_select_yql-group_by_optionality_null_/sql.yql" + "checksum": "afb4adde3f64281b96f3e558c3d09daa", + "size": 10934, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_optionality_null_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_optionality_pgint]": [ { - "checksum": "fbcd4a6623016b188f7e9a15a794a0fe", - "size": 11475, - "uri": "https://{canondata_backend}/1784117/3f0bf9bbff9508e6f345caaefebed54cf8bfac65/resource.tar.gz#test_sql2yql.test_select_yql-group_by_optionality_pgint_/sql.yql" + "checksum": "1714117fef9af7107da327bac8ea0667", + "size": 11628, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_optionality_pgint_/sql.yql" } ], "test_sql2yql.test[select_yql-group_by_sum_b]": [ { - "checksum": "de97aedaab0447fa39895f7d8f53d3fe", - "size": 995, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-group_by_sum_b_/sql.yql" + "checksum": "667e239030508cbece7eec281f703310", + "size": 1012, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-group_by_sum_b_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_a_cube_ab]": [ { - "checksum": "159d41205a7421d6f5cea1f81c89636c", - "size": 1519, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_a_cube_ab_/sql.yql" + "checksum": "44a2b51f2bd81699628d528573150ec2", + "size": 1536, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_a_cube_ab_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_a_explicit]": [ { - "checksum": "894f67068f8fbb7792996d980ab4a6b2", - "size": 1293, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_a_explicit_/sql.yql" + "checksum": "be4424a606a73c0b746f5ff6e70ea6f3", + "size": 1310, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_a_explicit_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_a_explicit_tuple]": [ { - "checksum": "4506d2e4ebf505eff26b634bea2ed8f6", - "size": 1440, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_a_explicit_tuple_/sql.yql" + "checksum": "3549d7df1697d6b46c70decbcc638e5e", + "size": 1474, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_a_explicit_tuple_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_a_implicit]": [ { - "checksum": "894f67068f8fbb7792996d980ab4a6b2", - "size": 1293, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_a_implicit_/sql.yql" + "checksum": "be4424a606a73c0b746f5ff6e70ea6f3", + "size": 1310, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_a_implicit_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_a_rollup_ab]": [ { - "checksum": "df658b4b164fefe6172aeb760899d70f", - "size": 1521, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_a_rollup_ab_/sql.yql" + "checksum": "aa4dbfc114909b9149a1900e7098e1f2", + "size": 1538, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_a_rollup_ab_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_ab_a]": [ { - "checksum": "ead074735b5305da24d5787191fc22fb", - "size": 1491, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_ab_a_/sql.yql" + "checksum": "c6d0d0f3cec3e5924408532e8f3e9d59", + "size": 1508, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_ab_a_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_ab_a_o]": [ { - "checksum": "db04c660de2fc07550decaeaad1d762e", - "size": 1495, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_ab_a_o_/sql.yql" + "checksum": "da23f6b6929f09fc0173bd2578498da7", + "size": 1512, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_ab_a_o_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_ab_a_o_optional]": [ { - "checksum": "b57516964ea723e56744a130587d0630", - "size": 1759, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_ab_a_o_optional_/sql.yql" + "checksum": "f8a88bdd058acc13c6a6df1c43bf819a", + "size": 1776, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_ab_a_o_optional_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_abc_a]": [ { - "checksum": "d73da09520920a8f5680bf4ac08dba55", - "size": 1660, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_abc_a_/sql.yql" + "checksum": "94ff8f77bb3dd2426052505e69e67fbf", + "size": 1677, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_abc_a_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_apb_c]": [ { - "checksum": "700c67d2cdd60fd7378c8033f80d0ab9", - "size": 1632, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_apb_c_/sql.yql" + "checksum": "5370f48ed7eb4820408f22d734488c40", + "size": 1666, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_apb_c_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_cube_ab]": [ { - "checksum": "c3b2d6a6971688b0ce3f17eeaa583a95", - "size": 1468, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_cube_ab_/sql.yql" + "checksum": "48018c797c92062ce9b64dd263c1064d", + "size": 1485, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_cube_ab_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_rollup_ab]": [ { - "checksum": "ddc22bc7da06e86ba334e42e0fca919a", - "size": 1470, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_rollup_ab_/sql.yql" + "checksum": "c0c050dc33874cf29a22621658c87bd4", + "size": 1487, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_rollup_ab_/sql.yql" } ], "test_sql2yql.test[select_yql-grouping_sets_rollup_grouping]": [ { - "checksum": "6d044060762c8c76208f1c20f9d33f9f", - "size": 1432, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_rollup_grouping_/sql.yql" + "checksum": "a1b55a857ac8a3bda8886c4bb0b4f156", + "size": 1449, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-grouping_sets_rollup_grouping_/sql.yql" } ], "test_sql2yql.test[select_yql-in_smart_parenthesis_context_3]": [ @@ -10032,9 +10032,9 @@ ], "test_sql2yql.test[select_yql-join_matrix]": [ { - "checksum": "23535b8191a726d7d951a1badaeff759", - "size": 11923, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-join_matrix_/sql.yql" + "checksum": "8b9ea3d0b6a49ebab03f7586b09800ab", + "size": 12093, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-join_matrix_/sql.yql" } ], "test_sql2yql.test[select_yql-join_on_2]": [ @@ -10074,23 +10074,23 @@ ], "test_sql2yql.test[select_yql-lambda_smart_parenthesis_context_4]": [ { - "checksum": "3011476103621298649a609d54f44c63", - "size": 815, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-lambda_smart_parenthesis_context_4_/sql.yql" + "checksum": "51c69227218727f0c5b3e037285a7a47", + "size": 832, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-lambda_smart_parenthesis_context_4_/sql.yql" } ], "test_sql2yql.test[select_yql-minimal]": [ { - "checksum": "ca720652ef30bfc7846fd91976ee2613", - "size": 725, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-minimal_/sql.yql" + "checksum": "a32e4c54033ad8ad026001c6c219e631", + "size": 742, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-minimal_/sql.yql" } ], "test_sql2yql.test[select_yql-minimal_ordered_columns]": [ { - "checksum": "9ae23c281a13afc99f18fe84ed18c0b5", - "size": 797, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-minimal_ordered_columns_/sql.yql" + "checksum": "9cf8f666aa4797ba2f3cb81c41a7e401", + "size": 814, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-minimal_ordered_columns_/sql.yql" } ], "test_sql2yql.test[select_yql-named_node_auto_subquery_flatten_by]": [ @@ -10109,16 +10109,16 @@ ], "test_sql2yql.test[select_yql-named_node_expression]": [ { - "checksum": "70f1b2552cb39ca07994bbced617e027", - "size": 868, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-named_node_expression_/sql.yql" + "checksum": "d66cb104bb0906fd089aefbb92297bee", + "size": 902, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-named_node_expression_/sql.yql" } ], "test_sql2yql.test[select_yql-named_node_subquery_external]": [ { - "checksum": "f685628cc099f9d180739ecce70e9932", - "size": 1434, - "uri": "https://{canondata_backend}/1903885/ac392fa3c931e976b881e7663233fb23a4666244/resource.tar.gz#test_sql2yql.test_select_yql-named_node_subquery_external_/sql.yql" + "checksum": "6a359eb6b7f7d8b5de710744e4620ec2", + "size": 1451, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-named_node_subquery_external_/sql.yql" } ], "test_sql2yql.test[select_yql-named_node_subquery_from]": [ @@ -10137,30 +10137,30 @@ ], "test_sql2yql.test[select_yql-named_node_subquery_in]": [ { - "checksum": "76a9a39241b37c5e5d75db621c677237", - "size": 1936, - "uri": "https://{canondata_backend}/1903885/ac392fa3c931e976b881e7663233fb23a4666244/resource.tar.gz#test_sql2yql.test_select_yql-named_node_subquery_in_/sql.yql" + "checksum": "38bf36a03ab44ec7bb463f9ff8104ede", + "size": 2004, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-named_node_subquery_in_/sql.yql" } ], "test_sql2yql.test[select_yql-named_node_subquery_in_chain]": [ { - "checksum": "1a7386356c6250e2ba39ac8333e9f09e", - "size": 2317, - "uri": "https://{canondata_backend}/1903885/ac392fa3c931e976b881e7663233fb23a4666244/resource.tar.gz#test_sql2yql.test_select_yql-named_node_subquery_in_chain_/sql.yql" + "checksum": "ef6f3d77a322ab0c8f003d572e69c0ff", + "size": 2385, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-named_node_subquery_in_chain_/sql.yql" } ], "test_sql2yql.test[select_yql-named_node_subquery_scalar]": [ { - "checksum": "7aa7a42682771894b5dcd23d41e6d49e", - "size": 1868, - "uri": "https://{canondata_backend}/1903885/ac392fa3c931e976b881e7663233fb23a4666244/resource.tar.gz#test_sql2yql.test_select_yql-named_node_subquery_scalar_/sql.yql" + "checksum": "a556d212a64078de3418cb2550407552", + "size": 1936, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-named_node_subquery_scalar_/sql.yql" } ], "test_sql2yql.test[select_yql-named_node_subquery_scalar_chain]": [ { - "checksum": "6cc7ec780d04ea60b3347e009889372c", - "size": 2215, - "uri": "https://{canondata_backend}/1903885/ac392fa3c931e976b881e7663233fb23a4666244/resource.tar.gz#test_sql2yql.test_select_yql-named_node_subquery_scalar_chain_/sql.yql" + "checksum": "9d6110ecd257fddc341b63f33e67ab2e", + "size": 2283, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-named_node_subquery_scalar_chain_/sql.yql" } ], "test_sql2yql.test[select_yql-projection_alias_case_sensivity]": [ @@ -10186,9 +10186,9 @@ ], "test_sql2yql.test[select_yql-projection_alias_conflict]": [ { - "checksum": "00f9972cbeedaad8134aada42231bc06", - "size": 796, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-projection_alias_conflict_/sql.yql" + "checksum": "bdea1d1c9abd78cd37e3cae7f0c885f9", + "size": 813, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-projection_alias_conflict_/sql.yql" } ], "test_sql2yql.test[select_yql-projection_column_order]": [ @@ -10207,9 +10207,9 @@ ], "test_sql2yql.test[select_yql-projection_in_expr]": [ { - "checksum": "375030b859f8e033310a4146a0a2ab3e", - "size": 1088, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-projection_in_expr_/sql.yql" + "checksum": "10e07bafeef424922c03e18e529d5fe1", + "size": 1105, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-projection_in_expr_/sql.yql" } ], "test_sql2yql.test[select_yql-select_op_except_all]": [ @@ -10298,107 +10298,107 @@ ], "test_sql2yql.test[select_yql-select_op_union_named_node_projection]": [ { - "checksum": "b40847bf1a3519d54798020fc38bbdb4", - "size": 1449, - "uri": "https://{canondata_backend}/1903885/ac392fa3c931e976b881e7663233fb23a4666244/resource.tar.gz#test_sql2yql.test_select_yql-select_op_union_named_node_projection_/sql.yql" + "checksum": "75ef0127df83376c3e2457c7fc987dcb", + "size": 1466, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-select_op_union_named_node_projection_/sql.yql" } ], "test_sql2yql.test[select_yql-select_op_union_subquery_2]": [ { - "checksum": "02176981e7424703642e3235bcb00ac8", - "size": 1880, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-select_op_union_subquery_2_/sql.yql" + "checksum": "3817ca6f8f00a5ca88aa603c94133551", + "size": 1914, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-select_op_union_subquery_2_/sql.yql" } ], "test_sql2yql.test[select_yql-select_op_union_subquery_3]": [ { - "checksum": "706df09760fbcc1f6ee4ca2487be16e4", - "size": 2955, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-select_op_union_subquery_3_/sql.yql" + "checksum": "771675437de18e77077d4c98186e2667", + "size": 3006, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-select_op_union_subquery_3_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_asterisk_asterisk_subquery_scalar]": [ { - "checksum": "bfa69e5e0d924bd222bbb2c853d4d37f", - "size": 1296, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_asterisk_subquery_scalar_/sql.yql" + "checksum": "6bfd08e929594f5353fb0648abe1fc25", + "size": 1313, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_asterisk_subquery_scalar_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_asterisk_subquery_exists]": [ { - "checksum": "277271ce36f1b4beb3bf95f2953df57e", - "size": 1101, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_exists_/sql.yql" + "checksum": "b4cecb64ab0ce8399e7f100a111d7cea", + "size": 1118, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_exists_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_asterisk_subquery_exists_with_source]": [ { - "checksum": "1fccb8af2e0edab9392c15a1af5571a7", - "size": 1261, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_exists_with_source_/sql.yql" + "checksum": "19731a09ccc21917825c4c9d7407c1cf", + "size": 1278, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_exists_with_source_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_asterisk_subquery_in]": [ { - "checksum": "beb34062223f102924411d746c53a6d2", - "size": 1133, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_in_/sql.yql" + "checksum": "337977f5ea8b1e1f79d7fa091bf22a92", + "size": 1150, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_in_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_asterisk_subquery_in_with_source]": [ { - "checksum": "763330cb9ca353889befbd7a113b1786", - "size": 1293, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_in_with_source_/sql.yql" + "checksum": "73d8c00e7ff8233d6658083b97b33fc6", + "size": 1310, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_in_with_source_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_asterisk_subquery_scalar]": [ { - "checksum": "97540482ab52ae3c4deb70a213e3b6b9", - "size": 1099, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_scalar_/sql.yql" + "checksum": "05b1e669bfe808fa8c64c87faedd0786", + "size": 1116, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_scalar_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_asterisk_subquery_scalar_with_join]": [ { - "checksum": "97b042de90109e5c71a9aa007604ac67", - "size": 1524, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_scalar_with_join_/sql.yql" + "checksum": "a646028daff2db90bbff890f234b221f", + "size": 1541, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_scalar_with_join_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_asterisk_subquery_scalar_with_source]": [ { - "checksum": "e2f85cea68ad76bcb5b74237ea5e5c03", - "size": 1259, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_scalar_with_source_/sql.yql" + "checksum": "fdc5c176aeeef6ddf43b7b8420e8f65a", + "size": 1276, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_asterisk_subquery_scalar_with_source_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_limit_trivial]": [ { - "checksum": "adafc5f76128f35f8ea16f6b01c11a30", - "size": 1203, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_limit_trivial_/sql.yql" + "checksum": "a802069a9b13ad70ea49797a57037943", + "size": 1220, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_limit_trivial_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_offset_trivial]": [ { - "checksum": "30ef9c37ed1f965f556c7493101623b8", - "size": 1227, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_offset_trivial_/sql.yql" + "checksum": "52ea96df1b0f5cee72eaf0b8df1bd189", + "size": 1244, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_offset_trivial_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_projection_in]": [ { - "checksum": "35fdd967e658d777232704972a121420", - "size": 1619, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_projection_in_/sql.yql" + "checksum": "58c2b2aed8d1cb3388a90743729695d9", + "size": 1653, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_projection_in_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_projection_trivial]": [ { - "checksum": "fa6cbdc296a5bba832798c0845cb19be", - "size": 908, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_projection_trivial_/sql.yql" + "checksum": "3f6010d399bdc1422d39fc64093779df", + "size": 942, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_projection_trivial_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_where_exists_fls]": [ @@ -10417,30 +10417,30 @@ ], "test_sql2yql.test[select_yql-uncorrelated_where_group_by]": [ { - "checksum": "306e67cb96d33faab2d6d803937f58e9", - "size": 2062, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_where_group_by_/sql.yql" + "checksum": "6aa844a7e044814c6a8d49df05b74211", + "size": 2079, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_where_group_by_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_where_in]": [ { - "checksum": "54ac95327c8bb80b2ef42778cc9c3bd8", - "size": 1602, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_where_in_/sql.yql" + "checksum": "0a1eba48a8a7fb6836b2042baa814855", + "size": 1619, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_where_in_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_where_in_group_by]": [ { - "checksum": "a4173b9d95919016a462be9754d5dcc6", - "size": 2638, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_where_in_group_by_/sql.yql" + "checksum": "1e3df9e0191eeb44cb33ed84be7fd35e", + "size": 2655, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_where_in_group_by_/sql.yql" } ], "test_sql2yql.test[select_yql-uncorrelated_where_trivial]": [ { - "checksum": "b9347185104d9569e3923de55bc64ed4", - "size": 1261, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_where_trivial_/sql.yql" + "checksum": "0780aa9f5f467b6a57410a92ae0d3ba1", + "size": 1278, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-uncorrelated_where_trivial_/sql.yql" } ], "test_sql2yql.test[select_yql-values]": [ @@ -10452,142 +10452,142 @@ ], "test_sql2yql.test[select_yql-window_fn_cumedist_0]": [ { - "checksum": "37c78736a5682152ac8324fc998ba1e6", - "size": 1421, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_cumedist_0_/sql.yql" + "checksum": "c1f6f15cd45634964896f676f19aa985", + "size": 1438, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_cumedist_0_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_denserank_0]": [ { - "checksum": "30d31a74af9a1d9677ea0aca34a9a745", - "size": 1422, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_denserank_0_/sql.yql" + "checksum": "5e23e2e2adf241e4b7d1083590651080", + "size": 1439, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_denserank_0_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_denserank_1]": [ { - "checksum": "b2cccdecc886cb80f495912ae5436832", - "size": 1442, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_denserank_1_/sql.yql" + "checksum": "3fcfec571976bac8b193aea106ebe482", + "size": 1459, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_denserank_1_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_lag_1]": [ { - "checksum": "f8a313a6b4cb19664419d395585b14f8", - "size": 1436, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_lag_1_/sql.yql" + "checksum": "4e3dde92e64dffe71da40fd93adb1b38", + "size": 1453, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_lag_1_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_lag_2]": [ { - "checksum": "e329f2fc52414de5d9ba5466b5ca9d5d", - "size": 1540, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_lag_2_/sql.yql" + "checksum": "7ef8448af62a11f7cff684953618d596", + "size": 1557, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_lag_2_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_lead_1]": [ { - "checksum": "9c81f627426d1e5e0508c930015dee54", - "size": 1437, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_lead_1_/sql.yql" + "checksum": "15dcb09ebe98ee204f7b8685edc5eeb0", + "size": 1454, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_lead_1_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_lead_2]": [ { - "checksum": "3cc3e0f58aa10de21869f1d681371ab0", - "size": 1541, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_lead_2_/sql.yql" + "checksum": "7a61cecab747841dd05ce3c2dc64660a", + "size": 1558, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_lead_2_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_ntile_2]": [ { - "checksum": "8e1ac8358abde9c446fb7c7dffa7686b", - "size": 1431, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_ntile_2_/sql.yql" + "checksum": "65bf0a29b1136214a4e17682f027bb75", + "size": 1448, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_ntile_2_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_ntile_3]": [ { - "checksum": "6869811895e6b91e381cdc5de62aefb3", - "size": 1557, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_ntile_3_/sql.yql" + "checksum": "70f75a4a92438a1f69b111be876340f2", + "size": 1574, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_ntile_3_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_percentrank_0]": [ { - "checksum": "9ab2348b4f83f1ac9e8f1b29791feee6", - "size": 1466, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_percentrank_0_/sql.yql" + "checksum": "9e219ada368b9826f5ce34a259adc0bf", + "size": 1483, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_percentrank_0_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_percentrank_1]": [ { - "checksum": "7163b49f9c48bf79ed7dc352083d4209", - "size": 1444, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_percentrank_1_/sql.yql" + "checksum": "7f2190f005331198b9cf580468b1ab22", + "size": 1461, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_percentrank_1_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_percentrank_1_desc]": [ { - "checksum": "2355ba3d5965313586adc01379cb9c77", - "size": 1487, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_percentrank_1_desc_/sql.yql" + "checksum": "9b6f247e37e871ed6925c6a08e8309b4", + "size": 1504, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_percentrank_1_desc_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_rank_0]": [ { - "checksum": "f91972ce96ffbce0827ab5cf4540468b", - "size": 1417, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_0_/sql.yql" + "checksum": "b519e19f85fea068282e4e6dd62c7154", + "size": 1434, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_0_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_rank_0_2]": [ { - "checksum": "1df32e7c274914e349418fdb6dd03bf2", - "size": 1479, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_0_2_/sql.yql" + "checksum": "4eb24a8ac7694a3ed877e995e0965427", + "size": 1496, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_0_2_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_rank_0_desc]": [ { - "checksum": "5d4232179e2c1ea4490696a3b003c5c7", - "size": 1418, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_0_desc_/sql.yql" + "checksum": "4bc2a64c3c68f74d0fdd409fff068b32", + "size": 1435, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_0_desc_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_rank_1]": [ { - "checksum": "8b591cc8df4ffcf72ef9f8507c917280", - "size": 1437, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_1_/sql.yql" + "checksum": "61193eeffe714a8428bf36e60f2881c7", + "size": 1454, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_1_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_rank_1_desc]": [ { - "checksum": "21ac09e0bd47c3174d2b622b680565bd", - "size": 1438, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_1_desc_/sql.yql" + "checksum": "d07a75e22ebb34fbde4f24ddce7d4599", + "size": 1455, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_1_desc_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_rank_2]": [ { - "checksum": "daedef26d9c4326faaf5c5dceebab5c6", - "size": 1460, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_2_/sql.yql" + "checksum": "bc3323c663a91aafae235bc2047b027f", + "size": 1477, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rank_2_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_rownumber_0]": [ { - "checksum": "1c6258630ebbeef2fd7dae4f0e043be2", - "size": 1506, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rownumber_0_/sql.yql" + "checksum": "55af9c08b8a18c9307a58ca87a7bf5c4", + "size": 1523, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rownumber_0_/sql.yql" } ], "test_sql2yql.test[select_yql-window_fn_rownumber_0_desc]": [ { - "checksum": "39bd2ecb8e585bc8047b607b098aadc4", - "size": 1507, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rownumber_0_desc_/sql.yql" + "checksum": "ca9094ba25984212382172c7a36acdbd", + "size": 1524, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_fn_rownumber_0_desc_/sql.yql" } ], "test_sql2yql.test[select_yql-window_order_by_range_frame]": [ @@ -10606,30 +10606,30 @@ ], "test_sql2yql.test[select_yql-window_pg_tutorial_1]": [ { - "checksum": "94b8277e7d0ecc220e5421cb106cb25d", - "size": 1933, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_pg_tutorial_1_/sql.yql" + "checksum": "a2975c7471143d23cc6850bf2403eea4", + "size": 1950, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_pg_tutorial_1_/sql.yql" } ], "test_sql2yql.test[select_yql-window_pg_tutorial_2]": [ { - "checksum": "fb67bbbb7fc767794169e926c73f01f6", - "size": 1984, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_pg_tutorial_2_/sql.yql" + "checksum": "4ab683fb491dadab446af105788fa01a", + "size": 2001, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_pg_tutorial_2_/sql.yql" } ], "test_sql2yql.test[select_yql-window_pg_tutorial_3]": [ { - "checksum": "3894c137ed48b14b20d799394bdf78df", - "size": 1715, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_pg_tutorial_3_/sql.yql" + "checksum": "7ecdae664a1e3275be44555c78ddb8ed", + "size": 1732, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_pg_tutorial_3_/sql.yql" } ], "test_sql2yql.test[select_yql-window_pg_tutorial_4]": [ { - "checksum": "95ecf0c061b9f73c8a4ad3c0f945f73c", - "size": 1821, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_pg_tutorial_4_/sql.yql" + "checksum": "5e1ee8314756ac2f132c8166e8c09aae", + "size": 1838, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_pg_tutorial_4_/sql.yql" } ], "test_sql2yql.test[select_yql-window_pg_tutorial_5]": [ @@ -10641,16 +10641,16 @@ ], "test_sql2yql.test[select_yql-window_pg_tutorial_6]": [ { - "checksum": "abf0525ec1bad4dbe55fbcbc4d32f557", - "size": 1903, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_pg_tutorial_6_/sql.yql" + "checksum": "a372cfef40cff8794952e2e5811cdc1e", + "size": 1937, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_pg_tutorial_6_/sql.yql" } ], "test_sql2yql.test[select_yql-window_sum]": [ { - "checksum": "e4067520d3e2f531a0ecc276556bdcb1", - "size": 1327, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_/sql.yql" + "checksum": "83642a3f482ef82a360c1cdc30d2834d", + "size": 1344, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_/sql.yql" } ], "test_sql2yql.test[select_yql-window_sum_agg]": [ @@ -10662,37 +10662,37 @@ ], "test_sql2yql.test[select_yql-window_sum_order_by]": [ { - "checksum": "8969ce02304d5b3b5d682f48b5b251a1", - "size": 1511, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_order_by_/sql.yql" + "checksum": "17b84f739c5e4e81129a24c2641ccbd9", + "size": 1528, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_order_by_/sql.yql" } ], "test_sql2yql.test[select_yql-window_sum_order_by_distinct_0]": [ { - "checksum": "e32239545b739a91c1e519c111f65a0d", - "size": 1507, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_order_by_distinct_0_/sql.yql" + "checksum": "ce0515f437add07985fef81d99be7e17", + "size": 1524, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_order_by_distinct_0_/sql.yql" } ], "test_sql2yql.test[select_yql-window_sum_order_by_distinct_1]": [ { - "checksum": "79e867117ade631013e8253c6dc3f2d0", - "size": 1519, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_order_by_distinct_1_/sql.yql" + "checksum": "61640bee6d4f4a6849022c5551012f82", + "size": 1536, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_order_by_distinct_1_/sql.yql" } ], "test_sql2yql.test[select_yql-window_sum_partition_by_1]": [ { - "checksum": "df807c8ff5952c2e1d1dda2d9a776bba", - "size": 1460, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_partition_by_1_/sql.yql" + "checksum": "64c39059946fffde8c91e6910ed7177e", + "size": 1477, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_partition_by_1_/sql.yql" } ], "test_sql2yql.test[select_yql-window_sum_partition_by_2]": [ { - "checksum": "dbcce6f8d41d8e9ec7954c500613ab9d", - "size": 1594, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_partition_by_2_/sql.yql" + "checksum": "136aaf95e4dd26ac4c5491b35dc62064", + "size": 1611, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_partition_by_2_/sql.yql" } ], "test_sql2yql.test[select_yql-window_sum_partition_by_agg]": [ @@ -10704,9 +10704,9 @@ ], "test_sql2yql.test[select_yql-window_sum_partition_by_order_by]": [ { - "checksum": "dc78ca42aca486c064267a18f8587770", - "size": 1656, - "uri": "https://{canondata_backend}/1942100/698d95eec1cbd4e4ada1a8da105c203a7a12bb85/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_partition_by_order_by_/sql.yql" + "checksum": "46b899ecb04bd6fbdcea9c9d3e959c66", + "size": 1673, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-window_sum_partition_by_order_by_/sql.yql" } ], "test_sql2yql.test[select_yql-with_cte_agg]": [ @@ -10718,30 +10718,65 @@ ], "test_sql2yql.test[select_yql-with_cte_column_order]": [ { - "checksum": "5f462197e5835b3e46c467231cbcd07e", - "size": 5020, - "uri": "https://{canondata_backend}/1600758/21871d7c036f105a5a9e6866b296e3810c413546/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_/sql.yql" + "checksum": "7f0e669ffeef2bbbae63a42f35aec16c", + "size": 4598, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_/sql.yql" } ], "test_sql2yql.test[select_yql-with_cte_column_order_less]": [ { - "checksum": "57d802f8b763817384f670ba4827c256", - "size": 959, - "uri": "https://{canondata_backend}/1773845/ab27a09a23bc26dd2d124ed3b07eac965514031e/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_less_/sql.yql" + "checksum": "4959f8e51e02361d2d2e7bc702d8ff34", + "size": 1004, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_less_/sql.yql" } ], "test_sql2yql.test[select_yql-with_cte_column_order_more]": [ { - "checksum": "b22843fb8e51e07ecce5644abeb92442", - "size": 1016, - "uri": "https://{canondata_backend}/1773845/ab27a09a23bc26dd2d124ed3b07eac965514031e/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_more_/sql.yql" + "checksum": "42d8f487d6cdcab363d749c6c9af0ae4", + "size": 1078, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_more_/sql.yql" + } + ], + "test_sql2yql.test[select_yql-with_cte_column_order_system_1]": [ + { + "checksum": "e11910dc0dd640107c04e84e923f3655", + "size": 1041, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_system_1_/sql.yql" + } + ], + "test_sql2yql.test[select_yql-with_cte_column_order_system_2]": [ + { + "checksum": "431d9711c9cc6c1677f12dba604a3169", + "size": 1046, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_system_2_/sql.yql" } ], "test_sql2yql.test[select_yql-with_cte_column_order_union_ambiguity]": [ { - "checksum": "18676cba7debedc48528dbde2e224312", - "size": 1100, - "uri": "https://{canondata_backend}/1925821/ff1bc09757f515ab922cf538c355b7fbfc7e5d52/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_union_ambiguity_/sql.yql" + "checksum": "1b402bd67dbee76352f615dfb6236a40", + "size": 1111, + "uri": "https://{canondata_backend}/1130705/f9aa8a4e0215f9fe8c0197008a49a731e7e74521/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_union_ambiguity_/sql.yql" + } + ], + "test_sql2yql.test[select_yql-with_cte_column_order_warning_1]": [ + { + "checksum": "fb28641039f99bf4a5fd04a436ede7b6", + "size": 1700, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_warning_1_/sql.yql" + } + ], + "test_sql2yql.test[select_yql-with_cte_column_order_warning_2]": [ + { + "checksum": "c950de780d5a3b1b4b577976b915170a", + "size": 2486, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_warning_2_/sql.yql" + } + ], + "test_sql2yql.test[select_yql-with_cte_column_order_warning_3]": [ + { + "checksum": "54770b8259855c274b2e609e31536eff", + "size": 1158, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_column_order_warning_3_/sql.yql" } ], "test_sql2yql.test[select_yql-with_cte_counter]": [ @@ -10767,9 +10802,9 @@ ], "test_sql2yql.test[select_yql-with_cte_graph]": [ { - "checksum": "32573b007fc82ae67937dc5b1fbc0df0", - "size": 2181, - "uri": "https://{canondata_backend}/1923547/f4f3258a5a86f0b9a1ffeef962a22189a35722c5/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_graph_/sql.yql" + "checksum": "8c6d678a5075ba9179faadb530948a73", + "size": 2198, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_graph_/sql.yql" } ], "test_sql2yql.test[select_yql-with_cte_multi_anchor]": [ @@ -10795,9 +10830,9 @@ ], "test_sql2yql.test[select_yql-with_cte_single]": [ { - "checksum": "f702987e6e4bffabd1cbc675e8feec3e", - "size": 2783, - "uri": "https://{canondata_backend}/1923547/3869dd5d63b2fe11b8afd98fd58476aa64c6ab45/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_single_/sql.yql" + "checksum": "4373af2e9d0f9e67683754fc9f596428", + "size": 2800, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql-with_cte_single_/sql.yql" } ], "test_sql2yql.test[select_yql_tpcds-q47]": [ @@ -10837,9 +10872,9 @@ ], "test_sql2yql.test[select_yql_tpch-q15]": [ { - "checksum": "e2bdef2adb975da51d2a5a820169b551", - "size": 3367, - "uri": "https://{canondata_backend}/1903885/ac392fa3c931e976b881e7663233fb23a4666244/resource.tar.gz#test_sql2yql.test_select_yql_tpch-q15_/sql.yql" + "checksum": "f35227f52d9e4e49dcde960cd16da14c", + "size": 3384, + "uri": "https://{canondata_backend}/1597364/fdeffca975c2d2d1bc708528a504feb47b36c5c9/resource.tar.gz#test_sql2yql.test_select_yql_tpch-q15_/sql.yql" } ], "test_sql2yql.test[seq_mode-shared_named_expr]": [ @@ -18844,11 +18879,36 @@ "uri": "file://test_sql_format.test_select_yql-with_cte_column_order_more_/formatted.sql" } ], + "test_sql_format.test[select_yql-with_cte_column_order_system_1]": [ + { + "uri": "file://test_sql_format.test_select_yql-with_cte_column_order_system_1_/formatted.sql" + } + ], + "test_sql_format.test[select_yql-with_cte_column_order_system_2]": [ + { + "uri": "file://test_sql_format.test_select_yql-with_cte_column_order_system_2_/formatted.sql" + } + ], "test_sql_format.test[select_yql-with_cte_column_order_union_ambiguity]": [ { "uri": "file://test_sql_format.test_select_yql-with_cte_column_order_union_ambiguity_/formatted.sql" } ], + "test_sql_format.test[select_yql-with_cte_column_order_warning_1]": [ + { + "uri": "file://test_sql_format.test_select_yql-with_cte_column_order_warning_1_/formatted.sql" + } + ], + "test_sql_format.test[select_yql-with_cte_column_order_warning_2]": [ + { + "uri": "file://test_sql_format.test_select_yql-with_cte_column_order_warning_2_/formatted.sql" + } + ], + "test_sql_format.test[select_yql-with_cte_column_order_warning_3]": [ + { + "uri": "file://test_sql_format.test_select_yql-with_cte_column_order_warning_3_/formatted.sql" + } + ], "test_sql_format.test[select_yql-with_cte_counter]": [ { "uri": "file://test_sql_format.test_select_yql-with_cte_counter_/formatted.sql" diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_/formatted.sql index dfc12d5d337..75d0aadf230 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_/formatted.sql +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_/formatted.sql @@ -24,18 +24,7 @@ FROM WITH x (a, b) AS ( SELECT 1 AS a, - '2' AS b -) -SELECT - * -FROM - x -; - -WITH x (a, b) AS ( - SELECT - 1 AS b, - '2' AS a + '2' ) SELECT * @@ -45,8 +34,8 @@ FROM WITH x (a, b) AS ( SELECT - 1 AS c, - '2' AS d + 1 AS a, + '2' AS b ) SELECT * diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_system_1_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_system_1_/formatted.sql new file mode 100644 index 00000000000..f11a8e91930 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_system_1_/formatted.sql @@ -0,0 +1,15 @@ +/* custom error: Uncompatible member column0 types: Int32 and String */ +PRAGMA YqlSelect = 'force'; + +WITH x (a) AS ( + SELECT + 1 + UNION ALL + SELECT + '2' +) +SELECT + * +FROM + x +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_system_2_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_system_2_/formatted.sql new file mode 100644 index 00000000000..d24693d6f54 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_system_2_/formatted.sql @@ -0,0 +1,15 @@ +/* custom error: Wrong number of columns, expected: 1, got: 2 */ +PRAGMA YqlSelect = 'force'; + +WITH x (a, b) AS ( + SELECT + 1 + UNION ALL + SELECT + '2' +) +SELECT + * +FROM + x +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_warning_1_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_warning_1_/formatted.sql new file mode 100644 index 00000000000..22153162cad --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_warning_1_/formatted.sql @@ -0,0 +1,23 @@ +PRAGMA YqlSelect = 'force'; + +WITH x (a, b) AS ( + SELECT + 1 AS b, + '2' AS a +) +SELECT + 1 +FROM + x +; + +WITH x (a, b) AS ( + SELECT + 1 AS c, + '2' AS d +) +SELECT + 1 +FROM + x +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_warning_2_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_warning_2_/formatted.sql new file mode 100644 index 00000000000..0410746271e --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_warning_2_/formatted.sql @@ -0,0 +1,34 @@ +PRAGMA YqlSelect = 'force'; + +WITH x (a, b) AS ( + SELECT + 1, + '2' +) +SELECT + 1 +FROM + x +; + +WITH x (a, b) AS ( + SELECT + 1, + '2' AS b +) +SELECT + 1 +FROM + x +; + +WITH x (a, b) AS ( + SELECT + 1, + '2' AS c +) +SELECT + 1 +FROM + x +; diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_warning_3_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_warning_3_/formatted.sql new file mode 100644 index 00000000000..a0ba4d3f5fc --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_select_yql-with_cte_column_order_warning_3_/formatted.sql @@ -0,0 +1,16 @@ +PRAGMA YqlSelect = 'force'; + +WITH x (a, b) AS ( + SELECT + 1 AS a, + '2' AS b + UNION + SELECT + 1 AS a, + '2' AS b +) +SELECT + 1 +FROM + x +; diff --git a/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order.yql b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order.yql index 0766bfc97e3..13d4fedcb5c 100644 --- a/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order.yql +++ b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order.yql @@ -6,13 +6,10 @@ SELECT * FROM x; WITH x (a, b) AS (SELECT 1, '2') SELECT * FROM x; -WITH x (a, b) AS (SELECT 1 AS a, '2' AS b) -SELECT * FROM x; - -WITH x (a, b) AS (SELECT 1 AS b, '2' AS a) +WITH x (a, b) AS (SELECT 1 AS a, '2') SELECT * FROM x; -WITH x (a, b) AS (SELECT 1 AS c, '2' AS d) +WITH x (a, b) AS (SELECT 1 AS a, '2' AS b) SELECT * FROM x; WITH x (a, b) AS ( diff --git a/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_1.cfg b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_1.cfg new file mode 100644 index 00000000000..5dae597903c --- /dev/null +++ b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_1.cfg @@ -0,0 +1 @@ +xfail diff --git a/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_1.yql b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_1.yql new file mode 100644 index 00000000000..42cabe6511f --- /dev/null +++ b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_1.yql @@ -0,0 +1,9 @@ +/* custom error: Uncompatible member column0 types: Int32 and String */ +PRAGMA YqlSelect = 'force'; + +WITH x (a) AS ( + SELECT 1 + UNION ALL + SELECT '2' +) +SELECT * FROM x; diff --git a/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_2.cfg b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_2.cfg new file mode 100644 index 00000000000..5dae597903c --- /dev/null +++ b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_2.cfg @@ -0,0 +1 @@ +xfail diff --git a/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_2.yql b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_2.yql new file mode 100644 index 00000000000..30f08c91c1d --- /dev/null +++ b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_system_2.yql @@ -0,0 +1,9 @@ +/* custom error: Wrong number of columns, expected: 1, got: 2 */ +PRAGMA YqlSelect = 'force'; + +WITH x (a, b) AS ( + SELECT 1 + UNION ALL + SELECT '2' +) +SELECT * FROM x; diff --git a/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_warning_1.yql b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_warning_1.yql new file mode 100644 index 00000000000..cda3f860084 --- /dev/null +++ b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_warning_1.yql @@ -0,0 +1,7 @@ +PRAGMA YqlSelect = 'force'; + +WITH x (a, b) AS (SELECT 1 AS b, '2' AS a) +SELECT 1 FROM x; + +WITH x (a, b) AS (SELECT 1 AS c, '2' AS d) +SELECT 1 FROM x; diff --git a/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_warning_2.yql b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_warning_2.yql new file mode 100644 index 00000000000..ffb11b136a2 --- /dev/null +++ b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_warning_2.yql @@ -0,0 +1,10 @@ +PRAGMA YqlSelect = 'force'; + +WITH x (a, b) AS (SELECT 1, '2') +SELECT 1 FROM x; + +WITH x (a, b) AS (SELECT 1, '2' AS b) +SELECT 1 FROM x; + +WITH x (a, b) AS (SELECT 1, '2' AS c) +SELECT 1 FROM x; diff --git a/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_warning_3.yql b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_warning_3.yql new file mode 100644 index 00000000000..2ab0d5735ac --- /dev/null +++ b/yql/essentials/tests/sql/suites/select_yql/with_cte_column_order_warning_3.yql @@ -0,0 +1,8 @@ +PRAGMA YqlSelect = 'force'; + +WITH x (a, b) AS ( + SELECT 1 AS a, '2' AS b + UNION + SELECT 1 AS a, '2' AS b +) +SELECT 1 FROM x; diff --git a/yql/essentials/tools/sql2yql/sql2yql.cpp b/yql/essentials/tools/sql2yql/sql2yql.cpp index 948cf86ef63..694be80034b 100644 --- a/yql/essentials/tools/sql2yql/sql2yql.cpp +++ b/yql/essentials/tools/sql2yql/sql2yql.cpp @@ -320,22 +320,6 @@ int BuildAST(int argc, char** argv) { opts.PrintUsage(argv[0], Cerr); } - NSQLTranslationV1::TLexers lexers; - lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); - lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); - NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( - res.Has("test-syntax-ambiguity"), - res.Has("debug-syntax-ambiguity")); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( - res.Has("test-syntax-ambiguity"), - res.Has("debug-syntax-ambiguity")); - - NSQLTranslation::TTranslators translators( - nullptr, - NSQLTranslationV1::MakeTranslator(lexers, parsers), - NSQLTranslationPG::MakeTranslator()); - TVector<TString> queries; int errors = 0; for (ui32 i = 0; i <= queryFiles.size(); ++i) { @@ -392,6 +376,27 @@ int BuildAST(int argc, char** argv) { settings.AssumeYdbOnClusterWithSlash = res.Has("assume-ydb-on-slash"); settings.TestAntlr4 = res.Has("test-antlr4"); + NSQLTranslationV1::TLexers lexers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(), + }; + + NSQLTranslationV1::TParsers parsers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + res.Has("test-syntax-ambiguity"), + res.Has("debug-syntax-ambiguity"), + settings.MaxParseTreeDepth), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + res.Has("test-syntax-ambiguity"), + res.Has("debug-syntax-ambiguity"), + settings.MaxParseTreeDepth), + }; + + NSQLTranslation::TTranslators translators( + nullptr, + NSQLTranslationV1::MakeTranslator(lexers, parsers), + NSQLTranslationPG::MakeTranslator()); + if (res.Has("lexer")) { NYql::TIssues issues; auto lexer = NSQLTranslation::SqlLexer(translators, query, issues, settings); diff --git a/yql/essentials/tools/sql_formatter/sql_formatter.cpp b/yql/essentials/tools/sql_formatter/sql_formatter.cpp index 0f471d062f2..03a8eab9aeb 100644 --- a/yql/essentials/tools/sql_formatter/sql_formatter.cpp +++ b/yql/essentials/tools/sql_formatter/sql_formatter.cpp @@ -57,17 +57,29 @@ int RunFormat(int argc, char** argv) { if (res.Has("print-query")) { out << queryString << Endl; } + google::protobuf::Arena arena; NSQLTranslation::TTranslationSettings settings; settings.LangVer = langver; settings.Arena = &arena; settings.AnsiLexer = res.Has("ansi-lexer"); - NSQLTranslationV1::TLexers lexers; - lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); - lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); - NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); + + NSQLTranslationV1::TLexers lexers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(), + }; + + NSQLTranslationV1::TParsers parsers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), + }; + auto formatter = NSQLFormat::MakeSqlFormatter(lexers, parsers, settings); TString frm_query; TString error; diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp index 6f55213a283..7d251c35b4a 100644 --- a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp +++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp @@ -603,14 +603,24 @@ int TFacadeRunner::DoMain(int argc, const char** argv) { NKikimr::NMiniKQL::FillStaticModules(*funcRegistry); FuncRegistry_ = funcRegistry; - NSQLTranslationV1::TLexers lexers; - lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); - lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); - NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( - /*isAmbiguityError=*/RunOptions_.TestSyntaxAmbiguities); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( - /*isAmbiguityError=*/RunOptions_.TestSyntaxAmbiguities); + NSQLTranslation::TTranslationSettings settings; + NSQLTranslation::ParseTranslationSettings(RunOptions_.SqlFlags, settings); + + NSQLTranslationV1::TLexers lexers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(), + }; + + NSQLTranslationV1::TParsers parsers = { + .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/RunOptions_.TestSyntaxAmbiguities, + /*isAmbiguityDebugging=*/false, + /*maxParseTreeDepth=*/settings.MaxParseTreeDepth), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/RunOptions_.TestSyntaxAmbiguities, + /*isAmbiguityDebugging=*/false, + /*maxParseTreeDepth=*/settings.MaxParseTreeDepth), + }; NSQLTranslation::TTranslators translators( nullptr, @@ -621,6 +631,7 @@ int TFacadeRunner::DoMain(int argc, const char** argv) { if (RunOptions_.PgSupport) { ctx.NextUniqueId = NPg::GetSqlLanguageParser()->GetContext().NextUniqueId; } + IModuleResolver::TPtr moduleResolver; TModuleResolver::TModuleChecker moduleChecker; if (RunOptions_.TestLexers || @@ -655,6 +666,7 @@ int TFacadeRunner::DoMain(int argc, const char** argv) { settings.ClusterMapping = clusters; settings.SyntaxVersion = 1; settings.AlwaysAllowExports = true; + settings.MaxParseTreeDepth = NSQLTranslation::SQL_MAX_PARSE_TREE_DEPTH; auto ast = NSQLTranslationV1::SqlToYql(lexers, parsers, query, settings); if (!ast.IsOk()) { @@ -899,9 +911,13 @@ int TFacadeRunner::DoRun(TProgramFactory& factory) { NSQLTranslationV1::TParsers parsers = { .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( - /*isAmbiguityError=*/true), + /*isAmbiguityError=*/true, + /*isAmbiguityDebugging=*/false, + /*maxParseTreeDepth=*/settings.MaxParseTreeDepth), .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( - /*isAmbiguityError=*/true), + /*isAmbiguityError=*/true, + /*isAmbiguityDebugging=*/false, + /*maxParseTreeDepth=*/settings.MaxParseTreeDepth), }; NSQLTranslation::TTranslators translators( diff --git a/yql/essentials/udfs/language/yql/sql2yql.cpp b/yql/essentials/udfs/language/yql/sql2yql.cpp index 0e468011be8..8a67b1fe5d6 100644 --- a/yql/essentials/udfs/language/yql/sql2yql.cpp +++ b/yql/essentials/udfs/language/yql/sql2yql.cpp @@ -50,14 +50,20 @@ void ParseTranslationSettings(const TSql2YqlInput& input, NSQLTranslation::TTran ParseGatewaysConfig(input.GatewaysCfg, settings); } -NSQLTranslation::TTranslators Translators() { +NSQLTranslation::TTranslators Translators(TMaybe<size_t> maxParseTreeDepth) { NSQLTranslationV1::TLexers lexers; lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); + parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + maxParseTreeDepth); + parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + maxParseTreeDepth); NSQLTranslation::TTranslators translators( /*v0=*/nullptr, @@ -76,7 +82,8 @@ TSql2YqlOutput Sql2Yql(const TSql2YqlInput& input) noexcept try { google::protobuf::Arena arena; settings.Arena = &arena; - NYql::TAstParseResult res = NSQLTranslation::SqlToYql(Translators(), input.Query, settings); + NYql::TAstParseResult res = NSQLTranslation::SqlToYql( + Translators(settings.MaxParseTreeDepth), input.Query, settings); TVector<TString> issues(Reserve(res.Issues.Size())); for (const auto& issue : res.Issues) { diff --git a/yql/tools/yqlrun/yqlrun.cpp b/yql/tools/yqlrun/yqlrun.cpp index 31b9e04b69a..36d906d99e7 100644 --- a/yql/tools/yqlrun/yqlrun.cpp +++ b/yql/tools/yqlrun/yqlrun.cpp @@ -224,8 +224,14 @@ int RunUI(int argc, const char* argv[]) lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); NSQLTranslationV1::TParsers parsers; - parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(); - parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(); + parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/true, + /*isAmbiguityDebugging=*/false, + /*maxParseTreeDepth=*/NSQLTranslation::SQL_MAX_PARSE_TREE_DEPTH); + parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/true, + /*isAmbiguityDebugging=*/false, + /*maxParseTreeDepth=*/NSQLTranslation::SQL_MAX_PARSE_TREE_DEPTH); NSQLTranslation::TTranslators translators( nullptr, diff --git a/yt/yql/providers/yt/lib/row_spec/yql_row_spec.cpp b/yt/yql/providers/yt/lib/row_spec/yql_row_spec.cpp index c6fc5a291e5..97cace5a03c 100644 --- a/yt/yql/providers/yt/lib/row_spec/yql_row_spec.cpp +++ b/yt/yql/providers/yt/lib/row_spec/yql_row_spec.cpp @@ -158,6 +158,24 @@ NYT::TNode FilterSchemaColumns(const NYT::TNode& origSchema, const NYT::TNode& f return filteredSchema; } +class TSingularTypesVisitor : public TDefaultTypeAnnotationVisitor { +public: + void Visit(const TVoidExprType&) override { + SingularTypeFlags_ |= NTCF_VOID; + } + + void Visit(const TNullExprType&) override { + SingularTypeFlags_ |= NTCF_NULL; + } + + ui64 SingularTypeFlags() const { + return SingularTypeFlags_; + } + +private: + ui64 SingularTypeFlags_ = 0ul; +}; + } ui64 GetItemNativeYtTypeFlags(const TTypeAnnotationNode& type) { @@ -1290,6 +1308,12 @@ void TYqlRowSpecInfo::FillDefValues(NYT::TNode& attrs, const NCommon::TStructMem void TYqlRowSpecInfo::FillFlags(NYT::TNode& attrs) const { attrs[RowSpecAttrStrictSchema] = StrictSchema; attrs[RowSpecAttrNativeYtTypeFlags] = NativeYtTypeFlags; + + // Backward compatibility with NTCF_VOID & NTCF_NULL presence in row spec + TSingularTypesVisitor visitor; + Type->Accept(visitor); + attrs[RowSpecAttrNativeYtTypeFlags] = NativeYtTypeFlags | visitor.SingularTypeFlags(); + // Backward compatibility. TODO: remove after releasing compatibility flags if (NativeYtTypeFlags != 0) { attrs[RowSpecAttrUseNativeYtTypes] = true; diff --git a/yt/yt/client/formats/config.cpp b/yt/yt/client/formats/config.cpp index 7f5d609f110..d80ae43a8dd 100644 --- a/yt/yt/client/formats/config.cpp +++ b/yt/yt/client/formats/config.cpp @@ -352,14 +352,6 @@ void TSkiffFormatConfig::Register(TRegistrar registrar) //////////////////////////////////////////////////////////////////////////////// -void TYamlFormatConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("write_uint_tag", &TThis::WriteUintTag) - .Default(false); -} - -//////////////////////////////////////////////////////////////////////////////// - void TArrowFormatConfig::Register(TRegistrar registrar) { registrar.Parameter("enable_tz_index", &TThis::EnableTzIndex) diff --git a/yt/yt/client/formats/config.h b/yt/yt/client/formats/config.h index c4cdcbe6a03..136b9c8267e 100644 --- a/yt/yt/client/formats/config.h +++ b/yt/yt/client/formats/config.h @@ -400,25 +400,6 @@ DEFINE_REFCOUNTED_TYPE(TSkiffFormatConfig) //////////////////////////////////////////////////////////////////////////////// -struct TYamlFormatConfig - : public NYTree::TYsonStruct -{ - //! Write explicit tag "!yt/uint64" for uint64 data type. - //! Use this option if you want to preserve information about - //! the original YT type (without it, numbers in range [0, 2^63-1] - //! will always be written as integers). - //! Option has no effect for parsing. - bool WriteUintTag; - - REGISTER_YSON_STRUCT(TYamlFormatConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TYamlFormatConfig) - -//////////////////////////////////////////////////////////////////////////////// - struct TArrowFormatConfig : public NYTree::TYsonStruct { diff --git a/yt/yt/client/formats/public.h b/yt/yt/client/formats/public.h index d73fc8f98ec..e2bfeb7716b 100644 --- a/yt/yt/client/formats/public.h +++ b/yt/yt/client/formats/public.h @@ -78,7 +78,6 @@ DECLARE_REFCOUNTED_STRUCT(TProtobufTableConfig) DECLARE_REFCOUNTED_STRUCT(TProtobufFormatConfig) DECLARE_REFCOUNTED_STRUCT(TWebJsonFormatConfig) DECLARE_REFCOUNTED_STRUCT(TSkiffFormatConfig) -DECLARE_REFCOUNTED_STRUCT(TYamlFormatConfig) DECLARE_REFCOUNTED_STRUCT(TArrowFormatConfig) DECLARE_REFCOUNTED_STRUCT(TBlobFormatConfig) diff --git a/yt/yt/core/misc/fair_share_hierarchical_queue-inl.h b/yt/yt/core/misc/fair_share_hierarchical_queue-inl.h index da9199a8e90..010825e7efa 100644 --- a/yt/yt/core/misc/fair_share_hierarchical_queue-inl.h +++ b/yt/yt/core/misc/fair_share_hierarchical_queue-inl.h @@ -365,7 +365,7 @@ bool TFairShareHierarchicalScheduler<TTag>::CompareSlots( } { - auto rhsGuard = ReaderGuard(lhsBucket->BucketLock); + auto rhsGuard = ReaderGuard(rhsBucket->BucketLock); auto rhsBucketChildIt = rhsBucket->Buckets.find(rhsLevel.GetTag()); newRhsBucket = rhsBucketChildIt == rhsBucket->Buckets.end() ? nullptr diff --git a/yt/yt/core/ya.make b/yt/yt/core/ya.make index d03837a55d9..c682bd8383c 100644 --- a/yt/yt/core/ya.make +++ b/yt/yt/core/ya.make @@ -424,6 +424,7 @@ END() RECURSE( http test_framework + yaml ) IF (NOT OPENSOURCE AND OS_LINUX) diff --git a/yt/yt/core/yaml/config.cpp b/yt/yt/core/yaml/config.cpp new file mode 100644 index 00000000000..a9f6d826cac --- /dev/null +++ b/yt/yt/core/yaml/config.cpp @@ -0,0 +1,15 @@ +#include "config.h" + +namespace NYT::NYaml { + +//////////////////////////////////////////////////////////////////////////////// + +void TYamlFormatConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("write_uint_tag", &TThis::WriteUintTag) + .Default(false); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYaml diff --git a/yt/yt/core/yaml/config.h b/yt/yt/core/yaml/config.h new file mode 100644 index 00000000000..b6c0d68eb81 --- /dev/null +++ b/yt/yt/core/yaml/config.h @@ -0,0 +1,30 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/core/ytree/yson_struct.h> + +namespace NYT::NYaml { + +//////////////////////////////////////////////////////////////////////////////// + +struct TYamlFormatConfig + : public NYTree::TYsonStruct +{ + //! Write explicit tag "!yt/uint64" for uint64 data type. + //! Use this option if you want to preserve information about + //! the original YT type (without it, numbers in range [0, 2^63-1] + //! will always be written as integers). + //! Option has no effect for parsing. + bool WriteUintTag; + + REGISTER_YSON_STRUCT(TYamlFormatConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TYamlFormatConfig) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYaml diff --git a/yt/yt/core/yaml/convert-inl.h b/yt/yt/core/yaml/convert-inl.h new file mode 100644 index 00000000000..6c4b808e7d2 --- /dev/null +++ b/yt/yt/core/yaml/convert-inl.h @@ -0,0 +1,31 @@ +#ifndef YAML_CONVERT_INL_H_ +#error "Direct inclusion of this file is not allowed, include convert.h" +#include "convert.h" +#endif + +#include "config.h" +#include "parser.h" + +#include <yt/yt/core/yson/yson_builder.h> + +#include <yt/yt/core/ytree/convert.h> + +#include <util/stream/mem.h> + +namespace NYT::NYaml { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T> +T ConvertFromYaml(TStringBuf yaml) +{ + TMemoryInput input(yaml); + auto config = New<TYamlFormatConfig>(); + NYson::TYsonStringBuilder builder(NYson::EYsonFormat::Binary, NYson::EYsonType::Node); + ParseYaml(&input, builder.GetConsumer(), config, NYson::EYsonType::Node); + return NYTree::ConvertTo<T>(NYTree::ConvertToNode(builder.Flush())); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYaml diff --git a/yt/yt/core/yaml/convert.h b/yt/yt/core/yaml/convert.h new file mode 100644 index 00000000000..b7c0da4e0ec --- /dev/null +++ b/yt/yt/core/yaml/convert.h @@ -0,0 +1,20 @@ +#pragma once + +#include <util/generic/strbuf.h> + +namespace NYT::NYaml { + +//////////////////////////////////////////////////////////////////////////////// + +//! Parses a YAML string and deserializes the result into type T +//! using the standard YsonStruct deserialization machinery. +template <class T> +T ConvertFromYaml(TStringBuf yaml); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYaml + +#define YAML_CONVERT_INL_H_ +#include "convert-inl.h" +#undef YAML_CONVERT_INL_H_ diff --git a/yt/yt/library/formats/yaml_helpers.cpp b/yt/yt/core/yaml/helpers.cpp index 33efe3fe22b..42f6443d5ac 100644 --- a/yt/yt/library/formats/yaml_helpers.cpp +++ b/yt/yt/core/yaml/helpers.cpp @@ -1,10 +1,10 @@ -#include "yaml_helpers.h" +#include "helpers.h" #include <yt/yt/core/ytree/fluent.h> #include <contrib/libs/re2/re2/re2.h> -namespace NYT::NFormats { +namespace NYT::NYaml { using namespace NYTree; @@ -197,7 +197,7 @@ std::string_view YamlLiteralToStringView(const yaml_char_t* literal) //////////////////////////////////////////////////////////////////////////////// -} // namespace NYT::NFormats +} // namespace NYT::NYaml void Serialize(const yaml_mark_t& mark, NYT::NYson::IYsonConsumer* consumer) { diff --git a/yt/yt/library/formats/yaml_helpers.h b/yt/yt/core/yaml/helpers.h index f9b3300ffc4..a1e044286b7 100644 --- a/yt/yt/library/formats/yaml_helpers.h +++ b/yt/yt/core/yaml/helpers.h @@ -1,14 +1,14 @@ #pragma once -#include "private.h" - #include <yt/yt/core/yson/public.h> #include <yt/yt/core/ytree/public.h> +#include <library/cpp/yt/misc/enum.h> + #include <contrib/libs/yaml/include/yaml.h> -namespace NYT::NFormats { +namespace NYT::NYaml { //////////////////////////////////////////////////////////////////////////////// @@ -106,7 +106,7 @@ std::string_view YamlLiteralToStringView(const yaml_char_t* literal); //////////////////////////////////////////////////////////////////////////////// -} // namespace NYT::NFormats +} // namespace NYT::NYaml // Note that ADL requires to put this function in the global namespace since // yaml_mark_t is defined in the global namespace from C++ POV diff --git a/yt/yt/library/formats/yaml_parser.cpp b/yt/yt/core/yaml/parser.cpp index c6e2c964a5c..63b2b3a2240 100644 --- a/yt/yt/library/formats/yaml_parser.cpp +++ b/yt/yt/core/yaml/parser.cpp @@ -1,20 +1,19 @@ -#include "yaml_parser.h" +#include "parser.h" -#include "yaml_helpers.h" - -#include <yt/yt/client/formats/parser.h> +#include "helpers.h" #include <yt/yt/core/yson/consumer.h> +#include <yt/yt/core/yson/writer.h> #include <yt/yt/core/ytree/convert.h> -#include <yt/yt/core/misc/coro_pipe.h> - #include <library/cpp/yt/string/stream.h> +#include <library/cpp/yt/yson_string/format.h> + #include <contrib/libs/yaml/include/yaml.h> -namespace NYT::NFormats { +namespace NYT::NYaml { using namespace NYson; using namespace NYTree; @@ -561,4 +560,4 @@ void ParseYaml( //////////////////////////////////////////////////////////////////////////////// -} // namespace NYT::NFormats +} // namespace NYT::NYaml diff --git a/yt/yt/library/formats/yaml_parser.h b/yt/yt/core/yaml/parser.h index 6cdfc4dab63..16e7ef371ba 100644 --- a/yt/yt/library/formats/yaml_parser.h +++ b/yt/yt/core/yaml/parser.h @@ -1,10 +1,12 @@ #pragma once -#include <yt/yt/client/formats/public.h> +#include "public.h" #include <yt/yt/core/yson/public.h> -namespace NYT::NFormats { +#include <util/stream/fwd.h> + +namespace NYT::NYaml { //////////////////////////////////////////////////////////////////////////////// @@ -17,4 +19,4 @@ void ParseYaml( //////////////////////////////////////////////////////////////////////////////// -} // namespace NYT::NFormats +} // namespace NYT::NYaml diff --git a/yt/yt/core/yaml/public.h b/yt/yt/core/yaml/public.h new file mode 100644 index 00000000000..384c1df0614 --- /dev/null +++ b/yt/yt/core/yaml/public.h @@ -0,0 +1,13 @@ +#pragma once + +#include <library/cpp/yt/memory/ref_counted.h> + +namespace NYT::NYaml { + +//////////////////////////////////////////////////////////////////////////////// + +DECLARE_REFCOUNTED_STRUCT(TYamlFormatConfig) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NYaml diff --git a/yt/yt/core/yaml/unittests/convert_ut.cpp b/yt/yt/core/yaml/unittests/convert_ut.cpp new file mode 100644 index 00000000000..ecb87a2c5ff --- /dev/null +++ b/yt/yt/core/yaml/unittests/convert_ut.cpp @@ -0,0 +1,61 @@ +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/core/yaml/convert.h> + +#include <yt/yt/core/ytree/fluent.h> +#include <yt/yt/core/ytree/yson_struct.h> + +namespace NYT::NYaml { +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +struct TTestConfig + : public NYTree::TYsonStruct +{ + std::string Name; + int Value; + + REGISTER_YSON_STRUCT(TTestConfig); + + static void Register(TRegistrar registrar) + { + registrar.Parameter("name", &TThis::Name) + .Default(); + registrar.Parameter("value", &TThis::Value) + .Default(0); + } +}; + +DEFINE_REFCOUNTED_TYPE(TTestConfig) + +//////////////////////////////////////////////////////////////////////////////// + +TEST(TYamlHelpersTest, StringList) +{ + auto result = ConvertFromYaml<std::vector<std::string>>("- foo\n- bar\n"); + EXPECT_EQ(result, (std::vector<std::string>{"foo", "bar"})); +} + +TEST(TYamlHelpersTest, Mapping) +{ + auto result = ConvertFromYaml<NYTree::INodePtr>("key: value\n"); + EXPECT_EQ(result->AsMap()->GetChildValueOrThrow<std::string>("key"), "value"); +} + +TEST(TYamlHelpersTest, YsonStruct) +{ + auto result = ConvertFromYaml<TIntrusivePtr<TTestConfig>>("name: hello\nvalue: 42\n"); + EXPECT_EQ(result->Name, "hello"); + EXPECT_EQ(result->Value, 42); +} + +TEST(TYamlHelpersTest, InvalidYaml) +{ + EXPECT_THROW(ConvertFromYaml<NYTree::INodePtr>("key: [unclosed"), std::exception); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NYaml diff --git a/yt/yt/core/yaml/unittests/parser_ut.cpp b/yt/yt/core/yaml/unittests/parser_ut.cpp new file mode 100644 index 00000000000..d45ec873dda --- /dev/null +++ b/yt/yt/core/yaml/unittests/parser_ut.cpp @@ -0,0 +1,600 @@ +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/core/yaml/config.h> + +#include <yt/yt/core/yaml/parser.h> + +#include <yt/yt/core/yson/writer.h> + +#include <library/cpp/yt/string/stream.h> + +#include <util/stream/mem.h> + +namespace NYT::NYaml { +namespace { + +using namespace NYson; + +//////////////////////////////////////////////////////////////////////////// + +std::string ParseYaml(const std::string& yaml, EYsonType ysonType) +{ + TMemoryInput inputStream(yaml); + TStdStringStream outputStream; + TYsonWriter writer(&outputStream, EYsonFormat::Pretty, ysonType); + auto config = New<TYamlFormatConfig>(); + ParseYaml(&inputStream, &writer, config, ysonType); + return outputStream.Str(); +} + +////////////////////////////////////////////////////////////////////////////// + +TEST(TYamlParserTest, Simple) +{ + std::string yaml = R"( +hello)"; + // Here and in the rest of the tests we introduce an extra leading \n for the better readabilty, which we later + // strip off in the comparison. + TStringBuf expectedYson = R"( +"hello")"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::Node), expectedYson.substr(1)); +} + +TEST(TYamlParserTest, Integers) +{ + std::string yaml = R"( +a: 1 +b: -1 +# Hex and oct +c: 0xDeAdBeEf +d: 0o42 +# Various non-normalized forms of numbers +e: -000 +f: +0 +g: +42 +# Would be oct in YAML 1.1, but not in YAML 1.2! +h: 0042 +i: -018 +# 2^63, should be unsigned +j: 9223372036854775808 +# 2^64 - 1, should be unsigned +k: 18446744073709551615 +l: -9223372036854775808 +m: !yt/uint64 1234 +n: !!int 23 +o: !!int -15)"; + TStringBuf expectedYson = R"( +{ + "a" = 1; + "b" = -1; + "c" = 3735928559u; + "d" = 34u; + "e" = 0; + "f" = 0; + "g" = 42; + "h" = 42; + "i" = -18; + "j" = 9223372036854775808u; + "k" = 18446744073709551615u; + "l" = -9223372036854775808; + "m" = 1234u; + "n" = 23; + "o" = -15; +})"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::Node), expectedYson.substr(1)); + + std::vector<std::string> invalidYamls = { + "!!int -0x42", + "!!int -0o23", + "!!int deadbeef", + "!!int 18446744073709551616", + "!!int -9223372036854775809" + "!yt/uint64 -1", + "!yt/uint64 18446744073709551616", + "!!int 0x", + // Examples below were integers in YAML 1.1, but not in YAML 1.2. + "!!int 123_456", + "!!int 190:20:30", + "!!int 0b1001", + "!!int \"\"", + }; + for (const auto& yaml : invalidYamls) { + EXPECT_THROW_MESSAGE_HAS_SUBSTR(ParseYaml(yaml, EYsonType::Node), std::exception, "is not an integer or does not fit") + << "For YAML: " << yaml << std::endl; + } +} + +TEST(TYamlParserTest, Floats) +{ + std::string yaml = R"( +a: 1. +b: .2 +c: +3.14 +d: -2.17 +e: .inf +f: -.Inf +g: +.INF +h: .nan +i: .NaN +j: .NAN +k: !!float 42 +l: 1e2 +m: 1e+2 +n: 1e-2 +)"; + TStringBuf expectedYson = R"( +{ + "a" = 1.; + "b" = 0.2; + "c" = 3.14; + "d" = -2.17; + "e" = %inf; + "f" = %-inf; + "g" = %inf; + "h" = %nan; + "i" = %nan; + "j" = %nan; + "k" = 42.; + "l" = 100.; + "m" = 100.; + "n" = 0.01; +})"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::Node), expectedYson.substr(1)); + + std::vector<std::string> invalidYamls = { + "!!float 0o23", + "!!float 1e", + "!!float 1e+", + "!!float 1e-", + "!!float 1e-2.3", + "!!float 1e2.3", + // Examples below were integers in YAML 1.1, but not in YAML 1.2. + "!!float 123_456", + "!!float 190:20:30.15", + "!!float inf", + "!!float .InF", + "!!float -+42.0", + "!!float .", + // For some reason arcadian FloatToString parses this, but it feels excessive to ban that + // despite not satisfying the regexp from the spec. + // "!!float 0x42", + }; + for (const auto& yaml : invalidYamls) { + EXPECT_THROW_MESSAGE_HAS_SUBSTR(ParseYaml(yaml, EYsonType::Node), std::exception, "is not a floating point") + << "For YAML: " << yaml << std::endl; + } +} + +TEST(TYamlParserTest, Booleans) +{ + std::string yaml = R"( +a: true +b: false +c: True +d: False +e: TRUE +f: FALSE +g: !!bool true +)"; + TStringBuf expectedYson = R"( +{ + "a" = %true; + "b" = %false; + "c" = %true; + "d" = %false; + "e" = %true; + "f" = %false; + "g" = %true; +})"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::Node), expectedYson.substr(1)); + + std::vector<std::string> invalidYamls = { + "!!bool 1", + "!!bool 0", + // Examples below were booleans in YAML 1.1, but not in YAML 1.2. + "!!bool yes", + "!!bool no", + "!!bool on", + "!!bool off", + "!!bool y", + "!!bool n", + "!!bool \"\"", + }; + for (const auto& yaml : invalidYamls) { + EXPECT_THROW_MESSAGE_HAS_SUBSTR(ParseYaml(yaml, EYsonType::Node), std::exception, "is not a boolean") + << "For YAML: " << yaml << std::endl; + } +} + +TEST(TYamlParserTest, Nulls) +{ + std::string yaml = R"( +a: null +b: Null +c: NULL +d: ~ +e: +f: !!null null +# This is not allowed by a regexp in a spec, but feels excessive to ban. +g: !!null foo +)"; + TStringBuf expectedYson = R"( +{ + "a" = #; + "b" = #; + "c" = #; + "d" = #; + "e" = #; + "f" = #; + "g" = #; +})"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::Node), expectedYson.substr(1)); +} + +TEST(TYamlParserTest, Strings) +{ + std::string yaml = R"( +a: "hello" +b: 'world' +c: of +d: !!str warcraft +e: !!str 42 +f: !!str ~ +g: ! hello +)"; + TStringBuf expectedYson = R"( +{ + "a" = "hello"; + "b" = "world"; + "c" = "of"; + "d" = "warcraft"; + "e" = "42"; + "f" = "~"; + "g" = "hello"; +})"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::Node), expectedYson.substr(1)); +} + +TEST(TYamlParserTest, Mappings) +{ + std::string yaml = R"( +a: + x: 1 + y: + foo: bar + bar: foo +42: + z: 3 +c: {} +)"; + TStringBuf expectedYson = R"( +{ + "a" = { + "x" = 1; + "y" = { + "foo" = "bar"; + "bar" = "foo"; + }; + }; + "42" = { + "z" = 3; + }; + "c" = {}; +})"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::Node), expectedYson.substr(1)); +} + +TEST(TYamlParserTest, Sequences) +{ + std::string yaml = R"( +- foo +- - 1 + - 2 + - 3 +- bar +- [] +- - - - null +)"; + TStringBuf expectedYson = R"( +[ + "foo"; + [ + 1; + 2; + 3; + ]; + "bar"; + []; + [ + [ + [ + #; + ]; + ]; + ]; +])"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::Node), expectedYson.substr(1)); +} + +TEST(TYamlParserTest, Attributes) +{ + std::string yaml = R"( +!yt/attrnode +- x: 1 + y: 2 +- a: !yt/attrnode + - {} + - 42 + b: !yt/attrnode + - x: null + - - 1 + - 2 + - 3 + c: !yt/attrnode + - foo: 1 + - null +)"; + // <x=1;y=2>{a=<>42; b=<x=#>[1;2;3]; c=<foo=1>#;} + TStringBuf expectedYson = R"( +< + "x" = 1; + "y" = 2; +> { + "a" = <> 42; + "b" = < + "x" = #; + > [ + 1; + 2; + 3; + ]; + "c" = < + "foo" = 1; + > #; +})"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::Node), expectedYson.substr(1)); + + std::vector<std::pair<std::string, std::string>> invalidYamlsAndErrors = { + {R"( +!yt/attrnode +- x: 1 +)", "Unexpected event type \"sequence_end\""}, + {R"( +!yt/attrnode +- foo +- bar +)", "Unexpected event type \"scalar\""}, + {R"( +!yt/attrnode +- x: 1 +- y: 2 +- z: 3 +)", "Unexpected event type \"mapping_start\""}, +}; + for (const auto& [yaml, error] : invalidYamlsAndErrors) { + EXPECT_THROW_MESSAGE_HAS_SUBSTR(ParseYaml(yaml, EYsonType::Node), std::exception, error) + << "For YAML: " << yaml << std::endl; + } +}; + +TEST(TYamlParserTest, MultiDocument) +{ + std::string yaml = R"( +a: 1 +--- +foo +--- +~ +--- +)"; + TStringBuf expectedYson = R"( +{ + "a" = 1; +}; +"foo"; +#; +#; +)"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::ListFragment), expectedYson.substr(1)); +} + +TEST(TYamlParserTest, Anchors) +{ + std::string yaml = R"( +a: &foo 1 +b: *foo +c: &bar + x: &baz + - False + - &qux True + y: 2 + z: *baz + t: *foo + w: *qux +d: *bar +e: *baz +f: *foo +g: *qux +)"; + TStringBuf expectedYson = R"( +{ + "a" = 1; + "b" = 1; + "c" = { + "x" = [ + %false; + %true; + ]; + "y" = 2; + "z" = [ + %false; + %true; + ]; + "t" = 1; + "w" = %true; + }; + "d" = { + "x" = [ + %false; + %true; + ]; + "y" = 2; + "z" = [ + %false; + %true; + ]; + "t" = 1; + "w" = %true; + }; + "e" = [ + %false; + %true; + ]; + "f" = 1; + "g" = %true; +})"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::Node), expectedYson.substr(1)); + + std::vector<std::pair<std::string, std::string>> invalidYamlsAndErrors = { + {R"( +a: *foo +)", "undefined or unfinished anchor"}, + {R"( +- &foo a +- &foo b +)", "already defined"}, + {R"( +a: &foo +- b: &foo + - c +)", "already defined"}, + {R"( +a: &foo + bar: *foo +)", "undefined or unfinished anchor"}, + {R"( +a: &foo bar +*foo: baz +)", "alias as a map key is not supported"}, + {R"( +&foo a: b +)", "anchors on map keys is not supported"}, + }; + for (const auto& [yaml, error] : invalidYamlsAndErrors) { + EXPECT_THROW_MESSAGE_HAS_SUBSTR(ParseYaml(yaml, EYsonType::Node), std::exception, error) + << "For YAML: " << yaml << std::endl; + } +} + +TEST(TYamlParserTest, Empty) +{ + std::string yaml = ""; + TStringBuf expectedYson = ""; + EXPECT_EQ(ParseYaml(yaml, EYsonType::ListFragment), expectedYson); +} + +//! There is a reverse test in writer_ut.cpp. +TEST(TYamlParserTest, RealExample) +{ + std::string yaml = R"( +mount_config: {} +schema: !yt/attrnode +- strict: true + unique_keys: false +- - name: lat + required: false + type: double + type_v3: + type_name: optional + item: double + - name: lon + required: false + type: double + type_v3: + type_name: optional + item: double +native_cell_tag: !yt/uint64 9991 +creation_time: 2024-08-15T11:17:59.314773Z +inherit_acl: true +revision: !yt/uint64 8233452423020 +resource_usage: + node_count: 1 + chunk_count: 1 + disk_space_per_medium: + default: 562182 + disk_space: 562182 + chunk_host_cell_master_memory: 0 + master_memory: 0 + tablet_count: 0 + tablet_static_memory: 0 +acl: [] +id: 77d-1c53a-27070191-e4d8f5ac +parent_id: 77d-1c0d3-2707012f-ddf40dd7 +foreign: false +type: table +sequoia: false +ref_counter: 1 +builtin: false +owner: max +compression_ratio: 0.3679379456925491 +)"; + TStringBuf expectedYson = R"( +{ + "mount_config" = {}; + "schema" = < + "strict" = %true; + "unique_keys" = %false; + > [ + { + "name" = "lat"; + "required" = %false; + "type" = "double"; + "type_v3" = { + "type_name" = "optional"; + "item" = "double"; + }; + }; + { + "name" = "lon"; + "required" = %false; + "type" = "double"; + "type_v3" = { + "type_name" = "optional"; + "item" = "double"; + }; + }; + ]; + "native_cell_tag" = 9991u; + "creation_time" = "2024-08-15T11:17:59.314773Z"; + "inherit_acl" = %true; + "revision" = 8233452423020u; + "resource_usage" = { + "node_count" = 1; + "chunk_count" = 1; + "disk_space_per_medium" = { + "default" = 562182; + }; + "disk_space" = 562182; + "chunk_host_cell_master_memory" = 0; + "master_memory" = 0; + "tablet_count" = 0; + "tablet_static_memory" = 0; + }; + "acl" = []; + "id" = "77d-1c53a-27070191-e4d8f5ac"; + "parent_id" = "77d-1c0d3-2707012f-ddf40dd7"; + "foreign" = %false; + "type" = "table"; + "sequoia" = %false; + "ref_counter" = 1; + "builtin" = %false; + "owner" = "max"; + "compression_ratio" = 0.3679379456925491; +})"; + EXPECT_EQ(ParseYaml(yaml, EYsonType::Node), expectedYson.substr(1)); +} + +//////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NYaml diff --git a/yt/yt/core/yaml/unittests/writer_ut.cpp b/yt/yt/core/yaml/unittests/writer_ut.cpp new file mode 100644 index 00000000000..29e7cb1a096 --- /dev/null +++ b/yt/yt/core/yaml/unittests/writer_ut.cpp @@ -0,0 +1,321 @@ +#include <yt/yt/core/test_framework/framework.h> + +#include <yt/yt/core/yaml/writer.h> + +#include <yt/yt/core/yaml/config.h> + +#include <yt/yt/core/yson/string.h> + +#include <yt/yt/core/ytree/convert.h> + +#include <library/cpp/yt/string/stream.h> + +namespace NYT::NYaml { +namespace { + +using namespace NYson; +using namespace NYTree; + +////////////////////////////////////////////////////////////////////////////// + +std::string YsonToYaml(const TYsonString& yson, const TYsonString& formatAttributes = TYsonString(TStringBuf("{}"))) +{ + TStdStringStream outputStream; + auto config = ConvertTo<TYamlFormatConfigPtr>(formatAttributes); + auto writer = CreateYamlWriter(&outputStream, yson.GetType(), config); + Serialize(yson, writer.get()); + writer->Flush(); + return outputStream.Str(); +} + +////////////////////////////////////////////////////////////////////////////// + +TEST(TYamlWriterTest, Simple) +{ + std::string yson = "hello"; + // Here and in the rest of the tests we introduce an extra leading \n for the better readabilty, which we later + // strip off in the comparison. + std::string expectedYaml = R"( +hello +)"; + + EXPECT_EQ(YsonToYaml(TYsonString(yson)), expectedYaml.substr(1)); +} + +TEST(TYamlWriterTest, IntegersWithoutUintTag) +{ + std::string yson = "{a=1; b=1u; c=-1; d=9223372036854775808u; e=-9223372036854775808; f=18446744073709551615u}"; + std::string expectedYaml = R"( +a: 1 +b: 1 +c: -1 +d: 9223372036854775808 +e: -9223372036854775808 +f: 18446744073709551615 +)"; + EXPECT_EQ(YsonToYaml(TYsonString(yson)), expectedYaml.substr(1)); +} + +TEST(TYamlWriterTest, IntegersWithUintTag) +{ + std::string formatAttributes = "{write_uint_tag=%true}"; + std::string yson = "{a=1; b=1u; c=-1; d=9223372036854775808u; e=-9223372036854775808; f=18446744073709551615u}"; + std::string expectedYaml = R"( +a: 1 +b: !yt/uint64 1 +c: -1 +d: !yt/uint64 9223372036854775808 +e: -9223372036854775808 +f: !yt/uint64 18446744073709551615 +)"; + EXPECT_EQ(YsonToYaml(TYsonString(yson), TYsonString(formatAttributes)), expectedYaml.substr(1)); +} + +TEST(TYamlWriterTest, Doubles) +{ + std::string yson = "{a=2.7; b=-3.14; c=0.0; d=4.; e=1e30; f=%nan; g=%inf; h=%-inf}"; + std::string expectedYaml = R"( +a: 2.7 +b: -3.14 +c: 0. +d: 4. +e: 1e+30 +f: .nan +g: .inf +h: -.inf +)"; + EXPECT_EQ(YsonToYaml(TYsonString(yson)), expectedYaml.substr(1)); +} + +TEST(TYamlWriterTest, Entity) +{ + std::string yson = "{a=#}"; + std::string expectedYaml = R"( +a: null +)"; + EXPECT_EQ(YsonToYaml(TYsonString(yson)), expectedYaml.substr(1)); +} + +TEST(TYamlWriterTest, Booleans) +{ + std::string yson = "{a=%true; b=%false}"; + std::string expectedYaml = R"( +a: true +b: false +)"; + EXPECT_EQ(YsonToYaml(TYsonString(yson)), expectedYaml.substr(1)); +} + +TEST(TYamlWriterTest, Strings) +{ + // a and b may be represented as plain scalars. + // c-e must be quoted on syntactical level, so libyaml chooses a single-quoted style. + // f-i must be quoted because they meet regexps for non-string types, so we force a double-quoted style. + std::string yson = R"({a=hello; b="23asd"; c=" "; d="foo\nbar"; e=""; f="42"; g="TRUE"; h="1e4000"; i="~";})"; + std::string expectedYaml = R"( +a: hello +b: 23asd +c: ' ' +d: 'foo + + bar' +e: "" +f: "42" +g: "TRUE" +h: "1e4000" +i: "~" +)"; + EXPECT_EQ(YsonToYaml(TYsonString(yson)), expectedYaml.substr(1)); +} + +TEST(TYamlWriterTest, Mappings) +{ + std::string yson("{a={x=1;y={foo=bar;bar=foo}};b={z=3};c={};}"); + std::string expectedYaml = R"( +a: + x: 1 + y: + foo: bar + bar: foo +b: + z: 3 +c: {} +)"; + EXPECT_EQ(YsonToYaml(TYsonString(yson)), expectedYaml.substr(1)); +} + +TEST(TYamlWriterTest, Sequences) +{ + std::string yson = "[foo; [1; 2; 3]; bar; []; [[[#]]]]"; + std::string expectedYaml = R"( +- foo +- - 1 + - 2 + - 3 +- bar +- [] +- - - - null +)"; + EXPECT_EQ(YsonToYaml(TYsonString(yson)), expectedYaml.substr(1)); +} + +TEST(TYamlWriterTest, MultiDocument) +{ + std::string yson = "foo;{a=1;b=2};[x;y];{};#;bar;[]"; + std::string expectedYaml = R"( +foo +--- +a: 1 +b: 2 +--- +- x +- y +--- {} +--- null +--- bar +--- [] +)"; + EXPECT_EQ(YsonToYaml(TYsonString(yson, EYsonType::ListFragment)), expectedYaml.substr(1)); +} + +TEST(TYamlWriterTest, Attributes) +{ + std::string yson = "<x=1;y=2>{a=<>42; b=<x=#>[1;2;3]; c=<foo=1>#;}"; + std::string expectedYaml = R"( +!yt/attrnode +- x: 1 + y: 2 +- a: !yt/attrnode + - {} + - 42 + b: !yt/attrnode + - x: null + - - 1 + - 2 + - 3 + c: !yt/attrnode + - foo: 1 + - null +)"; + EXPECT_EQ(YsonToYaml(TYsonString(yson)), expectedYaml.substr(1)); +}; + +////////////////////////////////////////////////////////////////////////////// + +TEST(TYamlWriterTest, EmptyStream) +{ + std::string yson = ""; + std::string expectedYaml = ""; + EXPECT_EQ(YsonToYaml(TYsonString(yson, EYsonType::ListFragment)), expectedYaml); +} + +////////////////////////////////////////////////////////////////////////////// + +//! There is a reverse test in yaml_reader_ut.cpp. +TEST(TYamlWriterTest, RealExample) +{ + std::string formatAttributes = "{write_uint_tag=%true}"; + std::string yson = R"( +{ + "mount_config" = {}; + "schema" = < + "strict" = %true; + "unique_keys" = %false; + > [ + { + "name" = "lat"; + "required" = %false; + "type" = "double"; + "type_v3" = { + "type_name" = "optional"; + "item" = "double"; + }; + }; + { + "name" = "lon"; + "required" = %false; + "type" = "double"; + "type_v3" = { + "type_name" = "optional"; + "item" = "double"; + }; + }; + ]; + "native_cell_tag" = 9991u; + "creation_time" = "2024-08-15T11:17:59.314773Z"; + "inherit_acl" = %true; + "revision" = 8233452423020u; + "resource_usage" = { + "node_count" = 1; + "chunk_count" = 1; + "disk_space_per_medium" = { + "default" = 562182; + }; + "disk_space" = 562182; + "chunk_host_cell_master_memory" = 0; + "master_memory" = 0; + "tablet_count" = 0; + "tablet_static_memory" = 0; + }; + "acl" = []; + "id" = "77d-1c53a-27070191-e4d8f5ac"; + "parent_id" = "77d-1c0d3-2707012f-ddf40dd7"; + "foreign" = %false; + "type" = "table"; + "sequoia" = %false; + "ref_counter" = 1; + "builtin" = %false; + "owner" = "max"; + "compression_ratio" = 0.3679379456925491; +} + )"; + + std::string expectedYaml = R"( +mount_config: {} +schema: !yt/attrnode +- strict: true + unique_keys: false +- - name: lat + required: false + type: double + type_v3: + type_name: optional + item: double + - name: lon + required: false + type: double + type_v3: + type_name: optional + item: double +native_cell_tag: !yt/uint64 9991 +creation_time: 2024-08-15T11:17:59.314773Z +inherit_acl: true +revision: !yt/uint64 8233452423020 +resource_usage: + node_count: 1 + chunk_count: 1 + disk_space_per_medium: + default: 562182 + disk_space: 562182 + chunk_host_cell_master_memory: 0 + master_memory: 0 + tablet_count: 0 + tablet_static_memory: 0 +acl: [] +id: 77d-1c53a-27070191-e4d8f5ac +parent_id: 77d-1c0d3-2707012f-ddf40dd7 +foreign: false +type: table +sequoia: false +ref_counter: 1 +builtin: false +owner: max +compression_ratio: 0.3679379456925491 +)"; + EXPECT_EQ(YsonToYaml(TYsonString(yson), TYsonString(formatAttributes)), expectedYaml.substr(1)); +} + +////////////////////////////////////////////////////////////////////////////// + +} // namespace +} // namespace NYT::NYaml diff --git a/yt/yt/core/yaml/unittests/ya.make b/yt/yt/core/yaml/unittests/ya.make new file mode 100644 index 00000000000..583ed1165c8 --- /dev/null +++ b/yt/yt/core/yaml/unittests/ya.make @@ -0,0 +1,18 @@ +GTEST(unittester-core-yaml) + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + convert_ut.cpp + parser_ut.cpp + writer_ut.cpp +) + +INCLUDE(${ARCADIA_ROOT}/yt/opensource.inc) + +PEERDIR( + yt/yt/core/test_framework + yt/yt/core/yaml +) + +END() diff --git a/yt/yt/library/formats/yaml_writer.cpp b/yt/yt/core/yaml/writer.cpp index b03e671f5b3..df3043db73b 100644 --- a/yt/yt/library/formats/yaml_writer.cpp +++ b/yt/yt/core/yaml/writer.cpp @@ -1,20 +1,21 @@ -#include "yaml_writer.h" +#include "writer.h" +#include "config.h" #include "helpers.h" -#include "yaml_helpers.h" -#include <yt/yt/client/formats/config.h> +#include <yt/yt/core/yson/consumer.h> #include <contrib/libs/yaml/include/yaml.h> -namespace NYT::NFormats { +namespace NYT::NYaml { using namespace NYson; //////////////////////////////////////////////////////////////////////////// class TYamlWriter - : public TFormatsConsumerBase + : public TYsonConsumerBase + , public virtual IFlushableYsonConsumer { public: TYamlWriter( @@ -385,4 +386,4 @@ std::unique_ptr<IFlushableYsonConsumer> CreateYamlWriter( //////////////////////////////////////////////////////////////////////////// -} // namespace NYT::NFormats +} // namespace NYT::NYaml diff --git a/yt/yt/library/formats/yaml_writer.h b/yt/yt/core/yaml/writer.h index c3e7fb76fd4..4fe676f88b3 100644 --- a/yt/yt/library/formats/yaml_writer.h +++ b/yt/yt/core/yaml/writer.h @@ -1,10 +1,12 @@ #pragma once -#include <yt/yt/core/yson/consumer.h> +#include "public.h" -#include <yt/yt/client/formats/public.h> +#include <yt/yt/core/yson/public.h> -namespace NYT::NFormats { +#include <util/stream/fwd.h> + +namespace NYT::NYaml { //////////////////////////////////////////////////////////////////////////////// @@ -15,4 +17,4 @@ std::unique_ptr<NYson::IFlushableYsonConsumer> CreateYamlWriter( //////////////////////////////////////////////////////////////////////////////// -} // namespace NYT::NFormats +} // namespace NYT::NYaml diff --git a/yt/yt/core/yaml/ya.make b/yt/yt/core/yaml/ya.make new file mode 100644 index 00000000000..b47db455c36 --- /dev/null +++ b/yt/yt/core/yaml/ya.make @@ -0,0 +1,23 @@ +LIBRARY() + +INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) + +SRCS( + config.cpp + helpers.cpp + parser.cpp + writer.cpp +) + +PEERDIR( + yt/yt/core + + contrib/libs/yaml + contrib/libs/re2 +) + +END() + +RECURSE_FOR_TESTS( + unittests +) diff --git a/yt/yt/library/formats/format.cpp b/yt/yt/library/formats/format.cpp index 954794bb00b..1aa8548919f 100644 --- a/yt/yt/library/formats/format.cpp +++ b/yt/yt/library/formats/format.cpp @@ -13,8 +13,6 @@ #include "schemaless_writer_adapter.h" #include "skiff_parser.h" #include "skiff_writer.h" -#include "yaml_parser.h" -#include "yaml_writer.h" #include "yamred_dsv_parser.h" #include "yamred_dsv_writer.h" #include "yamr_parser.h" @@ -41,12 +39,17 @@ #include <yt/yt/core/json/json_parser.h> #include <yt/yt/core/json/json_writer.h> +#include <yt/yt/core/yaml/config.h> +#include <yt/yt/core/yaml/parser.h> +#include <yt/yt/core/yaml/writer.h> + namespace NYT::NFormats { using namespace NConcurrency; using namespace NYTree; using namespace NYson; using namespace NJson; +using namespace NYaml; using namespace NTableClient; using namespace NSkiffExt; using namespace NComplexTypes; diff --git a/yt/yt/library/formats/ya.make b/yt/yt/library/formats/ya.make index 8259125d368..9c5c50b757c 100644 --- a/yt/yt/library/formats/ya.make +++ b/yt/yt/library/formats/ya.make @@ -24,9 +24,6 @@ SRCS( skiff_yson_converter.cpp unversioned_value_yson_writer.cpp web_json_writer.cpp - yaml_helpers.cpp - yaml_parser.cpp - yaml_writer.cpp yamred_dsv_parser.cpp yamred_dsv_writer.cpp yamr_parser_base.cpp @@ -41,10 +38,10 @@ SRCS( PEERDIR( yt/yt/client yt/yt/client/formats + yt/yt/core/yaml yt/yt/library/column_converters contrib/libs/apache/arrow_next - contrib/libs/yaml ) END() diff --git a/yt/yt/library/profiling/tag.cpp b/yt/yt/library/profiling/tag.cpp index 708766462cd..efdc79b37b6 100644 --- a/yt/yt/library/profiling/tag.cpp +++ b/yt/yt/library/profiling/tag.cpp @@ -186,7 +186,7 @@ TDynamicTagPtr TTagSet::AddDynamicTag(int index) return tag; } -void TTagSet::ApplyDynamicTag(TDynamicTagPtr dynamicTag) +void TTagSet::ApplyDynamicTag(const TDynamicTagPtr& dynamicTag) { for (auto& [tag, index] : DynamicTags_) { if (tag == dynamicTag) { diff --git a/yt/yt/library/profiling/tag.h b/yt/yt/library/profiling/tag.h index 835e1fab662..6d58006ceea 100644 --- a/yt/yt/library/profiling/tag.h +++ b/yt/yt/library/profiling/tag.h @@ -90,7 +90,7 @@ public: void Append(const TTagSet& other); TDynamicTagPtr AddDynamicTag(int index); - void ApplyDynamicTag(TDynamicTagPtr dynamicTag); + void ApplyDynamicTag(const TDynamicTagPtr& dynamicTag); const TTagList& Tags() const; diff --git a/yt/yt/library/profiling/unittests/solomon_ut.cpp b/yt/yt/library/profiling/unittests/solomon_ut.cpp index 2883f138352..363095f41e4 100644 --- a/yt/yt/library/profiling/unittests/solomon_ut.cpp +++ b/yt/yt/library/profiling/unittests/solomon_ut.cpp @@ -925,7 +925,7 @@ TEST(TSolomonRegistryTest, ExtensionTag) TEST_P(TRenameTest, RenameTag) { - bool recreate_sensor = GetParam(); + bool recreateSensor = GetParam(); auto impl = New<TSolomonRegistry>(); impl->SetWindowSize(12); @@ -955,7 +955,7 @@ TEST_P(TRenameTest, RenameTag) r.RenameDynamicTag(mediumTag, "medium", "default"); - if (recreate_sensor) { + if (recreateSensor) { c0 = r.Counter("/bytes_read"); } @@ -973,8 +973,7 @@ TEST_P(TRenameTest, RenameTag) INSTANTIATE_TEST_SUITE_P( TRenameTest, TRenameTest, - ::testing::Values(false, true) -); + ::testing::Values(false, true)); struct TBlinkingProducer : ISensorProducer diff --git a/yt/yt/library/tracing/example/main.cpp b/yt/yt/library/tracing/example/main.cpp index 99fcaa14c3a..8ad7ebe97ca 100644 --- a/yt/yt/library/tracing/example/main.cpp +++ b/yt/yt/library/tracing/example/main.cpp @@ -1,3 +1,4 @@ +#include <yt/yt/library/tracing/jaeger/config.h> #include <yt/yt/library/tracing/jaeger/tracer.h> #include <yt/yt/core/concurrency/scheduler_api.h> diff --git a/yt/yt/library/tracing/jaeger/config.cpp b/yt/yt/library/tracing/jaeger/config.cpp new file mode 100644 index 00000000000..f3b26f9fbe6 --- /dev/null +++ b/yt/yt/library/tracing/jaeger/config.cpp @@ -0,0 +1,120 @@ +#include "config.h" + +namespace NYT::NTracing { + +//////////////////////////////////////////////////////////////////////////////// + +void TSamplerConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("global_sample_rate", &TThis::GlobalSampleRate) + .Default(0.0); + registrar.Parameter("user_sample_rate", &TThis::UserSampleRate) + .Default(); + registrar.Parameter("user_endpoints", &TThis::UserEndpoint) + .Default(); + registrar.Parameter("clear_sampled_flag", &TThis::ClearSampledFlag) + .Default(); + + registrar.Parameter("min_per_user_samples", &TThis::MinPerUserSamples) + .Default(0); + registrar.Parameter("min_per_user_samples_period", &TThis::MinPerUserSamplesPeriod) + .Default(TDuration::Minutes(1)); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TJaegerTracerDynamicConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("collector_channel", &TThis::CollectorChannel) + .Alias("collector_channel_config") + .Optional(); + registrar.Parameter("max_request_size", &TThis::MaxRequestSize) + .Default(); + registrar.Parameter("max_memory", &TThis::MaxMemory) + .Default(); + registrar.Parameter("subsampling_rate", &TThis::SubsamplingRate) + .Default(); + registrar.Parameter("flush_period", &TThis::FlushPeriod) + .Default(); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TJaegerTracerConfig::Register(TRegistrar registrar) +{ + registrar.Parameter("collector_channel_config", &TThis::CollectorChannelConfig) + .Optional(); + + // 10K nodes x 128 KB / 15s == 85mb/s + registrar.Parameter("flush_period", &TThis::FlushPeriod) + .Default(TDuration::Seconds(15)); + registrar.Parameter("stop_timeout", &TThis::StopTimeout) + .Default(TDuration::Seconds(15)); + registrar.Parameter("rpc_timeout", &TThis::RpcTimeout) + .Default(TDuration::Seconds(15)); + registrar.Parameter("queue_stall_timeout", &TThis::QueueStallTimeout) + .Default(TDuration::Minutes(15)); + registrar.Parameter("max_request_size", &TThis::MaxRequestSize) + .Default(128_KB) + .LessThanOrEqual(4_MB); + registrar.Parameter("max_batch_size", &TThis::MaxBatchSize) + .Default(128); + registrar.Parameter("max_memory", &TThis::MaxMemory) + .Default(1_GB); + registrar.Parameter("subsampling_rate", &TThis::SubsamplingRate) + .Default(); + registrar.Parameter("reconnect_period", &TThis::ReconnectPeriod) + .Default(TDuration::Minutes(15)); + registrar.Parameter("endpoint_channel_timeout", &TThis::EndpointChannelTimeout) + .Default(TDuration::Hours(2)); + + registrar.Parameter("service_name", &TThis::ServiceName) + .Default(); + registrar.Parameter("process_tags", &TThis::ProcessTags) + .Default(); + registrar.Parameter("enable_pid_tag", &TThis::EnablePidTag) + .Default(false); + + registrar.Parameter("tvm_service", &TThis::TvmService) + .Optional(); + + registrar.Parameter("test_drop_spans", &TThis::TestDropSpans) + .Default(false); +} + +TJaegerTracerConfigPtr TJaegerTracerConfig::ApplyDynamic(const TJaegerTracerDynamicConfigPtr& dynamicConfig) const +{ + auto config = New<TJaegerTracerConfig>(); + config->CollectorChannelConfig = CollectorChannelConfig; + if (dynamicConfig->CollectorChannel) { + config->CollectorChannelConfig = dynamicConfig->CollectorChannel; + } + + config->FlushPeriod = dynamicConfig->FlushPeriod.value_or(FlushPeriod); + config->QueueStallTimeout = QueueStallTimeout; + config->MaxRequestSize = dynamicConfig->MaxRequestSize.value_or(MaxRequestSize); + config->MaxBatchSize = MaxBatchSize; + config->MaxMemory = dynamicConfig->MaxMemory.value_or(MaxMemory); + config->SubsamplingRate = SubsamplingRate; + if (dynamicConfig->SubsamplingRate) { + config->SubsamplingRate = dynamicConfig->SubsamplingRate; + } + + config->ServiceName = ServiceName; + config->ProcessTags = ProcessTags; + config->EnablePidTag = EnablePidTag; + config->TvmService = TvmService; + config->TestDropSpans = TestDropSpans; + + config->Postprocess(); + return config; +} + +bool TJaegerTracerConfig::IsEnabled() const +{ + return ServiceName && CollectorChannelConfig; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NTracing diff --git a/yt/yt/library/tracing/jaeger/config.h b/yt/yt/library/tracing/jaeger/config.h new file mode 100644 index 00000000000..f9d525a03df --- /dev/null +++ b/yt/yt/library/tracing/jaeger/config.h @@ -0,0 +1,116 @@ +#pragma once + +#include "public.h" + +#include <yt/yt/library/tvm/service/config.h> +#include <yt/yt/library/tvm/service/public.h> + +#include <yt/yt/core/rpc/grpc/config.h> + +#include <yt/yt/core/ytree/yson_struct.h> + +namespace NYT::NTracing { + +//////////////////////////////////////////////////////////////////////////////// + +struct TSamplerConfig + : public NYTree::TYsonStruct +{ + //! Request is sampled with probability P. + double GlobalSampleRate; + + //! Additionally, request is sampled with probability P(user). + THashMap<std::string, double> UserSampleRate; + + //! Spans are sent to specified endpoint. + THashMap<std::string, std::string> UserEndpoint; + + //! Additionally, sample first N requests for each user in the window. + ui64 MinPerUserSamples; + TDuration MinPerUserSamplesPeriod; + + //! Clear sampled from from incoming user request. + THashMap<std::string, bool> ClearSampledFlag; + + REGISTER_YSON_STRUCT(TSamplerConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TSamplerConfig) + +//////////////////////////////////////////////////////////////////////////////// + +struct TJaegerTracerDynamicConfig + : public NYTree::TYsonStruct +{ + NRpc::NGrpc::TChannelConfigPtr CollectorChannel; + + std::optional<i64> MaxRequestSize; + + std::optional<i64> MaxMemory; + + std::optional<double> SubsamplingRate; + + std::optional<TDuration> FlushPeriod; + + REGISTER_YSON_STRUCT(TJaegerTracerDynamicConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TJaegerTracerDynamicConfig) + +//////////////////////////////////////////////////////////////////////////////// + +struct TJaegerTracerConfig + : public NYTree::TYsonStruct +{ + NRpc::NGrpc::TChannelConfigPtr CollectorChannelConfig; + + TDuration FlushPeriod; + + TDuration StopTimeout; + + TDuration RpcTimeout; + + TDuration EndpointChannelTimeout; + + TDuration QueueStallTimeout; + + TDuration ReconnectPeriod; + + i64 MaxRequestSize; + + i64 MaxBatchSize; + + i64 MaxMemory; + + std::optional<double> SubsamplingRate; + + // ServiceName is required by jaeger. When ServiceName is missing, tracer is disabled. + std::optional<std::string> ServiceName; + + THashMap<std::string, std::string> ProcessTags; + + bool EnablePidTag; + + NAuth::TTvmServiceConfigPtr TvmService; + + // Does not send spans to a collector, but just drops them instead. Logs batch and span count. + bool TestDropSpans; + + TJaegerTracerConfigPtr ApplyDynamic(const TJaegerTracerDynamicConfigPtr& dynamicConfig) const; + + bool IsEnabled() const; + + REGISTER_YSON_STRUCT(TJaegerTracerConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TJaegerTracerConfig) + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NTracing diff --git a/yt/yt/library/tracing/jaeger/configure_tracer.cpp b/yt/yt/library/tracing/jaeger/configure_tracer.cpp index 376e8ea9a4c..5a74c4e64ff 100644 --- a/yt/yt/library/tracing/jaeger/configure_tracer.cpp +++ b/yt/yt/library/tracing/jaeger/configure_tracer.cpp @@ -1,7 +1,11 @@ #include "tracer.h" +#include "config.h" + #include <yt/yt/core/misc/configurable_singleton_def.h> +#include <yt/yt/core/tracing/trace_context.h> + namespace NYT::NTracing { using namespace NYTree; diff --git a/yt/yt/library/tracing/jaeger/private.h b/yt/yt/library/tracing/jaeger/private.h index fbc6373cda4..c3dcfc960f1 100644 --- a/yt/yt/library/tracing/jaeger/private.h +++ b/yt/yt/library/tracing/jaeger/private.h @@ -16,9 +16,4 @@ YT_DEFINE_GLOBAL(const NLogging::TLogger, JaegerLogger, "Jaeger"); //////////////////////////////////////////////////////////////////////////////// -DECLARE_REFCOUNTED_CLASS(TJaegerTracer) -DECLARE_REFCOUNTED_CLASS(TJaegerChannelManager) - -//////////////////////////////////////////////////////////////////////////////// - } // namespace NYT::NTracing diff --git a/yt/yt/library/tracing/jaeger/public.h b/yt/yt/library/tracing/jaeger/public.h index 873ec9702d9..a86bbf50cb1 100644 --- a/yt/yt/library/tracing/jaeger/public.h +++ b/yt/yt/library/tracing/jaeger/public.h @@ -11,6 +11,8 @@ namespace NYT::NTracing { DECLARE_REFCOUNTED_CLASS(TSampler) DECLARE_REFCOUNTED_STRUCT(TSamplerConfig) +DECLARE_REFCOUNTED_CLASS(TJaegerTracer) +DECLARE_REFCOUNTED_CLASS(TJaegerChannelManager) DECLARE_REFCOUNTED_STRUCT(TJaegerTracerDynamicConfig) DECLARE_REFCOUNTED_STRUCT(TJaegerTracerConfig) diff --git a/yt/yt/library/tracing/jaeger/sampler.cpp b/yt/yt/library/tracing/jaeger/sampler.cpp index cddeba81591..e95c6a5ea55 100644 --- a/yt/yt/library/tracing/jaeger/sampler.cpp +++ b/yt/yt/library/tracing/jaeger/sampler.cpp @@ -1,28 +1,15 @@ #include "sampler.h" -namespace NYT::NTracing { +#include "config.h" -using namespace NConcurrency; -using namespace NProfiling; +#include <yt/yt/core/profiling/timing.h> -//////////////////////////////////////////////////////////////////////////////// +#include <yt/yt/core/tracing/trace_context.h> -void TSamplerConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("global_sample_rate", &TThis::GlobalSampleRate) - .Default(0.0); - registrar.Parameter("user_sample_rate", &TThis::UserSampleRate) - .Default(); - registrar.Parameter("user_endpoints", &TThis::UserEndpoint) - .Default(); - registrar.Parameter("clear_sampled_flag", &TThis::ClearSampledFlag) - .Default(); +namespace NYT::NTracing { - registrar.Parameter("min_per_user_samples", &TThis::MinPerUserSamples) - .Default(0); - registrar.Parameter("min_per_user_samples_period", &TThis::MinPerUserSamplesPeriod) - .Default(TDuration::Minutes(1)); -} +using namespace NConcurrency; +using namespace NProfiling; //////////////////////////////////////////////////////////////////////////////// @@ -43,6 +30,10 @@ bool TSampler::TUserState::TrySampleByMinCount(ui64 minCount, TCpuDuration perio return Sampled.fetch_add(1) < minCount; } +TSampler::TSampler() + : TSampler(New<TSamplerConfig>()) +{ } + TSampler::TSampler( TSamplerConfigPtr config, const TProfiler& profiler) diff --git a/yt/yt/library/tracing/jaeger/sampler.h b/yt/yt/library/tracing/jaeger/sampler.h index d4ad3a9a1e9..1941585f9c9 100644 --- a/yt/yt/library/tracing/jaeger/sampler.h +++ b/yt/yt/library/tracing/jaeger/sampler.h @@ -1,42 +1,18 @@ #pragma once -#include "private.h" +#include "public.h" #include <yt/yt/library/syncmap/map.h> -#include <yt/yt/core/ytree/yson_struct.h> +#include <yt/yt/library/profiling/sensor.h> -#include <library/cpp/yt/memory/atomic_intrusive_ptr.h> - -namespace NYT::NTracing { - -//////////////////////////////////////////////////////////////////////////////// - -struct TSamplerConfig - : public NYTree::TYsonStruct -{ - //! Request is sampled with probability P. - double GlobalSampleRate; +#include <yt/yt/core/profiling/public.h> - //! Additionally, request is sampled with probability P(user). - THashMap<std::string, double> UserSampleRate; +#include <yt/yt/core/tracing/public.h> - //! Spans are sent to specified endpoint. - THashMap<std::string, std::string> UserEndpoint; - - //! Additionally, sample first N requests for each user in the window. - ui64 MinPerUserSamples; - TDuration MinPerUserSamplesPeriod; - - //! Clear sampled from from incoming user request. - THashMap<std::string, bool> ClearSampledFlag; - - REGISTER_YSON_STRUCT(TSamplerConfig); - - static void Register(TRegistrar registrar); -}; +#include <library/cpp/yt/memory/atomic_intrusive_ptr.h> -DEFINE_REFCOUNTED_TYPE(TSamplerConfig) +namespace NYT::NTracing { //////////////////////////////////////////////////////////////////////////////// @@ -44,8 +20,9 @@ class TSampler : public TRefCounted { public: + TSampler(); explicit TSampler( - TSamplerConfigPtr config = New<TSamplerConfig>(), + TSamplerConfigPtr config, const NProfiling::TProfiler& profiler = TracingProfiler()); void SampleTraceContext(const std::string& user, const TTraceContextPtr& traceContext); diff --git a/yt/yt/library/tracing/jaeger/tracer.cpp b/yt/yt/library/tracing/jaeger/tracer.cpp index 441e3779b02..d44ac1fe020 100644 --- a/yt/yt/library/tracing/jaeger/tracer.cpp +++ b/yt/yt/library/tracing/jaeger/tracer.cpp @@ -2,6 +2,8 @@ #include <algorithm> #include "tracer.h" + +#include "config.h" #include "private.h" #include <yt/yt/library/tracing/jaeger/model.pb.h> @@ -52,100 +54,6 @@ static const std::string TracingServiceAlias = "tracing"; //////////////////////////////////////////////////////////////////////////////// -void TJaegerTracerDynamicConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("collector_channel", &TThis::CollectorChannel) - .Alias("collector_channel_config") - .Optional(); - registrar.Parameter("max_request_size", &TThis::MaxRequestSize) - .Default(); - registrar.Parameter("max_memory", &TThis::MaxMemory) - .Default(); - registrar.Parameter("subsampling_rate", &TThis::SubsamplingRate) - .Default(); - registrar.Parameter("flush_period", &TThis::FlushPeriod) - .Default(); -} - -//////////////////////////////////////////////////////////////////////////////// - -void TJaegerTracerConfig::Register(TRegistrar registrar) -{ - registrar.Parameter("collector_channel_config", &TThis::CollectorChannelConfig) - .Optional(); - - // 10K nodes x 128 KB / 15s == 85mb/s - registrar.Parameter("flush_period", &TThis::FlushPeriod) - .Default(TDuration::Seconds(15)); - registrar.Parameter("stop_timeout", &TThis::StopTimeout) - .Default(TDuration::Seconds(15)); - registrar.Parameter("rpc_timeout", &TThis::RpcTimeout) - .Default(TDuration::Seconds(15)); - registrar.Parameter("queue_stall_timeout", &TThis::QueueStallTimeout) - .Default(TDuration::Minutes(15)); - registrar.Parameter("max_request_size", &TThis::MaxRequestSize) - .Default(128_KB) - .LessThanOrEqual(4_MB); - registrar.Parameter("max_batch_size", &TThis::MaxBatchSize) - .Default(128); - registrar.Parameter("max_memory", &TThis::MaxMemory) - .Default(1_GB); - registrar.Parameter("subsampling_rate", &TThis::SubsamplingRate) - .Default(); - registrar.Parameter("reconnect_period", &TThis::ReconnectPeriod) - .Default(TDuration::Minutes(15)); - registrar.Parameter("endpoint_channel_timeout", &TThis::EndpointChannelTimeout) - .Default(TDuration::Hours(2)); - - registrar.Parameter("service_name", &TThis::ServiceName) - .Default(); - registrar.Parameter("process_tags", &TThis::ProcessTags) - .Default(); - registrar.Parameter("enable_pid_tag", &TThis::EnablePidTag) - .Default(false); - - registrar.Parameter("tvm_service", &TThis::TvmService) - .Optional(); - - registrar.Parameter("test_drop_spans", &TThis::TestDropSpans) - .Default(false); -} - -TJaegerTracerConfigPtr TJaegerTracerConfig::ApplyDynamic(const TJaegerTracerDynamicConfigPtr& dynamicConfig) const -{ - auto config = New<TJaegerTracerConfig>(); - config->CollectorChannelConfig = CollectorChannelConfig; - if (dynamicConfig->CollectorChannel) { - config->CollectorChannelConfig = dynamicConfig->CollectorChannel; - } - - config->FlushPeriod = dynamicConfig->FlushPeriod.value_or(FlushPeriod); - config->QueueStallTimeout = QueueStallTimeout; - config->MaxRequestSize = dynamicConfig->MaxRequestSize.value_or(MaxRequestSize); - config->MaxBatchSize = MaxBatchSize; - config->MaxMemory = dynamicConfig->MaxMemory.value_or(MaxMemory); - config->SubsamplingRate = SubsamplingRate; - if (dynamicConfig->SubsamplingRate) { - config->SubsamplingRate = dynamicConfig->SubsamplingRate; - } - - config->ServiceName = ServiceName; - config->ProcessTags = ProcessTags; - config->EnablePidTag = EnablePidTag; - config->TvmService = TvmService; - config->TestDropSpans = TestDropSpans; - - config->Postprocess(); - return config; -} - -bool TJaegerTracerConfig::IsEnabled() const -{ - return ServiceName && CollectorChannelConfig; -} - -//////////////////////////////////////////////////////////////////////////////// - class TJaegerCollectorProxy : public TProxyBase { diff --git a/yt/yt/library/tracing/jaeger/tracer.h b/yt/yt/library/tracing/jaeger/tracer.h index 81350c50caa..4ffe389e4cb 100644 --- a/yt/yt/library/tracing/jaeger/tracer.h +++ b/yt/yt/library/tracing/jaeger/tracer.h @@ -1,19 +1,22 @@ #pragma once -#include "private.h" +#include "public.h" #include <yt/yt/library/tracing/tracer.h> #include <yt/yt/library/profiling/sensor.h> -#include <yt/yt/library/tvm/service/config.h> #include <yt/yt/library/tvm/service/public.h> +#include <yt/yt/core/actions/future.h> + +#include <yt/yt/core/concurrency/public.h> + #include <yt/yt/core/misc/mpsc_stack.h> -#include <yt/yt/core/rpc/grpc/config.h> +#include <yt/yt/core/rpc/grpc/public.h> -#include <yt/yt/core/ytree/yson_struct.h> +#include <yt/yt/core/rpc/public.h> #include <library/cpp/yt/threading/atomic_object.h> #include <library/cpp/yt/threading/spin_lock.h> @@ -24,78 +27,6 @@ namespace NYT::NTracing { //////////////////////////////////////////////////////////////////////////////// -struct TJaegerTracerDynamicConfig - : public NYTree::TYsonStruct -{ - NRpc::NGrpc::TChannelConfigPtr CollectorChannel; - - std::optional<i64> MaxRequestSize; - - std::optional<i64> MaxMemory; - - std::optional<double> SubsamplingRate; - - std::optional<TDuration> FlushPeriod; - - REGISTER_YSON_STRUCT(TJaegerTracerDynamicConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TJaegerTracerDynamicConfig) - -//////////////////////////////////////////////////////////////////////////////// - -struct TJaegerTracerConfig - : public NYTree::TYsonStruct -{ - NRpc::NGrpc::TChannelConfigPtr CollectorChannelConfig; - - TDuration FlushPeriod; - - TDuration StopTimeout; - - TDuration RpcTimeout; - - TDuration EndpointChannelTimeout; - - TDuration QueueStallTimeout; - - TDuration ReconnectPeriod; - - i64 MaxRequestSize; - - i64 MaxBatchSize; - - i64 MaxMemory; - - std::optional<double> SubsamplingRate; - - // ServiceName is required by jaeger. When ServiceName is missing, tracer is disabled. - std::optional<std::string> ServiceName; - - THashMap<std::string, std::string> ProcessTags; - - bool EnablePidTag; - - NAuth::TTvmServiceConfigPtr TvmService; - - // Does not send spans to a collector, but just drops them instead. Logs batch and span count. - bool TestDropSpans; - - TJaegerTracerConfigPtr ApplyDynamic(const TJaegerTracerDynamicConfigPtr& dynamicConfig) const; - - bool IsEnabled() const; - - REGISTER_YSON_STRUCT(TJaegerTracerConfig); - - static void Register(TRegistrar registrar); -}; - -DEFINE_REFCOUNTED_TYPE(TJaegerTracerConfig) - -//////////////////////////////////////////////////////////////////////////////// - class TBatchInfo { public: diff --git a/yt/yt/library/tracing/jaeger/ya.make b/yt/yt/library/tracing/jaeger/ya.make index 14f1877294e..e3edf6dad7a 100644 --- a/yt/yt/library/tracing/jaeger/ya.make +++ b/yt/yt/library/tracing/jaeger/ya.make @@ -11,6 +11,7 @@ PEERDIR( SRCS( model.proto + config.cpp sampler.cpp tracer.cpp GLOBAL configure_tracer.cpp diff --git a/yt/yt/library/tracing/unittests/sampler_ut.cpp b/yt/yt/library/tracing/unittests/sampler_ut.cpp index cf67d95736f..60dccb6322f 100644 --- a/yt/yt/library/tracing/unittests/sampler_ut.cpp +++ b/yt/yt/library/tracing/unittests/sampler_ut.cpp @@ -1,5 +1,6 @@ #include <gtest/gtest.h> +#include <yt/yt/library/tracing/jaeger/config.h> #include <yt/yt/library/tracing/jaeger/sampler.h> #include <yt/yt/core/tracing/trace_context.h> |