diff options
381 files changed, 31214 insertions, 17605 deletions
diff --git a/.github/TESTOWNERS b/.github/TESTOWNERS index 24df5b39587..c75a4079c54 100644 --- a/.github/TESTOWNERS +++ b/.github/TESTOWNERS @@ -163,6 +163,7 @@ /ydb/core/external_sources/s3 @ydb-platform/fq /ydb/core/resource_pools @ydb-platform/fq /ydb/services/workload_manager @ydb-platform/fq +/ydb/services/metadata @ydb-platform/fq #@ydb-platform/ui-backend /ydb/core/viewer @ydb-platform/ui-backend diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index b0cd38bd412..084722be4e3 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -32,7 +32,6 @@ ydb/core/kqp/ut/scheme KqpScheme.CreateDropTableMultipleTime ydb/core/load_test/ut GroupWriteTest.SimpleRdma ydb/core/tx/schemeshard/ut_compaction_reboots SchemeshardForcedCompactionTestReboots.ForceCompactWithIndexCreation[TabletRebootsBucket0] ydb/core/tx/schemeshard/ut_compaction_reboots unittest.[*/*] chunk -ydb/core/tx/schemeshard/ut_export_reboots_s3 TExportToS3WithRebootsTests.ShouldSucceedOnManyTablesWithParquet[TabletRebootsBucket0] ydb/library/actors/core/ut TestThreadContextQueueTimestamps.CurrentQueueTimestamps ydb/library/actors/http/ut Http2Integration.Http2UpgradeFrom101 ydb/library/actors/interconnect/ut InterconnectDirectSession.DoesNotReceiveUnexpectedReplies @@ -58,22 +57,16 @@ ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex. ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-2-1] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_stable-26-3-1_to_stable-26-3-1] ydb/tests/compatibility/indexes test_fulltext_index.py.TestFulltextIndex.test_fulltext_index[rolling_stable-26-3-1_to_prestable-26-4] -ydb/tests/compatibility/olap test_encoding.py.TestEncoding.test_encoding[rolling_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/result_set_format py3test.[test_result_set_arrow.py */*] chunk -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-False] -ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRestartToAnotherVersion.test_restart_to_another_version[restart_stable-26-3-1_to_stable-26-2-1-True] ydb/tests/compatibility/streaming test_streaming.py.TestStreamingRollingUpgradeAndDowngrade.test_rolling_upgrade[rolling_stable-26-2-1_to_stable-26-3-1-False] ydb/tests/fq/streaming py3test.[*/*] chunk -ydb/tests/fq/streaming test_streaming.py.TestStreamingInYdb.test_read_topic_shared_reading_limit[False] ydb/tests/fq/streaming test_watermarks.py.TestWatermarksInYdb.test_empty_partition[shared-False] ydb/tests/fq/streaming test_watermarks.py.TestWatermarksInYdb.test_idle_partition_gt_timeout[shared-False] ydb/tests/functional/blobstorage test_node_warden_cache.py.TestNodeWardenCacheFile.test_cache_file_lifecycle_and_restart -ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_disk_creation -ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_multiple_disks_creation ydb/tests/functional/query_cache/warmup test_warmup.py.TestCompileCacheViewPeerWarnings.test_peer_scan_warnings_increment_on_dead_peer ydb/tests/functional/query_cache/warmup test_warmup.py.TestWarmupRestart.test_warmup_basic +ydb/tests/functional/secrets test_old_secrets_usage.py.test_existing_objects_are_ok_if_old_secret_creation_is_disabled ydb/tests/functional/security/mon/canonical test_mon_endpoints_auth.py.test[require_healthcheck_authentication-with_schema_grants] -ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud[fifo] ydb/tests/functional/sqs/migration/compatibility/common ydb.tests.functional.sqs.common.test_queue_counters.py.TestSqsGettingCounters.test_sqs_action_counters ydb/tests/functional/sqs/migration/finished/common ydb.tests.functional.sqs.common.test_queue_counters.py.TestSqsGettingCounters.test_sqs_action_counters ydb/tests/functional/sqs/migration/topic_creation/multinode ydb.tests.functional.sqs.multinode.test_multinode_cluster.py.TestSqsMultinodeCluster.test_has_messages_counters[kick_tablets-fifo] @@ -84,9 +77,7 @@ ydb/tests/functional/tenants test_remove_storage_groups.py.test_remove_storage_g ydb/tests/stress/kafka_serverless/tests test_kafka_streams.py.TestYdbTopicWorkload.test ydb/tests/stress/oltp_workload/tests py3test.sole chunk ydb/tests/stress/oltp_workload/tests test_workload.py.TestYdbWorkload.test -ydb/tests/stress/scheme_board/pile_promotion/tests test_scheme_board_workload.py.TestSchemeBoard.test_scheme_board ydb/tests/stress/testshard_workload/tests test_workload.py.TestYdbTestShardWorkload.test -ydb/tests/stress/topic/tests test_workload_topic.py.TestYdbTopicWorkload.test_0 ydb/tests/stress/topic_sqs/tests test_workload.py.TestYdbTopicSqsWorkload.test ydb/tools/mnc/viewer/ut py3test.sole chunk ydb/tools/mnc/viewer/ut test_cluster_config_validation.py.ClusterConfigSelectionStateTest.test_operations_prepare_deploy_context_before_install diff --git a/.github/config/muted_ya_asan.txt b/.github/config/muted_ya_asan.txt index 55615b15b20..5692ab4cc0f 100644 --- a/.github/config/muted_ya_asan.txt +++ b/.github/config/muted_ya_asan.txt @@ -65,7 +65,6 @@ ydb/core/http_proxy/ut TestMalformedRequest.InvalidHttpStartLine2 ydb/core/http_proxy/ut TestMalformedRequest.InvalidHttpStartLine3 ydb/core/http_proxy/ut TestMalformedRequest.InvalidHttpStartLine4 ydb/core/http_proxy/ut TestMalformedRequest.InvalidHttpStartLine5 -ydb/core/http_proxy/ut TestMalformedRequest.InvalidHttpStartLine6 ydb/core/http_proxy/ut TestMalformedRequest.InvalidHttpStartLine7 ydb/core/http_proxy/ut/inside_ydb_ut TestKinesisHttpProxy.BadRequestUnknownMethod ydb/core/http_proxy/ut/inside_ydb_ut TestKinesisHttpProxy.CreateDeleteStream @@ -184,7 +183,6 @@ ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessage_Deduplicat ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessage_Deduplication_Compatibility_WithUserSettings ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessage_TableImplementation ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSendMessage_WithUserSettings -ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSetQueueAttributes ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestSetQueueAttributes_TableImplementation ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestTagQueue ydb/core/http_proxy/ut/inside_ydb_ut TestYmqHttpProxy.TestTagQueueMultipleQueriesInflight @@ -423,7 +421,6 @@ ydb/core/http_proxy/ut/sqs_topic_ut TestSqsTopicHttpProxyXml.TestSendMessageBadQ ydb/core/http_proxy/ut/sqs_topic_ut TestSqsTopicHttpProxyXml.TestSendMessageBatch ydb/core/http_proxy/ut/sqs_topic_ut TestSqsTopicHttpProxyXml.TestSendMessageBatchEmpty ydb/core/http_proxy/ut/sqs_topic_ut TestSqsTopicHttpProxyXml.TestSendMessageBatchLong -ydb/core/http_proxy/ut/sqs_topic_ut TestSqsTopicHttpProxyXml.TestSendMessageTooBig ydb/core/http_proxy/ut/sqs_topic_ut TestSqsTopicHttpProxyXml.TestSetQueueAttributesBasic ydb/core/http_proxy/ut/sqs_topic_ut TestSqsTopicHttpProxyXml.TestSetQueueAttributesFifoImmutable ydb/core/http_proxy/ut/sqs_topic_ut TestSqsTopicHttpProxyXml.TestSetQueueAttributesInvalidValues @@ -450,6 +447,7 @@ ydb/core/kqp/executer_actor/ut TKqpTasksGraphBuild.TpchQuery18 ydb/core/kqp/executer_actor/ut TKqpTasksGraphBuild.TpchQuery19 ydb/core/kqp/executer_actor/ut TKqpTasksGraphBuild.TpchQuery20 ydb/core/kqp/executer_actor/ut TKqpTasksGraphBuild.TpchQuery21 +ydb/core/kqp/executer_actor/ut TKqpTasksGraphBuild.TpchQuery22 ydb/core/kqp/executer_actor/ut unittest.sole chunk ydb/core/kqp/proxy_service/ut ScriptExecutionsTest.GetOperationDuringAtomicUploadFinalization ydb/core/kqp/ut/federated_query/datastreams KqpFederatedQueryDatastreams.ReadFromLocalTopicsWithAuth @@ -463,6 +461,7 @@ ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.StreamingQuer ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.StreamingQueryWithTwoGroupByHopsOnSameKey ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesDdl.WritingInLocalYdbTablesWithCheckpoints ydb/core/kqp/ut/federated_query/datastreams KqpStreamingQueriesSysView.ReadSysViewWithRowCountBackPressure +ydb/core/kqp/ut/indexes/prefixed_vector KqpPrefixedVectorIndexes.PrefixedVectorEmptyIndexedTableInsertWithOverlap-Covered ydb/core/kqp/ut/indexes/prefixed_vector KqpPrefixedVectorIndexes.PrefixedVectorIndexInsertNewPrefix-Nullable+Covered ydb/core/kqp/ut/indexes/prefixed_vector KqpPrefixedVectorIndexes.PrefixedVectorIndexInsertNewPrefixWithOverlap+Covered ydb/core/kqp/ut/indexes/prefixed_vector unittest.[*/*] chunk @@ -470,6 +469,9 @@ ydb/core/kqp/ut/join KqpJoinOrder.Chain65Nodes ydb/core/kqp/ut/olap KqpOlapOptimizer.SpecialSliceToOneLayer ydb/core/kqp/ut/olap KqpOlapOptimizer.TilingPlusPlusTtlDeletion ydb/core/kqp/ut/olap/indexes KqpOlapIndexes.IndexesActualization-QueryService +ydb/core/kqp/ut/olap/indexes KqpOlapIndexes.MinMaxIndexAppliedToDataAfterCompaction-QueryService-SchemeObjectDisabled +ydb/core/kqp/ut/olap/types KqpOlapJson.BloomMixIndexesOldSyntaxVariants[false,false,0,0,1000,0.5,CATEGORY_BLOOM_FILTER] +ydb/core/kqp/ut/olap/types KqpOlapJson.CompactionVariants[10,true,1024,1000,100,0.5] ydb/core/kqp/ut/olap/types KqpOlapJson.SimpleExistsVariants[10,false,1024,1000,0,0.5] ydb/core/kqp/ut/query KqpLimits.OutOfSpaceYQLUpsertFail ydb/core/kqp/ut/query unittest.[*/*] chunk @@ -492,7 +494,6 @@ ydb/core/kqp/ut/runtime unittest.[*/*] chunk ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs+IsOlap+UseAdmin ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs+IsOlap-UseAdmin ydb/core/kqp/ut/scheme KqpAcl.AclCreateTableAs-IsOlap+UseAdmin -ydb/core/kqp/ut/scheme KqpAcl.AclInterruptInheritance ydb/core/kqp/ut/scheme KqpAcl.AclTemporaryInterruptInheritance+IsOlap ydb/core/kqp/ut/scheme KqpAcl.AclTemporaryInterruptInheritance-IsOlap ydb/core/kqp/ut/scheme KqpConstraints.AlterTableSetDropDefaultAsyncIndexOnColumn-EnableCompileTimeDefaults @@ -569,7 +570,6 @@ ydb/core/statistics/service/ut ColumnStatistics.ManyColumns ydb/core/statistics/service/ut unittest.[*/*] chunk ydb/core/transfer/ut/column_table Transfer_ColumnTable.BigBatchSize_Local ydb/core/transfer/ut/column_table Transfer_ColumnTable.BigBatchSize_Remote -ydb/core/transfer/ut/column_table unittest.sole chunk ydb/core/transfer/ut/functional Transfer.CheckCommittedOffsetEmptyBatch_Local ydb/core/transfer/ut/functional Transfer.CheckCommittedOffsetEmptyBatch_Remote ydb/core/transfer/ut/functional Transfer.LocalTopic_BigMessage @@ -577,7 +577,6 @@ ydb/core/transfer/ut/large TransferLarge.Transfer100KM_10P_ColumnTable ydb/core/transfer/ut/large TransferLarge.Transfer100KM_10P_LocalRead_TopicAutoPartitioning ydb/core/transfer/ut/large unittest.sole chunk ydb/core/tx/columnshard/backup/async_jobs/ut unittest.sole chunk -ydb/core/tx/columnshard/backup/iscan/ut IScan.ShouldRejectParquetExportWithEncryption ydb/core/tx/columnshard/data_accessor/ut unittest.sole chunk ydb/core/tx/columnshard/ut_rw TColumnShardTestReadWrite.CompactionSplitGranuleStrKey_PKUtf8 ydb/core/tx/columnshard/ut_rw TColumnShardTestReadWrite.CompactionSplitGranule_PKDatetime @@ -602,7 +601,6 @@ ydb/core/tx/datashard/build_index/ut TTxDataShardSampleKScan.DimensionMismatchEr ydb/core/tx/datashard/build_index/ut TTxDataShardSampleKScan.NullEmbedding ydb/core/tx/datashard/build_index/ut TTxDataShardSampleKScan.RunScan ydb/core/tx/datashard/build_index/ut TTxDataShardSampleKScan.SkipForeign -ydb/core/tx/datashard/build_index/ut TTxDataShardValidateUniqueIndexScan.BadRequest ydb/core/tx/datashard/build_index/ut TTxDataShardValidateUniqueIndexScan.RunScan ydb/core/tx/datashard/build_index/ut unittest.sole chunk ydb/core/tx/datashard/ut_export ExportS3BufferTest.MinBufferSizeWithCompressionAndEncryption-parquet @@ -642,6 +640,7 @@ ydb/core/tx/tx_proxy/ut_ext_tenant TExtSubDomainTest.CreateTableInsideAndAlterDo ydb/core/viewer/tests test.py.TestViewer.test_viewer_acl_write ydb/core/viewer/tests test.py.TestViewer.test_viewer_query_long_multipart ydb/core/viewer/ut Viewer.PutRecordViewer +ydb/core/viewer/ut Viewer.QueryExecuteScript ydb/core/ymq/actor/cloud_events/cloud_events_ut unittest.sole chunk ydb/library/actors/interconnect/ut DynamicProxy.RaceCheck10 ydb/library/actors/interconnect/ut InterconnectDirectSession.DoesNotReceiveUnexpectedReplies @@ -658,6 +657,7 @@ ydb/library/actors/interconnect/ut_huge_cluster HugeCluster.AllToAll ydb/library/actors/interconnect/ut_huge_cluster unittest.sole chunk ydb/library/yql/providers/generic/actors/ut unittest.sole chunk ydb/public/sdk/cpp/src/client/topic/ut Describe.DescribePartitionPermissions +ydb/public/sdk/cpp/src/client/topic/ut/slow TxUsage.Write_Only_Big_Messages_In_Wide_Transactions_Table ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut BasicUsage.Producer_CloseTimeout ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut BasicUsage.ReadWithRestartsAndLargeData ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut BasicUsage.ReadWithRestartsAndLargeDataAndShuffle @@ -673,10 +673,8 @@ ydb/public/sdk/cpp/tests/integration/bulk_upsert BulkUpsert.RetryOverheadOnHappy ydb/public/sdk/cpp/tests/integration/sessions_pool YdbSdkSessionsPool.PeriodicTask/0 ydb/public/sdk/cpp/tests/integration/sessions_pool YdbSdkSessionsPool.PeriodicTask/1 ydb/services/persqueue_v1/ut TPersQueueTest.ConsumerAvailabilityPeriodCleanup -ydb/services/persqueue_v1/ut/new_schemecache_ut TPersQueueNewSchemeCacheTest.TestWriteStat1stClassTopicAPI ydb/services/scheme_secret/ut/fast DescribeSchemaSecretsService.FailWithoutGrants ydb/services/scheme_secret/ut/fast DescribeSchemaSecretsService.GetUpdatedValue -ydb/services/scheme_secret/ut/fast DescribeSchemaSecretsService.GroupGrants ydb/services/test_shard/ut unittest.sole chunk ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_NoConnectionStringRelativeTableTopicTransferPath ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_NoConnectionStringRelativeTopicAbsoluteTransferPath @@ -692,7 +690,6 @@ ydb/services/ydb/backup_ut BackupRestore.BackupRestoreTransfer_WithConnectionStr ydb/services/ydb/backup_ut BackupRestore.ReplicasAreNotBackedUp+UseSchemeSecret ydb/services/ydb/backup_ut BackupRestore.ReplicasAreNotBackedUp-UseSchemeSecret ydb/services/ydb/backup_ut BackupRestore.RestoreReplicationThatDoesNotUseSecret -ydb/tests/compatibility/common test_compatibility.py.TestCompatibility.test_simple[restart_stable-26-3-1_to_stable-26-3-1-row] ydb/tests/compatibility/common test_system_views.py.TestSystemViewsRegistry.test_domain_sys_dir[restart_stable-26-3-1_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_prestable-26-4] ydb/tests/compatibility/indexes test_bloom_filter_index.py.TestBloomFilterIndex.test_bloom_filter_survives_version_change[restart_prestable-26-4_to_stable-26-3-1] @@ -779,6 +776,7 @@ ydb/tests/fq/yt/kqp_yt_file/part9 test.py.test[aggregate-group_by_expr-] ydb/tests/fq/yt/kqp_yt_file/part9 test.py.test[aggregate-group_by_rollup_aggr_expr-] ydb/tests/functional/audit test_canonical_records.py.test_dstool_evict_vdisk_grpc ydb/tests/functional/blobstorage test_node_warden_cache.py.TestNodeWardenCacheFile.test_cache_file_lifecycle_and_restart +ydb/tests/functional/config test_distconf_sentinel_node_status.py.TestKiKiMRDistConfSelfHealReplicasSpecificVolume.test_state_storage ydb/tests/functional/dstool test_canonical_requests.py.Test.test_capacity_metrics ydb/tests/functional/kqp/kqp_indexes ConsistentIndexRead.InteractiveTx ydb/tests/functional/kqp/kqp_indexes unittest.sole chunk @@ -809,10 +807,10 @@ ydb/tests/functional/tpc/large py3test.[test_tpcds.py] chunk ydb/tests/functional/tpc/large py3test.[test_tpch_spilling.py] chunk ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[14] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[49] +ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[53] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1.test_tpcds[64] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1DecimalNative.test_tpcds[14] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1DecimalNative.test_tpcds[64] -ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS0_1DecimalNative.test_tpcds[85] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS1.test_tpcds[10] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS1.test_tpcds[11] ydb/tests/functional/tpc/large test_tpcds.py.TestTpcdsS1.test_tpcds[12] diff --git a/.github/config/muted_ya_msan.txt b/.github/config/muted_ya_msan.txt index dcd602704f2..1ac0fa6b08c 100644 --- a/.github/config/muted_ya_msan.txt +++ b/.github/config/muted_ya_msan.txt @@ -29,6 +29,7 @@ ydb/core/blobstorage/ut_blobstorage/ut_read_only_pdisk unittest.[*/*] chunk ydb/core/blobstorage/ut_blobstorage/ut_restart_pdisk BSCRestartPDisk.RestartOneByOne ydb/core/blobstorage/ut_blobstorage/ut_restart_pdisk BSCRestartPDisk.RestartOneByOneWithReconnects ydb/core/blobstorage/ut_blobstorage/ut_restart_pdisk unittest.[*/*] chunk +ydb/core/formats/arrow/accessor/sub_columns/ut SubColumnsSparsedBuilder.BinaryJsonMatchesPlain ydb/core/kqp/ut/service KqpQueryService.FlowControllOnHugeRealTable-LongRow ydb/core/load_test/ut GroupWriteTest.SimpleRdma ydb/core/load_test/ut GroupWriteTest.WriteHardRateDispatcher @@ -57,6 +58,7 @@ ydb/library/actors/interconnect/ut InterconnectSessionV2.LoadLikeRoundTripWithBa ydb/library/actors/interconnect/ut InterconnectSessionV2.LoadLikeRoundTripWithPayload ydb/library/actors/interconnect/ut InterconnectSessionV2.PeerStopTriggersDisconnect ydb/library/actors/interconnect/ut InterconnectSessionV2.RepeatedDisconnectChurnUnderLoad +ydb/public/sdk/cpp/src/client/topic/ut/with_direct_read_ut TxUsage.Sinks_Oltp_WriteToTopic_3_Query ydb/tests/functional/audit test_canonical_records.py.test_dstool_evict_vdisk_grpc ydb/tests/functional/nbs py3test.sole chunk ydb/tests/functional/nbs test_nbs.py.TestNbs.test_nbs_500gb_disk_read_write diff --git a/.github/config/muted_ya_tsan.txt b/.github/config/muted_ya_tsan.txt index 1b1d32b3f1c..2a7a1128ecf 100644 --- a/.github/config/muted_ya_tsan.txt +++ b/.github/config/muted_ya_tsan.txt @@ -53,7 +53,8 @@ ydb/core/blobstorage/ut_blobstorage/ut_vdisk_internals unittest.[*/*] chunk ydb/core/blobstorage/ut_mirror3of4 Mirror3of4.ReplicationHuge ydb/core/blobstorage/ut_mirror3of4 Mirror3of4.ReplicationSmall ydb/core/blobstorage/ut_mirror3of4 unittest.sole chunk -ydb/core/blobstorage/vdisk/synclog/ut_real_pdisk TBlobStorageSyncLogRealPDisk.LocalSyncDataCutWaitRetriesAfterInsufficientCut +ydb/core/blobstorage/ut_vdisk2 VDiskTest.HugeBlobWrite +ydb/core/blobstorage/ut_vdisk2 unittest.[*/*] chunk ydb/core/blobstorage/vdisk/synclog/ut_real_pdisk TBlobStorageSyncLogRealPDisk.LocalSyncDataUnsuccessfulFullRecoveryDoesNotExhaustLogChunks ydb/core/http_proxy/ut/sqs_topic_ut unittest.[*/*] chunk ydb/core/kqp/executer_actor/ut TKqpTasksGraphBuild.CustomQuery01 @@ -94,9 +95,12 @@ ydb/core/kqp/ut/rbo KqpRboYql.ColumnStatistics+ColumnStore ydb/core/kqp/ut/rbo KqpRboYql.TPCDS_YQL ydb/core/kqp/ut/rbo unittest.[*/*] chunk ydb/core/kqp/ut/runtime KqpScanLogs.GraceJoin-EnabledLogs +ydb/core/kqp/ut/runtime KqpScanLogs.WideCombine-EnabledLogs ydb/core/kqp/ut/runtime unittest.[*/*] chunk +ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsPublicApi+EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsPublicApi-EnableIndexStreamWrite +ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsSql+EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsSql-EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexDeniesTableModificationsPublicApi-EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexDeniesTableModificationsSql-EnableIndexStreamWrite @@ -110,7 +114,6 @@ ydb/core/kqp/ut/tli unittest.[*/*] chunk ydb/core/kqp/ut/tx KqpSnapshotIsolation.ConflictWrite+IsOlap+FillTables ydb/core/kqp/ut/tx unittest.[*/*] chunk ydb/core/kqp/workload_service/ut KqpWorkloadService.TestResourcePoolAfterTenantRecreation -ydb/core/load_test/ut GroupWriteTest.SimpleRdma ydb/core/load_test/ut GroupWriteTest.WriteHardRateDispatcher ydb/core/load_test/ut unittest.[*/*] chunk ydb/core/mind/hive/ut THiveTest.TestCheckSubHiveMigrationWithReboots @@ -119,7 +122,6 @@ ydb/core/mind/hive/ut THiveTestWithTenants.TestShrinkStoragePool ydb/core/mind/hive/ut unittest.[*/*] chunk ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/ut TVChunkTest.ShouldScheduleCleanup ydb/core/nbs/cloud/blockstore/libs/vhost/ut TServerTest.ShouldCancelRequestsInFlightWhenStopEndpointOrStopServer -ydb/core/persqueue/pqtablet/partition/mlp/ut TMLPConsumerFIFOWithSplitDedup.Dedup_WithGroup_SameIds_6min ydb/core/persqueue/public/describer/ut TDescriberTests.TopicNotCanonizedPath ydb/core/persqueue/public/describer/ut TDescriberTests.TopicWithoutDatabase ydb/core/persqueue/public/describer/ut unittest.sole chunk @@ -127,24 +129,17 @@ ydb/core/statistics/service/ut ColumnStatistics.ManyColumns ydb/core/statistics/service/ut unittest.[*/*] chunk ydb/core/transfer/ut/column_table Transfer_ColumnTable.BigBatchSize_Local ydb/core/transfer/ut/column_table Transfer_ColumnTable.BigBatchSize_Remote -ydb/core/transfer/ut/column_table Transfer_ColumnTable.ColumnType_Date -ydb/core/transfer/ut/column_table Transfer_ColumnTable.ColumnType_Int16 -ydb/core/transfer/ut/column_table Transfer_ColumnTable.ColumnType_Int32 -ydb/core/transfer/ut/column_table Transfer_ColumnTable.ColumnType_Int8 -ydb/core/transfer/ut/column_table Transfer_ColumnTable.ColumnType_Utf8_LongValue ydb/core/transfer/ut/column_table Transfer_ColumnTable.MessageField_WriteTimestamp ydb/core/transfer/ut/column_table unittest.sole chunk ydb/core/transfer/ut/functional Transfer.CheckCommittedOffsetEmptyBatch_Remote ydb/core/transfer/ut/functional Transfer.LocalTopic_BigMessage ydb/core/transfer/ut/functional Transfer.MessageField_Attributes_Remote -ydb/core/transfer/ut/row_table Transfer_RowTable.ComplexKey +ydb/core/transfer/ut/functional unittest.sole chunk ydb/core/transfer/ut/row_table unittest.sole chunk -ydb/core/tx/columnshard/backup/iscan/ut IScan.MultiExport -ydb/core/tx/columnshard/backup/iscan/ut IScan.ShouldRejectParquetExportWithEncryption -ydb/core/tx/columnshard/backup/iscan/ut IScan.UploaderExport ydb/core/tx/datashard/ut_truncate unittest.[*/*] chunk ydb/core/tx/schemeshard/ut_set_column_constraint unittest.[*/*] chunk ydb/core/tx/tx_proxy/ut_ext_tenant TExtSubDomainTest.CreateTableInsideAndAlterDomainAndTable-AlterDatabaseCreateHiveFirst-true +ydb/core/tx/tx_proxy/ut_ext_tenant TExtSubDomainTest.CreateTableInsideThenStopTenantAndForceDeleteSubDomain-AlterDatabaseCreateHiveFirst-false ydb/core/tx/tx_proxy/ut_storage_tenant TStorageTenantTest.CreateDummyTabletsInDifferentDomains ydb/tests/fq/http_api py3test.sole chunk ydb/tests/fq/http_api test_http_api.py.TestHttpApi.test_complex_results @@ -164,10 +159,10 @@ ydb/tests/functional/benchmarks_init test_generator.py.TestTpcdsGenerator.test_s ydb/tests/functional/dstool py3test.sole chunk ydb/tests/functional/dstool test_canonical_requests.py.Test.test_cluster_get_set ydb/tests/functional/dstool test_canonical_requests.py.Test.test_essential -ydb/tests/functional/dstool test_canonical_requests.py.Test.test_group_add ydb/tests/functional/dstool test_canonical_requests.py.Test.test_group_take_snapshot ydb/tests/functional/dstool test_canonical_requests.py.Test.test_infer_pdisk_slot_count ydb/tests/functional/dstool test_canonical_requests.py.Test.test_pdisk_set_status_inactive +ydb/tests/functional/encryption test_encryption.py.TestEncryption.test_simple_encryption ydb/tests/functional/kqp/kqp_indexes ConsistentIndexRead.InteractiveTx ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach ydb/tests/functional/kqp/kqp_query_svc NodeIdDescribe.HasDistribution @@ -624,6 +619,7 @@ ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpcdsDiffProcessing. ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpcdsDiffProcessing.test_tpcds[CheckCanonicalPolicy.WARNING] ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpchDiffProcessing.test_tpch[CheckCanonicalPolicy.ERROR] ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpchDiffProcessing.test_tpch[CheckCanonicalPolicy.NO] +ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpchDiffProcessing.test_tpch[CheckCanonicalPolicy.WARNING] ydb/tests/functional/tpc/medium test_import_csv.py.TestExternalImportCsv.test ydb/tests/functional/tpc/medium test_import_csv.py.TestExternalImportCsvArrow.test ydb/tests/functional/tpc/medium test_tpcc.py.TestTpccW10T4SdkCompaction.test @@ -638,6 +634,8 @@ ydb/tests/functional/tpc/medium test_upload.py.TestUploadTpchS0_1.test_tpch[18] ydb/tests/functional/tpc/medium test_upload.py.TestUploadTpchS0_1.test_tpch[19] ydb/tests/functional/tpc/medium test_upload.py.TestUploadTpchS0_1.test_tpch[4] ydb/tests/functional/tpc/medium test_upload.py.TestUploadTpchS0_1.test_tpch[5] +ydb/tests/functional/tpc/medium test_upload.py.TestUploadTpchS0_1.test_tpch[8] +ydb/tests/functional/tpc/medium test_upload.py.TestUploadTpchS0_1.test_tpch[9] ydb/tests/functional/tpc/medium test_workload_manager.py.TestClickbenchWM.test ydb/tests/functional/tpc/medium test_workload_manager.py.TestClickbenchWMComputeSchedulerP1T1.test ydb/tests/functional/tpc/medium test_workload_manager.py.TestTpchWMS0_1.test @@ -820,17 +818,15 @@ ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upl ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upload_download[1GB] ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upload_download[256MB] ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_upload_download[512MB] +ydb/tests/stress/kv_volume_tool/tests test_workload.py.TestKvVolumeTool.test_write_read ydb/tests/stress/log/tests test_workload.py.TestYdbLogWorkload.test[column] ydb/tests/stress/mixedpy/tests py3test.sole chunk ydb/tests/stress/mixedpy/tests test_workload.py.TestYdbMixedWorkload.test[column] ydb/tests/stress/mixedpy/tests test_workload.py.TestYdbMixedWorkload.test[row] -ydb/tests/stress/node_broker/tests py3test.sole chunk -ydb/tests/stress/node_broker/tests test_workload.py.TestDeltaProtocol.test ydb/tests/stress/olap_workload/tests py3test.sole chunk ydb/tests/stress/olap_workload/tests test_workload.py.TestYdbWorkload.test ydb/tests/stress/oltp_workload/tests py3test.sole chunk ydb/tests/stress/oltp_workload/tests test_workload.py.TestYdbWorkload.test -ydb/tests/stress/reconfig_state_storage_workload/tests test_scheme_board_workload.py.TestReconfigSchemeBoardWorkload.test_scheme_board ydb/tests/stress/result_set_format/tests py3test.sole chunk ydb/tests/stress/result_set_format/tests test_arrow_workload.py.TestResultSetArrowWorkload.test ydb/tests/stress/result_set_format/tests test_value_workload.py.TestResultSetValueWorkload.test diff --git a/.github/workflows/compare_index_performance.yml b/.github/workflows/compare_index_performance.yml index 7430938e174..cf573105efb 100644 --- a/.github/workflows/compare_index_performance.yml +++ b/.github/workflows/compare_index_performance.yml @@ -33,10 +33,9 @@ on: required: false type: choice options: - - all - vector - fulltext - default: all + default: vector rows: description: 'Number of rows in generated database' required: false @@ -74,6 +73,46 @@ on: required: false type: boolean default: true + vector_clusters: + description: 'Number of clusters in kmeans tree for vector index (empty = server auto-detect)' + required: false + default: '' + vector_levels: + description: 'Number of levels in kmeans tree for vector index (empty = server auto-detect)' + required: false + default: '' + dataset_source: + description: 'Source of the test dataset: generate (random data) or s3 (import fixed dataset from S3)' + required: false + type: choice + options: + - generate + - s3 + default: generate + s3_endpoint: + description: 'S3 endpoint for dataset import (required when dataset_source=s3)' + required: false + default: 'https://storage.yandexcloud.net' + s3_bucket: + description: 'S3 bucket for dataset import (required when dataset_source=s3)' + required: false + default: 'vector-index' + s3_source: + description: 'S3 object key prefix for dataset import (required when dataset_source=s3)' + required: false + default: '' + s3_destination: + description: 'Database destination path for dataset import (required when dataset_source=s3)' + required: false + default: '/Root/testdb/wikipedia' + s3_query_source: + description: 'S3 object key prefix for pre-computed queries table (optional when dataset_source=s3)' + required: false + default: '' + s3_query_destination: + description: 'Database destination path for queries table (optional, used with s3_query_source)' + required: false + default: '' jobs: compare: @@ -125,10 +164,17 @@ jobs: BASELINE_SHA: ${{ steps.resolve-ref.outputs.baseline_sha }} INPUT_BUILD_PRESET: ${{ inputs.build_preset || 'release' }} run: | + # Self-hosted runners reuse the workspace, so a ../baseline worktree + # (and its .git/worktrees admin entry) may linger from a prior run and + # make `git worktree add` fail with "'../baseline' already exists". + # Clean any stale copy first so the step is idempotent. + git worktree remove --force ../baseline 2>/dev/null || true + rm -rf ../baseline + git worktree prune git fetch --depth=1 origin "$BASELINE_SHA" git worktree add ../baseline "$BASELINE_SHA" cd ../baseline - ./ya make --build "$INPUT_BUILD_PRESET" ydb/apps/ydbd + ./ya make -DDEBUGINFO_LINES_ONLY --build "$INPUT_BUILD_PRESET" ydb/apps/ydbd BINARY=$(find . -name ydbd -path '*/ydb/apps/ydbd/ydbd' -not -path '*/ydb/apps/ydbd/ydbd/*' 2>/dev/null | head -1) BINARY=$(realpath "$BINARY") echo "binary=$BINARY" >> "$GITHUB_OUTPUT" @@ -151,14 +197,43 @@ jobs: INPUT_BASELINE_SHA: ${{ steps.resolve-ref.outputs.baseline_sha }} INPUT_CURRENT_SHA: ${{ steps.resolve-ref.outputs.checkout_ref }} INPUT_BASELINE_YDBD: ${{ steps.build-baseline.outputs.binary }} + INPUT_VECTOR_CLUSTERS: ${{ inputs.vector_clusters || '' }} + INPUT_VECTOR_LEVELS: ${{ inputs.vector_levels || '' }} + INPUT_DATASET_SOURCE: ${{ inputs.dataset_source || 'generate' }} + INPUT_S3_ENDPOINT: ${{ inputs.s3_endpoint || '' }} + INPUT_S3_BUCKET: ${{ inputs.s3_bucket || '' }} + INPUT_S3_SOURCE: ${{ inputs.s3_source || '' }} + INPUT_S3_DESTINATION: ${{ inputs.s3_destination || '' }} + INPUT_S3_QUERY_SOURCE: ${{ inputs.s3_query_source || '' }} + INPUT_S3_QUERY_DESTINATION: ${{ inputs.s3_query_destination || '' }} run: | # The feature-flag / table_service_config inputs are comma-separated # strings; the test splits them itself, so they are passed through as-is. EXTRA_PARAMS="" if [[ -n "$INPUT_BASELINE_YDBD" ]]; then - EXTRA_PARAMS="--test-param compare_baseline_ydbd=$INPUT_BASELINE_YDBD" + EXTRA_PARAMS="$EXTRA_PARAMS --test-param compare_baseline_ydbd=$INPUT_BASELINE_YDBD" + fi + if [[ -n "$INPUT_VECTOR_CLUSTERS" ]]; then + EXTRA_PARAMS="$EXTRA_PARAMS --test-param compare_vector_clusters=$INPUT_VECTOR_CLUSTERS" + fi + if [[ -n "$INPUT_VECTOR_LEVELS" ]]; then + EXTRA_PARAMS="$EXTRA_PARAMS --test-param compare_vector_levels=$INPUT_VECTOR_LEVELS" + fi + if [[ "$INPUT_DATASET_SOURCE" == "s3" ]]; then + EXTRA_PARAMS="$EXTRA_PARAMS --test-param compare_dataset_source=s3" + EXTRA_PARAMS="$EXTRA_PARAMS --test-param compare_s3_endpoint=$INPUT_S3_ENDPOINT" + EXTRA_PARAMS="$EXTRA_PARAMS --test-param compare_s3_bucket=$INPUT_S3_BUCKET" + EXTRA_PARAMS="$EXTRA_PARAMS --test-param compare_s3_source=$INPUT_S3_SOURCE" + EXTRA_PARAMS="$EXTRA_PARAMS --test-param compare_s3_destination=$INPUT_S3_DESTINATION" + if [[ -n "$INPUT_S3_QUERY_SOURCE" ]]; then + EXTRA_PARAMS="$EXTRA_PARAMS --test-param compare_s3_query_source=$INPUT_S3_QUERY_SOURCE" + if [[ -n "$INPUT_S3_QUERY_DESTINATION" ]]; then + EXTRA_PARAMS="$EXTRA_PARAMS --test-param compare_s3_query_destination=$INPUT_S3_QUERY_DESTINATION" + fi + fi fi - ./ya make --build "$INPUT_BUILD_PRESET" -tA --test-disable-timeout \ + + ./ya make -DDEBUGINFO_LINES_ONLY --build "$INPUT_BUILD_PRESET" -tA --test-disable-timeout \ ydb/tests/stress/compare_index_performance/tests \ --test-param compare_duration="$INPUT_DURATION" \ --test-param compare_build_preset="$INPUT_BUILD_PRESET" \ @@ -175,7 +250,7 @@ jobs: --test-param compare_current_table_service_config="$INPUT_CURRENT_TABLE_SERVICE_CONFIG" \ --test-param compare_baseline_sha="$INPUT_BASELINE_SHA" \ --test-param compare_current_sha="$INPUT_CURRENT_SHA" \ - $EXTRA_PARAMS + $EXTRA_PARAMS 2>&1 | tee ya_make.log; exit "${PIPESTATUS[0]}" - name: Post results to job summary if: always() @@ -191,6 +266,15 @@ jobs: done else echo "No report generated." >> "$GITHUB_STEP_SUMMARY" + if [[ -f ya_make.log ]]; then + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "<details><summary>Build/test log (last 50 lines)</summary>" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + tail -50 ya_make.log >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "</details>" >> "$GITHUB_STEP_SUMMARY" + fi fi - name: Post results to pull request @@ -214,6 +298,15 @@ jobs: done else echo "No report generated." + if [[ -f ya_make.log ]]; then + echo "" + echo "<details><summary>Build/test log (last 50 lines)</summary>" + echo "" + echo '```' + tail -50 ya_make.log + echo '```' + echo "</details>" + fi fi echo "[Workflow run](${run_url})" } | .github/scripts/tests/comment-pr.py --rewrite --no-timestamp diff --git a/.github/workflows/docs_build.yaml b/.github/workflows/docs_build.yaml index f478f7c836b..bcf6f8b3fcf 100644 --- a/.github/workflows/docs_build.yaml +++ b/.github/workflows/docs_build.yaml @@ -40,40 +40,6 @@ jobs: id: sha run: echo "value=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - # Branches listed below keep the old logic (use-ya-opensource: false). - # All other branches use the new logic (use-ya-opensource: true). - - name: Determine ya-opensource flag - id: ya - uses: actions/github-script@v8 - with: - script: | - const legacyBranches = [ - 'stable-25-1', - 'stable-25-2', - 'stable-25-2-1', - 'stable-25-3', - 'stable-25-3-1', - 'stable-25-4', - 'stable-25-4-1', - 'stable-26-1', - 'stable-26-1-1', - 'stable-26-2', - 'stable-26-2-1', - 'stable-26-1-1-enterprise', - ]; - let baseRef; - if (context.eventName === 'workflow_dispatch') { - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: Number(${{ inputs.pull_number }}), - }); - baseRef = pr.base.ref; - } else { - baseRef = context.payload.pull_request.base.ref; - } - core.setOutput('use', legacyBranches.includes(baseRef) ? 'false' : 'true'); - - name: Disable PDF for PR build run: yq -i '.["docs-viewer"].pdf = false | .singlePage = false' ydb/docs/.yfm @@ -83,7 +49,7 @@ jobs: revision: "pr-${{ github.event_name == 'workflow_dispatch' && inputs.pull_number || github.event.pull_request.number }}-${{ steps.sha.outputs.value }}" src-root: "./ydb/docs" cli-version: "4.59.12" - use-ya-opensource: ${{ steps.ya.outputs.use }} + use-ya-opensource: true - name: Fail on documentation build issues run: | diff --git a/.github/workflows/docs_build_rebuild.yaml b/.github/workflows/docs_build_rebuild.yaml index 786a777609a..d327829e867 100644 --- a/.github/workflows/docs_build_rebuild.yaml +++ b/.github/workflows/docs_build_rebuild.yaml @@ -64,33 +64,6 @@ jobs: if: steps.docs-changes.outputs.has_docs == 'true' run: echo "value=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - # Branches listed below keep the old logic (use-ya-opensource: false). - # All other branches use the new logic (use-ya-opensource: true). - # This workflow only runs on pull_request_target, so the relevant branch - # is the pull request base ref. - - name: Determine ya-opensource flag - id: ya - if: steps.docs-changes.outputs.has_docs == 'true' - uses: actions/github-script@v8 - with: - script: | - const legacyBranches = [ - 'stable-25-1', - 'stable-25-2', - 'stable-25-2-1', - 'stable-25-3', - 'stable-25-3-1', - 'stable-25-4', - 'stable-25-4-1', - 'stable-26-1', - 'stable-26-1-1', - 'stable-26-2', - 'stable-26-2-1', - ]; - const baseRef = context.payload.pull_request.base.ref; - core.setOutput('use', legacyBranches.includes(baseRef) ? 'false' : 'true'); - - - name: Build if: steps.docs-changes.outputs.has_docs == 'true' uses: diplodoc-platform/docs-build-action@v3 @@ -98,7 +71,7 @@ jobs: revision: "pr-${{ github.event.pull_request.number }}-${{ steps.sha.outputs.value }}" src-root: "./ydb/docs" cli-version: "4.59.12" - use-ya-opensource: ${{ steps.ya.outputs.use }} + use-ya-opensource: true - name: Fail on documentation build issues if: steps.docs-changes.outputs.has_docs == 'true' diff --git a/.github/workflows/docs_release.yaml b/.github/workflows/docs_release.yaml index d1f91972be2..e808f8887a0 100644 --- a/.github/workflows/docs_release.yaml +++ b/.github/workflows/docs_release.yaml @@ -20,38 +20,13 @@ jobs: - name: Checkout uses: actions/checkout@v5 - # Branches listed below keep the old logic (use-ya-opensource: false). - # All other branches use the new logic (use-ya-opensource: true). - # This workflow runs on push / workflow_dispatch, so the relevant branch - # is the one being built (context.ref), not a pull request base ref. - - name: Determine ya-opensource flag - id: ya - uses: actions/github-script@v8 - with: - script: | - const legacyBranches = [ - 'stable-25-1', - 'stable-25-2', - 'stable-25-2-1', - 'stable-25-3', - 'stable-25-3-1', - 'stable-25-4', - 'stable-25-4-1', - 'stable-26-1', - 'stable-26-1-1', - 'stable-26-2', - 'stable-26-2-1', - ]; - const branch = context.ref.replace('refs/heads/', ''); - core.setOutput('use', legacyBranches.includes(branch) ? 'false' : 'true'); - - name: Build uses: diplodoc-platform/docs-build-action@v3 with: revision: "${{ github.sha }}" src-root: ${{ vars.SRC_ROOT }} cli-version: "4.59.12" - use-ya-opensource: ${{ steps.ya.outputs.use }} + use-ya-opensource: true upload: needs: build diff --git a/.github/workflows/ydbdoc-review.yml b/.github/workflows/ydbdoc-review.yml index 88e943cc62c..32e0e93a119 100644 --- a/.github/workflows/ydbdoc-review.yml +++ b/.github/workflows/ydbdoc-review.yml @@ -21,11 +21,14 @@ jobs: group: ydbdoc-review-${{ github.event.pull_request.number }} cancel-in-progress: true steps: - - name: Checkout PR head + - name: Checkout translation source uses: actions/checkout@v5 with: - ref: ${{ github.event.pull_request.head.sha }} + # Merged PR (в т.ч. из форка): коммит уже в upstream (main). + # Open PR: head как раньше. + ref: ${{ github.event.pull_request.merged && github.event.pull_request.merge_commit_sha || github.event.pull_request.head.sha }} fetch-depth: 0 + allow-unsafe-pr-checkout: true - name: Fetch merge base ref run: | diff --git a/.github/workflows/ydbdoc-verify.yml b/.github/workflows/ydbdoc-verify.yml index 93b16919b9e..81160bd6316 100644 --- a/.github/workflows/ydbdoc-verify.yml +++ b/.github/workflows/ydbdoc-verify.yml @@ -21,11 +21,12 @@ jobs: group: ydbdoc-verify-${{ github.event.pull_request.number }} cancel-in-progress: true steps: - - name: Checkout PR head + - name: Checkout translation source uses: actions/checkout@v5 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.event.pull_request.merged && github.event.pull_request.merge_commit_sha || github.event.pull_request.head.sha }} fetch-depth: 0 + allow-unsafe-pr-checkout: true - name: Fetch merge base ref run: | diff --git a/contrib/python/asttokens/.dist-info/METADATA b/contrib/python/asttokens/.dist-info/METADATA index e733ebcdbe4..f8b294c35a8 100644 --- a/contrib/python/asttokens/.dist-info/METADATA +++ b/contrib/python/asttokens/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: asttokens -Version: 3.0.1 +Version: 3.0.2 Summary: Annotate AST trees with source code positions Home-page: https://github.com/gristlabs/asttokens Author: Dmitry Sagalovskiy, Grist Labs @@ -119,3 +119,23 @@ To contribute: 4. Run tests across all supported interpreters with the ``tox`` command. You will need to have the interpreters installed separately. We recommend ``pyenv`` for that. Use ``tox -p auto`` to run the tests in parallel. 5. By default certain tests which take a very long time to run are skipped, but they are run in CI. These are marked using the ``pytest`` marker ``slow`` and can be run on their own with ``pytest -m slow`` or as part of the full suite with ``pytest -m ''``. + +Documentation +------------- + +Documentation at https://asttokens.readthedocs.io/ is built by Read the Docs automatically on +every push, from the sources in ``docs/`` (configured in ``.readthedocs.yaml``). To preview +changes locally:: + + pip install -r docs/requirements.txt + sphinx-build -M html docs docs/_build + +Release +------- + +The version is derived from the git tag by ``setuptools_scm``; there is no version to update in +the source. To release, draft a new release at https://github.com/gristlabs/asttokens/releases +with a new tag named ``vX.Y.Z``, and use "Generate release notes" to fill in the description. +Publishing the release triggers the ``publish.yml`` workflow, which builds the package and +uploads it to PyPI via `trusted publishing +<https://docs.pypi.org/trusted-publishers/>`_. diff --git a/contrib/python/asttokens/README.rst b/contrib/python/asttokens/README.rst index 068d3960678..19272d6b9e0 100644 --- a/contrib/python/asttokens/README.rst +++ b/contrib/python/asttokens/README.rst @@ -80,3 +80,23 @@ To contribute: 4. Run tests across all supported interpreters with the ``tox`` command. You will need to have the interpreters installed separately. We recommend ``pyenv`` for that. Use ``tox -p auto`` to run the tests in parallel. 5. By default certain tests which take a very long time to run are skipped, but they are run in CI. These are marked using the ``pytest`` marker ``slow`` and can be run on their own with ``pytest -m slow`` or as part of the full suite with ``pytest -m ''``. + +Documentation +------------- + +Documentation at https://asttokens.readthedocs.io/ is built by Read the Docs automatically on +every push, from the sources in ``docs/`` (configured in ``.readthedocs.yaml``). To preview +changes locally:: + + pip install -r docs/requirements.txt + sphinx-build -M html docs docs/_build + +Release +------- + +The version is derived from the git tag by ``setuptools_scm``; there is no version to update in +the source. To release, draft a new release at https://github.com/gristlabs/asttokens/releases +with a new tag named ``vX.Y.Z``, and use "Generate release notes" to fill in the description. +Publishing the release triggers the ``publish.yml`` workflow, which builds the package and +uploads it to PyPI via `trusted publishing +<https://docs.pypi.org/trusted-publishers/>`_. diff --git a/contrib/python/asttokens/asttokens/asttokens.py b/contrib/python/asttokens/asttokens/asttokens.py index 6cbc5aad296..d49672be86c 100644 --- a/contrib/python/asttokens/asttokens/asttokens.py +++ b/contrib/python/asttokens/asttokens/asttokens.py @@ -18,17 +18,14 @@ import bisect import sys import token from ast import Module -from typing import Iterable, Iterator, List, Optional, Tuple, Any, cast, TYPE_CHECKING +from typing import Iterable, Iterator, List, Optional, Tuple, Any, cast from .line_numbers import LineNumbers from .util import ( - Token, match_token, is_non_coding_token, patched_generate_tokens, last_stmt, + AstNode, Token, TokenInfo, match_token, is_non_coding_token, patched_generate_tokens, last_stmt, annotate_fstring_nodes, generate_tokens, is_module, is_stmt ) -if TYPE_CHECKING: # pragma: no cover - from .util import AstNode, TokenInfo - class ASTTextBase(metaclass=abc.ABCMeta): def __init__(self, source_text: str, filename: str) -> None: @@ -43,8 +40,9 @@ class ASTTextBase(metaclass=abc.ABCMeta): self._line_numbers = LineNumbers(source_text) @abc.abstractmethod - def get_text_positions(self, node, padded): - # type: (AstNode, bool) -> Tuple[Tuple[int, int], Tuple[int, int]] + def get_text_positions( + self, node: AstNode, padded: bool + ) -> Tuple[Tuple[int, int], Tuple[int, int]]: """ Returns two ``(lineno, col_offset)`` tuples for the start and end of the given node. If the positions can't be determined, or the nodes don't correspond to any particular text, @@ -56,8 +54,7 @@ class ASTTextBase(metaclass=abc.ABCMeta): """ raise NotImplementedError # pragma: no cover - def get_text_range(self, node, padded=True): - # type: (AstNode, bool) -> Tuple[int, int] + def get_text_range(self, node: AstNode, padded: bool = True) -> Tuple[int, int]: """ Returns the (startpos, endpos) positions in source text corresponding to the given node. Returns (0, 0) for nodes (like `Load`) that don't correspond to any particular text. @@ -70,8 +67,7 @@ class ASTTextBase(metaclass=abc.ABCMeta): self._line_numbers.line_to_offset(*end), ) - def get_text(self, node, padded=True): - # type: (AstNode, bool) -> str + def get_text(self, node: AstNode, padded: bool = True) -> str: """ Returns the text corresponding to the given node. Returns '' for nodes (like `Load`) that don't correspond to any particular text. @@ -102,9 +98,15 @@ class ASTTokens(ASTTextBase): tree created separately. """ - def __init__(self, source_text, parse=False, tree=None, filename='<unknown>', tokens=None): - # type: (Any, bool, Optional[Module], str, Optional[Iterable[TokenInfo]]) -> None - super(ASTTokens, self).__init__(source_text, filename) + def __init__( + self, + source_text: Any, + parse: bool = False, + tree: Optional[Module] = None, + filename: str = '<unknown>', + tokens: Optional[Iterable[TokenInfo]] = None + ) -> None: + super().__init__(source_text, filename) self._tree = ast.parse(source_text, filename) if parse else tree @@ -119,8 +121,7 @@ class ASTTokens(ASTTextBase): if self._tree: self.mark_tokens(self._tree) - def mark_tokens(self, root_node): - # type: (Module) -> None + def mark_tokens(self, root_node: Module) -> None: """ Given the root of the AST or Astroid tree produced from source_text, visits all nodes marking them with token and position information by adding ``.first_token`` and @@ -131,8 +132,7 @@ class ASTTokens(ASTTextBase): from .mark_tokens import MarkTokens # to avoid import loops MarkTokens(self).visit_tree(root_node) - def _translate_tokens(self, original_tokens): - # type: (Iterable[TokenInfo]) -> Iterator[Token] + def _translate_tokens(self, original_tokens: Iterable[TokenInfo]) -> Iterator[Token]: """ Translates the given standard library tokens into our own representation. """ @@ -143,39 +143,33 @@ class ASTTokens(ASTTextBase): self._line_numbers.line_to_offset(end[0], end[1])) @property - def text(self): - # type: () -> str + def text(self) -> str: """The source code passed into the constructor.""" return self._text @property - def tokens(self): - # type: () -> List[Token] + def tokens(self) -> List[Token]: """The list of tokens corresponding to the source code from the constructor.""" return self._tokens @property - def tree(self): - # type: () -> Optional[Module] + def tree(self) -> Optional[Module]: """The root of the AST tree passed into the constructor or parsed from the source code.""" return self._tree @property - def filename(self): - # type: () -> str + def filename(self) -> str: """The filename that was parsed""" return self._filename - def get_token_from_offset(self, offset): - # type: (int) -> Token + def get_token_from_offset(self, offset: int) -> Token: """ Returns the token containing the given character offset (0-based position in source text), or the preceeding token if the position is between tokens. """ return self._tokens[bisect.bisect(self._token_offsets, offset) - 1] - def get_token(self, lineno, col_offset): - # type: (int, int) -> Token + def get_token(self, lineno: int, col_offset: int) -> Token: """ Returns the token containing the given (lineno, col_offset) position, or the preceeding token if the position is between tokens. @@ -185,15 +179,13 @@ class ASTTokens(ASTTextBase): # but isn't explicit. return self.get_token_from_offset(self._line_numbers.line_to_offset(lineno, col_offset)) - def get_token_from_utf8(self, lineno, col_offset): - # type: (int, int) -> Token + def get_token_from_utf8(self, lineno: int, col_offset: int) -> Token: """ Same as get_token(), but interprets col_offset as a UTF8 offset, which is what `ast` uses. """ return self.get_token(lineno, self._line_numbers.from_utf8_col(lineno, col_offset)) - def next_token(self, tok, include_extra=False): - # type: (Token, bool) -> Token + def next_token(self, tok: Token, include_extra: bool = False) -> Token: """ Returns the next token after the given one. If include_extra is True, includes non-coding tokens from the tokenize module, such as NL and COMMENT. @@ -204,8 +196,7 @@ class ASTTokens(ASTTextBase): i += 1 return self._tokens[i] - def prev_token(self, tok, include_extra=False): - # type: (Token, bool) -> Token + def prev_token(self, tok: Token, include_extra: bool = False) -> Token: """ Returns the previous token before the given one. If include_extra is True, includes non-coding tokens from the tokenize module, such as NL and COMMENT. @@ -216,8 +207,9 @@ class ASTTokens(ASTTextBase): i -= 1 return self._tokens[i] - def find_token(self, start_token, tok_type, tok_str=None, reverse=False): - # type: (Token, int, Optional[str], bool) -> Token + def find_token( + self, start_token: Token, tok_type: int, tok_str: Optional[str] = None, reverse: bool = False + ) -> Token: """ Looks for the first token, starting at start_token, that matches tok_type and, if given, the token string. Searches backwards if reverse is True. Returns ENDMARKER token if not found (you @@ -229,12 +221,12 @@ class ASTTokens(ASTTextBase): t = advance(t, include_extra=True) return t - def token_range(self, - first_token, # type: Token - last_token, # type: Token - include_extra=False, # type: bool - ): - # type: (...) -> Iterator[Token] + def token_range( + self, + first_token: Token, + last_token: Token, + include_extra: bool = False, + ) -> Iterator[Token]: """ Yields all tokens in order from first_token through and including last_token. If include_extra is True, includes non-coding tokens such as tokenize.NL and .COMMENT. @@ -243,16 +235,14 @@ class ASTTokens(ASTTextBase): if include_extra or not is_non_coding_token(self._tokens[i].type): yield self._tokens[i] - def get_tokens(self, node, include_extra=False): - # type: (AstNode, bool) -> Iterator[Token] + def get_tokens(self, node: AstNode, include_extra: bool = False) -> Iterator[Token]: """ Yields all tokens making up the given node. If include_extra is True, includes non-coding tokens such as tokenize.NL and .COMMENT. """ return self.token_range(node.first_token, node.last_token, include_extra=include_extra) - def get_text_positions(self, node, padded): - # type: (AstNode, bool) -> Tuple[Tuple[int, int], Tuple[int, int]] + def get_text_positions(self, node: AstNode, padded: bool) -> Tuple[Tuple[int, int], Tuple[int, int]]: """ Returns two ``(lineno, col_offset)`` tuples for the start and end of the given node. If the positions can't be determined, or the nodes don't correspond to any particular text, @@ -287,27 +277,24 @@ class ASTText(ASTTextBase): which incurs the usual setup cost the first time. If you want to avoid this, check ``supports_tokenless(node)`` before calling ``get_text*`` methods. """ - def __init__(self, source_text, tree=None, filename='<unknown>'): - # type: (Any, Optional[Module], str) -> None - super(ASTText, self).__init__(source_text, filename) + def __init__(self, source_text: Any, tree: Optional[Module] = None, filename: str = '<unknown>') -> None: + super().__init__(source_text, filename) self._tree = tree if self._tree is not None: annotate_fstring_nodes(self._tree) - self._asttokens = None # type: Optional[ASTTokens] + self._asttokens: Optional[ASTTokens] = None @property - def tree(self): - # type: () -> Module + def tree(self) -> Module: if self._tree is None: self._tree = ast.parse(self._text, self._filename) annotate_fstring_nodes(self._tree) return self._tree @property - def asttokens(self): - # type: () -> ASTTokens + def asttokens(self) -> ASTTokens: if self._asttokens is None: self._asttokens = ASTTokens( self._text, @@ -316,8 +303,9 @@ class ASTText(ASTTextBase): ) return self._asttokens - def _get_text_positions_tokenless(self, node, padded): - # type: (AstNode, bool) -> Tuple[Tuple[int, int], Tuple[int, int]] + def _get_text_positions_tokenless( + self, node: AstNode, padded: bool + ) -> Tuple[Tuple[int, int], Tuple[int, int]]: """ Version of ``get_text_positions()`` that doesn't use tokens. """ @@ -382,8 +370,7 @@ class ASTText(ASTTextBase): return start, end - def get_text_positions(self, node, padded): - # type: (AstNode, bool) -> Tuple[Tuple[int, int], Tuple[int, int]] + def get_text_positions(self, node: AstNode, padded: bool) -> Tuple[Tuple[int, int], Tuple[int, int]]: """ Returns two ``(lineno, col_offset)`` tuples for the start and end of the given node. If the positions can't be determined, or the nodes don't correspond to any particular text, @@ -405,7 +392,7 @@ class ASTText(ASTTextBase): # Node types that _get_text_positions_tokenless doesn't support. # These initial values are missing lineno. -_unsupported_tokenless_types = ("arguments", "Arguments", "withitem") # type: Tuple[str, ...] +_unsupported_tokenless_types: Tuple[str, ...] = ("arguments", "Arguments", "withitem") if sys.version_info[:2] == (3, 8): # _get_text_positions_tokenless works incorrectly for these types due to bugs in Python 3.8. _unsupported_tokenless_types += ("arg", "Starred") @@ -413,8 +400,7 @@ if sys.version_info[:2] == (3, 8): _unsupported_tokenless_types += ("Slice", "ExtSlice", "Index", "keyword") -def supports_tokenless(node=None): - # type: (Any) -> bool +def supports_tokenless(node: Any = None) -> bool: """ Returns True if the Python version and the node (if given) are supported by the ``get_text*`` methods of ``ASTText`` without falling back to ``ASTTokens``. @@ -440,10 +426,8 @@ def supports_tokenless(node=None): and not ( # astroid nodes not isinstance(node, ast.AST) and node is not None and ( - ( type(node).__name__ == "AssignName" and type(node.parent).__name__ in ("Arguments", "ExceptHandler") - ) ) ) and 'pypy' not in sys.version.lower() diff --git a/contrib/python/asttokens/asttokens/line_numbers.py b/contrib/python/asttokens/asttokens/line_numbers.py index 1441a53b787..3f1bc816fb2 100644 --- a/contrib/python/asttokens/asttokens/line_numbers.py +++ b/contrib/python/asttokens/asttokens/line_numbers.py @@ -16,7 +16,10 @@ import bisect import re from typing import Dict, List, Tuple -_line_start_re = re.compile(r'^', re.M) +# Matches the end-of-line sequences that Python treats as line boundaries in source code, i.e. +# "\r\n", "\r", or "\n". Using this (rather than a plain `re.M` `^`) means we recognise a lone +# "\r" as a line separator, matching how the tokenizer and ast module number lines. See issue #105. +_line_end_re = re.compile(r'\r\n|\r|\n') class LineNumbers: """ @@ -26,16 +29,15 @@ class LineNumbers: This class expects unicode for input and stores positions in unicode. But it supports translating to and from utf8 offsets, which are used by ast parsing. """ - def __init__(self, text): - # type: (str) -> None - # A list of character offsets of each line's first character. - self._line_offsets = [m.start(0) for m in _line_start_re.finditer(text)] + def __init__(self, text: str) -> None: + # A list of character offsets of each line's first character. The first line always starts at + # offset 0, and each subsequent line starts right after an end-of-line sequence. + self._line_offsets = [0] + [m.end(0) for m in _line_end_re.finditer(text)] self._text = text self._text_len = len(text) - self._utf8_offset_cache = {} # type: Dict[int, List[int]] # maps line num to list of char offset for each byte in line + self._utf8_offset_cache: Dict[int, List[int]] = {} # maps line num to list of char offset for each byte in line - def from_utf8_col(self, line, utf8_column): - # type: (int, int) -> int + def from_utf8_col(self, line: int, utf8_column: int) -> int: """ Given a 1-based line number and 0-based utf8 column, returns a 0-based unicode column. """ @@ -50,8 +52,7 @@ class LineNumbers: return offsets[max(0, min(len(offsets)-1, utf8_column))] - def line_to_offset(self, line, column): - # type: (int, int) -> int + def line_to_offset(self, line: int, column: int) -> int: """ Converts 1-based line number and 0-based column to 0-based character offset into text. """ @@ -63,8 +64,7 @@ class LineNumbers: else: return min(self._line_offsets[line] + max(0, column), self._text_len) - def offset_to_line(self, offset): - # type: (int) -> Tuple[int, int] + def offset_to_line(self, offset: int) -> Tuple[int, int]: """ Converts 0-based character offset to pair (line, col) of 1-based line and 0-based column numbers. diff --git a/contrib/python/asttokens/asttokens/mark_tokens.py b/contrib/python/asttokens/asttokens/mark_tokens.py index 62d38f8871b..00bc8a02c21 100644 --- a/contrib/python/asttokens/asttokens/mark_tokens.py +++ b/contrib/python/asttokens/asttokens/mark_tokens.py @@ -22,9 +22,7 @@ from typing import Callable, List, Union, cast, Optional, Tuple, TYPE_CHECKING from . import util from .asttokens import ASTTokens from .astroid_compat import astroid_node_classes as nc, BaseContainer as AstroidBaseContainer - -if TYPE_CHECKING: - from .util import AstNode +from .util import AstNode # Mapping of matching braces. To find a token here, look up token[:2]. @@ -46,19 +44,18 @@ class MarkTokens: Helper that visits all nodes in the AST tree and assigns .first_token and .last_token attributes to each of them. This is the heart of the token-marking logic. """ - def __init__(self, code): - # type: (ASTTokens) -> None + def __init__(self, code: ASTTokens) -> None: self._code = code self._methods = util.NodeMethods() - self._iter_children = None # type: Optional[Callable] + self._iter_children: Optional[Callable] = None - def visit_tree(self, node): - # type: (Module) -> None + def visit_tree(self, node: Module) -> None: self._iter_children = util.iter_children_func(node) util.visit_tree(node, self._visit_before_children, self._visit_after_children) - def _visit_before_children(self, node, parent_token): - # type: (AstNode, Optional[util.Token]) -> Tuple[Optional[util.Token], Optional[util.Token]] + def _visit_before_children( + self, node: AstNode, parent_token: Optional[util.Token] + ) -> Tuple[Optional[util.Token], Optional[util.Token]]: col = getattr(node, 'col_offset', None) token = self._code.get_token_from_utf8(node.lineno, col) if col is not None else None @@ -70,8 +67,9 @@ class MarkTokens: # parent_token argument. The second value becomes the token argument of _visit_after_children. return (token or parent_token, token) - def _visit_after_children(self, node, parent_token, token): - # type: (AstNode, Optional[util.Token], Optional[util.Token]) -> None + def _visit_after_children( + self, node: AstNode, parent_token: Optional[util.Token], token: Optional[util.Token] + ) -> None: # This processes the node generically first, after all children have been processed. # Get the first and last tokens that belong to children. Note how this doesn't assume that we @@ -112,8 +110,7 @@ class MarkTokens: node.first_token = nfirst node.last_token = nlast - def _find_last_in_stmt(self, start_token): - # type: (util.Token) -> util.Token + def _find_last_in_stmt(self, start_token: util.Token) -> util.Token: t = start_token while (not util.match_token(t, token.NEWLINE) and not util.match_token(t, token.OP, ';') and @@ -121,15 +118,16 @@ class MarkTokens: t = self._code.next_token(t, include_extra=True) return self._code.prev_token(t) - def _expand_to_matching_pairs(self, first_token, last_token, node): - # type: (util.Token, util.Token, AstNode) -> Tuple[util.Token, util.Token] + def _expand_to_matching_pairs( + self, first_token: util.Token, last_token: util.Token, node: AstNode + ) -> Tuple[util.Token, util.Token]: """ Scan tokens in [first_token, last_token] range that are between node's children, and for any unmatched brackets, adjust first/last tokens to include the closing pair. """ # We look for opening parens/braces among non-child tokens (i.e. tokens between our actual # child nodes). If we find any closing ones, we match them to the opens. - to_match_right = [] # type: List[Tuple[int, str]] + to_match_right: List[Tuple[int, str]] = [] to_match_left = [] for tok in self._code.token_range(first_token, last_token): tok_info = tok[:2] @@ -162,39 +160,43 @@ class MarkTokens: # Node visitors. Each takes a preliminary first and last tokens, and returns the adjusted pair # that will actually be assigned. - def visit_default(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_default( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: # pylint: disable=no-self-use # By default, we don't need to adjust the token we computed earlier. return (first_token, last_token) - def handle_comp(self, open_brace, node, first_token, last_token): - # type: (str, AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def handle_comp( + self, open_brace: str, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: # For list/set/dict comprehensions, we only get the token of the first child, so adjust it to # include the opening brace (the closing brace will be matched automatically). before = self._code.prev_token(first_token) util.expect_token(before, token.OP, open_brace) return (before, last_token) - def visit_comprehension(self, - node, # type: AstNode - first_token, # type: util.Token - last_token, # type: util.Token - ): - # type: (...) -> Tuple[util.Token, util.Token] + def visit_comprehension( + self, + node: AstNode, + first_token: util.Token, + last_token: util.Token, + ) -> Tuple[util.Token, util.Token]: # The 'comprehension' node starts with 'for' but we only get first child; we search backwards # to find the 'for' keyword. first = self._code.find_token(first_token, token.NAME, 'for', reverse=True) return (first, last_token) - def visit_if(self, node, first_token, last_token): - # type: (util.Token, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_if( + self, node: util.Token, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: while first_token.string not in ('if', 'elif'): first_token = self._code.prev_token(first_token) return first_token, last_token - def handle_attr(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def handle_attr( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: # Attribute node has ".attr" (2 tokens) after the last child. dot = self._code.find_token(last_token, token.OP, '.') name = self._code.next_token(dot) @@ -205,8 +207,9 @@ class MarkTokens: visit_assignattr = handle_attr visit_delattr = handle_attr - def handle_def(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def handle_def( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: # With astroid, nodes that start with a doc-string can have an empty body, in which case we # need to adjust the last token to include the doc string. if not node.body and (getattr(node, 'doc_node', None) or getattr(node, 'doc', None)): # type: ignore[union-attr] @@ -222,8 +225,9 @@ class MarkTokens: visit_classdef = handle_def visit_functiondef = handle_def - def handle_following_brackets(self, node, last_token, opening_bracket): - # type: (AstNode, util.Token, str) -> util.Token + def handle_following_brackets( + self, node: AstNode, last_token: util.Token, opening_bracket: str + ) -> util.Token: # This is for calls and subscripts, which have a pair of brackets # at the end which may contain no nodes, e.g. foo() or bar[:]. # We look for the opening bracket and then let the matching pair be found automatically @@ -235,8 +239,9 @@ class MarkTokens: last_token = call_start return last_token - def visit_call(self, node, first_token, last_token): - # type: (util.Token, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_call( + self, node: util.Token, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: last_token = self.handle_following_brackets(node, last_token, '(') # Handling a python bug with decorators with empty parens, e.g. @@ -246,22 +251,24 @@ class MarkTokens: first_token = self._code.next_token(first_token) return (first_token, last_token) - def visit_matchclass(self, node, first_token, last_token): - # type: (util.Token, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_matchclass( + self, node: util.Token, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: last_token = self.handle_following_brackets(node, last_token, '(') return (first_token, last_token) - def visit_subscript(self, - node, # type: AstNode - first_token, # type: util.Token - last_token, # type: util.Token - ): - # type: (...) -> Tuple[util.Token, util.Token] + def visit_subscript( + self, + node: AstNode, + first_token: util.Token, + last_token: util.Token, + ) -> Tuple[util.Token, util.Token]: last_token = self.handle_following_brackets(node, last_token, '[') return (first_token, last_token) - def visit_slice(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_slice( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: # consume `:` tokens to the left and right. In Python 3.9, Slice nodes are # given a col_offset, (and end_col_offset), so this will always start inside # the slice, even if it is the empty slice. However, in 3.8 and below, this @@ -280,8 +287,9 @@ class MarkTokens: last_token = next_ return (first_token, last_token) - def handle_bare_tuple(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def handle_bare_tuple( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: # A bare tuple doesn't include parens; if there is a trailing comma, make it part of the tuple. maybe_comma = self._code.next_token(last_token) if util.match_token(maybe_comma, token.OP, ','): @@ -289,8 +297,9 @@ class MarkTokens: return (first_token, last_token) # In Python3.8 parsed tuples include parentheses when present. - def handle_tuple_nonempty(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def handle_tuple_nonempty( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: assert isinstance(node, ast.Tuple) or isinstance(node, AstroidBaseContainer) # It's a bare tuple if the first token belongs to the first child. The first child may # include extraneous parentheses (which don't create new nodes), so account for those too. @@ -302,16 +311,18 @@ class MarkTokens: return self.handle_bare_tuple(node, first_token, last_token) return (first_token, last_token) - def visit_tuple(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_tuple( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: assert isinstance(node, ast.Tuple) or isinstance(node, AstroidBaseContainer) if not node.elts: # An empty tuple is just "()", and we need no further info. return (first_token, last_token) return self.handle_tuple_nonempty(node, first_token, last_token) - def _gobble_parens(self, first_token, last_token, include_all=False): - # type: (util.Token, util.Token, bool) -> Tuple[util.Token, util.Token] + def _gobble_parens( + self, first_token: util.Token, last_token: util.Token, include_all: bool = False + ) -> Tuple[util.Token, util.Token]: # Expands a range of tokens to include one or all pairs of surrounding parentheses, and # returns (first, last) tokens that include these parens. while first_token.index > 0: @@ -324,16 +335,17 @@ class MarkTokens: break return (first_token, last_token) - def visit_str(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_str( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: return self.handle_str(first_token, last_token) - def visit_joinedstr(self, - node, # type: AstNode - first_token, # type: util.Token - last_token, # type: util.Token - ): - # type: (...) -> Tuple[util.Token, util.Token] + def visit_joinedstr( + self, + node: AstNode, + first_token: util.Token, + last_token: util.Token, + ) -> Tuple[util.Token, util.Token]: if sys.version_info < (3, 12): # Older versions don't tokenize the contents of f-strings return self.handle_str(first_token, last_token) @@ -363,12 +375,14 @@ class MarkTokens: break return (first_token, last_token) - def visit_bytes(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_bytes( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: return self.handle_str(first_token, last_token) - def handle_str(self, first_token, last_token): - # type: (util.Token, util.Token) -> Tuple[util.Token, util.Token] + def handle_str( + self, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: # Multiple adjacent STRING tokens form a single string. last = self._code.next_token(last_token) while util.match_token(last, token.STRING): @@ -376,13 +390,13 @@ class MarkTokens: last = self._code.next_token(last_token) return (first_token, last_token) - def handle_num(self, - node, # type: AstNode - value, # type: Union[complex, int, numbers.Number] - first_token, # type: util.Token - last_token, # type: util.Token - ): - # type: (...) -> Tuple[util.Token, util.Token] + def handle_num( + self, + node: AstNode, + value: Union[complex, int, numbers.Number], + first_token: util.Token, + last_token: util.Token, + ) -> Tuple[util.Token, util.Token]: # A constant like '-1' gets turned into two tokens; this will skip the '-'. while util.match_token(last_token, token.OP): last_token = self._code.next_token(last_token) @@ -398,14 +412,16 @@ class MarkTokens: first_token = self._code.prev_token(first_token) return (first_token, last_token) - def visit_num(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_num( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: n = node.n # type: ignore[union-attr] # ast.Num has been removed in python 3.14 assert isinstance(n, (complex, int, numbers.Number)) return self.handle_num(node, n, first_token, last_token) - def visit_const(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_const( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: assert isinstance(node, ast.Constant) or isinstance(node, nc.Const) if isinstance(node.value, numbers.Number): return self.handle_num(node, node.value, first_token, last_token) @@ -415,8 +431,9 @@ class MarkTokens: visit_constant = visit_const - def visit_keyword(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_keyword( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: # Until python 3.9 (https://bugs.python.org/issue40141), # ast.keyword nodes didn't have line info. Astroid has lineno None. assert isinstance(node, ast.keyword) or isinstance(node, nc.Keyword) @@ -427,8 +444,9 @@ class MarkTokens: first_token = name return (first_token, last_token) - def visit_starred(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_starred( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: # Astroid has 'Starred' nodes (for "foo(*bar)" type args), but they need to be adjusted. if not util.match_token(first_token, token.OP, '*'): star = self._code.prev_token(first_token) @@ -436,8 +454,9 @@ class MarkTokens: first_token = star return (first_token, last_token) - def visit_assignname(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def visit_assignname( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: # Astroid may turn 'except' clause into AssignName, but we need to adjust it. if util.match_token(first_token, token.NAME, 'except'): colon = self._code.find_token(last_token, token.OP, ':') @@ -448,8 +467,9 @@ class MarkTokens: # but Python < 3.7 doesn't put the col_offset there # AsyncFunctionDef is slightly different because it might have # decorators before that, which visit_functiondef handles - def handle_async(self, node, first_token, last_token): - # type: (AstNode, util.Token, util.Token) -> Tuple[util.Token, util.Token] + def handle_async( + self, node: AstNode, first_token: util.Token, last_token: util.Token + ) -> Tuple[util.Token, util.Token]: if not first_token.string == 'async': first_token = self._code.prev_token(first_token) return (first_token, last_token) @@ -457,12 +477,12 @@ class MarkTokens: visit_asyncfor = handle_async visit_asyncwith = handle_async - def visit_asyncfunctiondef(self, - node, # type: AstNode - first_token, # type: util.Token - last_token, # type: util.Token - ): - # type: (...) -> Tuple[util.Token, util.Token] + def visit_asyncfunctiondef( + self, + node: AstNode, + first_token: util.Token, + last_token: util.Token, + ) -> Tuple[util.Token, util.Token]: if util.match_token(first_token, token.NAME, 'def'): # Include the 'async' token first_token = self._code.prev_token(first_token) diff --git a/contrib/python/asttokens/asttokens/util.py b/contrib/python/asttokens/asttokens/util.py index 74e0a7cdd30..9e3c15a4555 100644 --- a/contrib/python/asttokens/asttokens/util.py +++ b/contrib/python/asttokens/asttokens/util.py @@ -15,6 +15,7 @@ import ast import collections import io +import re import sys import token import tokenize @@ -38,27 +39,18 @@ from typing import ( if TYPE_CHECKING: # pragma: no cover from .astroid_compat import NodeNG +else: + NodeNG = Any - # Type class used to expand out the definition of AST to include fields added by this library - # It's not actually used for anything other than type checking though! - class EnhancedAST(AST): - # Additional attributes set by mark_tokens - first_token = None # type: Token - last_token = None # type: Token - lineno = 0 # type: int - end_lineno = 0 # type : int - end_col_offset = 0 # type : int - AstNode = Union[EnhancedAST, NodeNG] +TokenInfo = tokenize.TokenInfo +_lone_cr_re = re.compile(r'\r(?!\n)') - TokenInfo = tokenize.TokenInfo - -def token_repr(tok_type, string): - # type: (int, Optional[str]) -> str +def token_repr(tok_type: int, string: Optional[str]) -> str: """Returns a human-friendly representation of a token with the given type and string.""" # repr() prefixes unicode with 'u' on Python2 but not Python3; strip it out for consistency. - return '%s:%s' % (token.tok_name[tok_type], repr(string).lstrip('u')) + return f"{token.tok_name[tok_type]}:{repr(string).lstrip('u')}" class Token(collections.namedtuple('Token', 'type string start end line index startpos endpos')): @@ -75,50 +67,64 @@ class Token(collections.namedtuple('Token', 'type string start end line index st - [6] .startpos Starting character offset into the input text. - [7] .endpos Ending character offset into the input text. """ - def __str__(self): - # type: () -> str + def __str__(self) -> str: return token_repr(self.type, self.string) -def match_token(token, tok_type, tok_str=None): - # type: (Token, int, Optional[str]) -> bool +# Type class used to expand out the definition of AST to include fields added by this library +# It's not actually used for anything other than type checking though! +class EnhancedAST(AST): + # Additional attributes set by mark_tokens + first_token: Token = None # type: ignore + last_token: Token = None # type: ignore + lineno: int = 0 + end_lineno: int = 0 + end_col_offset: int = 0 + + +AstNode = Union[EnhancedAST, NodeNG] + + +def match_token(token: Token, tok_type: int, tok_str: Optional[str] = None) -> bool: """Returns true if token is of the given type and, if a string is given, has that string.""" return token.type == tok_type and (tok_str is None or token.string == tok_str) -def expect_token(token, tok_type, tok_str=None): - # type: (Token, int, Optional[str]) -> None +def expect_token(token: Token, tok_type: int, tok_str: Optional[str] = None) -> None: """ Verifies that the given token is of the expected type. If tok_str is given, the token string is verified too. If the token doesn't match, raises an informative ValueError. """ if not match_token(token, tok_type, tok_str): - raise ValueError("Expected token %s, got %s on line %s col %s" % ( - token_repr(tok_type, tok_str), str(token), - token.start[0], token.start[1] + 1)) + raise ValueError( + f"Expected token {token_repr(tok_type, tok_str)}, " + f"got {str(token)} on line {token.start[0]} col {token.start[1] + 1}" + ) -def is_non_coding_token(token_type): - # type: (int) -> bool +def is_non_coding_token(token_type: int) -> bool: """ These are considered non-coding tokens, as they don't affect the syntax tree. """ return token_type in (token.NL, token.COMMENT, token.ENCODING) -def generate_tokens(text): - # type: (str) -> Iterator[TokenInfo] +def generate_tokens(text: str) -> Iterator[TokenInfo]: """ Generates standard library tokens for the given code. """ + # Before Python 3.12, tokenize treats an entire line containing a lone carriage return as a + # non-coding NL token, even though ast.parse treats the carriage return as a line boundary. + # Replacing lone carriage returns is length-preserving, so token offsets still map to the + # original source. Keep CRLF intact because changing its length would shift later offsets. + text = _lone_cr_re.sub('\n', text) # tokenize.generate_tokens is technically an undocumented API for Python3, but allows us to use the same API as for # Python2. See https://stackoverflow.com/a/4952291/328565. # FIXME: Remove cast once https://github.com/python/typeshed/issues/7003 gets fixed return tokenize.generate_tokens(cast(Callable[[], str], io.StringIO(text).readline)) -def iter_children_func(node): - # type: (AST) -> Callable +def iter_children_func(node: AST) -> Callable: """ Returns a function which yields all direct children of a AST node, skipping children that are singleton nodes. @@ -127,8 +133,7 @@ def iter_children_func(node): return iter_children_astroid if hasattr(node, 'get_children') else iter_children_ast -def iter_children_astroid(node, include_joined_str=False): - # type: (NodeNG, bool) -> Union[Iterator, List] +def iter_children_astroid(node: NodeNG, include_joined_str: bool = False) -> Union[Iterator, list]: if not include_joined_str and is_joined_str(node): return [] @@ -139,8 +144,7 @@ SINGLETONS = {c for n, c in ast.__dict__.items() if isinstance(c, type) and issubclass(c, (ast.expr_context, ast.boolop, ast.operator, ast.unaryop, ast.cmpop))} -def iter_children_ast(node, include_joined_str=False): - # type: (AST, bool) -> Iterator[Union[AST, expr]] +def iter_children_ast(node: AST, include_joined_str: bool = False) -> Iterator[Union[AST, expr]]: if not include_joined_str and is_joined_str(node): return @@ -169,31 +173,26 @@ expr_class_names = ({n for n, c in ast.__dict__.items() # These feel hacky compared to isinstance() but allow us to work with both ast and astroid nodes # in the same way, and without even importing astroid. -def is_expr(node): - # type: (AstNode) -> bool +def is_expr(node: AstNode) -> bool: """Returns whether node is an expression node.""" return node.__class__.__name__ in expr_class_names -def is_stmt(node): - # type: (AstNode) -> bool +def is_stmt(node: AstNode) -> bool: """Returns whether node is a statement node.""" return node.__class__.__name__ in stmt_class_names -def is_module(node): - # type: (AstNode) -> bool +def is_module(node: AstNode) -> bool: """Returns whether node is a module node.""" return node.__class__.__name__ == 'Module' -def is_joined_str(node): - # type: (AstNode) -> bool +def is_joined_str(node: AstNode) -> bool: """Returns whether node is a JoinedStr node, used to represent f-strings.""" # At the moment, nodes below JoinedStr have wrong line/col info, and trying to process them only # leads to errors. return node.__class__.__name__ == 'JoinedStr' -def is_expr_stmt(node): - # type: (AstNode) -> bool +def is_expr_stmt(node: AstNode) -> bool: """Returns whether node is an `Expr` node, which is a statement that is an expression.""" return node.__class__.__name__ == 'Expr' @@ -207,26 +206,22 @@ except ImportError: # pragma: no cover # astroid is not available pass -def is_constant(node): - # type: (AstNode) -> bool +def is_constant(node: AstNode) -> bool: """Returns whether node is a Constant node.""" return isinstance(node, CONSTANT_CLASSES) -def is_ellipsis(node): - # type: (AstNode) -> bool +def is_ellipsis(node: AstNode) -> bool: """Returns whether node is an Ellipsis node.""" return is_constant(node) and node.value is Ellipsis # type: ignore -def is_starred(node): - # type: (AstNode) -> bool +def is_starred(node: AstNode) -> bool: """Returns whether node is a starred expression node.""" return node.__class__.__name__ == 'Starred' -def is_slice(node): - # type: (AstNode) -> bool +def is_slice(node: AstNode) -> bool: """Returns whether node represents a slice, e.g. `1:2` in `x[1:2]`""" # Before 3.9, a tuple containing a slice is an ExtSlice, # but this was removed in https://bugs.python.org/issue34822 @@ -239,8 +234,7 @@ def is_slice(node): ) -def is_empty_astroid_slice(node): - # type: (AstNode) -> bool +def is_empty_astroid_slice(node: AstNode) -> bool: return ( node.__class__.__name__ == "Slice" and not isinstance(node, ast.AST) @@ -251,8 +245,11 @@ def is_empty_astroid_slice(node): # Sentinel value used by visit_tree(). _PREVISIT = object() -def visit_tree(node, previsit, postvisit): - # type: (Module, Callable[[AstNode, Optional[Token]], Tuple[Optional[Token], Optional[Token]]], Optional[Callable[[AstNode, Optional[Token], Optional[Token]], None]]) -> None +def visit_tree( + node: Module, + previsit: Callable[[AstNode, Optional[Token]], Tuple[Optional[Token], Optional[Token]]], + postvisit: Optional[Callable[[AstNode, Optional[Token], Optional[Token]], None]] +) -> None: """ Scans the tree under the node depth-first using an explicit stack. It avoids implicit recursion via the function call stack to avoid hitting 'maximum recursion depth exceeded' error. @@ -275,7 +272,7 @@ def visit_tree(node, previsit, postvisit): iter_children = iter_children_func(node) done = set() ret = None - stack = [(node, None, _PREVISIT)] # type: List[Tuple[AstNode, Optional[Token], Union[Optional[Token], object]]] + stack: List[Tuple[AstNode, Optional[Token], object]] = [(node, None, _PREVISIT)] while stack: current, par_value, value = stack.pop() if value is _PREVISIT: @@ -294,8 +291,7 @@ def visit_tree(node, previsit, postvisit): return ret -def walk(node, include_joined_str=False): - # type: (AST, bool) -> Iterator[Union[Module, AstNode]] +def walk(node: AST, include_joined_str: bool = False) -> Iterator[Union[Module, AstNode]]: """ Recursively yield all descendant nodes in the tree starting at ``node`` (including ``node`` itself), using depth-first pre-order traversal (yieling parents before their children). @@ -323,8 +319,7 @@ def walk(node, include_joined_str=False): stack.insert(ins, c) -def replace(text, replacements): - # type: (str, List[Tuple[int, int, str]]) -> str +def replace(text: str, replacements: List[Tuple[int, int, str]]) -> str: """ Replaces multiple slices of text with new values. This is a convenience method for making code modifications of ranges e.g. as identified by ``ASTTokens.get_text_range(node)``. Replacements is @@ -347,12 +342,10 @@ class NodeMethods: """ Helper to get `visit_{node_type}` methods given a node's class and cache the results. """ - def __init__(self): - # type: () -> None - self._cache = {} # type: Dict[Union[ABCMeta, type], Callable[[AstNode, Token, Token], Tuple[Token, Token]]] + def __init__(self) -> None: + self._cache: Dict[Union[ABCMeta, type], Callable[[AstNode, Token, Token], Tuple[Token, Token]]] = {} - def get(self, obj, cls): - # type: (Any, Union[ABCMeta, type]) -> Callable + def get(self, obj: Any, cls: Union[ABCMeta, type]) -> Callable: """ Using the lowercase name of the class as node_type, returns `obj.visit_{node_type}`, or `obj.visit_default` if the type-specific method is not found. @@ -365,8 +358,7 @@ class NodeMethods: return method -def patched_generate_tokens(original_tokens): - # type: (Iterable[TokenInfo]) -> Iterator[TokenInfo] +def patched_generate_tokens(original_tokens: Iterable[TokenInfo]) -> Iterator[TokenInfo]: """ Fixes tokens yielded by `tokenize.generate_tokens` to handle more non-ASCII characters in identifiers. Workaround for https://github.com/python/cpython/issues/68382. @@ -374,7 +366,7 @@ def patched_generate_tokens(original_tokens): because it assumes that error tokens are not actually errors. Combines groups of consecutive NAME, NUMBER, and/or ERRORTOKEN tokens into a single NAME token. """ - group = [] # type: List[tokenize.TokenInfo] + group: List[tokenize.TokenInfo] = [] for tok in original_tokens: if ( tok.type in (tokenize.NAME, tokenize.ERRORTOKEN, tokenize.NUMBER) @@ -390,8 +382,7 @@ def patched_generate_tokens(original_tokens): for combined_token in combine_tokens(group): yield combined_token -def combine_tokens(group): - # type: (List[tokenize.TokenInfo]) -> List[tokenize.TokenInfo] +def combine_tokens(group: List[tokenize.TokenInfo]) -> List[tokenize.TokenInfo]: if not any(tok.type == tokenize.ERRORTOKEN for tok in group) or len({tok.line for tok in group}) != 1: return group return [ @@ -405,8 +396,7 @@ def combine_tokens(group): ] -def last_stmt(node): - # type: (AstNode) -> AstNode +def last_stmt(node: AstNode) -> AstNode: """ If the given AST node contains multiple statements, return the last one. Otherwise, just return the node. @@ -429,8 +419,7 @@ def last_stmt(node): @lru_cache(maxsize=None) -def fstring_positions_work(): - # type: () -> bool +def fstring_positions_work() -> bool: """ The positions attached to nodes inside f-string FormattedValues have some bugs that were fixed in Python 3.9.7 in https://github.com/python/cpython/pull/27729. @@ -460,8 +449,7 @@ def fstring_positions_work(): ) return positions_are_unique and correct_source_segments -def annotate_fstring_nodes(tree): - # type: (ast.AST) -> None +def annotate_fstring_nodes(tree: ast.AST) -> None: """ Add a special attribute `_broken_positions` to nodes inside f-strings if the lineno/col_offset cannot be trusted. diff --git a/contrib/python/asttokens/asttokens/version.py b/contrib/python/asttokens/asttokens/version.py index 05527687810..131942e76a2 100644 --- a/contrib/python/asttokens/asttokens/version.py +++ b/contrib/python/asttokens/asttokens/version.py @@ -1 +1 @@ -__version__ = "3.0.1" +__version__ = "3.0.2" diff --git a/contrib/python/asttokens/ya.make b/contrib/python/asttokens/ya.make index dcb609db1a3..c3e4f49817d 100644 --- a/contrib/python/asttokens/ya.make +++ b/contrib/python/asttokens/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.0.1) +VERSION(3.0.2) LICENSE(Apache-2.0) diff --git a/contrib/python/cffi/py3/.dist-info/METADATA b/contrib/python/cffi/py3/.dist-info/METADATA index 67508e56a53..6f3e2a45b7a 100644 --- a/contrib/python/cffi/py3/.dist-info/METADATA +++ b/contrib/python/cffi/py3/.dist-info/METADATA @@ -1,30 +1,29 @@ Metadata-Version: 2.4 Name: cffi -Version: 2.0.0 +Version: 2.1.0 Summary: Foreign Function Interface for Python calling C code. Author: Armin Rigo, Maciej Fijalkowski -Maintainer: Matt Davis, Matt Clay, Matti Picus -License-Expression: MIT +Maintainer: Matt Davis, Matt Clay +License-Expression: MIT-0 Project-URL: Documentation, https://cffi.readthedocs.io/ Project-URL: Changelog, https://cffi.readthedocs.io/en/latest/whatsnew.html -Project-URL: Downloads, https://github.com/python-cffi/cffi/releases +Project-URL: Download, https://github.com/python-cffi/cffi/releases Project-URL: Contact, https://groups.google.com/forum/#!forum/python-cffi Project-URL: Source Code, https://github.com/python-cffi/cffi Project-URL: Issue Tracker, https://github.com/python-cffi/cffi/issues Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: 3.15 Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta Classifier: Programming Language :: Python :: Implementation :: CPython -Requires-Python: >=3.9 +Requires-Python: >=3.10 Description-Content-Type: text/markdown License-File: LICENSE -License-File: AUTHORS Requires-Dist: pycparser; implementation_name != "PyPy" Dynamic: license-file @@ -59,7 +58,7 @@ Contact Testing/development tips ------------------------ -After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. we call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: +After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. We call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: pip install pytest pip install -e . # editable install of CFFI for local development diff --git a/contrib/python/cffi/py3/.dist-info/entry_points.txt b/contrib/python/cffi/py3/.dist-info/entry_points.txt index 4b0274f2333..46bb21b8418 100644 --- a/contrib/python/cffi/py3/.dist-info/entry_points.txt +++ b/contrib/python/cffi/py3/.dist-info/entry_points.txt @@ -1,2 +1,5 @@ +[console_scripts] +cffi-gen-src = cffi._cffi_gen_src:run + [distutils.setup_keywords] cffi_modules = cffi.setuptools_ext:cffi_modules diff --git a/contrib/python/cffi/py3/README.md b/contrib/python/cffi/py3/README.md index 723624d743d..fc7a235dc12 100644 --- a/contrib/python/cffi/py3/README.md +++ b/contrib/python/cffi/py3/README.md @@ -29,7 +29,7 @@ Contact Testing/development tips ------------------------ -After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. we call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: +After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. We call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: pip install pytest pip install -e . # editable install of CFFI for local development diff --git a/contrib/python/cffi/py3/c/_cffi_backend.c b/contrib/python/cffi/py3/c/_cffi_backend.c index 2aaccede0d1..74176b5746c 100644 --- a/contrib/python/cffi/py3/c/_cffi_backend.c +++ b/contrib/python/cffi/py3/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include <Python.h> #include "structmember.h" #include "misc_thread_common.h" -#define CFFI_VERSION "2.0.0" +#define CFFI_VERSION "2.1.0" #ifdef MS_WIN32 #include <windows.h> @@ -142,69 +142,10 @@ #include "malloc_closure.h" -#if PY_MAJOR_VERSION >= 3 -# define STR_OR_BYTES "bytes" -# define PyText_Type PyUnicode_Type -# define PyText_Check PyUnicode_Check -# define PyTextAny_Check PyUnicode_Check -# define PyText_FromFormat PyUnicode_FromFormat -# define PyText_AsUTF8 PyUnicode_AsUTF8 -# define PyText_AS_UTF8 PyUnicode_AsUTF8 -# if PY_VERSION_HEX >= 0x03030000 -# define PyText_GetSize PyUnicode_GetLength -# else -# define PyText_GetSize PyUnicode_GetSize -# endif -# define PyText_FromString PyUnicode_FromString -# define PyText_FromStringAndSize PyUnicode_FromStringAndSize -# define PyText_InternInPlace PyUnicode_InternInPlace -# define PyText_InternFromString PyUnicode_InternFromString -# define PyIntOrLong_Check PyLong_Check -#else -# define STR_OR_BYTES "str" -# define PyText_Type PyString_Type -# define PyText_Check PyString_Check -# define PyTextAny_Check(op) (PyString_Check(op) || PyUnicode_Check(op)) -# define PyText_FromFormat PyString_FromFormat -# define PyText_AsUTF8 PyString_AsString -# define PyText_AS_UTF8 PyString_AS_STRING -# define PyText_GetSize PyString_Size -# define PyText_FromString PyString_FromString -# define PyText_FromStringAndSize PyString_FromStringAndSize -# define PyText_InternInPlace PyString_InternInPlace -# define PyText_InternFromString PyString_InternFromString -# define PyIntOrLong_Check(op) (PyInt_Check(op) || PyLong_Check(op)) -#endif - -#if PY_MAJOR_VERSION >= 3 -# define PyInt_FromLong PyLong_FromLong -# define PyInt_FromSsize_t PyLong_FromSsize_t -# define PyInt_AsSsize_t PyLong_AsSsize_t -# define PyInt_AsLong PyLong_AsLong -#endif - -#if PY_MAJOR_VERSION >= 3 -/* This is the default on Python3 and constant has been removed. */ -# define Py_TPFLAGS_CHECKTYPES 0 -#endif - -#if PY_MAJOR_VERSION < 3 -# undef PyCapsule_GetPointer -# undef PyCapsule_New -# define PyCapsule_GetPointer(capsule, name) \ - (PyCObject_AsVoidPtr(capsule)) -# define PyCapsule_New(pointer, name, destructor) \ - (PyCObject_FromVoidPtr(pointer, destructor)) -#endif - #if PY_VERSION_HEX < 0x030900a4 # define Py_SET_REFCNT(obj, val) (Py_REFCNT(obj) = (val)) #endif -#if PY_VERSION_HEX >= 0x03080000 -# define USE_WRITEUNRAISABLEMSG -#endif - #if PY_VERSION_HEX <= 0x030d00a1 static int PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result) { @@ -435,10 +376,7 @@ typedef struct { #endif #include "minibuffer.h" - -#if PY_MAJOR_VERSION >= 3 -# include "file_emulator.h" -#endif +#include "file_emulator.h" #ifdef PyUnicode_KIND /* Python >= 3.3 */ # include "wchar_helper_3.h" @@ -512,7 +450,7 @@ ctypedescr_new_on_top(CTypeDescrObject *ct_base, const char *extra_text, static PyObject * ctypedescr_repr(CTypeDescrObject *ct) { - return PyText_FromFormat("<ctype '%s'>", ct->ct_name); + return PyUnicode_FromFormat("<ctype '%s'>", ct->ct_name); } static void remove_dead_unique_reference(PyObject *unique_key); @@ -588,12 +526,12 @@ static PyObject *ctypeget_kind(CTypeDescrObject *ct, void *context) else result = "?"; - return PyText_FromString(result); + return PyUnicode_FromString(result); } static PyObject *ctypeget_cname(CTypeDescrObject *ct, void *context) { - return PyText_FromString(ct->ct_name); + return PyUnicode_FromString(ct->ct_name); } static PyObject *ctypeget_item(CTypeDescrObject *ct, void *context) @@ -609,7 +547,7 @@ static PyObject *ctypeget_length(CTypeDescrObject *ct, void *context) { if (ct->ct_flags & CT_ARRAY) { if (ct->ct_length >= 0) { - return PyInt_FromSsize_t(ct->ct_length); + return PyLong_FromSsize_t(ct->ct_length); } else { Py_INCREF(Py_None); @@ -763,7 +701,7 @@ ctypedescr_dir(PyObject *ct, PyObject *noarg) } else { Py_DECREF(x); - x = PyText_FromString(gsdef->name); + x = PyUnicode_FromString(gsdef->name); err = (x != NULL) ? PyList_Append(res, x) : -1; Py_XDECREF(x); if (err < 0) { @@ -897,12 +835,6 @@ _my_PyLong_AsLongLong(PyObject *ob) Like PyLong_AsLongLong(), this version accepts a Python int too, and does conversions from other types of objects. The difference is that this version refuses floats. */ -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(ob)) { - return PyInt_AS_LONG(ob); - } - else -#endif if (PyLong_Check(ob)) { return PyLong_AsLongLong(ob); } @@ -920,7 +852,7 @@ _my_PyLong_AsLongLong(PyObject *ob) if (io == NULL) return -1; - if (PyIntOrLong_Check(io)) { + if (PyLong_Check(io)) { res = _my_PyLong_AsLongLong(io); } else { @@ -940,15 +872,6 @@ _my_PyLong_AsUnsignedLongLong(PyObject *ob, int strict) does conversions from other types of objects. If 'strict', complains with OverflowError and refuses floats. If '!strict', rounds floats and masks the result. */ -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(ob)) { - long value1 = PyInt_AS_LONG(ob); - if (strict && value1 < 0) - goto negative; - return (unsigned PY_LONG_LONG)(PY_LONG_LONG)value1; - } - else -#endif if (PyLong_Check(ob)) { if (strict) { if (_PyLong_Sign(ob) < 0) @@ -973,7 +896,7 @@ _my_PyLong_AsUnsignedLongLong(PyObject *ob, int strict) if (io == NULL) return (unsigned PY_LONG_LONG)-1; - if (PyIntOrLong_Check(io)) { + if (PyLong_Check(io)) { res = _my_PyLong_AsUnsignedLongLong(io, strict); } else { @@ -1192,7 +1115,7 @@ convert_to_object(char *data, CTypeDescrObject *ct) /*READ(data, ct->ct_size)*/ value = read_raw_signed_data(data, ct->ct_size); if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromLongLong(value); } @@ -1216,7 +1139,7 @@ convert_to_object(char *data, CTypeDescrObject *ct) Py_INCREF(x); return x; } - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); } else return PyLong_FromUnsignedLongLong(value); @@ -1273,7 +1196,7 @@ convert_to_object_bitfield(char *data, CFieldObject *cf) result = ((PY_LONG_LONG)value) - (PY_LONG_LONG)shiftforsign; if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) - return PyInt_FromLong((long)result); + return PyLong_FromLong((long)result); else return PyLong_FromLongLong(result); } @@ -1285,7 +1208,7 @@ convert_to_object_bitfield(char *data, CFieldObject *cf) value = (value >> cf->cf_bitshift) & valuemask; if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromUnsignedLongLong(value); } @@ -1300,7 +1223,7 @@ static int _convert_overflow(PyObject *init, const char *ct_name) if (s == NULL) return -1; PyErr_Format(PyExc_OverflowError, "integer %s does not fit '%s'", - PyText_AS_UTF8(s), ct_name); + PyUnicode_AsUTF8(s), ct_name); Py_DECREF(s); return -1; } @@ -1318,7 +1241,7 @@ static int _convert_to_char(PyObject *init) return *(unsigned char *)data; } PyErr_Format(PyExc_TypeError, - "initializer for ctype 'char' must be a "STR_OR_BYTES + "initializer for ctype 'char' must be a bytes" " of length 1, not %.200s", Py_TYPE(init)->tp_name); return -1; } @@ -1582,13 +1505,13 @@ convert_array_from_object(char *data, CTypeDescrObject *ct, PyObject *init) char *srcdata; Py_ssize_t n; if (!PyBytes_Check(init)) { - expected = STR_OR_BYTES" or list or tuple"; + expected = "bytes or list or tuple"; goto cannot_convert; } n = PyBytes_GET_SIZE(init); if (ct->ct_length >= 0 && n > ct->ct_length) { PyErr_Format(PyExc_IndexError, - "initializer "STR_OR_BYTES" is too long for '%s' " + "initializer bytes is too long for '%s' " "(got %zd characters)", ct->ct_name, n); return -1; } @@ -1923,9 +1846,9 @@ convert_from_object_bitfield(char *data, CFieldObject *cf, PyObject *init) PyErr_Format(PyExc_OverflowError, "value %s outside the range allowed by the " "bit field width: %s <= x <= %s", - PyText_AS_UTF8(svalue), - PyText_AS_UTF8(sfmin), - PyText_AS_UTF8(sfmax)); + PyUnicode_AsUTF8(svalue), + PyUnicode_AsUTF8(sfmin), + PyUnicode_AsUTF8(sfmax)); skip: Py_XDECREF(svalue); Py_XDECREF(sfmin); @@ -2181,9 +2104,9 @@ static PyObject *convert_cdata_to_enum_string(CDataObject *cd, int both) if (o == NULL) d_value = NULL; else { - d_value = PyText_FromFormat("%s: %s", - PyText_AS_UTF8(o), - PyText_AS_UTF8(d_value)); + d_value = PyUnicode_FromFormat("%s: %s", + PyUnicode_AsUTF8(o), + PyUnicode_AsUTF8(d_value)); Py_DECREF(o); } } @@ -2211,7 +2134,7 @@ static PyObject *cdata_repr(CDataObject *cd) /*READ(cd->c_data, sizeof(long double)*/ lvalue = read_raw_longdouble_data(cd->c_data); sprintf(buffer, "%LE", lvalue); - s = PyText_FromString(buffer); + s = PyUnicode_FromString(buffer); } else { PyObject *o = convert_to_object(cd->c_data, cd->c_type); @@ -2222,14 +2145,14 @@ static PyObject *cdata_repr(CDataObject *cd) } } else if ((cd->c_type->ct_flags & CT_ARRAY) && cd->c_type->ct_length < 0) { - s = PyText_FromFormat("sliced length %zd", get_array_length(cd)); + s = PyUnicode_FromFormat("sliced length %zd", get_array_length(cd)); } else { if (cd->c_data != NULL) { - s = PyText_FromFormat("%p", cd->c_data); + s = PyUnicode_FromFormat("%p", cd->c_data); } else - s = PyText_FromString("NULL"); + s = PyUnicode_FromString("NULL"); } if (s == NULL) return NULL; @@ -2240,9 +2163,9 @@ static PyObject *cdata_repr(CDataObject *cd) extra = " &"; else extra = ""; - result = PyText_FromFormat("<cdata '%s%s' %s>", + result = PyUnicode_FromFormat("<cdata '%s%s' %s>", cd->c_type->ct_name, extra, - PyText_AsUTF8(s)); + PyUnicode_AsUTF8(s)); Py_DECREF(s); return result; } @@ -2252,8 +2175,8 @@ static PyObject *_cdata_repr2(CDataObject *cd, char *text, PyObject *x) PyObject *res, *s = PyObject_Repr(x); if (s == NULL) return NULL; - res = PyText_FromFormat("<cdata '%s' %s %s>", - cd->c_type->ct_name, text, PyText_AsUTF8(s)); + res = PyUnicode_FromFormat("<cdata '%s' %s %s>", + cd->c_type->ct_name, text, PyUnicode_AsUTF8(s)); Py_DECREF(s); return res; } @@ -2280,7 +2203,7 @@ static PyObject *_frombuf_repr(CDataObject *cd, const char *cd_type_name) Py_buffer *view = ((CDataObject_frombuf *)cd)->bufferview; const char *obj_tp_name; if (view->obj == NULL) { - return PyText_FromFormat( + return PyUnicode_FromFormat( "<cdata '%s' buffer RELEASED>", cd_type_name); } @@ -2289,7 +2212,7 @@ static PyObject *_frombuf_repr(CDataObject *cd, const char *cd_type_name) if (cd->c_type->ct_flags & CT_ARRAY) { Py_ssize_t buflen = get_array_length(cd); - return PyText_FromFormat( + return PyUnicode_FromFormat( "<cdata '%s' buffer len %zd from '%.200s' object>", cd_type_name, buflen, @@ -2297,7 +2220,7 @@ static PyObject *_frombuf_repr(CDataObject *cd, const char *cd_type_name) } else { - return PyText_FromFormat( + return PyUnicode_FromFormat( "<cdata '%s' buffer from '%.200s' object>", cd_type_name, obj_tp_name); @@ -2321,7 +2244,7 @@ static Py_ssize_t cdataowning_size_bytes(CDataObject *cd) static PyObject *cdataowning_repr(CDataObject *cd) { Py_ssize_t size = cdataowning_size_bytes(cd); - return PyText_FromFormat("<cdata '%s' owning %zd bytes>", + return PyUnicode_FromFormat("<cdata '%s' owning %zd bytes>", cd->c_type->ct_name, size); } @@ -2378,37 +2301,33 @@ static PyObject *cdata_int(CDataObject *cd) long value; /*READ(cd->c_data, cd->c_type->ct_size)*/ value = (long)read_raw_signed_data(cd->c_data, cd->c_type->ct_size); - return PyInt_FromLong(value); + return PyLong_FromLong(value); } if (cd->c_type->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED)) { PyObject *result = convert_to_object(cd->c_data, cd->c_type); if (result != NULL && PyBool_Check(result)) - result = PyInt_FromLong(PyInt_AsLong(result)); + result = PyLong_FromLong(PyLong_AsLong(result)); return result; } else if (cd->c_type->ct_flags & CT_PRIMITIVE_CHAR) { /*READ(cd->c_data, cd->c_type->ct_size)*/ switch (cd->c_type->ct_size) { case sizeof(char): - return PyInt_FromLong((unsigned char)cd->c_data[0]); + return PyLong_FromLong((unsigned char)cd->c_data[0]); case 2: - return PyInt_FromLong((long)*(cffi_char16_t *)cd->c_data); + return PyLong_FromLong((long)*(cffi_char16_t *)cd->c_data); case 4: if (cd->c_type->ct_flags & CT_IS_SIGNED_WCHAR) - return PyInt_FromLong((long)*(int32_t *)cd->c_data); + return PyLong_FromLong((long)*(int32_t *)cd->c_data); else if (sizeof(long) > 4) - return PyInt_FromLong(*(uint32_t *)cd->c_data); + return PyLong_FromLong(*(uint32_t *)cd->c_data); else return PyLong_FromUnsignedLong(*(uint32_t *)cd->c_data); } } else if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) { PyObject *o = cdata_float(cd); -#if PY_MAJOR_VERSION < 3 - PyObject *r = o ? PyNumber_Int(o) : NULL; -#else PyObject *r = o ? PyNumber_Long(o) : NULL; -#endif Py_XDECREF(o); return r; } @@ -2417,19 +2336,6 @@ static PyObject *cdata_int(CDataObject *cd) return NULL; } -#if PY_MAJOR_VERSION < 3 -static PyObject *cdata_long(CDataObject *cd) -{ - PyObject *res = cdata_int(cd); - if (res != NULL && PyInt_CheckExact(res)) { - PyObject *o = PyLong_FromLong(PyInt_AS_LONG(res)); - Py_DECREF(res); - res = o; - } - return res; -} -#endif - static PyObject *cdata_float(CDataObject *cd) { if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) { @@ -2526,10 +2432,6 @@ static PyObject *cdata_richcompare(PyObject *v, PyObject *w, int op) return pyres; } -#if PY_MAJOR_VERSION < 3 -typedef long Py_hash_t; -#endif - static Py_hash_t cdata_hash(PyObject *v) { if (((CDataObject *)v)->c_type->ct_flags & CT_PRIMITIVE_ANY) { @@ -2618,13 +2520,13 @@ _cdata_getslicearg(CDataObject *cd, PySliceObject *slice, Py_ssize_t bounds[]) Py_ssize_t start, stop; CTypeDescrObject *ct; - start = PyInt_AsSsize_t(slice->start); + start = PyLong_AsSsize_t(slice->start); if (start == -1 && PyErr_Occurred()) { if (slice->start == Py_None) PyErr_SetString(PyExc_IndexError, "slice start must be specified"); return NULL; } - stop = PyInt_AsSsize_t(slice->stop); + stop = PyLong_AsSsize_t(slice->stop); if (stop == -1 && PyErr_Occurred()) { if (slice->stop == Py_None) PyErr_SetString(PyExc_IndexError, "slice stop must be specified"); @@ -2706,6 +2608,11 @@ cdata_ass_slice(CDataObject *cd, PySliceObject *slice, PyObject *v) cdata = cd->c_data + itemsize * bounds[0]; length = bounds[1]; + if (v == NULL) { + PyErr_SetString(PyExc_TypeError, + "'del x[n]' not supported for cdata objects"); + return -1; + } if (CData_Check(v)) { CTypeDescrObject *ctv = ((CDataObject *)v)->c_type; if ((ctv->ct_flags & CT_ARRAY) && (ctv->ct_itemdescr == ct) && @@ -2926,11 +2833,7 @@ cdata_sub(PyObject *v, PyObject *w) } diff = diff / itemsize; } -#if PY_MAJOR_VERSION < 3 - return PyInt_FromSsize_t(diff); -#else return PyLong_FromSsize_t(diff); -#endif } return _cdata_add_or_sub(v, w, -1); @@ -2943,7 +2846,7 @@ _cdata_attr_errmsg(char *errmsg, CDataObject *cd, PyObject *attr) if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return; PyErr_Clear(); - text = PyText_AsUTF8(attr); + text = PyUnicode_AsUTF8(attr); if (text == NULL) return; PyErr_Format(PyExc_AttributeError, errmsg, cd->c_type->ct_name, text); @@ -3233,11 +3136,7 @@ cdata_call(CDataObject *cd, PyObject *args, PyObject *kwds) } PyTuple_SET_ITEM(fvarargs, i, (PyObject *)ct); } -#if PY_MAJOR_VERSION < 3 - fabi = PyInt_AS_LONG(PyTuple_GET_ITEM(signature, 0)); -#else fabi = PyLong_AS_LONG(PyTuple_GET_ITEM(signature, 0)); -#endif cif_descr = fb_prepare_cif(fvarargs, fresult, nargs_declared, fabi); if (cif_descr == NULL) goto error; @@ -3460,9 +3359,6 @@ static PyNumberMethods CData_as_number = { (binaryfunc)cdata_add, /*nb_add*/ (binaryfunc)cdata_sub, /*nb_subtract*/ 0, /*nb_multiply*/ -#if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ -#endif 0, /*nb_remainder*/ 0, /*nb_divmod*/ 0, /*nb_power*/ @@ -3476,15 +3372,8 @@ static PyNumberMethods CData_as_number = { 0, /*nb_and*/ 0, /*nb_xor*/ 0, /*nb_or*/ -#if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ -#endif (unaryfunc)cdata_int, /*nb_int*/ -#if PY_MAJOR_VERSION < 3 - (unaryfunc)cdata_long, /*nb_long*/ -#else - 0, -#endif + 0, /*nb_reserved*/ (unaryfunc)cdata_float, /*nb_float*/ 0, /*nb_oct*/ 0, /*nb_hex*/ @@ -3530,7 +3419,7 @@ static PyTypeObject CData_Type = { (getattrofunc)cdata_getattro, /* tp_getattro */ (setattrofunc)cdata_setattro, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ "The internal base type for CData objects. Use FFI.CData to access " "it. Always check with isinstance(): subtypes are sometimes returned " "on CPython, for performance reasons.", /* tp_doc */ @@ -3574,7 +3463,7 @@ static PyTypeObject CDataOwning_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ "This is an internal subtype of _CDataBase for performance only on " "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ 0, /* tp_traverse */ @@ -3617,8 +3506,7 @@ static PyTypeObject CDataOwningGC_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES /* tp_flags */ - | Py_TPFLAGS_HAVE_GC, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ "This is an internal subtype of _CDataBase for performance only on " "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ (traverseproc)cdataowninggc_traverse, /* tp_traverse */ @@ -3661,8 +3549,7 @@ static PyTypeObject CDataFromBuf_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES /* tp_flags */ - | Py_TPFLAGS_HAVE_GC, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ "This is an internal subtype of _CDataBase for performance only on " "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ (traverseproc)cdatafrombuf_traverse, /* tp_traverse */ @@ -3705,7 +3592,7 @@ static PyTypeObject CDataGCP_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES /* tp_flags */ + Py_TPFLAGS_DEFAULT /* tp_flags */ #ifdef Py_TPFLAGS_HAVE_FINALIZE | Py_TPFLAGS_HAVE_FINALIZE #endif @@ -4072,12 +3959,6 @@ _my_PyObject_AsBool(PyObject *ob) PyNumberMethods *nb; int res; -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(ob)) { - return PyInt_AS_LONG(ob) != 0; - } - else -#endif if (PyLong_Check(ob)) { return _PyLong_Sign(ob) != 0; } @@ -4111,7 +3992,7 @@ _my_PyObject_AsBool(PyObject *ob) if (io == NULL) return -1; - if (PyIntOrLong_Check(io) || PyFloat_Check(io)) { + if (PyLong_Check(io) || PyFloat_Check(io)) { res = _my_PyObject_AsBool(io); } else { @@ -4145,17 +4026,6 @@ static CDataObject *cast_to_integer_or_char(CTypeDescrObject *ct, PyObject *ob) (CT_POINTER|CT_FUNCTIONPTR|CT_ARRAY)) { value = (Py_intptr_t)((CDataObject *)ob)->c_data; } -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(ob)) { - if (PyString_GET_SIZE(ob) != 1) { - PyErr_Format(PyExc_TypeError, - "cannot cast string of length %zd to ctype '%s'", - PyString_GET_SIZE(ob), ct->ct_name); - return NULL; - } - value = (unsigned char)PyString_AS_STRING(ob)[0]; - } -#endif else if (PyUnicode_Check(ob)) { char err_buf[80]; cffi_char32_t ordinal; @@ -4421,7 +4291,7 @@ static void dl_dealloc(DynLibObject *dlobj) static PyObject *dl_repr(DynLibObject *dlobj) { - return PyText_FromFormat("<clibrary '%s'>", dlobj->dl_name); + return PyUnicode_FromFormat("<clibrary '%s'>", dlobj->dl_name); } static int dl_check_closed(DynLibObject *dlobj) @@ -4616,8 +4486,8 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, PyErr_Format(PyExc_RuntimeError, "cannot call dlopen(NULL)"); return NULL; } - *p_temp = PyText_FromFormat("%p", handle); - *p_printable_filename = PyText_AsUTF8(*p_temp); + *p_temp = PyUnicode_FromFormat("%p", handle); + *p_printable_filename = PyUnicode_AsUTF8(*p_temp); *auto_close = 0; return handle; } @@ -4630,17 +4500,11 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, { Py_ssize_t sz1; wchar_t *w1; -#if PY_MAJOR_VERSION < 3 - s = PyUnicode_AsUTF8String(s); - if (s == NULL) - return NULL; - *p_temp = s; -#endif - *p_printable_filename = PyText_AsUTF8(s); + *p_printable_filename = PyUnicode_AsUTF8(s); if (*p_printable_filename == NULL) return NULL; - sz1 = PyText_GetSize(filename_unicode) + 1; + sz1 = PyUnicode_GetLength(filename_unicode) + 1; sz1 *= 2; /* should not be needed, but you never know */ w1 = alloca(sizeof(wchar_t) * sz1); sz1 = PyUnicode_AsWideChar(filename_unicode, @@ -4656,18 +4520,7 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, if (!PyArg_ParseTuple(args, "et|i:load_library", Py_FileSystemDefaultEncoding, &filename_or_null, &flags)) return NULL; -#if PY_MAJOR_VERSION < 3 - if (PyUnicode_Check(s)) - { - s = PyUnicode_AsUTF8String(s); - if (s == NULL) { - PyMem_Free(filename_or_null); - return NULL; - } - *p_temp = s; - } -#endif - *p_printable_filename = PyText_AsUTF8(s); + *p_printable_filename = PyUnicode_AsUTF8(s); if (*p_printable_filename == NULL) { PyMem_Free(filename_or_null); return NULL; @@ -5206,7 +5059,7 @@ _add_field(PyObject *interned_fields, PyObject *fname, CTypeDescrObject *ftype, cf->cf_flags = flags; Py_INCREF(fname); - PyText_InternInPlace(&fname); + PyUnicode_InternInPlace(&fname); prev_size = PyDict_Size(interned_fields); err = PyDict_SetItem(interned_fields, fname, (PyObject *)cf); Py_DECREF(fname); @@ -5216,7 +5069,7 @@ _add_field(PyObject *interned_fields, PyObject *fname, CTypeDescrObject *ftype, if (PyDict_Size(interned_fields) != prev_size + 1) { PyErr_Format(PyExc_KeyError, "duplicate field name '%s'", - PyText_AS_UTF8(fname)); + PyUnicode_AsUTF8(fname)); return NULL; } return cf; /* borrowed reference */ @@ -5303,7 +5156,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, Py_ssize_t byteoffsetorg; CFieldObject **previous; int prev_bitfield_size, prev_bitfield_free; - PyObject *interned_fields; + PyObject *interned_fields = NULL; sflags = complete_sflags(sflags); if (sflags & SF_PACKED) @@ -5344,7 +5197,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, Py_ssize_t foffset = -1; if (!PyArg_ParseTuple(PyList_GET_ITEM(fields, i), "O!O!|in:list item", - &PyText_Type, &fname, + &PyUnicode_Type, &fname, &CTypeDescr_Type, &ftype, &fbitsize, &foffset)) goto finally; @@ -5365,7 +5218,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, else { PyErr_Format(PyExc_TypeError, "field '%s.%s' has ctype '%s' of unknown size", - ct->ct_name, PyText_AS_UTF8(fname), + ct->ct_name, PyUnicode_AsUTF8(fname), ftype->ct_name); goto finally; } @@ -5396,7 +5249,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, if (!(sflags & SF_GCC_ARM_BITFIELDS) && fbitsize >= 0) { if (!(sflags & SF_MSVC_BITFIELDS)) { /* GCC: anonymous bitfields (of any size) don't cause alignment */ - do_align = PyText_GetSize(fname) > 0; + do_align = PyUnicode_GetLength(fname) > 0; } else { /* MSVC: zero-sized bitfields don't cause alignment */ @@ -5435,12 +5288,12 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, except to know if we must set CT_CUSTOM_FIELD_POS */ if (detect_custom_layout(ct, sflags, byteoffset, foffset, "wrong offset for field '", - PyText_AS_UTF8(fname), "'") < 0) + PyUnicode_AsUTF8(fname), "'") < 0) goto finally; byteoffset = foffset; } - if (PyText_GetSize(fname) == 0 && + if (PyUnicode_GetLength(fname) == 0 && ftype->ct_flags & (CT_STRUCT|CT_UNION)) { /* a nested anonymous struct or union */ CFieldObject *cfsrc = (CFieldObject *)ftype->ct_extra; @@ -5482,7 +5335,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, PyErr_Format(PyExc_TypeError, "field '%s.%s' is a bitfield, " "but a fixed offset is specified", - ct->ct_name, PyText_AS_UTF8(fname)); + ct->ct_name, PyUnicode_AsUTF8(fname)); goto finally; } @@ -5491,7 +5344,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, CT_PRIMITIVE_CHAR))) { PyErr_Format(PyExc_TypeError, "field '%s.%s' declared as '%s' cannot be a bit field", - ct->ct_name, PyText_AS_UTF8(fname), + ct->ct_name, PyUnicode_AsUTF8(fname), ftype->ct_name); goto finally; } @@ -5499,7 +5352,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, PyErr_Format(PyExc_TypeError, "bit field '%s.%s' is declared '%s:%d', which " "exceeds the width of the type", - ct->ct_name, PyText_AS_UTF8(fname), + ct->ct_name, PyUnicode_AsUTF8(fname), ftype->ct_name, fbitsize); goto finally; } @@ -5511,10 +5364,10 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, field_offset_bytes &= ~(falign - 1); if (fbitsize == 0) { - if (PyText_GetSize(fname) > 0) { + if (PyUnicode_GetLength(fname) > 0) { PyErr_Format(PyExc_TypeError, "field '%s.%s' is declared with :0", - ct->ct_name, PyText_AS_UTF8(fname)); + ct->ct_name, PyUnicode_AsUTF8(fname)); goto finally; } if (!(sflags & SF_MSVC_BITFIELDS)) { @@ -5554,7 +5407,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, PyErr_Format(PyExc_NotImplementedError, "with 'packed', gcc would compile field " "'%s.%s' to reuse some bits in the previous " - "field", ct->ct_name, PyText_AS_UTF8(fname)); + "field", ct->ct_name, PyUnicode_AsUTF8(fname)); goto finally; } field_offset_bytes += falign; @@ -5600,7 +5453,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, if (sflags & SF_GCC_BIG_ENDIAN) bitshift = 8 * ftype->ct_size - fbitsize - bitshift; - if (PyText_GetSize(fname) > 0) { + if (PyUnicode_GetLength(fname) > 0) { *previous = _add_field(interned_fields, fname, ftype, field_offset_bytes, bitshift, fbitsize, @@ -6173,7 +6026,7 @@ static PyObject *new_function_type(PyObject *fargs, /* tuple */ fct->ct_stuff = PyTuple_New(2 + funcbuilder.nargs); if (fct->ct_stuff == NULL) goto error; - fabiobj = PyInt_FromLong(fabi); + fabiobj = PyLong_FromLong(fabi); if (fabiobj == NULL) goto error; PyTuple_SET_ITEM(fct->ct_stuff, 0, fabiobj); @@ -6284,7 +6137,6 @@ static void _my_PyErr_WriteUnraisable(PyObject *t, PyObject *v, PyObject *tb, char *extra_error_line) { /* like PyErr_WriteUnraisable(), but write a full traceback */ -#ifdef USE_WRITEUNRAISABLEMSG /* PyErr_WriteUnraisable actually writes the full traceback anyway from Python 3.4, but we can't really get the formatting of the @@ -6314,43 +6166,13 @@ static void _my_PyErr_WriteUnraisable(PyObject *t, PyObject *v, PyObject *tb, #if PY_VERSION_HEX >= 0x030D0000 PyErr_FormatUnraisable("Exception ignored %S", s); #else - _PyErr_WriteUnraisableMsg(PyText_AS_UTF8(s), NULL); + _PyErr_WriteUnraisableMsg(PyUnicode_AsUTF8(s), NULL); #endif Py_DECREF(s); } else PyErr_WriteUnraisable(obj); /* best effort */ PyErr_Clear(); - -#else - - /* version for Python 2.7 and < 3.8 */ - PyObject *f; -#if PY_MAJOR_VERSION >= 3 - /* jump through hoops to ensure the tb is attached to v, on Python 3 */ - PyErr_NormalizeException(&t, &v, &tb); - if (tb == NULL) { - tb = Py_None; - Py_INCREF(tb); - } - PyException_SetTraceback(v, tb); -#endif - f = PySys_GetObject("stderr"); - if (f != NULL) { - if (obj != NULL) { - PyFile_WriteString(objdescr, f); - PyFile_WriteObject(obj, f, 0); - PyFile_WriteString(":\n", f); - } - if (extra_error_line != NULL) - PyFile_WriteString(extra_error_line, f); - PyErr_Display(t, v, tb); - } - Py_XDECREF(t); - Py_XDECREF(v); - Py_XDECREF(tb); - -#endif } static void general_invoke_callback(int decode_args_from_libffi, @@ -6400,11 +6222,7 @@ static void general_invoke_callback(int decode_args_from_libffi, goto error; if (convert_from_object_fficallback(result, SIGNATURE(1), py_res, decode_args_from_libffi) < 0) { -#ifdef USE_WRITEUNRAISABLEMSG extra_error_line = ", trying to convert the result back to C"; -#else - extra_error_line = "Trying to convert the result back to C:\n"; -#endif goto error; } done: @@ -6456,16 +6274,9 @@ static void general_invoke_callback(int decode_args_from_libffi, _my_PyErr_WriteUnraisable(exc1, val1, tb1, "From cffi callback ", py_ob, extra_error_line); -#ifdef USE_WRITEUNRAISABLEMSG _my_PyErr_WriteUnraisable(exc2, val2, tb2, "during handling of the above exception by 'onerror'", NULL, NULL); -#else - extra_error_line = ("\nDuring the call to 'onerror', " - "another exception occurred:\n\n"); - _my_PyErr_WriteUnraisable(exc2, val2, tb2, - NULL, NULL, extra_error_line); -#endif _cffi_stop_error_capture(ecap); } } @@ -6533,14 +6344,6 @@ static PyObject *prepare_callback_info_tuple(CTypeDescrObject *ct, infotuple = Py_BuildValue("OOOO", ct, ob, py_rawerr, onerror_ob); Py_DECREF(py_rawerr); -#if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000 - /* We must setup the GIL here, in case the callback is invoked in - some other non-Pythonic thread. This is the same as ctypes. - But PyEval_InitThreads() is always a no-op from CPython 3.7 - (the call from ctypes was removed some time later I think). */ - PyEval_InitThreads(); -#endif - return infotuple; } @@ -6714,19 +6517,7 @@ static PyObject *b_new_enum_type(PyObject *self, PyObject *args) PyObject *value = PyTuple_GET_ITEM(enumvalues, i); tmpkey = PyTuple_GET_ITEM(enumerators, i); Py_INCREF(tmpkey); - if (!PyText_Check(tmpkey)) { -#if PY_MAJOR_VERSION < 3 - if (PyUnicode_Check(tmpkey)) { - const char *text = PyText_AsUTF8(tmpkey); - if (text == NULL) - goto error; - Py_DECREF(tmpkey); - tmpkey = PyString_FromString(text); - if (tmpkey == NULL) - goto error; - } - else -#endif + if (!PyUnicode_Check(tmpkey)) { { PyErr_SetString(PyExc_TypeError, "enumerators must be a list of strings"); @@ -6782,7 +6573,7 @@ static PyObject *b_alignof(PyObject *self, PyObject *arg) align = get_alignment((CTypeDescrObject *)arg); if (align < 0) return NULL; - return PyInt_FromLong(align); + return PyLong_FromLong(align); } static Py_ssize_t direct_sizeof_cdata(CDataObject *cd) @@ -6820,7 +6611,7 @@ static PyObject *b_sizeof(PyObject *self, PyObject *arg) "expected a 'cdata' or 'ctype' object"); return NULL; } - return PyInt_FromSsize_t(size); + return PyLong_FromSsize_t(size); } static PyObject *b_typeof(PyObject *self, PyObject *arg) @@ -6844,7 +6635,7 @@ static CTypeDescrObject *direct_typeoffsetof(CTypeDescrObject *ct, CTypeDescrObject *res; CFieldObject *cf; - if (PyTextAny_Check(fieldname)) { + if (PyUnicode_Check(fieldname)) { if (!following && (ct->ct_flags & CT_POINTER)) ct = ct->ct_itemdescr; if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) { @@ -6871,7 +6662,7 @@ static CTypeDescrObject *direct_typeoffsetof(CTypeDescrObject *ct, *offset = cf->cf_offset; } else { - Py_ssize_t index = PyInt_AsSsize_t(fieldname); + Py_ssize_t index = PyLong_AsSsize_t(fieldname); if (index < 0 && PyErr_Occurred()) { PyErr_SetString(PyExc_TypeError, "field name or array index expected"); @@ -6961,7 +6752,7 @@ static PyObject *b_getcname(PyObject *self, PyObject *args) memcpy(p, ct->ct_name + ct->ct_name_position, namelen - ct->ct_name_position); - return PyText_FromStringAndSize(s, namelen + replacelen); + return PyUnicode_FromStringAndSize(s, namelen + replacelen); } static PyObject *b_string(PyObject *self, PyObject *args, PyObject *kwds) @@ -6985,7 +6776,7 @@ static PyObject *b_string(PyObject *self, PyObject *args, PyObject *kwds) if (s != NULL) { PyErr_Format(PyExc_RuntimeError, "cannot use string() on %s", - PyText_AS_UTF8(s)); + PyUnicode_AsUTF8(s)); Py_DECREF(s); } return NULL; @@ -7105,7 +6896,7 @@ static PyObject *b_unpack(PyObject *self, PyObject *args, PyObject *kwds) if (s != NULL) { PyErr_Format(PyExc_RuntimeError, "cannot use unpack() on %s", - PyText_AS_UTF8(s)); + PyUnicode_AsUTF8(s)); Py_DECREF(s); } return NULL; @@ -7193,13 +6984,13 @@ static PyObject *b_unpack(PyObject *self, PyObject *args, PyObject *kwds) default: x = convert_to_object(src, ctitem); break; /* special cases for performance only */ - case 0: x = PyInt_FromLong(*(signed char *)src); break; - case 1: x = PyInt_FromLong(*(short *)src); break; - case 2: x = PyInt_FromLong(*(int *)src); break; - case 3: x = PyInt_FromLong(*(long *)src); break; - case 4: x = PyInt_FromLong(*(unsigned char *)src); break; - case 5: x = PyInt_FromLong(*(unsigned short *)src); break; - case 6: x = PyInt_FromLong((long)*(unsigned int *)src); break; + case 0: x = PyLong_FromLong(*(signed char *)src); break; + case 1: x = PyLong_FromLong(*(short *)src); break; + case 2: x = PyLong_FromLong(*(int *)src); break; + case 3: x = PyLong_FromLong(*(long *)src); break; + case 4: x = PyLong_FromLong(*(unsigned char *)src); break; + case 5: x = PyLong_FromLong(*(unsigned short *)src); break; + case 6: x = PyLong_FromLong((long)*(unsigned int *)src); break; case 7: x = PyLong_FromUnsignedLong(*(unsigned long *)src); break; case 8: x = PyFloat_FromDouble(*(float *)src); break; case 9: x = PyFloat_FromDouble(*(double *)src); break; @@ -7283,12 +7074,12 @@ static PyObject *b_get_errno(PyObject *self, PyObject *noarg) restore_errno_only(); err = errno; errno = 0; - return PyInt_FromLong(err); + return PyLong_FromLong(err); } static PyObject *b_set_errno(PyObject *self, PyObject *arg) { - long ival = PyInt_AsLong(arg); + long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) return NULL; else if (ival < INT_MIN || ival > INT_MAX) { @@ -7370,46 +7161,6 @@ static PyObject *b_from_handle(PyObject *self, PyObject *arg) static int _my_PyObject_GetContiguousBuffer(PyObject *x, Py_buffer *view, int writable_only) { -#if PY_MAJOR_VERSION < 3 - /* Some objects only support the buffer interface and CPython doesn't - translate it into the memoryview interface, mess. Hack a very - minimal content for 'view'. Don't care if the other fields are - uninitialized: we only call PyBuffer_Release(), which only reads - 'view->obj'. */ - PyBufferProcs *pb = x->ob_type->tp_as_buffer; - if (pb && !pb->bf_releasebuffer) { - /* we used to try all three in some vaguely sensible order, - i.e. first the write. But trying to call the write on a - read-only buffer fails with TypeError. So we use a less- - sensible order now. See test_from_buffer_more_cases. - - If 'writable_only', we only try bf_getwritebuffer. - */ - readbufferproc proc = NULL; - if (!writable_only) { - proc = (readbufferproc)pb->bf_getreadbuffer; - if (!proc) - proc = (readbufferproc)pb->bf_getcharbuffer; - } - if (!proc) - proc = (readbufferproc)pb->bf_getwritebuffer; - - if (proc && pb->bf_getsegcount) { - if ((*pb->bf_getsegcount)(x, NULL) != 1) { - PyErr_SetString(PyExc_TypeError, - "expected a single-segment buffer object"); - return -1; - } - view->len = (*proc)(x, 0, &view->buf); - if (view->len < 0) - return -1; - view->obj = x; - Py_INCREF(x); - return 0; - } - } -#endif - if (PyObject_GetBuffer(x, view, writable_only ? PyBUF_WRITABLE : PyBUF_SIMPLE) < 0) return -1; @@ -7857,30 +7608,6 @@ static PyObject *b__testfunc(PyObject *self, PyObject *args) return PyLong_FromVoidPtr(f); } -#if PY_MAJOR_VERSION < 3 -static Py_ssize_t _test_segcountproc(PyObject *o, Py_ssize_t *ignored) -{ - return 1; -} -static Py_ssize_t _test_getreadbuf(PyObject *o, Py_ssize_t i, void **r) -{ - static char buf[] = "RDB"; - *r = buf; - return 3; -} -static Py_ssize_t _test_getwritebuf(PyObject *o, Py_ssize_t i, void **r) -{ - static char buf[] = "WRB"; - *r = buf; - return 3; -} -static Py_ssize_t _test_getcharbuf(PyObject *o, Py_ssize_t i, char **r) -{ - static char buf[] = "CHB"; - *r = buf; - return 3; -} -#endif static int _test_getbuf(PyObject *self, Py_buffer *view, int flags) { static char buf[] = "GTB"; @@ -7903,14 +7630,6 @@ static PyObject *b__testbuff(PyObject *self, PyObject *args) assert(obj->tp_as_buffer != NULL); -#if PY_MAJOR_VERSION < 3 - obj->tp_as_buffer->bf_getsegcount = &_test_segcountproc; - obj->tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER; - obj->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; - if (methods & 1) obj->tp_as_buffer->bf_getreadbuffer = &_test_getreadbuf; - if (methods & 2) obj->tp_as_buffer->bf_getwritebuffer = &_test_getwritebuf; - if (methods & 4) obj->tp_as_buffer->bf_getcharbuffer = &_test_getcharbuf; -#endif if (methods & 8) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf; if (methods & 16) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf_ro; @@ -8052,7 +7771,7 @@ static PyObject *_cffi_get_struct_layout(Py_ssize_t nums[]) return NULL; while (--count >= 0) { - PyObject *o = PyInt_FromSsize_t(nums[count]); + PyObject *o = PyLong_FromSsize_t(nums[count]); if (o == NULL) { Py_DECREF(result); return NULL; @@ -8173,7 +7892,6 @@ static struct { const char *name; int value; } all_dlopen_flags[] = { /************************************************************/ -#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef FFIBackendModuleDef = { PyModuleDef_HEAD_INIT, "_cffi_backend", @@ -8186,12 +7904,6 @@ static struct PyModuleDef FFIBackendModuleDef = { PyMODINIT_FUNC PyInit__cffi_backend(void) -#else -#define INITERROR return - -PyMODINIT_FUNC -init_cffi_backend(void) -#endif { PyObject *m, *v; int i; @@ -8214,19 +7926,15 @@ init_cffi_backend(void) }; v = PySys_GetObject("version"); - if (v == NULL || !PyText_Check(v) || - strncmp(PyText_AS_UTF8(v), PY_VERSION, 3) != 0) { + if (v == NULL || !PyUnicode_Check(v) || + strncmp(PyUnicode_AsUTF8(v), PY_VERSION, 3) != 0) { PyErr_Format(PyExc_ImportError, "this module was compiled for Python %c%c%c", PY_VERSION[0], PY_VERSION[1], PY_VERSION[2]); INITERROR; } -#if PY_MAJOR_VERSION >= 3 m = PyModule_Create(&FFIBackendModuleDef); -#else - m = Py_InitModule("_cffi_backend", FFIBackendMethods); -#endif if (m == NULL) INITERROR; @@ -8259,11 +7967,11 @@ init_cffi_backend(void) } if (!init_done) { - v = PyText_FromString("_cffi_backend"); + v = PyUnicode_FromString("_cffi_backend"); if (v == NULL || PyDict_SetItemString(CData_Type.tp_dict, "__module__", v) < 0) INITERROR; - v = PyText_FromString("<cdata>"); + v = PyUnicode_FromString("<cdata>"); if (v == NULL || PyDict_SetItemString(CData_Type.tp_dict, "__name__", v) < 0) INITERROR; @@ -8275,7 +7983,7 @@ init_cffi_backend(void) if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0) INITERROR; - v = PyText_FromString(CFFI_VERSION); + v = PyUnicode_FromString(CFFI_VERSION); if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0) INITERROR; @@ -8312,9 +8020,7 @@ init_cffi_backend(void) if (init_ffi_lib(m) < 0) INITERROR; -#if PY_MAJOR_VERSION >= 3 if (init_file_emulator() < 0) INITERROR; return m; -#endif } diff --git a/contrib/python/cffi/py3/c/call_python.c b/contrib/python/cffi/py3/c/call_python.c index 8c6703cc26b..57e5f31402f 100644 --- a/contrib/python/cffi/py3/c/call_python.c +++ b/contrib/python/cffi/py3/c/call_python.c @@ -1,16 +1,7 @@ -#if PY_VERSION_HEX >= 0x03080000 -# define HAVE_PYINTERPSTATE_GETDICT -#endif - - static PyObject *_current_interp_key(void) { PyInterpreterState *interp = PyThreadState_GET()->interp; -#ifdef HAVE_PYINTERPSTATE_GETDICT return PyInterpreterState_GetDict(interp); /* shared reference */ -#else - return interp->modules; -#endif } static PyObject *_get_interpstate_dict(void) @@ -33,11 +24,7 @@ static PyObject *_get_interpstate_dict(void) } interp = tstate->interp; -#ifdef HAVE_PYINTERPSTATE_GETDICT interpdict = PyInterpreterState_GetDict(interp); /* shared reference */ -#else - interpdict = interp->builtins; -#endif if (interpdict == NULL) { /* subinterpreter was cleared already, or is being cleared right now, to a point that is too much for us to continue */ @@ -47,7 +34,7 @@ static PyObject *_get_interpstate_dict(void) /* from there on, we know the (sub-)interpreter is still valid */ if (attr_name == NULL) { - attr_name = PyText_InternFromString("__cffi_backend_extern_py"); + attr_name = PyUnicode_InternFromString("__cffi_backend_extern_py"); if (attr_name == NULL) goto error; } @@ -90,7 +77,7 @@ static PyObject *_ffi_def_extern_decorator(PyObject *outer_args, PyObject *fn) name = PyObject_GetAttrString(fn, "__name__"); if (name == NULL) return NULL; - s = PyText_AsUTF8(name); + s = PyUnicode_AsUTF8(name); if (s == NULL) { Py_DECREF(name); return NULL; diff --git a/contrib/python/cffi/py3/c/cdlopen.c b/contrib/python/cffi/py3/c/cdlopen.c index 7db7ae280fb..d07fcd25ae6 100644 --- a/contrib/python/cffi/py3/c/cdlopen.c +++ b/contrib/python/cffi/py3/c/cdlopen.c @@ -7,7 +7,7 @@ static void *cdlopen_fetch(PyObject *libname, void *libhandle, if (libhandle == NULL) { PyErr_Format(FFIError, "library '%s' has been closed", - PyText_AS_UTF8(libname)); + PyUnicode_AsUTF8(libname)); return NULL; } @@ -16,7 +16,7 @@ static void *cdlopen_fetch(PyObject *libname, void *libhandle, if (address == NULL) { const char *error = dlerror(); PyErr_Format(FFIError, "symbol '%s' not found in library '%s': %s", - symbol, PyText_AS_UTF8(libname), error); + symbol, PyUnicode_AsUTF8(libname), error); } return address; } @@ -32,7 +32,7 @@ static int cdlopen_close(PyObject *libname, void *libhandle) if (libhandle != NULL && dlclose(libhandle) != 0) { const char *error = dlerror(); PyErr_Format(FFIError, "closing library '%s': %s", - PyText_AS_UTF8(libname), error); + PyUnicode_AsUTF8(libname), error); return -1; } return 0; @@ -195,13 +195,6 @@ static int ffiobj_init(PyObject *self, PyObject *args, PyObject *kwds) _CFFI_GETOP(nglobs[i].type_op) == _CFFI_OP_ENUM) { PyObject *o = PyTuple_GET_ITEM(globals, i * 2 + 1); nglobs[i].address = &_cdl_realize_global_int; -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(o)) { - nintconsts[i].neg = PyInt_AS_LONG(o) <= 0; - nintconsts[i].value = (long long)PyInt_AS_LONG(o); - } - else -#endif { nintconsts[i].neg = PyObject_RichCompareBool(o, Py_False, Py_LE); diff --git a/contrib/python/cffi/py3/c/cffi1_module.c b/contrib/python/cffi/py3/c/cffi1_module.c index 70d9ee8e0fe..7c16b2b7073 100644 --- a/contrib/python/cffi/py3/c/cffi1_module.c +++ b/contrib/python/cffi/py3/c/cffi1_module.c @@ -46,7 +46,7 @@ static int init_ffi_lib(PyObject *m) return -1; for (i = 0; all_dlopen_flags[i].name != NULL; i++) { - x = PyInt_FromLong(all_dlopen_flags[i].value); + x = PyLong_FromLong(all_dlopen_flags[i].value); if (x == NULL) return -1; res = PyDict_SetItemString(FFI_Type.tp_dict, @@ -116,7 +116,6 @@ static int make_included_tuples(char *module_name, static PyObject *_my_Py_InitModule(char *module_name) { -#if PY_MAJOR_VERSION >= 3 struct PyModuleDef *module_def, local_module_def = { PyModuleDef_HEAD_INIT, module_name, @@ -137,9 +136,6 @@ static PyObject *_my_Py_InitModule(char *module_name) } # endif return m; -#else - return Py_InitModule(module_name, NULL); -#endif } static PyObject *b_init_cffi_1_0_external_module(PyObject *self, PyObject *arg) @@ -211,12 +207,10 @@ static PyObject *b_init_cffi_1_0_external_module(PyObject *self, PyObject *arg) (PyObject *)lib) < 0) return NULL; -#if PY_MAJOR_VERSION >= 3 /* add manually 'module_name' in sys.modules: it seems that Py_InitModule() is not enough to do that */ if (PyDict_SetItemString(modules_dict, module_name, m) < 0) return NULL; -#endif return m; } diff --git a/contrib/python/cffi/py3/c/cglob.c b/contrib/python/cffi/py3/c/cglob.c index e97767c9d75..6c6fd4bae2e 100644 --- a/contrib/python/cffi/py3/c/cglob.c +++ b/contrib/python/cffi/py3/c/cglob.c @@ -74,7 +74,7 @@ static void *fetch_global_var_addr(GlobSupportObject *gs) } if (data == NULL) { PyErr_Format(FFIError, "global variable '%s' is at address NULL", - PyText_AS_UTF8(gs->gs_name)); + PyUnicode_AsUTF8(gs->gs_name)); return NULL; } return data; diff --git a/contrib/python/cffi/py3/c/commontypes.c b/contrib/python/cffi/py3/c/commontypes.c index 2337bf99447..a15c3ae87b2 100644 --- a/contrib/python/cffi/py3/c/commontypes.c +++ b/contrib/python/cffi/py3/c/commontypes.c @@ -205,7 +205,7 @@ static PyObject *b__get_common_types(PyObject *self, PyObject *arg) size_t i; for (i = 0; i < num_common_simple_types; i++) { const char *s = common_simple_types[i]; - PyObject *o = PyText_FromString(s + strlen(s) + 1); + PyObject *o = PyUnicode_FromString(s + strlen(s) + 1); if (o == NULL) return NULL; err = PyDict_SetItemString(arg, s, o); diff --git a/contrib/python/cffi/py3/c/ffi_obj.c b/contrib/python/cffi/py3/c/ffi_obj.c index 0c4933eeff6..fbba28941ce 100644 --- a/contrib/python/cffi/py3/c/ffi_obj.c +++ b/contrib/python/cffi/py3/c/ffi_obj.c @@ -185,13 +185,13 @@ static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg, /* Returns the CTypeDescrObject from the user-supplied 'arg'. Does not return a new reference! */ - if ((accept & ACCEPT_STRING) && PyText_Check(arg)) { + if ((accept & ACCEPT_STRING) && PyUnicode_Check(arg)) { PyObject *types_dict = ffi->types_builder.types_dict; /* The types_dict keeps the reference alive. Items are never removed */ PyObject *x = PyDict_GetItem(types_dict, arg); if (x == NULL) { - const char *input_text = PyText_AS_UTF8(arg); + const char *input_text = PyUnicode_AsUTF8(arg); struct _cffi_parse_info_s info; info.ctx = &ffi->types_builder.ctx; info.output_size = FFI_COMPLEXITY_OUTPUT; @@ -241,17 +241,6 @@ static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg, else if ((accept & ACCEPT_CDATA) && CData_Check(arg)) { return ((CDataObject *)arg)->c_type; } -#if PY_MAJOR_VERSION < 3 - else if (PyUnicode_Check(arg)) { - CTypeDescrObject *result; - arg = PyUnicode_AsASCIIString(arg); - if (arg == NULL) - return NULL; - result = _ffi_type(ffi, arg, accept); - Py_DECREF(arg); - return result; - } -#endif else { const char *m1 = (accept & ACCEPT_STRING) ? "string" : ""; const char *m2 = (accept & ACCEPT_CTYPE) ? "ctype object" : ""; @@ -291,7 +280,7 @@ static PyObject *ffi_sizeof(FFIObject *self, PyObject *arg) return NULL; } } - return PyInt_FromSsize_t(size); + return PyLong_FromSsize_t(size); } PyDoc_STRVAR(ffi_alignof_doc, @@ -308,7 +297,7 @@ static PyObject *ffi_alignof(FFIObject *self, PyObject *arg) align = get_alignment(ct); if (align < 0) return NULL; - return PyInt_FromLong(align); + return PyLong_FromLong(align); } PyDoc_STRVAR(ffi_typeof_doc, @@ -526,7 +515,7 @@ static PyObject *ffi_offsetof(FFIObject *self, PyObject *args) return NULL; offset += ofs1; } - return PyInt_FromSsize_t(offset); + return PyLong_FromSsize_t(offset); } PyDoc_STRVAR(ffi_addressof_doc, @@ -639,9 +628,7 @@ static PyObject *ffi_getctype(FFIObject *self, PyObject *args, PyObject *kwds) CTypeDescrObject *ct; size_t replace_with_len; static char *keywords[] = {"cdecl", "replace_with", NULL}; -#if PY_MAJOR_VERSION >= 3 PyObject *u; -#endif if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:getctype", keywords, &c_decl, &replace_with)) @@ -675,14 +662,12 @@ static PyObject *ffi_getctype(FFIObject *self, PyObject *args, PyObject *kwds) if (add_paren) p[replace_with_len] = ')'; -#if PY_MAJOR_VERSION >= 3 /* bytes -> unicode string */ u = PyUnicode_DecodeLatin1(PyBytes_AS_STRING(res), PyBytes_GET_SIZE(res), NULL); Py_DECREF(res); res = u; -#endif return res; } @@ -931,7 +916,7 @@ static PyObject *ffi_list_types(FFIObject *self, PyObject *noargs) goto error; for (i = 0; i < n1; i++) { - o = PyText_FromString(self->types_builder.ctx.typenames[i].name); + o = PyUnicode_FromString(self->types_builder.ctx.typenames[i].name); if (o == NULL) goto error; PyList_SET_ITEM(lst[0], i, o); @@ -945,7 +930,7 @@ static PyObject *ffi_list_types(FFIObject *self, PyObject *noargs) if (s->name[0] == '$') continue; - o = PyText_FromString(s->name); + o = PyUnicode_FromString(s->name); if (o == NULL) goto error; index = (s->flags & _CFFI_F_UNION) ? 2 : 1; @@ -990,14 +975,6 @@ PyDoc_STRVAR(ffi_init_once_doc, "of function() is done. If function() raises an exception, it is\n" "propagated and nothing is cached."); -#if PY_MAJOR_VERSION < 3 -/* PyCapsule_New is redefined to be PyCObject_FromVoidPtr in _cffi_backend, - which gives 2.6 compatibility; but the destructor signature is different */ -static void _free_init_once_lock(void *lock) -{ - PyThread_free_lock((PyThread_type_lock)lock); -} -#else static void _free_init_once_lock(PyObject *capsule) { PyThread_type_lock lock; @@ -1005,7 +982,6 @@ static void _free_init_once_lock(PyObject *capsule) if (lock != NULL) PyThread_free_lock(lock); } -#endif static PyObject *ffi_init_once(FFIObject *self, PyObject *args, PyObject *kwds) { diff --git a/contrib/python/cffi/py3/c/file_emulator.h b/contrib/python/cffi/py3/c/file_emulator.h index 82a34c0c72f..bfdcdeceac2 100644 --- a/contrib/python/cffi/py3/c/file_emulator.h +++ b/contrib/python/cffi/py3/c/file_emulator.h @@ -51,7 +51,7 @@ static FILE *PyFile_AsFile(PyObject *ob_file) ob_mode = PyObject_GetAttrString(ob_file, "mode"); if (ob_mode == NULL) goto fail; - mode = PyText_AsUTF8(ob_mode); + mode = PyUnicode_AsUTF8(ob_mode); if (mode == NULL) goto fail; diff --git a/contrib/python/cffi/py3/c/lib_obj.c b/contrib/python/cffi/py3/c/lib_obj.c index 446b225cb5d..946db3b39dd 100644 --- a/contrib/python/cffi/py3/c/lib_obj.c +++ b/contrib/python/cffi/py3/c/lib_obj.c @@ -123,8 +123,8 @@ static int lib_traverse(LibObject *lib, visitproc visit, void *arg) static PyObject *lib_repr(LibObject *lib) { - return PyText_FromFormat("<Lib object for '%.200s'>", - PyText_AS_UTF8(lib->l_libname)); + return PyUnicode_FromFormat("<Lib object for '%.200s'>", + PyUnicode_AsUTF8(lib->l_libname)); } static PyObject *lib_build_cpython_func(LibObject *lib, @@ -143,7 +143,7 @@ static PyObject *lib_build_cpython_func(LibObject *lib, int i, type_index = _CFFI_GETARG(g->type_op); _cffi_opcode_t *opcodes = lib->l_types_builder->ctx.types; static const char *const format = ";\n\nCFFI C function from %s.lib"; - const char *libname = PyText_AS_UTF8(lib->l_libname); + const char *libname = PyUnicode_AsUTF8(lib->l_libname); struct funcbuilder_s funcbuilder; /* return type: */ @@ -226,7 +226,7 @@ static PyObject *lib_build_and_cache_attr(LibObject *lib, PyObject *name, const struct _cffi_global_s *g; CTypeDescrObject *ct; builder_c_t *types_builder = lib->l_types_builder; - const char *s = PyText_AsUTF8(name); + const char *s = PyUnicode_AsUTF8(name); if (s == NULL) return NULL; @@ -281,7 +281,7 @@ static PyObject *lib_build_and_cache_attr(LibObject *lib, PyObject *name, PyErr_Format(PyExc_AttributeError, "cffi library '%.200s' has no function, constant " "or global variable named '%.200s'", - PyText_AS_UTF8(lib->l_libname), s); + PyUnicode_AsUTF8(lib->l_libname), s); return NULL; } @@ -478,7 +478,7 @@ static PyObject *_lib_dir1(LibObject *lib, int ignore_global_vars) if (op == _CFFI_OP_GLOBAL_VAR || op == _CFFI_OP_GLOBAL_VAR_F) continue; } - s = PyText_FromString(g[i].name); + s = PyUnicode_FromString(g[i].name); if (s == NULL) goto error; PyList_SET_ITEM(lst, count, s); @@ -502,7 +502,7 @@ static PyObject *_lib_dict(LibObject *lib) return NULL; for (i = 0; i < total; i++) { - name = PyText_FromString(g[i].name); + name = PyUnicode_FromString(g[i].name); if (name == NULL) goto error; @@ -534,7 +534,7 @@ static PyObject *lib_getattr(LibObject *lib, PyObject *name) missing: /*** ATTRIBUTEERROR IS SET HERE ***/ - p = PyText_AsUTF8(name); + p = PyUnicode_AsUTF8(name); if (p == NULL) return NULL; if (strcmp(p, "__all__") == 0) { @@ -558,16 +558,14 @@ static PyObject *lib_getattr(LibObject *lib, PyObject *name) module-like behavior */ if (strcmp(p, "__name__") == 0) { PyErr_Clear(); - return PyText_FromFormat("%s.lib", PyText_AS_UTF8(lib->l_libname)); + return PyUnicode_FromFormat("%s.lib", PyUnicode_AsUTF8(lib->l_libname)); } -#if PY_MAJOR_VERSION >= 3 if (strcmp(p, "__loader__") == 0 || strcmp(p, "__spec__") == 0) { /* some more module-like behavior hacks */ PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } -#endif return NULL; } @@ -587,7 +585,7 @@ static int lib_setattr(LibObject *lib, PyObject *name, PyObject *val) PyErr_Format(PyExc_AttributeError, "cannot write to function or constant '%.200s'", - PyText_Check(name) ? PyText_AS_UTF8(name) : "?"); + PyUnicode_Check(name) ? PyUnicode_AsUTF8(name) : "?"); return -1; } @@ -645,7 +643,7 @@ static LibObject *lib_internal_new(FFIObject *ffi, const char *module_name, LibObject *lib; PyObject *libname, *dict; - libname = PyText_FromString(module_name); + libname = PyUnicode_FromString(module_name); if (libname == NULL) goto err1; @@ -712,7 +710,7 @@ static PyObject *address_of_global_var(PyObject *args) /* rebuild a string from 'varname', to do typechecks and to force a unicode back to a plain string (on python 2) */ - o_varname = PyText_FromString(varname); + o_varname = PyUnicode_FromString(varname); if (o_varname == NULL) return NULL; diff --git a/contrib/python/cffi/py3/c/minibuffer.h b/contrib/python/cffi/py3/c/minibuffer.h index d92da317ade..89e400fa069 100644 --- a/contrib/python/cffi/py3/c/minibuffer.h +++ b/contrib/python/cffi/py3/c/minibuffer.h @@ -50,7 +50,7 @@ static int mb_ass_item(MiniBufferObj *self, Py_ssize_t idx, PyObject *other) } else { PyErr_Format(PyExc_TypeError, - "must assign a "STR_OR_BYTES + "must assign a bytes" " of length 1, not %.200s", Py_TYPE(other)->tp_name); return -1; } @@ -85,29 +85,6 @@ static int mb_ass_slice(MiniBufferObj *self, return 0; } -#if PY_MAJOR_VERSION < 3 -static Py_ssize_t mb_getdata(MiniBufferObj *self, Py_ssize_t idx, void **pp) -{ - *pp = self->mb_data; - return self->mb_size; -} - -static Py_ssize_t mb_getsegcount(MiniBufferObj *self, Py_ssize_t *lenp) -{ - if (lenp) - *lenp = self->mb_size; - return 1; -} - -static PyObject *mb_str(MiniBufferObj *self) -{ - /* Python 2: we want str(buffer) to behave like buffer[:], because - that's what bytes(buffer) does on Python 3 and there is no way - we can prevent this. */ - return PyString_FromStringAndSize(self->mb_data, self->mb_size); -} -#endif - static int mb_getbuf(MiniBufferObj *self, Py_buffer *view, int flags) { return PyBuffer_FillInfo(view, (PyObject *)self, @@ -126,12 +103,6 @@ static PySequenceMethods mb_as_sequence = { }; static PyBufferProcs mb_as_buffer = { -#if PY_MAJOR_VERSION < 3 - (readbufferproc)mb_getdata, - (writebufferproc)mb_getdata, - (segcountproc)mb_getsegcount, - (charbufferproc)mb_getdata, -#endif (getbufferproc)mb_getbuf, (releasebufferproc)0, }; @@ -234,16 +205,8 @@ mb_richcompare(PyObject *self, PyObject *other, int op) return res; } -#if PY_MAJOR_VERSION >= 3 /* pfffffffffffff pages of copy-paste from listobject.c */ -/* pfffffffffffff#2: the PySlice_GetIndicesEx() *macro* should not - be called, because C extension modules compiled with it differ - on ABI between 3.6.0, 3.6.1 and 3.6.2. */ -#if PY_VERSION_HEX < 0x03070000 && defined(PySlice_GetIndicesEx) && !defined(PYPY_VERSION) -#undef PySlice_GetIndicesEx -#endif - static PyObject *mb_subscript(MiniBufferObj *self, PyObject *item) { if (PyIndex_Check(item)) { @@ -280,6 +243,11 @@ static PyObject *mb_subscript(MiniBufferObj *self, PyObject *item) static int mb_ass_subscript(MiniBufferObj* self, PyObject* item, PyObject* value) { + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "'del x[n]' not supported for buffer objects"); + return -1; + } if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) @@ -317,13 +285,6 @@ static PyMappingMethods mb_as_mapping = { (binaryfunc)mb_subscript, /*mp_subscript*/ (objobjargproc)mb_ass_subscript, /*mp_ass_subscript*/ }; -#endif - -#if PY_MAJOR_VERSION >= 3 -# define MINIBUF_TPFLAGS 0 -#else -# define MINIBUF_TPFLAGS (Py_TPFLAGS_HAVE_GETCHARBUFFER | Py_TPFLAGS_HAVE_NEWBUFFER) -#endif PyDoc_STRVAR(ffi_buffer_doc, "ffi.buffer(cdata[, byte_size]):\n" @@ -354,23 +315,14 @@ static PyTypeObject MiniBuffer_Type = { 0, /* tp_repr */ 0, /* tp_as_number */ &mb_as_sequence, /* tp_as_sequence */ -#if PY_MAJOR_VERSION < 3 - 0, /* tp_as_mapping */ -#else &mb_as_mapping, /* tp_as_mapping */ -#endif 0, /* tp_hash */ 0, /* tp_call */ -#if PY_MAJOR_VERSION < 3 - (reprfunc)mb_str, /* tp_str */ -#else 0, /* tp_str */ -#endif PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &mb_as_buffer, /* tp_as_buffer */ - (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - MINIBUF_TPFLAGS), /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ ffi_buffer_doc, /* tp_doc */ (traverseproc)mb_traverse, /* tp_traverse */ (inquiry)mb_clear, /* tp_clear */ diff --git a/contrib/python/cffi/py3/c/misc_thread_common.h b/contrib/python/cffi/py3/c/misc_thread_common.h index b0710acceec..766dcb2ad7d 100644 --- a/contrib/python/cffi/py3/c/misc_thread_common.h +++ b/contrib/python/cffi/py3/c/misc_thread_common.h @@ -328,20 +328,12 @@ static void restore_errno_only(void) It was added in 3.5.2 but should never be used in 3.5.x because it is not available in 3.5.0 or 3.5.1. */ -#if PY_VERSION_HEX >= 0x03050100 && PY_VERSION_HEX < 0x03060000 -PyAPI_DATA(void *volatile) _PyThreadState_Current; -#endif - static PyThreadState *get_current_ts(void) { #if PY_VERSION_HEX >= 0x030D0000 return PyThreadState_GetUnchecked(); -#elif PY_VERSION_HEX >= 0x03060000 - return _PyThreadState_UncheckedGet(); -#elif defined(_Py_atomic_load_relaxed) - return (PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current); #else - return (PyThreadState*)_PyThreadState_Current; /* assume atomic read */ + return _PyThreadState_UncheckedGet(); #endif } diff --git a/contrib/python/cffi/py3/c/misc_win32.h b/contrib/python/cffi/py3/c/misc_win32.h index 90c63adc68c..4fe723a727d 100644 --- a/contrib/python/cffi/py3/c/misc_win32.h +++ b/contrib/python/cffi/py3/c/misc_win32.h @@ -64,7 +64,6 @@ static void restore_errno(void) /************************************************************/ -#if PY_MAJOR_VERSION >= 3 static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds) { int err = -1; @@ -113,54 +112,6 @@ static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds) LocalFree(s_buf); return v; } -#else -static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds) -{ - int err = -1; - int len; - char *s; - char *s_buf = NULL; /* Free via LocalFree */ - char s_small_buf[40]; /* Room for "Windows Error 0xFFFFFFFFFFFFFFFF" */ - PyObject *v; - static char *keywords[] = {"code", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i", keywords, &err)) - return NULL; - - if (err == -1) { - struct cffi_tls_s *p = get_cffi_tls(); - if (p == NULL) - return PyErr_NoMemory(); - err = p->saved_lasterror; - } - - len = FormatMessage( - /* Error API error */ - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, /* no message source */ - err, - MAKELANGID(LANG_NEUTRAL, - SUBLANG_DEFAULT), /* Default language */ - (LPTSTR) &s_buf, - 0, /* size not used */ - NULL); /* no args */ - if (len==0) { - /* Only seen this in out of mem situations */ - sprintf(s_small_buf, "Windows Error 0x%X", err); - s = s_small_buf; - } else { - s = s_buf; - /* remove trailing cr/lf and dots */ - while (len > 0 && (s[len-1] <= ' ' || s[len-1] == '.')) - s[--len] = '\0'; - } - v = Py_BuildValue("(is)", err, s); - LocalFree(s_buf); - return v; -} -#endif /************************************************************/ @@ -251,7 +202,9 @@ static int cffi_atomic_compare_exchange(void **ptr, void **expected, static void *cffi_atomic_load(void **ptr) { -#if defined(_M_X64) || defined(_M_IX86) +#if defined(__GNUC__) || defined(__clang__) + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +#elif defined(_M_X64) || defined(_M_IX86) return *(volatile void **)ptr; #elif defined(_M_ARM64) return (void *)__ldar64((volatile unsigned __int64 *)ptr); @@ -262,7 +215,9 @@ static void *cffi_atomic_load(void **ptr) static uint8_t cffi_atomic_load_uint8(uint8_t *ptr) { -#if defined(_M_X64) || defined(_M_IX86) +#if defined(__GNUC__) || defined(__clang__) + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +#elif defined(_M_X64) || defined(_M_IX86) return *(volatile uint8_t *)ptr; #elif defined(_M_ARM64) return (uint8_t)__ldar8((volatile uint8_t *)ptr); @@ -273,7 +228,9 @@ static uint8_t cffi_atomic_load_uint8(uint8_t *ptr) static Py_ssize_t cffi_atomic_load_ssize(Py_ssize_t *ptr) { -#if defined(_M_X64) || defined(_M_IX86) +#if defined(__GNUC__) || defined(__clang__) + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +#elif defined(_M_X64) || defined(_M_IX86) return *(volatile Py_ssize_t *)ptr; #elif defined(_M_ARM64) return (Py_ssize_t)__ldar64((volatile unsigned __int64 *)ptr); @@ -284,17 +241,29 @@ static Py_ssize_t cffi_atomic_load_ssize(Py_ssize_t *ptr) static void cffi_atomic_store_ssize(Py_ssize_t *ptr, Py_ssize_t value) { +#if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +#else _InterlockedExchangePointer(ptr, value); +#endif } static void cffi_atomic_store(void **ptr, void *value) { +#if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +#else _InterlockedExchangePointer(ptr, value); +#endif } static void cffi_atomic_store_uint8(uint8_t *ptr, uint8_t value) { +#if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +#else _InterlockedExchange8(ptr, value); +#endif } #endif /* CFFI_MISC_WIN32_H */ diff --git a/contrib/python/cffi/py3/c/realize_c_type.c b/contrib/python/cffi/py3/c/realize_c_type.c index fbfa68ca748..993a07d122f 100644 --- a/contrib/python/cffi/py3/c/realize_c_type.c +++ b/contrib/python/cffi/py3/c/realize_c_type.c @@ -29,7 +29,6 @@ static PyObject *build_primitive_type(int num); /* forward */ static int init_global_types_dict(PyObject *ffi_type_dict) { int err; - Py_ssize_t i; PyObject *ct_void, *ct_char, *ct2, *pnull; /* XXX some leaks in case these functions fail, but well, MemoryErrors during importing an extension module are kind @@ -78,7 +77,7 @@ static int init_global_types_dict(PyObject *ffi_type_dict) #ifdef Py_GIL_DISABLED // Ensure that all primitive types are initialised to avoid race conditions // on the first access. - for (i = 0; i < _CFFI__NUM_PRIM; i++) { + for (Py_ssize_t i = 0; i < _CFFI__NUM_PRIM; i++) { ct2 = get_primitive_type(i); if (ct2 == NULL) return -1; @@ -247,13 +246,13 @@ static PyObject *realize_global_int(builder_c_t *builder, int gindex) case 0: if (value <= (unsigned long long)LONG_MAX) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromUnsignedLongLong(value); case 1: if ((long long)value >= (long long)LONG_MIN) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromLongLong((long long)value); @@ -551,7 +550,7 @@ realize_c_type_or_func_now(builder_c_t *builder, _cffi_opcode_t op, j = 0; while (p[j] != ',' && p[j] != '\0') j++; - tmp = PyText_FromStringAndSize(p, j); + tmp = PyUnicode_FromStringAndSize(p, j); if (tmp == NULL) break; PyTuple_SET_ITEM(enumerators, i, tmp); diff --git a/contrib/python/cffi/py3/cffi/__init__.py b/contrib/python/cffi/py3/cffi/__init__.py index c99ec3d4817..da1875cc253 100644 --- a/contrib/python/cffi/py3/cffi/__init__.py +++ b/contrib/python/cffi/py3/cffi/__init__.py @@ -5,8 +5,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "2.0.0" -__version_info__ = (2, 0, 0) +__version__ = "2.1.0" +__version_info__ = (2, 1, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/contrib/python/cffi/py3/cffi/_cffi_errors.h b/contrib/python/cffi/py3/cffi/_cffi_errors.h index 158e0590346..0931379ccd2 100644 --- a/contrib/python/cffi/py3/cffi/_cffi_errors.h +++ b/contrib/python/cffi/py3/cffi/_cffi_errors.h @@ -36,11 +36,7 @@ static PyObject *_cffi_start_error_capture(void) if (result == NULL) goto error; -#if PY_MAJOR_VERSION >= 3 bi = PyImport_ImportModule("builtins"); -#else - bi = PyImport_ImportModule("__builtin__"); -#endif if (bi == NULL) goto error; PyDict_SetItemString(result, "__builtins__", bi); @@ -81,15 +77,9 @@ static PyObject *_cffi_start_error_capture(void) static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored) { Sleep(666); /* may be interrupted if the whole process is closing */ -#if PY_MAJOR_VERSION >= 3 MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text, L"Python-CFFI error", MB_OK | MB_ICONERROR); -#else - MessageBoxA(NULL, (char *)_cffi_bootstrap_text, - "Python-CFFI error", - MB_OK | MB_ICONERROR); -#endif _cffi_bootstrap_text = NULL; return 0; } @@ -111,11 +101,7 @@ static void _cffi_stop_error_capture(PyObject *ecap) /* Show a dialog box, but in a background thread, and never show multiple dialog boxes at once. */ -#if PY_MAJOR_VERSION >= 3 text = PyUnicode_AsWideCharString(s, NULL); -#else - text = PyString_AsString(s); -#endif _cffi_bootstrap_text = text; diff --git a/contrib/python/cffi/py3/cffi/_cffi_gen_src.py b/contrib/python/cffi/py3/cffi/_cffi_gen_src.py new file mode 100644 index 00000000000..3d4fb7e3d9c --- /dev/null +++ b/contrib/python/cffi/py3/cffi/_cffi_gen_src.py @@ -0,0 +1,216 @@ +# Integrated from the cffi-buildtool project by Rose Davidson +# (https://github.com/inklesspen/cffi-buildtool), under the following +# license: +# +# MIT License +# +# Copyright (c) 2024, Rose Davidson +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice (including the +# next paragraph) shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Implementation of the ``cffi-gen-src`` command-line tool. + +This module is private; the command line is the only supported +interface. Two subcommands: + +``exec-python`` + Execute a Python script that constructs a :class:`cffi.FFI` + (the same kind of script that the CFFI docs' "Main mode of usage" + describes) and emit the generated C source. + +``read-sources`` + Create the :class:`cffi.FFI` from a separate ``cdef`` file and C + source prelude, then emit the generated C source. +""" + +import argparse +import io +import os +import sys + +from .api import FFI + + +def _execfile(pysrc, filename, globs): + compiled = compile(source=pysrc, filename=filename, mode='exec') + exec(compiled, globs, globs) + + +def find_ffi_in_python_script(pysrc, filename, ffivar): + """Execute ``pysrc`` and return the :class:`FFI` object it defines. + + The script is executed with ``__name__`` set to ``"cffi.gen_src"``, + so a trailing ``if __name__ == "__main__": ffibuilder.compile()`` + block in the script is skipped. + + ``ffivar`` is the name bound by the script to the :class:`FFI` + object, or to a callable that returns one. + + Raises :class:`NameError` if the name is not bound by the script, + or :class:`TypeError` if the name does not resolve to an + :class:`FFI` instance. + """ + filename = os.path.abspath(filename) + globs = {'__name__': 'cffi.gen_src', '__file__': filename} + old_path = sys.path[:] + sys.path.insert(0, os.path.dirname(filename)) + try: + _execfile(pysrc, filename, globs) + if ffivar not in globs: + raise NameError( + "Expected to find the FFI object with the name %r, " + "but it was not found." % (ffivar,) + ) + ffi = globs[ffivar] + if not isinstance(ffi, FFI) and callable(ffi): + # Maybe it's a callable that returns a FFI + ffi = ffi() + if not isinstance(ffi, FFI): + raise TypeError( + "Found an object with the name %r but it was not an " + "instance of cffi.api.FFI" % (ffivar,) + ) + return ffi + finally: + sys.path[:] = old_path + + +def make_ffi_from_sources(modulename, cdef, csrc): + """Create an :class:`FFI` from ``cdef`` text and a C source prelude.""" + ffibuilder = FFI() + ffibuilder.cdef(cdef) + ffibuilder.set_source(modulename, csrc) + return ffibuilder + + +def generate_c_source(ffi): + """Return the C source that :meth:`FFI.emit_c_code` would write.""" + output = io.StringIO() + ffi.emit_c_code(output) + return output.getvalue() + + +def write_c_source(output, generated): + if output == '-': + sys.stdout.write(generated) + return + with open(output, 'w', encoding='utf-8') as f: + f.write(generated) + + +def same_input_file(file1, file2): + if file1 is file2: + return True + try: + return os.path.samefile(file1.name, file2.name) + except OSError: + return False + + +def exec_python(*, output, pyfile, ffi_var): + with pyfile: + ffi = find_ffi_in_python_script(pyfile.read(), pyfile.name, ffi_var) + generated = generate_c_source(ffi) + write_c_source(output, generated) + + +def read_sources(*, output, module_name, cdef_input, csrc_input): + with csrc_input, cdef_input: + csrc = csrc_input.read() + cdef = cdef_input.read() + ffi = make_ffi_from_sources(module_name, cdef, csrc) + generated = generate_c_source(ffi) + write_c_source(output, generated) + + +def _prog(): + # The same parser serves both documented invocations; make --help + # and usage messages show the one that was actually used. + argv0 = os.path.basename(sys.argv[0]) if sys.argv else '' + if argv0.startswith('cffi-gen-src'): + return 'cffi-gen-src' + return 'python -m cffi.gen_src' + + +parser = argparse.ArgumentParser( + prog=_prog(), + description='Generate CFFI C source for extension modules.', +) +subparsers = parser.add_subparsers(dest='mode') + +exec_python_parser = subparsers.add_parser( + 'exec-python', + help='Execute a Python script that defines an FFI object', +) +exec_python_parser.add_argument( + '--ffi-var', + default='ffibuilder', + help="Name of the FFI object in the Python script; defaults to 'ffibuilder'.", +) +exec_python_parser.add_argument( + 'pyfile', + type=argparse.FileType('r', encoding='utf-8'), + help='Path to the Python script', +) +exec_python_parser.add_argument( + 'output', + help='Output path for the C source', +) + +read_sources_parser = subparsers.add_parser( + 'read-sources', + help='Read cdef and C source prelude files that define an FFI object', +) +read_sources_parser.add_argument( + 'module_name', + help='Full name of the generated module, including packages', +) +read_sources_parser.add_argument( + 'cdef', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C definitions', +) +read_sources_parser.add_argument( + 'csrc', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C source prelude', +) +read_sources_parser.add_argument( + 'output', + help='Output path for the C source', +) + + +def run(args=None): + args = parser.parse_args(args=args) + if args.mode == 'exec-python': + exec_python(output=args.output, pyfile=args.pyfile, ffi_var=args.ffi_var) + elif args.mode == 'read-sources': + if same_input_file(args.cdef, args.csrc): + parser.error('cdef and csrc are the same file and should not be') + read_sources( + output=args.output, + module_name=args.module_name, + cdef_input=args.cdef, + csrc_input=args.csrc, + ) + else: + parser.error('a subcommand is required: exec-python or read-sources') + parser.exit(0) diff --git a/contrib/python/cffi/py3/cffi/_cffi_include.h b/contrib/python/cffi/py3/cffi/_cffi_include.h index 908a1d7343f..c8c6d343bcc 100644 --- a/contrib/python/cffi/py3/cffi/_cffi_include.h +++ b/contrib/python/cffi/py3/cffi/_cffi_include.h @@ -28,8 +28,13 @@ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) # ifdef _MSC_VER # if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) -# define Py_LIMITED_API +# if !defined(Py_GIL_DISABLED) +# define Py_LIMITED_API +# else +# define Py_LIMITED_API 0x030f0000 +# endif # endif + # include <pyconfig.h> /* sanity-check: Py_LIMITED_API will cause crashes if any of these are also defined. Normally, the Python file PC/pyconfig.h does not @@ -49,7 +54,11 @@ # else # include <pyconfig.h> # if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) -# define Py_LIMITED_API +# if !defined(Py_GIL_DISABLED) +# define Py_LIMITED_API +# else +# define Py_LIMITED_API 0x030f0000 +# endif # endif # endif #endif @@ -59,6 +68,9 @@ extern "C" { #endif #include <stddef.h> +#include <stdlib.h> +#include <string.h> + #include "parse_c_type.h" /* this block of #ifs should be kept exactly identical between @@ -128,13 +140,9 @@ extern "C" { #ifndef PYPY_VERSION -#if PY_MAJOR_VERSION >= 3 -# define PyInt_FromLong PyLong_FromLong -#endif - #define _cffi_from_c_double PyFloat_FromDouble #define _cffi_from_c_float PyFloat_FromDouble -#define _cffi_from_c_long PyInt_FromLong +#define _cffi_from_c_long PyLong_FromLong #define _cffi_from_c_ulong PyLong_FromUnsignedLong #define _cffi_from_c_longlong PyLong_FromLongLong #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong @@ -146,12 +154,12 @@ extern "C" { #define _cffi_from_c_int(x, type) \ (((type)-1) > 0 ? /* unsigned */ \ (sizeof(type) < sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ sizeof(type) == sizeof(long) ? \ PyLong_FromUnsignedLong((unsigned long)x) : \ PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ (sizeof(type) <= sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ PyLong_FromLongLong((long long)x))) #define _cffi_to_c_int(o, type) \ diff --git a/contrib/python/cffi/py3/cffi/_embedding.h b/contrib/python/cffi/py3/cffi/_embedding.h index 64c04f67cac..12ff017fb83 100644 --- a/contrib/python/cffi/py3/cffi/_embedding.h +++ b/contrib/python/cffi/py3/cffi/_embedding.h @@ -177,9 +177,6 @@ static int _cffi_initialize_python(void) if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0) goto error; x = PyEval_EvalCode( -#if PY_MAJOR_VERSION < 3 - (PyCodeObject *) -#endif pycode, global_dict, global_dict); if (x == NULL) goto error; @@ -225,7 +222,7 @@ static int _cffi_initialize_python(void) if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 2.0.0" + "\ncompiled with cffi version: 2.1.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); @@ -247,10 +244,6 @@ static int _cffi_initialize_python(void) goto done; } -#if PY_VERSION_HEX < 0x03080000 -PyAPI_DATA(char *) _PyParser_TokenNames[]; /* from CPython */ -#endif - static int _cffi_carefully_make_gil(void) { /* This does the basic initialization of Python. It can be called @@ -295,27 +288,6 @@ static int _cffi_carefully_make_gil(void) */ #ifdef WITH_THREAD -# if PY_VERSION_HEX < 0x03080000 - char *volatile *lock = (char *volatile *)_PyParser_TokenNames; - char *old_value, *locked_value; - - while (1) { /* spin loop */ - old_value = *lock; - locked_value = old_value + 1; - if (old_value[0] == 'E') { - assert(old_value[1] == 'N'); - if (cffi_compare_and_swap(lock, old_value, locked_value)) - break; - } - else { - assert(old_value[0] == 'N'); - /* should ideally do a spin loop instruction here, but - hard to do it portably and doesn't really matter I - think: PyEval_InitThreads() should be very fast, and - this is only run at start-up anyway. */ - } - } -# else # if PY_VERSION_HEX < 0x030C0000 int volatile *lock = (int volatile *)&PyCapsule_Type.tp_version_tag; int old_value, locked_value = -42; @@ -348,26 +320,16 @@ static int _cffi_carefully_make_gil(void) this is only run at start-up anyway. */ } } -# endif #endif /* call Py_InitializeEx() */ if (!Py_IsInitialized()) { _cffi_py_initialize(); -#if PY_VERSION_HEX < 0x03070000 - PyEval_InitThreads(); -#endif PyEval_SaveThread(); /* release the GIL */ /* the returned tstate must be the one that has been stored into the autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */ } else { -#if PY_VERSION_HEX < 0x03070000 - /* PyEval_InitThreads() is always a no-op from CPython 3.7 */ - PyGILState_STATE state = PyGILState_Ensure(); - PyEval_InitThreads(); - PyGILState_Release(state); -#endif } #ifdef WITH_THREAD diff --git a/contrib/python/cffi/py3/cffi/api.py b/contrib/python/cffi/py3/cffi/api.py index 5a474f3da92..ec8ee7dfd2b 100644 --- a/contrib/python/cffi/py3/cffi/api.py +++ b/contrib/python/cffi/py3/cffi/api.py @@ -4,13 +4,6 @@ from .error import CDefError from . import model try: - callable -except NameError: - # Python 3.1 - from collections import Callable - callable = lambda x: isinstance(x, Callable) - -try: basestring except NameError: # Python 3.x @@ -20,7 +13,7 @@ _unspecified = object() -class FFI(object): +class FFI: r''' The main top-level class that you instantiate once, or once per module. @@ -414,7 +407,7 @@ class FFI(object): if (replace_with.startswith('*') and '&[' in self._backend.getcname(cdecl, '&')): replace_with = '(%s)' % replace_with - elif replace_with and not replace_with[0] in '[(': + elif replace_with and replace_with[0] not in '[(': replace_with = ' ' + replace_with return self._backend.getcname(cdecl, replace_with) @@ -909,7 +902,7 @@ def _make_ffi_library(ffi, libname, flags): raise AttributeError(name) accessors[name](name) # - class FFILibrary(object): + class FFILibrary: def __getattr__(self, name): make_accessor(name) return getattr(self, name) diff --git a/contrib/python/cffi/py3/cffi/backend_ctypes.py b/contrib/python/cffi/py3/cffi/backend_ctypes.py index e7956a79cfb..12cd16259db 100644 --- a/contrib/python/cffi/py3/cffi/backend_ctypes.py +++ b/contrib/python/cffi/py3/cffi/backend_ctypes.py @@ -12,7 +12,7 @@ else: class CTypesType(type): pass -class CTypesData(object): +class CTypesData: __metaclass__ = CTypesType __slots__ = ['__weakref__'] __name__ = '<cdata>' @@ -270,7 +270,7 @@ class CTypesBaseStructOrUnion(CTypesData): return CTypesData.__repr__(self, c_name or self._get_c_name(' &')) -class CTypesBackend(object): +class CTypesBackend: PRIMITIVE_TYPES = { 'char': ctypes.c_char, @@ -336,7 +336,6 @@ class CTypesBackend(object): if novalue is not None: raise TypeError("None expected, got %s object" % (type(novalue).__name__,)) - return None CTypesVoid._fix_class() return CTypesVoid @@ -387,7 +386,7 @@ class CTypesBackend(object): return ctype() return ctype(CTypesPrimitive._to_ctypes(init)) - if kind == 'int' or kind == 'byte': + if kind in {'int', 'byte'}: @classmethod def _cast_from(cls, source): source = _cast_source_to_int(source) @@ -435,7 +434,7 @@ class CTypesBackend(object): _cast_to_integer = __int__ - if kind == 'int' or kind == 'byte' or kind == 'bool': + if kind in {'int', 'byte', 'bool'}: @staticmethod def _to_ctypes(x): if not isinstance(x, (int, long)): @@ -558,15 +557,15 @@ class CTypesBackend(object): def __setitem__(self, index, value): self._as_ctype_ptr[index] = BItem._to_ctypes(value) - if kind == 'charp' or kind == 'voidp': + if kind in {'charp', 'voidp'}: @classmethod def _arg_to_ctypes(cls, *value): if value and isinstance(value[0], bytes): return ctypes.c_char_p(value[0]) else: - return super(CTypesPtr, cls)._arg_to_ctypes(*value) + return super()._arg_to_ctypes(*value) - if kind == 'charp' or kind == 'bytep': + if kind in {'charp', 'bytep'}: def _to_string(self, maxlen): if maxlen < 0: maxlen = sys.maxsize @@ -581,7 +580,7 @@ class CTypesBackend(object): if getattr(self, '_own', False): return 'owning %d bytes' % ( ctypes.sizeof(self._as_ctype_ptr.contents),) - return super(CTypesPtr, self)._get_own_repr() + return super()._get_own_repr() # if (BItem is self.ffi._get_cached_btype(model.void_type) or BItem is self.ffi._get_cached_btype(model.PrimitiveType('char'))): @@ -663,7 +662,7 @@ class CTypesBackend(object): raise IndexError self._blob[index] = BItem._to_ctypes(value) - if kind == 'char' or kind == 'byte': + if kind in {'char', 'byte'}: def _to_string(self, maxlen): if maxlen < 0: maxlen = len(self._blob) @@ -677,7 +676,7 @@ class CTypesBackend(object): def _get_own_repr(self): if getattr(self, '_own', False): return 'owning %d bytes' % (ctypes.sizeof(self._blob),) - return super(CTypesArray, self)._get_own_repr() + return super()._get_own_repr() def _convert_to_address(self, BClass): if BClass in (CTypesPtr, None) or BClass._automatic_casts: @@ -917,7 +916,7 @@ class CTypesBackend(object): def _get_own_repr(self): if getattr(self, '_own_callback', None) is not None: return 'calling %r' % (self._own_callback,) - return super(CTypesFunctionPtr, self)._get_own_repr() + return super()._get_own_repr() def __call__(self, *args): if has_varargs: @@ -1094,7 +1093,7 @@ class CTypesBackend(object): return BTypePtr._from_ctypes(ptr) -class CTypesLibrary(object): +class CTypesLibrary: def __init__(self, backend, cdll): self.backend = backend diff --git a/contrib/python/cffi/py3/cffi/cffi_opcode.py b/contrib/python/cffi/py3/cffi/cffi_opcode.py index 6421df62134..c964ba13e69 100644 --- a/contrib/python/cffi/py3/cffi/cffi_opcode.py +++ b/contrib/python/cffi/py3/cffi/cffi_opcode.py @@ -1,6 +1,6 @@ from .error import VerificationError -class CffiOp(object): +class CffiOp: def __init__(self, op, arg): self.op = op self.arg = arg diff --git a/contrib/python/cffi/py3/cffi/cparser.py b/contrib/python/cffi/py3/cffi/cparser.py index dd590d8743d..1896b7ef5dc 100644 --- a/contrib/python/cffi/py3/cffi/cparser.py +++ b/contrib/python/cffi/py3/cffi/cparser.py @@ -292,7 +292,7 @@ def _common_type_names(csource): return words_used -class Parser(object): +class Parser: def __init__(self): self._declarations = {} @@ -804,11 +804,10 @@ class Parser(object): raise AssertionError("kind = %r" % (kind,)) if name is not None: self._declare(key, tp) - else: - if kind == 'enum' and type.values is not None: - raise NotImplementedError( - "enum %s: the '{}' declaration should appear on the first " - "time the enum is mentioned, not later" % explicit_name) + elif kind == 'enum' and type.values is not None: + raise NotImplementedError( + "enum %s: the '{}' declaration should appear on the first " + "time the enum is mentioned, not later" % explicit_name) if not tp.forcename: tp.force_the_name(force_name) if tp.forcename and '$' in tp.name: diff --git a/contrib/python/cffi/py3/cffi/gen_src.py b/contrib/python/cffi/py3/cffi/gen_src.py new file mode 100644 index 00000000000..8eb57b16952 --- /dev/null +++ b/contrib/python/cffi/py3/cffi/gen_src.py @@ -0,0 +1,5 @@ +"""Entry point so ``python -m cffi.gen_src`` works.""" + +if __name__ == '__main__': + from cffi._cffi_gen_src import run + run() diff --git a/contrib/python/cffi/py3/cffi/model.py b/contrib/python/cffi/py3/cffi/model.py index e5f4cae3e84..8bade1d6dca 100644 --- a/contrib/python/cffi/py3/cffi/model.py +++ b/contrib/python/cffi/py3/cffi/model.py @@ -22,7 +22,7 @@ def qualify(quals, replace_with): return replace_with -class BaseTypeByIdentity(object): +class BaseTypeByIdentity: is_array_type = False is_raw_function = False @@ -34,7 +34,7 @@ class BaseTypeByIdentity(object): if replace_with: if replace_with.startswith('*') and '&[' in result: replace_with = '(%s)' % replace_with - elif not replace_with[0] in '[(': + elif replace_with[0] not in '[(': replace_with = ' ' + replace_with replace_with = qualify(quals, replace_with) result = result.replace('&', replace_with) @@ -371,8 +371,7 @@ class StructOrUnion(StructOrUnionOrEnum): if (name == '' and isinstance(type, StructOrUnion) and expand_anonymous_struct_union): # nested anonymous struct/union - for result in type.enumfields(): - yield result + yield from type.enumfields() else: yield (name, type, bitsize, quals) diff --git a/contrib/python/cffi/py3/cffi/pkgconfig.py b/contrib/python/cffi/py3/cffi/pkgconfig.py index 5c93f15a60e..e1034ad68a6 100644 --- a/contrib/python/cffi/py3/cffi/pkgconfig.py +++ b/contrib/python/cffi/py3/cffi/pkgconfig.py @@ -31,7 +31,7 @@ def call(libname, flag, encoding=sys.getfilesystemencoding()): a.append(libname) try: pc = subprocess.Popen(a, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except EnvironmentError as e: + except OSError as e: raise PkgConfigError("cannot run pkg-config: %s" % (str(e).strip(),)) bout, berr = pc.communicate() diff --git a/contrib/python/cffi/py3/cffi/recompiler.py b/contrib/python/cffi/py3/cffi/recompiler.py index 25bf9e91d9e..fbafc011c97 100644 --- a/contrib/python/cffi/py3/cffi/recompiler.py +++ b/contrib/python/cffi/py3/cffi/recompiler.py @@ -7,9 +7,11 @@ VERSION_BASE = 0x2601 VERSION_EMBEDDED = 0x2701 VERSION_CHAR16CHAR32 = 0x2801 +FREE_THREADED_BUILD = sysconfig.get_config_var("Py_GIL_DISABLED") USE_LIMITED_API = ((sys.platform != 'win32' or sys.version_info < (3, 0) or sys.version_info >= (3, 5)) and - not sysconfig.get_config_var("Py_GIL_DISABLED")) # free-threaded doesn't yet support limited API + # free-threaded build doesn't support the stable ABI until Python 3.15 + (not FREE_THREADED_BUILD or sys.version_info >= (3, 15))) class GlobalExpr: def __init__(self, name, address, type_op, size=0, check_value=0): @@ -93,9 +95,18 @@ class EnumExpr: self.allenums = allenums def as_c_expr(self): - return (' { "%s", %d, _cffi_prim_int(%s, %s),\n' - ' "%s" },' % (self.name, self.type_index, - self.size, self.signed, self.allenums)) + lines = [' { "%s", %d, _cffi_prim_int(%s, %s),' % (self.name, self.type_index, + self.size, self.signed,)] + pending = 0 + while len(self.allenums) > pending + 110: + j = self.allenums.find(',', pending + 100) + if j < 0: + break + j += 1 + lines.append(' "%s"' % (self.allenums[pending:j],)) + pending = j + lines.append(' "%s" },' % (self.allenums[pending:],)) + return '\n'.join(lines) def as_python_expr(self): prim_index = { @@ -315,11 +326,8 @@ class Recompiler: prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) - prnt('#elif PY_MAJOR_VERSION >= 3') - prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % ( - base_module_name,)) prnt('#else') - prnt('# define _CFFI_PYTHON_STARTUP_FUNC init%s' % ( + prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % ( base_module_name,)) prnt('#endif') lines = self._rel_readlines('_embedding.h') @@ -427,35 +435,22 @@ class Recompiler: prnt(' }') prnt(' p[0] = (const void *)0x%x;' % self._version) prnt(' p[1] = &_cffi_type_context;') - prnt('#if PY_MAJOR_VERSION >= 3') prnt(' return NULL;') - prnt('#endif') prnt('}') # on Windows, distutils insists on putting init_cffi_xyz in # 'export_symbols', so instead of fighting it, just give up and # give it one prnt('# ifdef _MSC_VER') prnt(' PyMODINIT_FUNC') - prnt('# if PY_MAJOR_VERSION >= 3') prnt(' PyInit_%s(void) { return NULL; }' % (base_module_name,)) - prnt('# else') - prnt(' init%s(void) { }' % (base_module_name,)) prnt('# endif') - prnt('# endif') - prnt('#elif PY_MAJOR_VERSION >= 3') + prnt('#else') prnt('PyMODINIT_FUNC') prnt('PyInit_%s(void)' % (base_module_name,)) prnt('{') prnt(' return _cffi_init("%s", 0x%x, &_cffi_type_context);' % ( self.module_name, self._version)) prnt('}') - prnt('#else') - prnt('PyMODINIT_FUNC') - prnt('init%s(void)' % (base_module_name,)) - prnt('{') - prnt(' _cffi_init("%s", 0x%x, &_cffi_type_context);' % ( - self.module_name, self._version)) - prnt('}') prnt('#endif') prnt() prnt('#ifdef __GNUC__') @@ -552,8 +547,7 @@ class Recompiler: tovar, errcode) return # - elif (isinstance(tp, model.StructOrUnionOrEnum) or - isinstance(tp, model.BasePrimitiveType)): + elif (isinstance(tp, (model.StructOrUnionOrEnum, model.BasePrimitiveType))): # a struct (not a struct pointer) as a function argument; # or, a complex (the same code works) self._prnt(' if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)' @@ -1313,10 +1307,6 @@ class Recompiler: s = s.encode('utf-8') # -> bytes else: s.decode('utf-8') # got bytes, check for valid utf-8 - try: - s.decode('ascii') - except UnicodeDecodeError: - s = b'# -*- encoding: utf8 -*-\n' + s for line in s.splitlines(True): comment = line if type('//') is bytes: # python2 @@ -1415,7 +1405,7 @@ else: def write(self, s): if isinstance(s, unicode): s = s.encode('ascii') - super(NativeIO, self).write(s) + super().write(s) def _is_file_like(maybefile): # compare to xml.etree.ElementTree._get_writer @@ -1437,11 +1427,11 @@ def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose): try: with open(target_file, 'r') as f1: if f1.read(len(output) + 1) != output: - raise IOError + raise OSError if verbose: print("(already up-to-date)") return False # already up-to-date - except IOError: + except OSError: tmp_file = '%s.~%d' % (target_file, os.getpid()) with open(tmp_file, 'w') as f1: f1.write(output) diff --git a/contrib/python/cffi/py3/cffi/setuptools_ext.py b/contrib/python/cffi/py3/cffi/setuptools_ext.py index 5cdd246fb4d..946afb45a6b 100644 --- a/contrib/python/cffi/py3/cffi/setuptools_ext.py +++ b/contrib/python/cffi/py3/cffi/setuptools_ext.py @@ -105,10 +105,12 @@ def _set_py_limited_api(Extension, kwds): kwds['py_limited_api'] = True if sysconfig.get_config_var("Py_GIL_DISABLED"): - if kwds.get('py_limited_api'): - log.info("Ignoring py_limited_api=True for free-threaded build.") - - kwds['py_limited_api'] = False + if sys.version_info < (3, 15): + if kwds.get('py_limited_api'): + log.info("Ignoring py_limited_api=True for free-threaded build.") + kwds['py_limited_api'] = False + else: + kwds.setdefault("define_macros", []).append(("Py_TARGET_ABI3T", "0x030f0000")) if kwds.get('py_limited_api') is False: # avoid setting Py_LIMITED_API if py_limited_api=False diff --git a/contrib/python/cffi/py3/cffi/vengine_cpy.py b/contrib/python/cffi/py3/cffi/vengine_cpy.py index 02e6a471d04..4d2ea43e5d9 100644 --- a/contrib/python/cffi/py3/cffi/vengine_cpy.py +++ b/contrib/python/cffi/py3/cffi/vengine_cpy.py @@ -7,7 +7,7 @@ from .error import VerificationError from . import _imp_emulation as imp -class VCPythonEngine(object): +class VCPythonEngine: _class_key = 'x' _gen_python_module = True @@ -102,8 +102,6 @@ class VCPythonEngine(object): # standard init. modname = self.verifier.get_module_name() constants = self._chained_list_constants[False] - prnt('#if PY_MAJOR_VERSION >= 3') - prnt() prnt('static struct PyModuleDef _cffi_module_def = {') prnt(' PyModuleDef_HEAD_INIT,') prnt(' "%s",' % modname) @@ -130,21 +128,6 @@ class VCPythonEngine(object): prnt(' return lib;') prnt('}') prnt() - prnt('#else') - prnt() - prnt('PyMODINIT_FUNC') - prnt('init%s(void)' % modname) - prnt('{') - prnt(' PyObject *lib;') - prnt(' lib = Py_InitModule("%s", _cffi_methods);' % modname) - prnt(' if (lib == NULL)') - prnt(' return;') - prnt(' if (%s < 0 || _cffi_init() < 0)' % (constants,)) - prnt(' return;') - prnt(' return;') - prnt('}') - prnt() - prnt('#endif') def load_library(self, flags=None): # XXX review all usages of 'self' here! @@ -183,7 +166,7 @@ class VCPythonEngine(object): # it will invoke the chained list of functions that will really # build (notably) the constant objects, as <cdata> if they are # pointers, and store them as attributes on the 'library' object. - class FFILibrary(object): + class FFILibrary: _cffi_python_module = module _cffi_ffi = self.ffi _cffi_dir = [] @@ -873,21 +856,9 @@ cffimod_header = r''' # define _cffi_double_complex_t double _Complex #endif -#if PY_MAJOR_VERSION < 3 -# undef PyCapsule_CheckExact -# undef PyCapsule_GetPointer -# define PyCapsule_CheckExact(capsule) (PyCObject_Check(capsule)) -# define PyCapsule_GetPointer(capsule, name) \ - (PyCObject_AsVoidPtr(capsule)) -#endif - -#if PY_MAJOR_VERSION >= 3 -# define PyInt_FromLong PyLong_FromLong -#endif - #define _cffi_from_c_double PyFloat_FromDouble #define _cffi_from_c_float PyFloat_FromDouble -#define _cffi_from_c_long PyInt_FromLong +#define _cffi_from_c_long PyLong_FromLong #define _cffi_from_c_ulong PyLong_FromUnsignedLong #define _cffi_from_c_longlong PyLong_FromLongLong #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong @@ -899,21 +870,21 @@ cffimod_header = r''' #define _cffi_from_c_int_const(x) \ (((x) > 0) ? \ ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ? \ - PyInt_FromLong((long)(x)) : \ + PyLong_FromLong((long)(x)) : \ PyLong_FromUnsignedLongLong((unsigned long long)(x)) : \ ((long long)(x) >= (long long)LONG_MIN) ? \ - PyInt_FromLong((long)(x)) : \ + PyLong_FromLong((long)(x)) : \ PyLong_FromLongLong((long long)(x))) #define _cffi_from_c_int(x, type) \ (((type)-1) > 0 ? /* unsigned */ \ (sizeof(type) < sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ sizeof(type) == sizeof(long) ? \ PyLong_FromUnsignedLong((unsigned long)x) : \ PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ (sizeof(type) <= sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ PyLong_FromLongLong((long long)x))) #define _cffi_to_c_int(o, type) \ diff --git a/contrib/python/cffi/py3/cffi/vengine_gen.py b/contrib/python/cffi/py3/cffi/vengine_gen.py index bffc82122c3..0209c794ac7 100644 --- a/contrib/python/cffi/py3/cffi/vengine_gen.py +++ b/contrib/python/cffi/py3/cffi/vengine_gen.py @@ -8,7 +8,7 @@ from . import model from .error import VerificationError -class VGenericEngine(object): +class VGenericEngine: _class_key = 'g' _gen_python_module = False diff --git a/contrib/python/cffi/py3/cffi/verifier.py b/contrib/python/cffi/py3/cffi/verifier.py index e392a2b7fda..96baadb6f76 100644 --- a/contrib/python/cffi/py3/cffi/verifier.py +++ b/contrib/python/cffi/py3/cffi/verifier.py @@ -24,10 +24,10 @@ else: def write(self, s): if isinstance(s, unicode): s = s.encode('ascii') - super(NativeIO, self).write(s) + super().write(s) -class Verifier(object): +class Verifier: def __init__(self, ffi, preamble, tmpdir=None, modulename=None, ext_package=None, tag='', force_generic_engine=False, @@ -182,7 +182,7 @@ class Verifier(object): # Determine if this matches the current file if os.path.exists(self.sourcefilename): with open(self.sourcefilename, "r") as fp: - needs_written = not (fp.read() == source_data) + needs_written = fp.read() != source_data else: needs_written = True diff --git a/contrib/python/cffi/py3/patches/01-arcadia.patch b/contrib/python/cffi/py3/patches/01-arcadia.patch index 23331873652..8ab50be29d7 100644 --- a/contrib/python/cffi/py3/patches/01-arcadia.patch +++ b/contrib/python/cffi/py3/patches/01-arcadia.patch @@ -92,16 +92,8 @@ if (*w > 0xFFFF) { --- contrib/python/cffi/py3/cffi/recompiler.py (index) +++ contrib/python/cffi/py3/cffi/recompiler.py (working tree) -@@ -287,10 +287,8 @@ class Recompiler: - self.write_c_source_to_f(f, preamble) - - def _rel_readlines(self, filename): -- g = open(os.path.join(os.path.dirname(__file__), filename), 'r') -- lines = g.readlines() -- g.close() -- return lines +@@ -287,2 +287,2 @@ class Recompiler: +- with open(os.path.join(os.path.dirname(__file__), filename), 'r') as g: +- return g.readlines() + import pkgutil + return pkgutil.get_data('cffi', filename).decode('utf-8').splitlines(True) - - def write_c_source_to_f(self, f, preamble): - self._f = f diff --git a/contrib/python/cffi/py3/ya.make b/contrib/python/cffi/py3/ya.make index d2eb8e11ad8..a526c12160c 100644 --- a/contrib/python/cffi/py3/ya.make +++ b/contrib/python/cffi/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.0.0) +VERSION(2.1.0) LICENSE(MIT) @@ -31,6 +31,7 @@ PY_REGISTER( PY_SRCS( TOP_LEVEL cffi/__init__.py + cffi/_cffi_gen_src.py cffi/_imp_emulation.py cffi/_shimmed_dist_utils.py cffi/api.py @@ -40,6 +41,7 @@ PY_SRCS( cffi/cparser.py cffi/error.py cffi/ffiplatform.py + cffi/gen_src.py cffi/lock.py cffi/model.py cffi/pkgconfig.py diff --git a/contrib/python/idna/py3/.dist-info/METADATA b/contrib/python/idna/py3/.dist-info/METADATA index 3dd388cb837..6c4bf89b25c 100644 --- a/contrib/python/idna/py3/.dist-info/METADATA +++ b/contrib/python/idna/py3/.dist-info/METADATA @@ -1,9 +1,9 @@ Metadata-Version: 2.4 Name: idna -Version: 3.15 +Version: 3.18 Summary: Internationalized Domain Names in Applications (IDNA) Author-email: Kim Davies <[email protected]> -Requires-Python: >=3.8 +Requires-Python: >=3.9 Description-Content-Type: text/markdown License-Expression: BSD-3-Clause Classifier: Development Status :: 5 - Production/Stable @@ -13,7 +13,6 @@ Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 @@ -36,40 +35,18 @@ Provides-Extra: all # Internationalized Domain Names in Applications (IDNA) -Support for [Internationalized Domain Names in -Applications (IDNA)](https://tools.ietf.org/html/rfc5891) -and [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). - -The latest versions of these standards supplied here provide -more comprehensive language coverage and reduce the potential of -allowing domains with known security vulnerabilities. This library -is a suitable replacement for the "encodings.idna" -module that comes with the Python standard library, but which -only supports an older superseded IDNA specification from 2003. - -Basic functions are simply executed: - -```pycon ->>> import idna ->>> idna.encode('ドメイン.テスト') -b'xn--eckwd4c7c.xn--zckzah' ->>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) -ドメイン.テスト -``` - - -## Installation - -This package is available for installation from PyPI via the -typical mechanisms, such as: - -```bash -$ python3 -m pip install idna -``` - +Support for [Internationalized Domain Names in Applications +(IDNA)](https://tools.ietf.org/html/rfc5891) and [Unicode IDNA +Compatibility Processing](https://unicode.org/reports/tr46/). It +supersedes the standard library's `encodings.idna`, which only +implements the 2003 specification, offering broader script coverage and +limiting domains with known security vulnerabilities. ## Usage +Package may be installed from [PyPI](https://pypi.org/project/idna/) via +the typical methods (e.g. `python3 -m pip install idna`) + For typical usage, the `encode` and `decode` functions will take a domain name argument and perform a conversion to ASCII-compatible encoding (known as A-labels), or to Unicode strings (known as U-labels) @@ -84,12 +61,7 @@ b'xn--eckwd4c7c.xn--zckzah' ``` Conversions can be applied at a per-label basis using the `ulabel` or -`alabel` functions if necessary: - -```pycon ->>> idna.alabel('测试') -b'xn--0zwm56d' -``` +`alabel` functions for specialized use cases. ### Compatibility Mapping (UTS #46) @@ -102,10 +74,9 @@ conversion operations. This functionality, known as a specification to be a local user-interface issue distinct from IDNA conversion functionality. -For example, "Königsgäßchen" is not a permissible label as *LATIN -CAPITAL LETTER K* is not allowed (nor are capital letters in general). -UTS 46 will convert this into lower case prior to applying the IDNA -conversion. +For example, "Königsgäßchen" is not a permissible label as capital letters +are not allowed. UTS 46 will convert this into lower case prior to applying +the IDNA conversion. ```pycon >>> import idna @@ -114,81 +85,71 @@ conversion. idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed >>> idna.encode('Königsgäßchen', uts46=True) b'xn--knigsgchen-b4a3dun' ->>> print(idna.decode('xn--knigsgchen-b4a3dun')) -königsgäßchen +>>> idna.decode('xn--knigsgchen-b4a3dun') +'königsgäßchen' ``` +When performing a decode operation for display purposes, `decode()` +accepts a `display=True` argument that leaves any `xn--` label that +fails to decode unchanged. This is useful for user interface display +where a domain is in use, the A-label form can be presented when it +is not a valid IDN. -## Exceptions -All errors raised during the conversion following the specification -should raise an exception derived from the `idna.IDNAError` base -class. +## Exceptions -More specific exceptions that may be generated as `idna.IDNABidiError` -when the error reflects an illegal combination of left-to-right and -right-to-left characters in a label; `idna.InvalidCodepoint` when -a specific codepoint is an illegal character in an IDN label (i.e. -INVALID); and `idna.InvalidCodepointContext` when the codepoint is -illegal based on its position in the string (i.e. it is CONTEXTO or CONTEXTJ -but the contextual requirements are not satisfied.) +All errors raised during conversion derive from the `idna.IDNAError` +base class. The more specific exceptions are: -## Building and Diagnostics +* `idna.IDNABidiError` — raised when a label contains an illegal + combination of left-to-right and right-to-left characters. +* `idna.InvalidCodepoint` — raised when a label contains a codepoint + that is INVALID for IDNA. +* `idna.InvalidCodepointContext` — raised when a CONTEXTO or CONTEXTJ + codepoint appears in a position whose contextual requirements are + not satisfied. -The IDNA and UTS 46 functionality relies upon pre-calculated lookup -tables for performance. These tables are derived from computing against -eligibility criteria in the respective standards using the command-line -script `tools/idna-data`. -This tool will fetch relevant codepoint data from the Unicode repository -and perform the required calculations to identify eligibility. There are -three main modes: +## Command-line tool -* `idna-data make-libdata`. Generates `idnadata.py` and - `uts46data.py`, the pre-calculated lookup tables used for IDNA and - UTS 46 conversions. Implementers who wish to track this library against - a different Unicode version may use this tool to manually generate a - different version of the `idnadata.py` and `uts46data.py` files. +The package supports command-line usage to convert domain names +between their Unicode and ASCII-compatible forms. It can be run either +as a module (`python3 -m idna`) or, once installed (such as with `uv +tool` or `pipx`), via the `idna` script: -* `idna-data make-table`. Generate a table of the IDNA disposition - (e.g. PVALID, CONTEXTJ, CONTEXTO) in the format found in Appendix - B.1 of RFC 5892 and the pre-computed tables published by [IANA](https://www.iana.org/). +```bash +$ uv tool install idna +$ idna xn--e1afmkfd.xn--p1ai +пример.рф +$ idna пример.рф +xn--e1afmkfd.xn--p1ai +``` -* `idna-data U+0061`. Prints debugging output on the various - properties associated with an individual Unicode codepoint (in this - case, U+0061), that are used to assess the IDNA and UTS 46 status of a - codepoint. This is helpful in debugging or analysis. +Mode can be specified with `-e`/`--encode` or `-d`/`--decode`, otherwise +it will be chosen automatically based on the first input. Multiple +domains can be supplied either as arguments or through standard input. +UTS #46 mapping is applied by default, which lets the tool accept +inputs that aren't strictly valid IDNA 2008 by normalising them first, +pass `--strict` to disable UTS #46. -The tool accepts a number of arguments, described using `idna-data -h`. -Most notably, the `--version` argument allows the specification -of the version of Unicode to be used in computing the table data. For -example, `idna-data --version 9.0.0 make-libdata` will generate -library data against Unicode 9.0.0. +Conversion failures are reported on stderr together with the +offending input; processing continues with the remaining domains and +the tool exits with a non-zero status if any conversion failed. ## Additional Notes -* **Packages**. The latest tagged release version is published in the - [Python Package Index](https://pypi.org/project/idna/). - -* **Version support**. This library supports Python 3.8 and higher. +* **Version support**. This library supports Python 3.9 and higher. As this library serves as a low-level toolkit for a variety of - applications, many of which strive for broad compatibility with older - Python versions, there is no rush to remove older interpreter support. - Support for older versions are likely to be removed from new releases - as automated tests can no longer easily be run, i.e. once the Python - version is officially end-of-life. - -* **Testing**. The library has a test suite based on each rule of the - IDNA specification, as well as tests that are provided as part of the - Unicode Technical Standard 46, [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). + applications, we strive to support all versions of Python that are + not beyond end-of-life. * **Emoji**. It is an occasional request to support emoji domains in this library. Encoding of symbols like emoji is expressly prohibited by the IDNA technical standard, and emoji domains are broadly phased out across the domain industry due to associated security risks. -* **Transitional processing**. Unicode 16.0.0 removed transitional - processing so the `transitional` argument for the encode() method - no longer has any effect and will be removed at a later date. +* **Regenerating lookup tables**. The IDNA and UTS 46 functionality + relies upon pre-calculated lookup tables, generated using the + `idna-data` script in [`tools/`](tools/README.md). diff --git a/contrib/python/idna/py3/.dist-info/entry_points.txt b/contrib/python/idna/py3/.dist-info/entry_points.txt new file mode 100644 index 00000000000..59ca7ac02ea --- /dev/null +++ b/contrib/python/idna/py3/.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +idna=idna.cli:main + diff --git a/contrib/python/idna/py3/README.md b/contrib/python/idna/py3/README.md index 5772219126b..7671ae22937 100644 --- a/contrib/python/idna/py3/README.md +++ b/contrib/python/idna/py3/README.md @@ -1,39 +1,17 @@ # Internationalized Domain Names in Applications (IDNA) -Support for [Internationalized Domain Names in -Applications (IDNA)](https://tools.ietf.org/html/rfc5891) -and [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). - -The latest versions of these standards supplied here provide -more comprehensive language coverage and reduce the potential of -allowing domains with known security vulnerabilities. This library -is a suitable replacement for the "encodings.idna" -module that comes with the Python standard library, but which -only supports an older superseded IDNA specification from 2003. - -Basic functions are simply executed: - -```pycon ->>> import idna ->>> idna.encode('ドメイン.テスト') -b'xn--eckwd4c7c.xn--zckzah' ->>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) -ドメイン.テスト -``` - - -## Installation - -This package is available for installation from PyPI via the -typical mechanisms, such as: - -```bash -$ python3 -m pip install idna -``` - +Support for [Internationalized Domain Names in Applications +(IDNA)](https://tools.ietf.org/html/rfc5891) and [Unicode IDNA +Compatibility Processing](https://unicode.org/reports/tr46/). It +supersedes the standard library's `encodings.idna`, which only +implements the 2003 specification, offering broader script coverage and +limiting domains with known security vulnerabilities. ## Usage +Package may be installed from [PyPI](https://pypi.org/project/idna/) via +the typical methods (e.g. `python3 -m pip install idna`) + For typical usage, the `encode` and `decode` functions will take a domain name argument and perform a conversion to ASCII-compatible encoding (known as A-labels), or to Unicode strings (known as U-labels) @@ -48,12 +26,7 @@ b'xn--eckwd4c7c.xn--zckzah' ``` Conversions can be applied at a per-label basis using the `ulabel` or -`alabel` functions if necessary: - -```pycon ->>> idna.alabel('测试') -b'xn--0zwm56d' -``` +`alabel` functions for specialized use cases. ### Compatibility Mapping (UTS #46) @@ -66,10 +39,9 @@ conversion operations. This functionality, known as a specification to be a local user-interface issue distinct from IDNA conversion functionality. -For example, "Königsgäßchen" is not a permissible label as *LATIN -CAPITAL LETTER K* is not allowed (nor are capital letters in general). -UTS 46 will convert this into lower case prior to applying the IDNA -conversion. +For example, "Königsgäßchen" is not a permissible label as capital letters +are not allowed. UTS 46 will convert this into lower case prior to applying +the IDNA conversion. ```pycon >>> import idna @@ -78,80 +50,70 @@ conversion. idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed >>> idna.encode('Königsgäßchen', uts46=True) b'xn--knigsgchen-b4a3dun' ->>> print(idna.decode('xn--knigsgchen-b4a3dun')) -königsgäßchen +>>> idna.decode('xn--knigsgchen-b4a3dun') +'königsgäßchen' ``` +When performing a decode operation for display purposes, `decode()` +accepts a `display=True` argument that leaves any `xn--` label that +fails to decode unchanged. This is useful for user interface display +where a domain is in use, the A-label form can be presented when it +is not a valid IDN. -## Exceptions -All errors raised during the conversion following the specification -should raise an exception derived from the `idna.IDNAError` base -class. +## Exceptions -More specific exceptions that may be generated as `idna.IDNABidiError` -when the error reflects an illegal combination of left-to-right and -right-to-left characters in a label; `idna.InvalidCodepoint` when -a specific codepoint is an illegal character in an IDN label (i.e. -INVALID); and `idna.InvalidCodepointContext` when the codepoint is -illegal based on its position in the string (i.e. it is CONTEXTO or CONTEXTJ -but the contextual requirements are not satisfied.) +All errors raised during conversion derive from the `idna.IDNAError` +base class. The more specific exceptions are: -## Building and Diagnostics +* `idna.IDNABidiError` — raised when a label contains an illegal + combination of left-to-right and right-to-left characters. +* `idna.InvalidCodepoint` — raised when a label contains a codepoint + that is INVALID for IDNA. +* `idna.InvalidCodepointContext` — raised when a CONTEXTO or CONTEXTJ + codepoint appears in a position whose contextual requirements are + not satisfied. -The IDNA and UTS 46 functionality relies upon pre-calculated lookup -tables for performance. These tables are derived from computing against -eligibility criteria in the respective standards using the command-line -script `tools/idna-data`. -This tool will fetch relevant codepoint data from the Unicode repository -and perform the required calculations to identify eligibility. There are -three main modes: +## Command-line tool -* `idna-data make-libdata`. Generates `idnadata.py` and - `uts46data.py`, the pre-calculated lookup tables used for IDNA and - UTS 46 conversions. Implementers who wish to track this library against - a different Unicode version may use this tool to manually generate a - different version of the `idnadata.py` and `uts46data.py` files. +The package supports command-line usage to convert domain names +between their Unicode and ASCII-compatible forms. It can be run either +as a module (`python3 -m idna`) or, once installed (such as with `uv +tool` or `pipx`), via the `idna` script: -* `idna-data make-table`. Generate a table of the IDNA disposition - (e.g. PVALID, CONTEXTJ, CONTEXTO) in the format found in Appendix - B.1 of RFC 5892 and the pre-computed tables published by [IANA](https://www.iana.org/). +```bash +$ uv tool install idna +$ idna xn--e1afmkfd.xn--p1ai +пример.рф +$ idna пример.рф +xn--e1afmkfd.xn--p1ai +``` -* `idna-data U+0061`. Prints debugging output on the various - properties associated with an individual Unicode codepoint (in this - case, U+0061), that are used to assess the IDNA and UTS 46 status of a - codepoint. This is helpful in debugging or analysis. +Mode can be specified with `-e`/`--encode` or `-d`/`--decode`, otherwise +it will be chosen automatically based on the first input. Multiple +domains can be supplied either as arguments or through standard input. +UTS #46 mapping is applied by default, which lets the tool accept +inputs that aren't strictly valid IDNA 2008 by normalising them first, +pass `--strict` to disable UTS #46. -The tool accepts a number of arguments, described using `idna-data -h`. -Most notably, the `--version` argument allows the specification -of the version of Unicode to be used in computing the table data. For -example, `idna-data --version 9.0.0 make-libdata` will generate -library data against Unicode 9.0.0. +Conversion failures are reported on stderr together with the +offending input; processing continues with the remaining domains and +the tool exits with a non-zero status if any conversion failed. ## Additional Notes -* **Packages**. The latest tagged release version is published in the - [Python Package Index](https://pypi.org/project/idna/). - -* **Version support**. This library supports Python 3.8 and higher. +* **Version support**. This library supports Python 3.9 and higher. As this library serves as a low-level toolkit for a variety of - applications, many of which strive for broad compatibility with older - Python versions, there is no rush to remove older interpreter support. - Support for older versions are likely to be removed from new releases - as automated tests can no longer easily be run, i.e. once the Python - version is officially end-of-life. - -* **Testing**. The library has a test suite based on each rule of the - IDNA specification, as well as tests that are provided as part of the - Unicode Technical Standard 46, [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). + applications, we strive to support all versions of Python that are + not beyond end-of-life. * **Emoji**. It is an occasional request to support emoji domains in this library. Encoding of symbols like emoji is expressly prohibited by the IDNA technical standard, and emoji domains are broadly phased out across the domain industry due to associated security risks. -* **Transitional processing**. Unicode 16.0.0 removed transitional - processing so the `transitional` argument for the encode() method - no longer has any effect and will be removed at a later date. +* **Regenerating lookup tables**. The IDNA and UTS 46 functionality + relies upon pre-calculated lookup tables, generated using the + `idna-data` script in [`tools/`](tools/README.md). diff --git a/contrib/python/idna/py3/idna/__main__.py b/contrib/python/idna/py3/idna/__main__.py new file mode 100644 index 00000000000..dbdd0661721 --- /dev/null +++ b/contrib/python/idna/py3/idna/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/python/idna/py3/idna/cli.py b/contrib/python/idna/py3/idna/cli.py new file mode 100644 index 00000000000..4acda2c0f03 --- /dev/null +++ b/contrib/python/idna/py3/idna/cli.py @@ -0,0 +1,128 @@ +"""Command-line interface for the :mod:`idna` package. + +Invoked via ``python -m idna``. See :func:`main` for the entry point. +""" + +import argparse +import sys +from collections.abc import Iterable +from itertools import chain +from typing import IO, Optional + +from . import IDNAError, decode, encode +from .core import _alabel_prefix, _unicode_dots_re +from .package_data import __version__ + + +def _looks_like_alabel(s: str) -> bool: + """Return True if any label in ``s`` carries the ``xn--`` ACE prefix.""" + prefix = _alabel_prefix.decode("ascii") + return any(label.lower().startswith(prefix) for label in _unicode_dots_re.split(s)) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m idna", + description=( + "Convert a domain name between its Unicode (U-label) and " + "ASCII-compatible (A-label) forms. With no mode flag, the " + "direction is chosen from the first input — if it contains " + "an xn-- label the stream is decoded, otherwise it is " + "encoded — and the same mode is applied to every remaining " + "input. UTS #46 mapping is applied by default; pass " + "--strict to disable it. When no domains are given on the " + "command line and stdin is piped, one domain per line is " + "read from stdin." + ), + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + "-e", + "--encode", + dest="mode", + action="store_const", + const="encode", + help="Encode the input to its ASCII A-label form.", + ) + mode.add_argument( + "-d", + "--decode", + dest="mode", + action="store_const", + const="decode", + help="Decode the input from its ASCII A-label form.", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Disable the default UTS #46 mapping and apply IDNA 2008 rules verbatim.", + ) + parser.add_argument( + "--version", + action="version", + version=f"idna {__version__}", + ) + parser.add_argument( + "domain", + nargs="*", + help="One or more domain names to convert. Omit to read from stdin.", + ) + return parser + + +def _iter_stdin(stream: IO[str]) -> Iterable[str]: + """Yield non-empty stripped lines from ``stream``, ignoring blanks.""" + for line in stream: + stripped = line.strip() + if stripped: + yield stripped + + +def _convert_one(domain: str, mode: str, uts46: bool) -> bool: + """Convert ``domain`` and write the result; return ``False`` on failure.""" + try: + if mode == "decode": + print(decode(domain, uts46=uts46)) + else: + print(encode(domain, uts46=uts46).decode("ascii")) + except IDNAError as err: + print(f"idna: {mode} failed for {domain!r}: {err}", file=sys.stderr) + return False + return True + + +def main(argv: Optional[list[str]] = None) -> int: + """Entry point for ``python -m idna``. + + When more than one domain is supplied (via positional arguments or + piped stdin) and no mode flag is given, the first input determines + the direction and that mode is applied uniformly to the rest. + + :param argv: Argument list excluding the program name. Defaults to + :data:`sys.argv` when ``None``. + :returns: ``0`` on success, ``1`` if any conversion fails. + """ + parser = _build_parser() + args = parser.parse_args(argv) + uts46 = not args.strict + + if args.domain: + domains: Iterable[str] = args.domain + elif not sys.stdin.isatty(): + domains = _iter_stdin(sys.stdin) + else: + parser.error("a domain argument is required when stdin is a terminal") + + iterator = iter(domains) + first = next(iterator, None) + if first is None: + return 0 + + mode = args.mode or ("decode" if _looks_like_alabel(first) else "encode") + + results = [_convert_one(domain, mode, uts46) for domain in chain([first], iterator)] + return 0 if all(results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/python/idna/py3/idna/codec.py b/contrib/python/idna/py3/idna/codec.py index 280dc3972cb..83b42fe42b5 100644 --- a/contrib/python/idna/py3/idna/codec.py +++ b/contrib/python/idna/py3/idna/codec.py @@ -1,5 +1,5 @@ import codecs -from typing import Any, Optional, Tuple +from typing import Any, Optional from .core import IDNAError, _unicode_dots_re, alabel, decode, encode, ulabel @@ -15,7 +15,7 @@ class Codec(codecs.Codec): raises :exc:`~idna.IDNAError`. """ - def encode(self, data: str, errors: str = "strict") -> Tuple[bytes, int]: # ty: ignore[invalid-method-override] + def encode(self, data: str, errors: str = "strict") -> tuple[bytes, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') @@ -24,7 +24,7 @@ class Codec(codecs.Codec): return encode(data), len(data) - def decode(self, data: bytes, errors: str = "strict") -> Tuple[str, int]: # ty: ignore[invalid-method-override] + def decode(self, data: bytes, errors: str = "strict") -> tuple[str, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') @@ -46,7 +46,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder): Only the ``"strict"`` error handler is supported. """ - def _buffer_encode(self, data: str, errors: str, final: bool) -> Tuple[bytes, int]: # ty: ignore[invalid-method-override] + def _buffer_encode(self, data: str, errors: str, final: bool) -> tuple[bytes, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') @@ -89,7 +89,7 @@ class IncrementalDecoder(codecs.BufferedIncrementalDecoder): Only the ``"strict"`` error handler is supported. """ - def _buffer_decode(self, data: Any, errors: str, final: bool) -> Tuple[str, int]: # ty: ignore[invalid-method-override] + def _buffer_decode(self, data: Any, errors: str, final: bool) -> tuple[str, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') diff --git a/contrib/python/idna/py3/idna/core.py b/contrib/python/idna/py3/idna/core.py index b6f9442deb3..1ccbd1f3356 100644 --- a/contrib/python/idna/py3/idna/core.py +++ b/contrib/python/idna/py3/idna/core.py @@ -9,6 +9,7 @@ from .intranges import intranges_contain _virama_combining_class = 9 _alabel_prefix = b"xn--" +_max_input_length = 1024 _unicode_dots_re = re.compile("[\u002e\u3002\uff0e\uff61]") @@ -20,33 +21,32 @@ _bidi_rtl_valid_ending = frozenset({"R", "AL", "EN", "AN"}) _bidi_rtl_numeric = frozenset({"AN", "EN"}) _bidi_ltr_allowed = frozenset({"L", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"}) _bidi_ltr_valid_ending = frozenset({"L", "EN"}) -_bidi_joiner_l_or_d = frozenset({ord("L"), ord("D")}) -_bidi_joiner_r_or_d = frozenset({ord("R"), ord("D")}) +_bidi_joiner_l_or_d = frozenset({"L", "D"}) +_bidi_joiner_r_or_d = frozenset({"R", "D"}) + + +def _joining_type(cp: int) -> Optional[str]: + for jt, ranges in idnadata.joining_types.items(): + if intranges_contain(cp, ranges): + return jt + return None class IDNAError(UnicodeError): """Base exception for all IDNA-encoding related problems""" - pass - class IDNABidiError(IDNAError): """Exception when bidirectional requirements are not satisfied""" - pass - class InvalidCodepoint(IDNAError): """Exception when a disallowed or unallocated codepoint is used""" - pass - class InvalidCodepointContext(IDNAError): """Exception when the codepoint is not valid in the context it is used""" - pass - def _combining_class(cp: int) -> int: v = unicodedata.combining(chr(cp)) @@ -112,13 +112,15 @@ def check_bidi(label: str, check_ltr: bool = False) -> bool: :raises IDNABidiError: If any of Bidi Rule conditions 1-6 are violated, or if the directional category of a codepoint cannot be determined. """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") # Bidi rules should only be applied if string contains RTL characters bidi_label = False for idx, cp in enumerate(label, 1): direction = unicodedata.bidirectional(cp) if direction == "": # String likely comes from a newer version of Unicode - raise IDNABidiError(f"Unknown directionality in label {repr(label)} at position {idx}") + raise IDNABidiError(f"Unknown directionality in label {label!r} at position {idx}") if direction in _bidi_rtl_categories: bidi_label = True if not bidi_label and not check_ltr: @@ -131,7 +133,7 @@ def check_bidi(label: str, check_ltr: bool = False) -> bool: elif direction == "L": rtl = False else: - raise IDNABidiError(f"First codepoint in label {repr(label)} must be directionality L, R or AL") + raise IDNABidiError(f"First codepoint in label {label!r} must be directionality L, R or AL") valid_ending = False number_type: Optional[str] = None @@ -151,9 +153,8 @@ def check_bidi(label: str, check_ltr: bool = False) -> bool: if direction in _bidi_rtl_numeric: if not number_type: number_type = direction - else: - if number_type != direction: - raise IDNABidiError("Can not mix numeral types in a right-to-left label") + elif number_type != direction: + raise IDNABidiError("Can not mix numeral types in a right-to-left label") else: # Bidi rule 5 if direction not in _bidi_ltr_allowed: @@ -209,6 +210,8 @@ def check_nfc(label: str) -> None: :param label: The label to check. :raises IDNAError: If ``label`` differs from its NFC normalisation. """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") if unicodedata.normalize("NFC", label) != label: raise IDNAError("Label must be in Normalization Form C") @@ -227,7 +230,10 @@ def valid_contextj(label: str, pos: int) -> bool: ``pos`` is not a recognised joiner). :raises ValueError: If an adjacent codepoint has no Unicode name when determining its combining class. + :raises IDNAError: If ``label`` exceeds the defensive input length limit. """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") cp_value = ord(label[pos]) if cp_value == 0x200C: @@ -236,35 +242,32 @@ def valid_contextj(label: str, pos: int) -> bool: ok = False for i in range(pos - 1, -1, -1): - joining_type = idnadata.joining_types().get(ord(label[i])) - if joining_type == ord("T"): + joining_type = _joining_type(ord(label[i])) + if joining_type == "T": continue - elif joining_type in _bidi_joiner_l_or_d: + if joining_type in _bidi_joiner_l_or_d: ok = True break - else: - break + break if not ok: return False ok = False for i in range(pos + 1, len(label)): - joining_type = idnadata.joining_types().get(ord(label[i])) - if joining_type == ord("T"): + joining_type = _joining_type(ord(label[i])) + if joining_type == "T": continue - elif joining_type in _bidi_joiner_r_or_d: + if joining_type in _bidi_joiner_r_or_d: ok = True break - else: - break + break return ok if cp_value == 0x200D: return pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class - else: - return False + return False def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: @@ -280,23 +283,26 @@ def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: :returns: ``True`` if the codepoint at ``pos`` satisfies its CONTEXTO rule, ``False`` otherwise (including when the codepoint is not a recognised CONTEXTO codepoint). + :raises IDNAError: If ``label`` exceeds the defensive input length limit. """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") cp_value = ord(label[pos]) if cp_value == 0x00B7: return 0 < pos < len(label) - 1 and ord(label[pos - 1]) == 0x006C and ord(label[pos + 1]) == 0x006C - elif cp_value == 0x0375: + if cp_value == 0x0375: if pos < len(label) - 1 and len(label) > 1: return _is_script(label[pos + 1], "Greek") return False - elif cp_value == 0x05F3 or cp_value == 0x05F4: + if cp_value in {0x05F3, 0x05F4}: if pos > 0: return _is_script(label[pos - 1], "Hebrew") return False - elif cp_value == 0x30FB: + if cp_value == 0x30FB: for cp in label: if cp == "\u30fb": continue @@ -304,10 +310,10 @@ def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: return True return False - elif 0x660 <= cp_value <= 0x669: + if 0x660 <= cp_value <= 0x669: return not any(0x6F0 <= ord(cp) <= 0x06F9 for cp in label) - elif 0x6F0 <= cp_value <= 0x6F9: + if 0x6F0 <= cp_value <= 0x6F9: return not any(0x660 <= ord(cp) <= 0x0669 for cp in label) return False @@ -331,6 +337,8 @@ def check_label(label: Union[str, bytes, bytearray]) -> None: is not valid in its context. :raises IDNABidiError: If the Bidi Rule is violated. """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") if isinstance(label, (bytes, bytearray)): label = label.decode("utf-8") if len(label) == 0: @@ -349,23 +357,19 @@ def check_label(label: Union[str, bytes, bytearray]) -> None: cp_value = ord(cp) if intranges_contain(cp_value, idnadata.codepoint_classes["PVALID"]): continue - elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]): + if intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]): try: if not valid_contextj(label, pos): - raise InvalidCodepointContext( - f"Joiner {_unot(cp_value)} not allowed at position {pos + 1} in {repr(label)}" - ) + raise InvalidCodepointContext(f"Joiner {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") except ValueError as err: raise IDNAError( - f"Unknown codepoint adjacent to joiner {_unot(cp_value)} at position {pos + 1} in {repr(label)}" + f"Unknown codepoint adjacent to joiner {_unot(cp_value)} at position {pos + 1} in {label!r}" ) from err elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTO"]): if not valid_contexto(label, pos): - raise InvalidCodepointContext( - f"Codepoint {_unot(cp_value)} not allowed at position {pos + 1} in {repr(label)}" - ) + raise InvalidCodepointContext(f"Codepoint {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") else: - raise InvalidCodepoint(f"Codepoint {_unot(cp_value)} at position {pos + 1} of {repr(label)} not allowed") + raise InvalidCodepoint(f"Codepoint {_unot(cp_value)} at position {pos + 1} of {label!r} not allowed") check_bidi(label) @@ -383,14 +387,17 @@ def alabel(label: str) -> bytes: :raises IDNAError: If the label is invalid or the resulting A-label exceeds 63 octets. """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") try: label_bytes = label.encode("ascii") + except UnicodeEncodeError: + pass + else: ulabel(label_bytes) if not valid_label_length(label_bytes): raise IDNAError("Label too long") return label_bytes - except UnicodeEncodeError: - pass check_label(label) label_bytes = _alabel_prefix + _punycode(label) @@ -414,6 +421,8 @@ def ulabel(label: Union[str, bytes, bytearray]) -> str: :returns: The U-label as a Unicode string. :raises IDNAError: If the label is malformed or fails validation. """ + if len(label) > _max_input_length: + raise IDNAError("Label too long") if not isinstance(label, (bytes, bytearray)): try: label_bytes = label.encode("ascii") @@ -460,18 +469,19 @@ def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False :returns: The remapped domain, in Normalisation Form C. :raises InvalidCodepoint: If the domain contains a disallowed codepoint under the chosen rules. + :raises IDNAError: If ``domain`` exceeds the defensive input length limit. """ - from .uts46data import uts46data + if len(domain) > _max_input_length: + raise IDNAError("Domain too long") + from .uts46data import uts46_replacements, uts46_starts, uts46_statuses output = "" for pos, char in enumerate(domain): code_point = ord(char) - uts46row = uts46data[code_point if code_point < 256 else bisect.bisect_left(uts46data, (code_point, "Z")) - 1] - status = uts46row[1] - replacement: Optional[str] = None - if len(uts46row) == 3: - replacement = uts46row[2] # ty: ignore[index-out-of-bounds] + i = code_point if code_point < 256 else bisect.bisect_right(uts46_starts, code_point) - 1 + status = chr(uts46_statuses[i]) + replacement: Optional[str] = uts46_replacements[i] # UTS #46 §4: V is always valid, D is deviation (kept unless transitional), # 3 is disallowed-STD3 (kept unmapped if std3_rules is off and no mapping). @@ -492,7 +502,7 @@ def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False elif status == "I": continue else: - raise InvalidCodepoint(f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {repr(domain)}") + raise InvalidCodepoint(f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {domain!r}") return unicodedata.normalize("NFC", output) @@ -537,6 +547,8 @@ def encode( s = str(s, "ascii") except (UnicodeDecodeError, TypeError) as err: raise IDNAError("should pass a unicode string to the function rather than a byte string.") from err + if len(s) > _max_input_length: + raise IDNAError("Domain too long") if uts46: s = uts46_remap(s, std3_rules, transitional) @@ -572,6 +584,7 @@ def decode( strict: bool = False, uts46: bool = False, std3_rules: bool = False, + display: bool = False, ) -> str: """Decode an A-label-encoded domain name back to Unicode. @@ -586,6 +599,13 @@ def decode( :param uts46: If ``True``, apply UTS #46 mapping before decoding. :param std3_rules: Forwarded to :func:`uts46_remap` when ``uts46`` is ``True``. + :param display: If ``True``, any ``xn--`` label that fails IDNA + validation is passed through unchanged (lowercased) rather than + aborting the whole call. Intended for "decode for display" + consumers (e.g. URL libraries, HTTP clients) that want to show + the user the label as it appears on the wire when it cannot be + rendered as Unicode. Matches the per-label recovery prescribed + by UTS #46 §4 and the WHATWG URL "domain to Unicode" algorithm. :returns: The decoded domain as a Unicode string. :raises IDNAError: If the input is not valid ASCII, contains an invalid label, or is empty. @@ -595,6 +615,8 @@ def decode( s = str(s, "ascii") except (UnicodeDecodeError, TypeError) as err: raise IDNAError("Invalid ASCII in A-label") from err + if len(s) > _max_input_length: + raise IDNAError("Domain too long") if uts46: s = uts46_remap(s, std3_rules, False) # Reject inputs that exceed the maximum DNS domain length up-front @@ -610,9 +632,15 @@ def decode( del labels[-1] trailing_dot = True for label in labels: - s = ulabel(label) - if s: - result.append(s) + try: + u = ulabel(label) + except IDNAError: + if display and label[:4].lower() == "xn--": + u = label.lower() + else: + raise + if u: + result.append(u) else: raise IDNAError("Empty label") if trailing_dot: diff --git a/contrib/python/idna/py3/idna/idnadata.py b/contrib/python/idna/py3/idna/idnadata.py index 8335f26059c..f2ab38897bf 100644 --- a/contrib/python/idna/py3/idna/idnadata.py +++ b/contrib/python/idna/py3/idna/idnadata.py @@ -1,8 +1,5 @@ # This file is automatically generated by tools/idna-data -from functools import lru_cache -from typing import Dict - __version__ = "17.0.0" scripts = { @@ -105,3014 +102,547 @@ scripts = { } -@lru_cache(None) -def joining_types() -> Dict[int, int]: - return { - 0xAD: 84, - 0x300: 84, - 0x301: 84, - 0x302: 84, - 0x303: 84, - 0x304: 84, - 0x305: 84, - 0x306: 84, - 0x307: 84, - 0x308: 84, - 0x309: 84, - 0x30A: 84, - 0x30B: 84, - 0x30C: 84, - 0x30D: 84, - 0x30E: 84, - 0x30F: 84, - 0x310: 84, - 0x311: 84, - 0x312: 84, - 0x313: 84, - 0x314: 84, - 0x315: 84, - 0x316: 84, - 0x317: 84, - 0x318: 84, - 0x319: 84, - 0x31A: 84, - 0x31B: 84, - 0x31C: 84, - 0x31D: 84, - 0x31E: 84, - 0x31F: 84, - 0x320: 84, - 0x321: 84, - 0x322: 84, - 0x323: 84, - 0x324: 84, - 0x325: 84, - 0x326: 84, - 0x327: 84, - 0x328: 84, - 0x329: 84, - 0x32A: 84, - 0x32B: 84, - 0x32C: 84, - 0x32D: 84, - 0x32E: 84, - 0x32F: 84, - 0x330: 84, - 0x331: 84, - 0x332: 84, - 0x333: 84, - 0x334: 84, - 0x335: 84, - 0x336: 84, - 0x337: 84, - 0x338: 84, - 0x339: 84, - 0x33A: 84, - 0x33B: 84, - 0x33C: 84, - 0x33D: 84, - 0x33E: 84, - 0x33F: 84, - 0x340: 84, - 0x341: 84, - 0x342: 84, - 0x343: 84, - 0x344: 84, - 0x345: 84, - 0x346: 84, - 0x347: 84, - 0x348: 84, - 0x349: 84, - 0x34A: 84, - 0x34B: 84, - 0x34C: 84, - 0x34D: 84, - 0x34E: 84, - 0x34F: 84, - 0x350: 84, - 0x351: 84, - 0x352: 84, - 0x353: 84, - 0x354: 84, - 0x355: 84, - 0x356: 84, - 0x357: 84, - 0x358: 84, - 0x359: 84, - 0x35A: 84, - 0x35B: 84, - 0x35C: 84, - 0x35D: 84, - 0x35E: 84, - 0x35F: 84, - 0x360: 84, - 0x361: 84, - 0x362: 84, - 0x363: 84, - 0x364: 84, - 0x365: 84, - 0x366: 84, - 0x367: 84, - 0x368: 84, - 0x369: 84, - 0x36A: 84, - 0x36B: 84, - 0x36C: 84, - 0x36D: 84, - 0x36E: 84, - 0x36F: 84, - 0x483: 84, - 0x484: 84, - 0x485: 84, - 0x486: 84, - 0x487: 84, - 0x488: 84, - 0x489: 84, - 0x591: 84, - 0x592: 84, - 0x593: 84, - 0x594: 84, - 0x595: 84, - 0x596: 84, - 0x597: 84, - 0x598: 84, - 0x599: 84, - 0x59A: 84, - 0x59B: 84, - 0x59C: 84, - 0x59D: 84, - 0x59E: 84, - 0x59F: 84, - 0x5A0: 84, - 0x5A1: 84, - 0x5A2: 84, - 0x5A3: 84, - 0x5A4: 84, - 0x5A5: 84, - 0x5A6: 84, - 0x5A7: 84, - 0x5A8: 84, - 0x5A9: 84, - 0x5AA: 84, - 0x5AB: 84, - 0x5AC: 84, - 0x5AD: 84, - 0x5AE: 84, - 0x5AF: 84, - 0x5B0: 84, - 0x5B1: 84, - 0x5B2: 84, - 0x5B3: 84, - 0x5B4: 84, - 0x5B5: 84, - 0x5B6: 84, - 0x5B7: 84, - 0x5B8: 84, - 0x5B9: 84, - 0x5BA: 84, - 0x5BB: 84, - 0x5BC: 84, - 0x5BD: 84, - 0x5BF: 84, - 0x5C1: 84, - 0x5C2: 84, - 0x5C4: 84, - 0x5C5: 84, - 0x5C7: 84, - 0x610: 84, - 0x611: 84, - 0x612: 84, - 0x613: 84, - 0x614: 84, - 0x615: 84, - 0x616: 84, - 0x617: 84, - 0x618: 84, - 0x619: 84, - 0x61A: 84, - 0x61C: 84, - 0x620: 68, - 0x622: 82, - 0x623: 82, - 0x624: 82, - 0x625: 82, - 0x626: 68, - 0x627: 82, - 0x628: 68, - 0x629: 82, - 0x62A: 68, - 0x62B: 68, - 0x62C: 68, - 0x62D: 68, - 0x62E: 68, - 0x62F: 82, - 0x630: 82, - 0x631: 82, - 0x632: 82, - 0x633: 68, - 0x634: 68, - 0x635: 68, - 0x636: 68, - 0x637: 68, - 0x638: 68, - 0x639: 68, - 0x63A: 68, - 0x63B: 68, - 0x63C: 68, - 0x63D: 68, - 0x63E: 68, - 0x63F: 68, - 0x640: 67, - 0x641: 68, - 0x642: 68, - 0x643: 68, - 0x644: 68, - 0x645: 68, - 0x646: 68, - 0x647: 68, - 0x648: 82, - 0x649: 68, - 0x64A: 68, - 0x64B: 84, - 0x64C: 84, - 0x64D: 84, - 0x64E: 84, - 0x64F: 84, - 0x650: 84, - 0x651: 84, - 0x652: 84, - 0x653: 84, - 0x654: 84, - 0x655: 84, - 0x656: 84, - 0x657: 84, - 0x658: 84, - 0x659: 84, - 0x65A: 84, - 0x65B: 84, - 0x65C: 84, - 0x65D: 84, - 0x65E: 84, - 0x65F: 84, - 0x66E: 68, - 0x66F: 68, - 0x670: 84, - 0x671: 82, - 0x672: 82, - 0x673: 82, - 0x675: 82, - 0x676: 82, - 0x677: 82, - 0x678: 68, - 0x679: 68, - 0x67A: 68, - 0x67B: 68, - 0x67C: 68, - 0x67D: 68, - 0x67E: 68, - 0x67F: 68, - 0x680: 68, - 0x681: 68, - 0x682: 68, - 0x683: 68, - 0x684: 68, - 0x685: 68, - 0x686: 68, - 0x687: 68, - 0x688: 82, - 0x689: 82, - 0x68A: 82, - 0x68B: 82, - 0x68C: 82, - 0x68D: 82, - 0x68E: 82, - 0x68F: 82, - 0x690: 82, - 0x691: 82, - 0x692: 82, - 0x693: 82, - 0x694: 82, - 0x695: 82, - 0x696: 82, - 0x697: 82, - 0x698: 82, - 0x699: 82, - 0x69A: 68, - 0x69B: 68, - 0x69C: 68, - 0x69D: 68, - 0x69E: 68, - 0x69F: 68, - 0x6A0: 68, - 0x6A1: 68, - 0x6A2: 68, - 0x6A3: 68, - 0x6A4: 68, - 0x6A5: 68, - 0x6A6: 68, - 0x6A7: 68, - 0x6A8: 68, - 0x6A9: 68, - 0x6AA: 68, - 0x6AB: 68, - 0x6AC: 68, - 0x6AD: 68, - 0x6AE: 68, - 0x6AF: 68, - 0x6B0: 68, - 0x6B1: 68, - 0x6B2: 68, - 0x6B3: 68, - 0x6B4: 68, - 0x6B5: 68, - 0x6B6: 68, - 0x6B7: 68, - 0x6B8: 68, - 0x6B9: 68, - 0x6BA: 68, - 0x6BB: 68, - 0x6BC: 68, - 0x6BD: 68, - 0x6BE: 68, - 0x6BF: 68, - 0x6C0: 82, - 0x6C1: 68, - 0x6C2: 68, - 0x6C3: 82, - 0x6C4: 82, - 0x6C5: 82, - 0x6C6: 82, - 0x6C7: 82, - 0x6C8: 82, - 0x6C9: 82, - 0x6CA: 82, - 0x6CB: 82, - 0x6CC: 68, - 0x6CD: 82, - 0x6CE: 68, - 0x6CF: 82, - 0x6D0: 68, - 0x6D1: 68, - 0x6D2: 82, - 0x6D3: 82, - 0x6D5: 82, - 0x6D6: 84, - 0x6D7: 84, - 0x6D8: 84, - 0x6D9: 84, - 0x6DA: 84, - 0x6DB: 84, - 0x6DC: 84, - 0x6DF: 84, - 0x6E0: 84, - 0x6E1: 84, - 0x6E2: 84, - 0x6E3: 84, - 0x6E4: 84, - 0x6E7: 84, - 0x6E8: 84, - 0x6EA: 84, - 0x6EB: 84, - 0x6EC: 84, - 0x6ED: 84, - 0x6EE: 82, - 0x6EF: 82, - 0x6FA: 68, - 0x6FB: 68, - 0x6FC: 68, - 0x6FF: 68, - 0x70F: 84, - 0x710: 82, - 0x711: 84, - 0x712: 68, - 0x713: 68, - 0x714: 68, - 0x715: 82, - 0x716: 82, - 0x717: 82, - 0x718: 82, - 0x719: 82, - 0x71A: 68, - 0x71B: 68, - 0x71C: 68, - 0x71D: 68, - 0x71E: 82, - 0x71F: 68, - 0x720: 68, - 0x721: 68, - 0x722: 68, - 0x723: 68, - 0x724: 68, - 0x725: 68, - 0x726: 68, - 0x727: 68, - 0x728: 82, - 0x729: 68, - 0x72A: 82, - 0x72B: 68, - 0x72C: 82, - 0x72D: 68, - 0x72E: 68, - 0x72F: 82, - 0x730: 84, - 0x731: 84, - 0x732: 84, - 0x733: 84, - 0x734: 84, - 0x735: 84, - 0x736: 84, - 0x737: 84, - 0x738: 84, - 0x739: 84, - 0x73A: 84, - 0x73B: 84, - 0x73C: 84, - 0x73D: 84, - 0x73E: 84, - 0x73F: 84, - 0x740: 84, - 0x741: 84, - 0x742: 84, - 0x743: 84, - 0x744: 84, - 0x745: 84, - 0x746: 84, - 0x747: 84, - 0x748: 84, - 0x749: 84, - 0x74A: 84, - 0x74D: 82, - 0x74E: 68, - 0x74F: 68, - 0x750: 68, - 0x751: 68, - 0x752: 68, - 0x753: 68, - 0x754: 68, - 0x755: 68, - 0x756: 68, - 0x757: 68, - 0x758: 68, - 0x759: 82, - 0x75A: 82, - 0x75B: 82, - 0x75C: 68, - 0x75D: 68, - 0x75E: 68, - 0x75F: 68, - 0x760: 68, - 0x761: 68, - 0x762: 68, - 0x763: 68, - 0x764: 68, - 0x765: 68, - 0x766: 68, - 0x767: 68, - 0x768: 68, - 0x769: 68, - 0x76A: 68, - 0x76B: 82, - 0x76C: 82, - 0x76D: 68, - 0x76E: 68, - 0x76F: 68, - 0x770: 68, - 0x771: 82, - 0x772: 68, - 0x773: 82, - 0x774: 82, - 0x775: 68, - 0x776: 68, - 0x777: 68, - 0x778: 82, - 0x779: 82, - 0x77A: 68, - 0x77B: 68, - 0x77C: 68, - 0x77D: 68, - 0x77E: 68, - 0x77F: 68, - 0x7A6: 84, - 0x7A7: 84, - 0x7A8: 84, - 0x7A9: 84, - 0x7AA: 84, - 0x7AB: 84, - 0x7AC: 84, - 0x7AD: 84, - 0x7AE: 84, - 0x7AF: 84, - 0x7B0: 84, - 0x7CA: 68, - 0x7CB: 68, - 0x7CC: 68, - 0x7CD: 68, - 0x7CE: 68, - 0x7CF: 68, - 0x7D0: 68, - 0x7D1: 68, - 0x7D2: 68, - 0x7D3: 68, - 0x7D4: 68, - 0x7D5: 68, - 0x7D6: 68, - 0x7D7: 68, - 0x7D8: 68, - 0x7D9: 68, - 0x7DA: 68, - 0x7DB: 68, - 0x7DC: 68, - 0x7DD: 68, - 0x7DE: 68, - 0x7DF: 68, - 0x7E0: 68, - 0x7E1: 68, - 0x7E2: 68, - 0x7E3: 68, - 0x7E4: 68, - 0x7E5: 68, - 0x7E6: 68, - 0x7E7: 68, - 0x7E8: 68, - 0x7E9: 68, - 0x7EA: 68, - 0x7EB: 84, - 0x7EC: 84, - 0x7ED: 84, - 0x7EE: 84, - 0x7EF: 84, - 0x7F0: 84, - 0x7F1: 84, - 0x7F2: 84, - 0x7F3: 84, - 0x7FA: 67, - 0x7FD: 84, - 0x816: 84, - 0x817: 84, - 0x818: 84, - 0x819: 84, - 0x81B: 84, - 0x81C: 84, - 0x81D: 84, - 0x81E: 84, - 0x81F: 84, - 0x820: 84, - 0x821: 84, - 0x822: 84, - 0x823: 84, - 0x825: 84, - 0x826: 84, - 0x827: 84, - 0x829: 84, - 0x82A: 84, - 0x82B: 84, - 0x82C: 84, - 0x82D: 84, - 0x840: 82, - 0x841: 68, - 0x842: 68, - 0x843: 68, - 0x844: 68, - 0x845: 68, - 0x846: 82, - 0x847: 82, - 0x848: 68, - 0x849: 82, - 0x84A: 68, - 0x84B: 68, - 0x84C: 68, - 0x84D: 68, - 0x84E: 68, - 0x84F: 68, - 0x850: 68, - 0x851: 68, - 0x852: 68, - 0x853: 68, - 0x854: 82, - 0x855: 68, - 0x856: 82, - 0x857: 82, - 0x858: 82, - 0x859: 84, - 0x85A: 84, - 0x85B: 84, - 0x860: 68, - 0x862: 68, - 0x863: 68, - 0x864: 68, - 0x865: 68, - 0x867: 82, - 0x868: 68, - 0x869: 82, - 0x86A: 82, - 0x870: 82, - 0x871: 82, - 0x872: 82, - 0x873: 82, - 0x874: 82, - 0x875: 82, - 0x876: 82, - 0x877: 82, - 0x878: 82, - 0x879: 82, - 0x87A: 82, - 0x87B: 82, - 0x87C: 82, - 0x87D: 82, - 0x87E: 82, - 0x87F: 82, - 0x880: 82, - 0x881: 82, - 0x882: 82, - 0x883: 67, - 0x884: 67, - 0x885: 67, - 0x886: 68, - 0x889: 68, - 0x88A: 68, - 0x88B: 68, - 0x88C: 68, - 0x88D: 68, - 0x88E: 82, - 0x88F: 68, - 0x897: 84, - 0x898: 84, - 0x899: 84, - 0x89A: 84, - 0x89B: 84, - 0x89C: 84, - 0x89D: 84, - 0x89E: 84, - 0x89F: 84, - 0x8A0: 68, - 0x8A1: 68, - 0x8A2: 68, - 0x8A3: 68, - 0x8A4: 68, - 0x8A5: 68, - 0x8A6: 68, - 0x8A7: 68, - 0x8A8: 68, - 0x8A9: 68, - 0x8AA: 82, - 0x8AB: 82, - 0x8AC: 82, - 0x8AE: 82, - 0x8AF: 68, - 0x8B0: 68, - 0x8B1: 82, - 0x8B2: 82, - 0x8B3: 68, - 0x8B4: 68, - 0x8B5: 68, - 0x8B6: 68, - 0x8B7: 68, - 0x8B8: 68, - 0x8B9: 82, - 0x8BA: 68, - 0x8BB: 68, - 0x8BC: 68, - 0x8BD: 68, - 0x8BE: 68, - 0x8BF: 68, - 0x8C0: 68, - 0x8C1: 68, - 0x8C2: 68, - 0x8C3: 68, - 0x8C4: 68, - 0x8C5: 68, - 0x8C6: 68, - 0x8C7: 68, - 0x8C8: 68, - 0x8CA: 84, - 0x8CB: 84, - 0x8CC: 84, - 0x8CD: 84, - 0x8CE: 84, - 0x8CF: 84, - 0x8D0: 84, - 0x8D1: 84, - 0x8D2: 84, - 0x8D3: 84, - 0x8D4: 84, - 0x8D5: 84, - 0x8D6: 84, - 0x8D7: 84, - 0x8D8: 84, - 0x8D9: 84, - 0x8DA: 84, - 0x8DB: 84, - 0x8DC: 84, - 0x8DD: 84, - 0x8DE: 84, - 0x8DF: 84, - 0x8E0: 84, - 0x8E1: 84, - 0x8E3: 84, - 0x8E4: 84, - 0x8E5: 84, - 0x8E6: 84, - 0x8E7: 84, - 0x8E8: 84, - 0x8E9: 84, - 0x8EA: 84, - 0x8EB: 84, - 0x8EC: 84, - 0x8ED: 84, - 0x8EE: 84, - 0x8EF: 84, - 0x8F0: 84, - 0x8F1: 84, - 0x8F2: 84, - 0x8F3: 84, - 0x8F4: 84, - 0x8F5: 84, - 0x8F6: 84, - 0x8F7: 84, - 0x8F8: 84, - 0x8F9: 84, - 0x8FA: 84, - 0x8FB: 84, - 0x8FC: 84, - 0x8FD: 84, - 0x8FE: 84, - 0x8FF: 84, - 0x900: 84, - 0x901: 84, - 0x902: 84, - 0x93A: 84, - 0x93C: 84, - 0x941: 84, - 0x942: 84, - 0x943: 84, - 0x944: 84, - 0x945: 84, - 0x946: 84, - 0x947: 84, - 0x948: 84, - 0x94D: 84, - 0x951: 84, - 0x952: 84, - 0x953: 84, - 0x954: 84, - 0x955: 84, - 0x956: 84, - 0x957: 84, - 0x962: 84, - 0x963: 84, - 0x981: 84, - 0x9BC: 84, - 0x9C1: 84, - 0x9C2: 84, - 0x9C3: 84, - 0x9C4: 84, - 0x9CD: 84, - 0x9E2: 84, - 0x9E3: 84, - 0x9FE: 84, - 0xA01: 84, - 0xA02: 84, - 0xA3C: 84, - 0xA41: 84, - 0xA42: 84, - 0xA47: 84, - 0xA48: 84, - 0xA4B: 84, - 0xA4C: 84, - 0xA4D: 84, - 0xA51: 84, - 0xA70: 84, - 0xA71: 84, - 0xA75: 84, - 0xA81: 84, - 0xA82: 84, - 0xABC: 84, - 0xAC1: 84, - 0xAC2: 84, - 0xAC3: 84, - 0xAC4: 84, - 0xAC5: 84, - 0xAC7: 84, - 0xAC8: 84, - 0xACD: 84, - 0xAE2: 84, - 0xAE3: 84, - 0xAFA: 84, - 0xAFB: 84, - 0xAFC: 84, - 0xAFD: 84, - 0xAFE: 84, - 0xAFF: 84, - 0xB01: 84, - 0xB3C: 84, - 0xB3F: 84, - 0xB41: 84, - 0xB42: 84, - 0xB43: 84, - 0xB44: 84, - 0xB4D: 84, - 0xB55: 84, - 0xB56: 84, - 0xB62: 84, - 0xB63: 84, - 0xB82: 84, - 0xBC0: 84, - 0xBCD: 84, - 0xC00: 84, - 0xC04: 84, - 0xC3C: 84, - 0xC3E: 84, - 0xC3F: 84, - 0xC40: 84, - 0xC46: 84, - 0xC47: 84, - 0xC48: 84, - 0xC4A: 84, - 0xC4B: 84, - 0xC4C: 84, - 0xC4D: 84, - 0xC55: 84, - 0xC56: 84, - 0xC62: 84, - 0xC63: 84, - 0xC81: 84, - 0xCBC: 84, - 0xCBF: 84, - 0xCC6: 84, - 0xCCC: 84, - 0xCCD: 84, - 0xCE2: 84, - 0xCE3: 84, - 0xD00: 84, - 0xD01: 84, - 0xD3B: 84, - 0xD3C: 84, - 0xD41: 84, - 0xD42: 84, - 0xD43: 84, - 0xD44: 84, - 0xD4D: 84, - 0xD62: 84, - 0xD63: 84, - 0xD81: 84, - 0xDCA: 84, - 0xDD2: 84, - 0xDD3: 84, - 0xDD4: 84, - 0xDD6: 84, - 0xE31: 84, - 0xE34: 84, - 0xE35: 84, - 0xE36: 84, - 0xE37: 84, - 0xE38: 84, - 0xE39: 84, - 0xE3A: 84, - 0xE47: 84, - 0xE48: 84, - 0xE49: 84, - 0xE4A: 84, - 0xE4B: 84, - 0xE4C: 84, - 0xE4D: 84, - 0xE4E: 84, - 0xEB1: 84, - 0xEB4: 84, - 0xEB5: 84, - 0xEB6: 84, - 0xEB7: 84, - 0xEB8: 84, - 0xEB9: 84, - 0xEBA: 84, - 0xEBB: 84, - 0xEBC: 84, - 0xEC8: 84, - 0xEC9: 84, - 0xECA: 84, - 0xECB: 84, - 0xECC: 84, - 0xECD: 84, - 0xECE: 84, - 0xF18: 84, - 0xF19: 84, - 0xF35: 84, - 0xF37: 84, - 0xF39: 84, - 0xF71: 84, - 0xF72: 84, - 0xF73: 84, - 0xF74: 84, - 0xF75: 84, - 0xF76: 84, - 0xF77: 84, - 0xF78: 84, - 0xF79: 84, - 0xF7A: 84, - 0xF7B: 84, - 0xF7C: 84, - 0xF7D: 84, - 0xF7E: 84, - 0xF80: 84, - 0xF81: 84, - 0xF82: 84, - 0xF83: 84, - 0xF84: 84, - 0xF86: 84, - 0xF87: 84, - 0xF8D: 84, - 0xF8E: 84, - 0xF8F: 84, - 0xF90: 84, - 0xF91: 84, - 0xF92: 84, - 0xF93: 84, - 0xF94: 84, - 0xF95: 84, - 0xF96: 84, - 0xF97: 84, - 0xF99: 84, - 0xF9A: 84, - 0xF9B: 84, - 0xF9C: 84, - 0xF9D: 84, - 0xF9E: 84, - 0xF9F: 84, - 0xFA0: 84, - 0xFA1: 84, - 0xFA2: 84, - 0xFA3: 84, - 0xFA4: 84, - 0xFA5: 84, - 0xFA6: 84, - 0xFA7: 84, - 0xFA8: 84, - 0xFA9: 84, - 0xFAA: 84, - 0xFAB: 84, - 0xFAC: 84, - 0xFAD: 84, - 0xFAE: 84, - 0xFAF: 84, - 0xFB0: 84, - 0xFB1: 84, - 0xFB2: 84, - 0xFB3: 84, - 0xFB4: 84, - 0xFB5: 84, - 0xFB6: 84, - 0xFB7: 84, - 0xFB8: 84, - 0xFB9: 84, - 0xFBA: 84, - 0xFBB: 84, - 0xFBC: 84, - 0xFC6: 84, - 0x102D: 84, - 0x102E: 84, - 0x102F: 84, - 0x1030: 84, - 0x1032: 84, - 0x1033: 84, - 0x1034: 84, - 0x1035: 84, - 0x1036: 84, - 0x1037: 84, - 0x1039: 84, - 0x103A: 84, - 0x103D: 84, - 0x103E: 84, - 0x1058: 84, - 0x1059: 84, - 0x105E: 84, - 0x105F: 84, - 0x1060: 84, - 0x1071: 84, - 0x1072: 84, - 0x1073: 84, - 0x1074: 84, - 0x1082: 84, - 0x1085: 84, - 0x1086: 84, - 0x108D: 84, - 0x109D: 84, - 0x135D: 84, - 0x135E: 84, - 0x135F: 84, - 0x1712: 84, - 0x1713: 84, - 0x1714: 84, - 0x1732: 84, - 0x1733: 84, - 0x1752: 84, - 0x1753: 84, - 0x1772: 84, - 0x1773: 84, - 0x17B4: 84, - 0x17B5: 84, - 0x17B7: 84, - 0x17B8: 84, - 0x17B9: 84, - 0x17BA: 84, - 0x17BB: 84, - 0x17BC: 84, - 0x17BD: 84, - 0x17C6: 84, - 0x17C9: 84, - 0x17CA: 84, - 0x17CB: 84, - 0x17CC: 84, - 0x17CD: 84, - 0x17CE: 84, - 0x17CF: 84, - 0x17D0: 84, - 0x17D1: 84, - 0x17D2: 84, - 0x17D3: 84, - 0x17DD: 84, - 0x1807: 68, - 0x180A: 67, - 0x180B: 84, - 0x180C: 84, - 0x180D: 84, - 0x180F: 84, - 0x1820: 68, - 0x1821: 68, - 0x1822: 68, - 0x1823: 68, - 0x1824: 68, - 0x1825: 68, - 0x1826: 68, - 0x1827: 68, - 0x1828: 68, - 0x1829: 68, - 0x182A: 68, - 0x182B: 68, - 0x182C: 68, - 0x182D: 68, - 0x182E: 68, - 0x182F: 68, - 0x1830: 68, - 0x1831: 68, - 0x1832: 68, - 0x1833: 68, - 0x1834: 68, - 0x1835: 68, - 0x1836: 68, - 0x1837: 68, - 0x1838: 68, - 0x1839: 68, - 0x183A: 68, - 0x183B: 68, - 0x183C: 68, - 0x183D: 68, - 0x183E: 68, - 0x183F: 68, - 0x1840: 68, - 0x1841: 68, - 0x1842: 68, - 0x1843: 68, - 0x1844: 68, - 0x1845: 68, - 0x1846: 68, - 0x1847: 68, - 0x1848: 68, - 0x1849: 68, - 0x184A: 68, - 0x184B: 68, - 0x184C: 68, - 0x184D: 68, - 0x184E: 68, - 0x184F: 68, - 0x1850: 68, - 0x1851: 68, - 0x1852: 68, - 0x1853: 68, - 0x1854: 68, - 0x1855: 68, - 0x1856: 68, - 0x1857: 68, - 0x1858: 68, - 0x1859: 68, - 0x185A: 68, - 0x185B: 68, - 0x185C: 68, - 0x185D: 68, - 0x185E: 68, - 0x185F: 68, - 0x1860: 68, - 0x1861: 68, - 0x1862: 68, - 0x1863: 68, - 0x1864: 68, - 0x1865: 68, - 0x1866: 68, - 0x1867: 68, - 0x1868: 68, - 0x1869: 68, - 0x186A: 68, - 0x186B: 68, - 0x186C: 68, - 0x186D: 68, - 0x186E: 68, - 0x186F: 68, - 0x1870: 68, - 0x1871: 68, - 0x1872: 68, - 0x1873: 68, - 0x1874: 68, - 0x1875: 68, - 0x1876: 68, - 0x1877: 68, - 0x1878: 68, - 0x1885: 84, - 0x1886: 84, - 0x1887: 68, - 0x1888: 68, - 0x1889: 68, - 0x188A: 68, - 0x188B: 68, - 0x188C: 68, - 0x188D: 68, - 0x188E: 68, - 0x188F: 68, - 0x1890: 68, - 0x1891: 68, - 0x1892: 68, - 0x1893: 68, - 0x1894: 68, - 0x1895: 68, - 0x1896: 68, - 0x1897: 68, - 0x1898: 68, - 0x1899: 68, - 0x189A: 68, - 0x189B: 68, - 0x189C: 68, - 0x189D: 68, - 0x189E: 68, - 0x189F: 68, - 0x18A0: 68, - 0x18A1: 68, - 0x18A2: 68, - 0x18A3: 68, - 0x18A4: 68, - 0x18A5: 68, - 0x18A6: 68, - 0x18A7: 68, - 0x18A8: 68, - 0x18A9: 84, - 0x18AA: 68, - 0x1920: 84, - 0x1921: 84, - 0x1922: 84, - 0x1927: 84, - 0x1928: 84, - 0x1932: 84, - 0x1939: 84, - 0x193A: 84, - 0x193B: 84, - 0x1A17: 84, - 0x1A18: 84, - 0x1A1B: 84, - 0x1A56: 84, - 0x1A58: 84, - 0x1A59: 84, - 0x1A5A: 84, - 0x1A5B: 84, - 0x1A5C: 84, - 0x1A5D: 84, - 0x1A5E: 84, - 0x1A60: 84, - 0x1A62: 84, - 0x1A65: 84, - 0x1A66: 84, - 0x1A67: 84, - 0x1A68: 84, - 0x1A69: 84, - 0x1A6A: 84, - 0x1A6B: 84, - 0x1A6C: 84, - 0x1A73: 84, - 0x1A74: 84, - 0x1A75: 84, - 0x1A76: 84, - 0x1A77: 84, - 0x1A78: 84, - 0x1A79: 84, - 0x1A7A: 84, - 0x1A7B: 84, - 0x1A7C: 84, - 0x1A7F: 84, - 0x1AB0: 84, - 0x1AB1: 84, - 0x1AB2: 84, - 0x1AB3: 84, - 0x1AB4: 84, - 0x1AB5: 84, - 0x1AB6: 84, - 0x1AB7: 84, - 0x1AB8: 84, - 0x1AB9: 84, - 0x1ABA: 84, - 0x1ABB: 84, - 0x1ABC: 84, - 0x1ABD: 84, - 0x1ABE: 84, - 0x1ABF: 84, - 0x1AC0: 84, - 0x1AC1: 84, - 0x1AC2: 84, - 0x1AC3: 84, - 0x1AC4: 84, - 0x1AC5: 84, - 0x1AC6: 84, - 0x1AC7: 84, - 0x1AC8: 84, - 0x1AC9: 84, - 0x1ACA: 84, - 0x1ACB: 84, - 0x1ACC: 84, - 0x1ACD: 84, - 0x1ACE: 84, - 0x1ACF: 84, - 0x1AD0: 84, - 0x1AD1: 84, - 0x1AD2: 84, - 0x1AD3: 84, - 0x1AD4: 84, - 0x1AD5: 84, - 0x1AD6: 84, - 0x1AD7: 84, - 0x1AD8: 84, - 0x1AD9: 84, - 0x1ADA: 84, - 0x1ADB: 84, - 0x1ADC: 84, - 0x1ADD: 84, - 0x1AE0: 84, - 0x1AE1: 84, - 0x1AE2: 84, - 0x1AE3: 84, - 0x1AE4: 84, - 0x1AE5: 84, - 0x1AE6: 84, - 0x1AE7: 84, - 0x1AE8: 84, - 0x1AE9: 84, - 0x1AEA: 84, - 0x1AEB: 84, - 0x1B00: 84, - 0x1B01: 84, - 0x1B02: 84, - 0x1B03: 84, - 0x1B34: 84, - 0x1B36: 84, - 0x1B37: 84, - 0x1B38: 84, - 0x1B39: 84, - 0x1B3A: 84, - 0x1B3C: 84, - 0x1B42: 84, - 0x1B6B: 84, - 0x1B6C: 84, - 0x1B6D: 84, - 0x1B6E: 84, - 0x1B6F: 84, - 0x1B70: 84, - 0x1B71: 84, - 0x1B72: 84, - 0x1B73: 84, - 0x1B80: 84, - 0x1B81: 84, - 0x1BA2: 84, - 0x1BA3: 84, - 0x1BA4: 84, - 0x1BA5: 84, - 0x1BA8: 84, - 0x1BA9: 84, - 0x1BAB: 84, - 0x1BAC: 84, - 0x1BAD: 84, - 0x1BE6: 84, - 0x1BE8: 84, - 0x1BE9: 84, - 0x1BED: 84, - 0x1BEF: 84, - 0x1BF0: 84, - 0x1BF1: 84, - 0x1C2C: 84, - 0x1C2D: 84, - 0x1C2E: 84, - 0x1C2F: 84, - 0x1C30: 84, - 0x1C31: 84, - 0x1C32: 84, - 0x1C33: 84, - 0x1C36: 84, - 0x1C37: 84, - 0x1CD0: 84, - 0x1CD1: 84, - 0x1CD2: 84, - 0x1CD4: 84, - 0x1CD5: 84, - 0x1CD6: 84, - 0x1CD7: 84, - 0x1CD8: 84, - 0x1CD9: 84, - 0x1CDA: 84, - 0x1CDB: 84, - 0x1CDC: 84, - 0x1CDD: 84, - 0x1CDE: 84, - 0x1CDF: 84, - 0x1CE0: 84, - 0x1CE2: 84, - 0x1CE3: 84, - 0x1CE4: 84, - 0x1CE5: 84, - 0x1CE6: 84, - 0x1CE7: 84, - 0x1CE8: 84, - 0x1CED: 84, - 0x1CF4: 84, - 0x1CF8: 84, - 0x1CF9: 84, - 0x1DC0: 84, - 0x1DC1: 84, - 0x1DC2: 84, - 0x1DC3: 84, - 0x1DC4: 84, - 0x1DC5: 84, - 0x1DC6: 84, - 0x1DC7: 84, - 0x1DC8: 84, - 0x1DC9: 84, - 0x1DCA: 84, - 0x1DCB: 84, - 0x1DCC: 84, - 0x1DCD: 84, - 0x1DCE: 84, - 0x1DCF: 84, - 0x1DD0: 84, - 0x1DD1: 84, - 0x1DD2: 84, - 0x1DD3: 84, - 0x1DD4: 84, - 0x1DD5: 84, - 0x1DD6: 84, - 0x1DD7: 84, - 0x1DD8: 84, - 0x1DD9: 84, - 0x1DDA: 84, - 0x1DDB: 84, - 0x1DDC: 84, - 0x1DDD: 84, - 0x1DDE: 84, - 0x1DDF: 84, - 0x1DE0: 84, - 0x1DE1: 84, - 0x1DE2: 84, - 0x1DE3: 84, - 0x1DE4: 84, - 0x1DE5: 84, - 0x1DE6: 84, - 0x1DE7: 84, - 0x1DE8: 84, - 0x1DE9: 84, - 0x1DEA: 84, - 0x1DEB: 84, - 0x1DEC: 84, - 0x1DED: 84, - 0x1DEE: 84, - 0x1DEF: 84, - 0x1DF0: 84, - 0x1DF1: 84, - 0x1DF2: 84, - 0x1DF3: 84, - 0x1DF4: 84, - 0x1DF5: 84, - 0x1DF6: 84, - 0x1DF7: 84, - 0x1DF8: 84, - 0x1DF9: 84, - 0x1DFA: 84, - 0x1DFB: 84, - 0x1DFC: 84, - 0x1DFD: 84, - 0x1DFE: 84, - 0x1DFF: 84, - 0x200B: 84, - 0x200D: 67, - 0x200E: 84, - 0x200F: 84, - 0x202A: 84, - 0x202B: 84, - 0x202C: 84, - 0x202D: 84, - 0x202E: 84, - 0x2060: 84, - 0x2061: 84, - 0x2062: 84, - 0x2063: 84, - 0x2064: 84, - 0x206A: 84, - 0x206B: 84, - 0x206C: 84, - 0x206D: 84, - 0x206E: 84, - 0x206F: 84, - 0x20D0: 84, - 0x20D1: 84, - 0x20D2: 84, - 0x20D3: 84, - 0x20D4: 84, - 0x20D5: 84, - 0x20D6: 84, - 0x20D7: 84, - 0x20D8: 84, - 0x20D9: 84, - 0x20DA: 84, - 0x20DB: 84, - 0x20DC: 84, - 0x20DD: 84, - 0x20DE: 84, - 0x20DF: 84, - 0x20E0: 84, - 0x20E1: 84, - 0x20E2: 84, - 0x20E3: 84, - 0x20E4: 84, - 0x20E5: 84, - 0x20E6: 84, - 0x20E7: 84, - 0x20E8: 84, - 0x20E9: 84, - 0x20EA: 84, - 0x20EB: 84, - 0x20EC: 84, - 0x20ED: 84, - 0x20EE: 84, - 0x20EF: 84, - 0x20F0: 84, - 0x2CEF: 84, - 0x2CF0: 84, - 0x2CF1: 84, - 0x2D7F: 84, - 0x2DE0: 84, - 0x2DE1: 84, - 0x2DE2: 84, - 0x2DE3: 84, - 0x2DE4: 84, - 0x2DE5: 84, - 0x2DE6: 84, - 0x2DE7: 84, - 0x2DE8: 84, - 0x2DE9: 84, - 0x2DEA: 84, - 0x2DEB: 84, - 0x2DEC: 84, - 0x2DED: 84, - 0x2DEE: 84, - 0x2DEF: 84, - 0x2DF0: 84, - 0x2DF1: 84, - 0x2DF2: 84, - 0x2DF3: 84, - 0x2DF4: 84, - 0x2DF5: 84, - 0x2DF6: 84, - 0x2DF7: 84, - 0x2DF8: 84, - 0x2DF9: 84, - 0x2DFA: 84, - 0x2DFB: 84, - 0x2DFC: 84, - 0x2DFD: 84, - 0x2DFE: 84, - 0x2DFF: 84, - 0x302A: 84, - 0x302B: 84, - 0x302C: 84, - 0x302D: 84, - 0x3099: 84, - 0x309A: 84, - 0xA66F: 84, - 0xA670: 84, - 0xA671: 84, - 0xA672: 84, - 0xA674: 84, - 0xA675: 84, - 0xA676: 84, - 0xA677: 84, - 0xA678: 84, - 0xA679: 84, - 0xA67A: 84, - 0xA67B: 84, - 0xA67C: 84, - 0xA67D: 84, - 0xA69E: 84, - 0xA69F: 84, - 0xA6F0: 84, - 0xA6F1: 84, - 0xA802: 84, - 0xA806: 84, - 0xA80B: 84, - 0xA825: 84, - 0xA826: 84, - 0xA82C: 84, - 0xA840: 68, - 0xA841: 68, - 0xA842: 68, - 0xA843: 68, - 0xA844: 68, - 0xA845: 68, - 0xA846: 68, - 0xA847: 68, - 0xA848: 68, - 0xA849: 68, - 0xA84A: 68, - 0xA84B: 68, - 0xA84C: 68, - 0xA84D: 68, - 0xA84E: 68, - 0xA84F: 68, - 0xA850: 68, - 0xA851: 68, - 0xA852: 68, - 0xA853: 68, - 0xA854: 68, - 0xA855: 68, - 0xA856: 68, - 0xA857: 68, - 0xA858: 68, - 0xA859: 68, - 0xA85A: 68, - 0xA85B: 68, - 0xA85C: 68, - 0xA85D: 68, - 0xA85E: 68, - 0xA85F: 68, - 0xA860: 68, - 0xA861: 68, - 0xA862: 68, - 0xA863: 68, - 0xA864: 68, - 0xA865: 68, - 0xA866: 68, - 0xA867: 68, - 0xA868: 68, - 0xA869: 68, - 0xA86A: 68, - 0xA86B: 68, - 0xA86C: 68, - 0xA86D: 68, - 0xA86E: 68, - 0xA86F: 68, - 0xA870: 68, - 0xA871: 68, - 0xA872: 76, - 0xA8C4: 84, - 0xA8C5: 84, - 0xA8E0: 84, - 0xA8E1: 84, - 0xA8E2: 84, - 0xA8E3: 84, - 0xA8E4: 84, - 0xA8E5: 84, - 0xA8E6: 84, - 0xA8E7: 84, - 0xA8E8: 84, - 0xA8E9: 84, - 0xA8EA: 84, - 0xA8EB: 84, - 0xA8EC: 84, - 0xA8ED: 84, - 0xA8EE: 84, - 0xA8EF: 84, - 0xA8F0: 84, - 0xA8F1: 84, - 0xA8FF: 84, - 0xA926: 84, - 0xA927: 84, - 0xA928: 84, - 0xA929: 84, - 0xA92A: 84, - 0xA92B: 84, - 0xA92C: 84, - 0xA92D: 84, - 0xA947: 84, - 0xA948: 84, - 0xA949: 84, - 0xA94A: 84, - 0xA94B: 84, - 0xA94C: 84, - 0xA94D: 84, - 0xA94E: 84, - 0xA94F: 84, - 0xA950: 84, - 0xA951: 84, - 0xA980: 84, - 0xA981: 84, - 0xA982: 84, - 0xA9B3: 84, - 0xA9B6: 84, - 0xA9B7: 84, - 0xA9B8: 84, - 0xA9B9: 84, - 0xA9BC: 84, - 0xA9BD: 84, - 0xA9E5: 84, - 0xAA29: 84, - 0xAA2A: 84, - 0xAA2B: 84, - 0xAA2C: 84, - 0xAA2D: 84, - 0xAA2E: 84, - 0xAA31: 84, - 0xAA32: 84, - 0xAA35: 84, - 0xAA36: 84, - 0xAA43: 84, - 0xAA4C: 84, - 0xAA7C: 84, - 0xAAB0: 84, - 0xAAB2: 84, - 0xAAB3: 84, - 0xAAB4: 84, - 0xAAB7: 84, - 0xAAB8: 84, - 0xAABE: 84, - 0xAABF: 84, - 0xAAC1: 84, - 0xAAEC: 84, - 0xAAED: 84, - 0xAAF6: 84, - 0xABE5: 84, - 0xABE8: 84, - 0xABED: 84, - 0xFB1E: 84, - 0xFE00: 84, - 0xFE01: 84, - 0xFE02: 84, - 0xFE03: 84, - 0xFE04: 84, - 0xFE05: 84, - 0xFE06: 84, - 0xFE07: 84, - 0xFE08: 84, - 0xFE09: 84, - 0xFE0A: 84, - 0xFE0B: 84, - 0xFE0C: 84, - 0xFE0D: 84, - 0xFE0E: 84, - 0xFE0F: 84, - 0xFE20: 84, - 0xFE21: 84, - 0xFE22: 84, - 0xFE23: 84, - 0xFE24: 84, - 0xFE25: 84, - 0xFE26: 84, - 0xFE27: 84, - 0xFE28: 84, - 0xFE29: 84, - 0xFE2A: 84, - 0xFE2B: 84, - 0xFE2C: 84, - 0xFE2D: 84, - 0xFE2E: 84, - 0xFE2F: 84, - 0xFEFF: 84, - 0xFFF9: 84, - 0xFFFA: 84, - 0xFFFB: 84, - 0x101FD: 84, - 0x102E0: 84, - 0x10376: 84, - 0x10377: 84, - 0x10378: 84, - 0x10379: 84, - 0x1037A: 84, - 0x10A01: 84, - 0x10A02: 84, - 0x10A03: 84, - 0x10A05: 84, - 0x10A06: 84, - 0x10A0C: 84, - 0x10A0D: 84, - 0x10A0E: 84, - 0x10A0F: 84, - 0x10A38: 84, - 0x10A39: 84, - 0x10A3A: 84, - 0x10A3F: 84, - 0x10AC0: 68, - 0x10AC1: 68, - 0x10AC2: 68, - 0x10AC3: 68, - 0x10AC4: 68, - 0x10AC5: 82, - 0x10AC7: 82, - 0x10AC9: 82, - 0x10ACA: 82, - 0x10ACD: 76, - 0x10ACE: 82, - 0x10ACF: 82, - 0x10AD0: 82, - 0x10AD1: 82, - 0x10AD2: 82, - 0x10AD3: 68, - 0x10AD4: 68, - 0x10AD5: 68, - 0x10AD6: 68, - 0x10AD7: 76, - 0x10AD8: 68, - 0x10AD9: 68, - 0x10ADA: 68, - 0x10ADB: 68, - 0x10ADC: 68, - 0x10ADD: 82, - 0x10ADE: 68, - 0x10ADF: 68, - 0x10AE0: 68, - 0x10AE1: 82, - 0x10AE4: 82, - 0x10AE5: 84, - 0x10AE6: 84, - 0x10AEB: 68, - 0x10AEC: 68, - 0x10AED: 68, - 0x10AEE: 68, - 0x10AEF: 82, - 0x10B80: 68, - 0x10B81: 82, - 0x10B82: 68, - 0x10B83: 82, - 0x10B84: 82, - 0x10B85: 82, - 0x10B86: 68, - 0x10B87: 68, - 0x10B88: 68, - 0x10B89: 82, - 0x10B8A: 68, - 0x10B8B: 68, - 0x10B8C: 82, - 0x10B8D: 68, - 0x10B8E: 82, - 0x10B8F: 82, - 0x10B90: 68, - 0x10B91: 82, - 0x10BA9: 82, - 0x10BAA: 82, - 0x10BAB: 82, - 0x10BAC: 82, - 0x10BAD: 68, - 0x10BAE: 68, - 0x10D00: 76, - 0x10D01: 68, - 0x10D02: 68, - 0x10D03: 68, - 0x10D04: 68, - 0x10D05: 68, - 0x10D06: 68, - 0x10D07: 68, - 0x10D08: 68, - 0x10D09: 68, - 0x10D0A: 68, - 0x10D0B: 68, - 0x10D0C: 68, - 0x10D0D: 68, - 0x10D0E: 68, - 0x10D0F: 68, - 0x10D10: 68, - 0x10D11: 68, - 0x10D12: 68, - 0x10D13: 68, - 0x10D14: 68, - 0x10D15: 68, - 0x10D16: 68, - 0x10D17: 68, - 0x10D18: 68, - 0x10D19: 68, - 0x10D1A: 68, - 0x10D1B: 68, - 0x10D1C: 68, - 0x10D1D: 68, - 0x10D1E: 68, - 0x10D1F: 68, - 0x10D20: 68, - 0x10D21: 68, - 0x10D22: 82, - 0x10D23: 68, - 0x10D24: 84, - 0x10D25: 84, - 0x10D26: 84, - 0x10D27: 84, - 0x10D69: 84, - 0x10D6A: 84, - 0x10D6B: 84, - 0x10D6C: 84, - 0x10D6D: 84, - 0x10EAB: 84, - 0x10EAC: 84, - 0x10EC2: 82, - 0x10EC3: 68, - 0x10EC4: 68, - 0x10EC6: 68, - 0x10EC7: 68, - 0x10EFA: 84, - 0x10EFB: 84, - 0x10EFC: 84, - 0x10EFD: 84, - 0x10EFE: 84, - 0x10EFF: 84, - 0x10F30: 68, - 0x10F31: 68, - 0x10F32: 68, - 0x10F33: 82, - 0x10F34: 68, - 0x10F35: 68, - 0x10F36: 68, - 0x10F37: 68, - 0x10F38: 68, - 0x10F39: 68, - 0x10F3A: 68, - 0x10F3B: 68, - 0x10F3C: 68, - 0x10F3D: 68, - 0x10F3E: 68, - 0x10F3F: 68, - 0x10F40: 68, - 0x10F41: 68, - 0x10F42: 68, - 0x10F43: 68, - 0x10F44: 68, - 0x10F46: 84, - 0x10F47: 84, - 0x10F48: 84, - 0x10F49: 84, - 0x10F4A: 84, - 0x10F4B: 84, - 0x10F4C: 84, - 0x10F4D: 84, - 0x10F4E: 84, - 0x10F4F: 84, - 0x10F50: 84, - 0x10F51: 68, - 0x10F52: 68, - 0x10F53: 68, - 0x10F54: 82, - 0x10F70: 68, - 0x10F71: 68, - 0x10F72: 68, - 0x10F73: 68, - 0x10F74: 82, - 0x10F75: 82, - 0x10F76: 68, - 0x10F77: 68, - 0x10F78: 68, - 0x10F79: 68, - 0x10F7A: 68, - 0x10F7B: 68, - 0x10F7C: 68, - 0x10F7D: 68, - 0x10F7E: 68, - 0x10F7F: 68, - 0x10F80: 68, - 0x10F81: 68, - 0x10F82: 84, - 0x10F83: 84, - 0x10F84: 84, - 0x10F85: 84, - 0x10FB0: 68, - 0x10FB2: 68, - 0x10FB3: 68, - 0x10FB4: 82, - 0x10FB5: 82, - 0x10FB6: 82, - 0x10FB8: 68, - 0x10FB9: 82, - 0x10FBA: 82, - 0x10FBB: 68, - 0x10FBC: 68, - 0x10FBD: 82, - 0x10FBE: 68, - 0x10FBF: 68, - 0x10FC1: 68, - 0x10FC2: 82, - 0x10FC3: 82, - 0x10FC4: 68, - 0x10FC9: 82, - 0x10FCA: 68, - 0x10FCB: 76, - 0x11001: 84, - 0x11038: 84, - 0x11039: 84, - 0x1103A: 84, - 0x1103B: 84, - 0x1103C: 84, - 0x1103D: 84, - 0x1103E: 84, - 0x1103F: 84, - 0x11040: 84, - 0x11041: 84, - 0x11042: 84, - 0x11043: 84, - 0x11044: 84, - 0x11045: 84, - 0x11046: 84, - 0x11070: 84, - 0x11073: 84, - 0x11074: 84, - 0x1107F: 84, - 0x11080: 84, - 0x11081: 84, - 0x110B3: 84, - 0x110B4: 84, - 0x110B5: 84, - 0x110B6: 84, - 0x110B9: 84, - 0x110BA: 84, - 0x110C2: 84, - 0x11100: 84, - 0x11101: 84, - 0x11102: 84, - 0x11127: 84, - 0x11128: 84, - 0x11129: 84, - 0x1112A: 84, - 0x1112B: 84, - 0x1112D: 84, - 0x1112E: 84, - 0x1112F: 84, - 0x11130: 84, - 0x11131: 84, - 0x11132: 84, - 0x11133: 84, - 0x11134: 84, - 0x11173: 84, - 0x11180: 84, - 0x11181: 84, - 0x111B6: 84, - 0x111B7: 84, - 0x111B8: 84, - 0x111B9: 84, - 0x111BA: 84, - 0x111BB: 84, - 0x111BC: 84, - 0x111BD: 84, - 0x111BE: 84, - 0x111C9: 84, - 0x111CA: 84, - 0x111CB: 84, - 0x111CC: 84, - 0x111CF: 84, - 0x1122F: 84, - 0x11230: 84, - 0x11231: 84, - 0x11234: 84, - 0x11236: 84, - 0x11237: 84, - 0x1123E: 84, - 0x11241: 84, - 0x112DF: 84, - 0x112E3: 84, - 0x112E4: 84, - 0x112E5: 84, - 0x112E6: 84, - 0x112E7: 84, - 0x112E8: 84, - 0x112E9: 84, - 0x112EA: 84, - 0x11300: 84, - 0x11301: 84, - 0x1133B: 84, - 0x1133C: 84, - 0x11340: 84, - 0x11366: 84, - 0x11367: 84, - 0x11368: 84, - 0x11369: 84, - 0x1136A: 84, - 0x1136B: 84, - 0x1136C: 84, - 0x11370: 84, - 0x11371: 84, - 0x11372: 84, - 0x11373: 84, - 0x11374: 84, - 0x113BB: 84, - 0x113BC: 84, - 0x113BD: 84, - 0x113BE: 84, - 0x113BF: 84, - 0x113C0: 84, - 0x113CE: 84, - 0x113D0: 84, - 0x113D2: 84, - 0x113E1: 84, - 0x113E2: 84, - 0x11438: 84, - 0x11439: 84, - 0x1143A: 84, - 0x1143B: 84, - 0x1143C: 84, - 0x1143D: 84, - 0x1143E: 84, - 0x1143F: 84, - 0x11442: 84, - 0x11443: 84, - 0x11444: 84, - 0x11446: 84, - 0x1145E: 84, - 0x114B3: 84, - 0x114B4: 84, - 0x114B5: 84, - 0x114B6: 84, - 0x114B7: 84, - 0x114B8: 84, - 0x114BA: 84, - 0x114BF: 84, - 0x114C0: 84, - 0x114C2: 84, - 0x114C3: 84, - 0x115B2: 84, - 0x115B3: 84, - 0x115B4: 84, - 0x115B5: 84, - 0x115BC: 84, - 0x115BD: 84, - 0x115BF: 84, - 0x115C0: 84, - 0x115DC: 84, - 0x115DD: 84, - 0x11633: 84, - 0x11634: 84, - 0x11635: 84, - 0x11636: 84, - 0x11637: 84, - 0x11638: 84, - 0x11639: 84, - 0x1163A: 84, - 0x1163D: 84, - 0x1163F: 84, - 0x11640: 84, - 0x116AB: 84, - 0x116AD: 84, - 0x116B0: 84, - 0x116B1: 84, - 0x116B2: 84, - 0x116B3: 84, - 0x116B4: 84, - 0x116B5: 84, - 0x116B7: 84, - 0x1171D: 84, - 0x1171F: 84, - 0x11722: 84, - 0x11723: 84, - 0x11724: 84, - 0x11725: 84, - 0x11727: 84, - 0x11728: 84, - 0x11729: 84, - 0x1172A: 84, - 0x1172B: 84, - 0x1182F: 84, - 0x11830: 84, - 0x11831: 84, - 0x11832: 84, - 0x11833: 84, - 0x11834: 84, - 0x11835: 84, - 0x11836: 84, - 0x11837: 84, - 0x11839: 84, - 0x1183A: 84, - 0x1193B: 84, - 0x1193C: 84, - 0x1193E: 84, - 0x11943: 84, - 0x119D4: 84, - 0x119D5: 84, - 0x119D6: 84, - 0x119D7: 84, - 0x119DA: 84, - 0x119DB: 84, - 0x119E0: 84, - 0x11A01: 84, - 0x11A02: 84, - 0x11A03: 84, - 0x11A04: 84, - 0x11A05: 84, - 0x11A06: 84, - 0x11A07: 84, - 0x11A08: 84, - 0x11A09: 84, - 0x11A0A: 84, - 0x11A33: 84, - 0x11A34: 84, - 0x11A35: 84, - 0x11A36: 84, - 0x11A37: 84, - 0x11A38: 84, - 0x11A3B: 84, - 0x11A3C: 84, - 0x11A3D: 84, - 0x11A3E: 84, - 0x11A47: 84, - 0x11A51: 84, - 0x11A52: 84, - 0x11A53: 84, - 0x11A54: 84, - 0x11A55: 84, - 0x11A56: 84, - 0x11A59: 84, - 0x11A5A: 84, - 0x11A5B: 84, - 0x11A8A: 84, - 0x11A8B: 84, - 0x11A8C: 84, - 0x11A8D: 84, - 0x11A8E: 84, - 0x11A8F: 84, - 0x11A90: 84, - 0x11A91: 84, - 0x11A92: 84, - 0x11A93: 84, - 0x11A94: 84, - 0x11A95: 84, - 0x11A96: 84, - 0x11A98: 84, - 0x11A99: 84, - 0x11B60: 84, - 0x11B62: 84, - 0x11B63: 84, - 0x11B64: 84, - 0x11B66: 84, - 0x11C30: 84, - 0x11C31: 84, - 0x11C32: 84, - 0x11C33: 84, - 0x11C34: 84, - 0x11C35: 84, - 0x11C36: 84, - 0x11C38: 84, - 0x11C39: 84, - 0x11C3A: 84, - 0x11C3B: 84, - 0x11C3C: 84, - 0x11C3D: 84, - 0x11C3F: 84, - 0x11C92: 84, - 0x11C93: 84, - 0x11C94: 84, - 0x11C95: 84, - 0x11C96: 84, - 0x11C97: 84, - 0x11C98: 84, - 0x11C99: 84, - 0x11C9A: 84, - 0x11C9B: 84, - 0x11C9C: 84, - 0x11C9D: 84, - 0x11C9E: 84, - 0x11C9F: 84, - 0x11CA0: 84, - 0x11CA1: 84, - 0x11CA2: 84, - 0x11CA3: 84, - 0x11CA4: 84, - 0x11CA5: 84, - 0x11CA6: 84, - 0x11CA7: 84, - 0x11CAA: 84, - 0x11CAB: 84, - 0x11CAC: 84, - 0x11CAD: 84, - 0x11CAE: 84, - 0x11CAF: 84, - 0x11CB0: 84, - 0x11CB2: 84, - 0x11CB3: 84, - 0x11CB5: 84, - 0x11CB6: 84, - 0x11D31: 84, - 0x11D32: 84, - 0x11D33: 84, - 0x11D34: 84, - 0x11D35: 84, - 0x11D36: 84, - 0x11D3A: 84, - 0x11D3C: 84, - 0x11D3D: 84, - 0x11D3F: 84, - 0x11D40: 84, - 0x11D41: 84, - 0x11D42: 84, - 0x11D43: 84, - 0x11D44: 84, - 0x11D45: 84, - 0x11D47: 84, - 0x11D90: 84, - 0x11D91: 84, - 0x11D95: 84, - 0x11D97: 84, - 0x11EF3: 84, - 0x11EF4: 84, - 0x11F00: 84, - 0x11F01: 84, - 0x11F36: 84, - 0x11F37: 84, - 0x11F38: 84, - 0x11F39: 84, - 0x11F3A: 84, - 0x11F40: 84, - 0x11F42: 84, - 0x11F5A: 84, - 0x13430: 84, - 0x13431: 84, - 0x13432: 84, - 0x13433: 84, - 0x13434: 84, - 0x13435: 84, - 0x13436: 84, - 0x13437: 84, - 0x13438: 84, - 0x13439: 84, - 0x1343A: 84, - 0x1343B: 84, - 0x1343C: 84, - 0x1343D: 84, - 0x1343E: 84, - 0x1343F: 84, - 0x13440: 84, - 0x13447: 84, - 0x13448: 84, - 0x13449: 84, - 0x1344A: 84, - 0x1344B: 84, - 0x1344C: 84, - 0x1344D: 84, - 0x1344E: 84, - 0x1344F: 84, - 0x13450: 84, - 0x13451: 84, - 0x13452: 84, - 0x13453: 84, - 0x13454: 84, - 0x13455: 84, - 0x1611E: 84, - 0x1611F: 84, - 0x16120: 84, - 0x16121: 84, - 0x16122: 84, - 0x16123: 84, - 0x16124: 84, - 0x16125: 84, - 0x16126: 84, - 0x16127: 84, - 0x16128: 84, - 0x16129: 84, - 0x1612D: 84, - 0x1612E: 84, - 0x1612F: 84, - 0x16AF0: 84, - 0x16AF1: 84, - 0x16AF2: 84, - 0x16AF3: 84, - 0x16AF4: 84, - 0x16B30: 84, - 0x16B31: 84, - 0x16B32: 84, - 0x16B33: 84, - 0x16B34: 84, - 0x16B35: 84, - 0x16B36: 84, - 0x16F4F: 84, - 0x16F8F: 84, - 0x16F90: 84, - 0x16F91: 84, - 0x16F92: 84, - 0x16FE4: 84, - 0x1BC9D: 84, - 0x1BC9E: 84, - 0x1BCA0: 84, - 0x1BCA1: 84, - 0x1BCA2: 84, - 0x1BCA3: 84, - 0x1CF00: 84, - 0x1CF01: 84, - 0x1CF02: 84, - 0x1CF03: 84, - 0x1CF04: 84, - 0x1CF05: 84, - 0x1CF06: 84, - 0x1CF07: 84, - 0x1CF08: 84, - 0x1CF09: 84, - 0x1CF0A: 84, - 0x1CF0B: 84, - 0x1CF0C: 84, - 0x1CF0D: 84, - 0x1CF0E: 84, - 0x1CF0F: 84, - 0x1CF10: 84, - 0x1CF11: 84, - 0x1CF12: 84, - 0x1CF13: 84, - 0x1CF14: 84, - 0x1CF15: 84, - 0x1CF16: 84, - 0x1CF17: 84, - 0x1CF18: 84, - 0x1CF19: 84, - 0x1CF1A: 84, - 0x1CF1B: 84, - 0x1CF1C: 84, - 0x1CF1D: 84, - 0x1CF1E: 84, - 0x1CF1F: 84, - 0x1CF20: 84, - 0x1CF21: 84, - 0x1CF22: 84, - 0x1CF23: 84, - 0x1CF24: 84, - 0x1CF25: 84, - 0x1CF26: 84, - 0x1CF27: 84, - 0x1CF28: 84, - 0x1CF29: 84, - 0x1CF2A: 84, - 0x1CF2B: 84, - 0x1CF2C: 84, - 0x1CF2D: 84, - 0x1CF30: 84, - 0x1CF31: 84, - 0x1CF32: 84, - 0x1CF33: 84, - 0x1CF34: 84, - 0x1CF35: 84, - 0x1CF36: 84, - 0x1CF37: 84, - 0x1CF38: 84, - 0x1CF39: 84, - 0x1CF3A: 84, - 0x1CF3B: 84, - 0x1CF3C: 84, - 0x1CF3D: 84, - 0x1CF3E: 84, - 0x1CF3F: 84, - 0x1CF40: 84, - 0x1CF41: 84, - 0x1CF42: 84, - 0x1CF43: 84, - 0x1CF44: 84, - 0x1CF45: 84, - 0x1CF46: 84, - 0x1D167: 84, - 0x1D168: 84, - 0x1D169: 84, - 0x1D173: 84, - 0x1D174: 84, - 0x1D175: 84, - 0x1D176: 84, - 0x1D177: 84, - 0x1D178: 84, - 0x1D179: 84, - 0x1D17A: 84, - 0x1D17B: 84, - 0x1D17C: 84, - 0x1D17D: 84, - 0x1D17E: 84, - 0x1D17F: 84, - 0x1D180: 84, - 0x1D181: 84, - 0x1D182: 84, - 0x1D185: 84, - 0x1D186: 84, - 0x1D187: 84, - 0x1D188: 84, - 0x1D189: 84, - 0x1D18A: 84, - 0x1D18B: 84, - 0x1D1AA: 84, - 0x1D1AB: 84, - 0x1D1AC: 84, - 0x1D1AD: 84, - 0x1D242: 84, - 0x1D243: 84, - 0x1D244: 84, - 0x1DA00: 84, - 0x1DA01: 84, - 0x1DA02: 84, - 0x1DA03: 84, - 0x1DA04: 84, - 0x1DA05: 84, - 0x1DA06: 84, - 0x1DA07: 84, - 0x1DA08: 84, - 0x1DA09: 84, - 0x1DA0A: 84, - 0x1DA0B: 84, - 0x1DA0C: 84, - 0x1DA0D: 84, - 0x1DA0E: 84, - 0x1DA0F: 84, - 0x1DA10: 84, - 0x1DA11: 84, - 0x1DA12: 84, - 0x1DA13: 84, - 0x1DA14: 84, - 0x1DA15: 84, - 0x1DA16: 84, - 0x1DA17: 84, - 0x1DA18: 84, - 0x1DA19: 84, - 0x1DA1A: 84, - 0x1DA1B: 84, - 0x1DA1C: 84, - 0x1DA1D: 84, - 0x1DA1E: 84, - 0x1DA1F: 84, - 0x1DA20: 84, - 0x1DA21: 84, - 0x1DA22: 84, - 0x1DA23: 84, - 0x1DA24: 84, - 0x1DA25: 84, - 0x1DA26: 84, - 0x1DA27: 84, - 0x1DA28: 84, - 0x1DA29: 84, - 0x1DA2A: 84, - 0x1DA2B: 84, - 0x1DA2C: 84, - 0x1DA2D: 84, - 0x1DA2E: 84, - 0x1DA2F: 84, - 0x1DA30: 84, - 0x1DA31: 84, - 0x1DA32: 84, - 0x1DA33: 84, - 0x1DA34: 84, - 0x1DA35: 84, - 0x1DA36: 84, - 0x1DA3B: 84, - 0x1DA3C: 84, - 0x1DA3D: 84, - 0x1DA3E: 84, - 0x1DA3F: 84, - 0x1DA40: 84, - 0x1DA41: 84, - 0x1DA42: 84, - 0x1DA43: 84, - 0x1DA44: 84, - 0x1DA45: 84, - 0x1DA46: 84, - 0x1DA47: 84, - 0x1DA48: 84, - 0x1DA49: 84, - 0x1DA4A: 84, - 0x1DA4B: 84, - 0x1DA4C: 84, - 0x1DA4D: 84, - 0x1DA4E: 84, - 0x1DA4F: 84, - 0x1DA50: 84, - 0x1DA51: 84, - 0x1DA52: 84, - 0x1DA53: 84, - 0x1DA54: 84, - 0x1DA55: 84, - 0x1DA56: 84, - 0x1DA57: 84, - 0x1DA58: 84, - 0x1DA59: 84, - 0x1DA5A: 84, - 0x1DA5B: 84, - 0x1DA5C: 84, - 0x1DA5D: 84, - 0x1DA5E: 84, - 0x1DA5F: 84, - 0x1DA60: 84, - 0x1DA61: 84, - 0x1DA62: 84, - 0x1DA63: 84, - 0x1DA64: 84, - 0x1DA65: 84, - 0x1DA66: 84, - 0x1DA67: 84, - 0x1DA68: 84, - 0x1DA69: 84, - 0x1DA6A: 84, - 0x1DA6B: 84, - 0x1DA6C: 84, - 0x1DA75: 84, - 0x1DA84: 84, - 0x1DA9B: 84, - 0x1DA9C: 84, - 0x1DA9D: 84, - 0x1DA9E: 84, - 0x1DA9F: 84, - 0x1DAA1: 84, - 0x1DAA2: 84, - 0x1DAA3: 84, - 0x1DAA4: 84, - 0x1DAA5: 84, - 0x1DAA6: 84, - 0x1DAA7: 84, - 0x1DAA8: 84, - 0x1DAA9: 84, - 0x1DAAA: 84, - 0x1DAAB: 84, - 0x1DAAC: 84, - 0x1DAAD: 84, - 0x1DAAE: 84, - 0x1DAAF: 84, - 0x1E000: 84, - 0x1E001: 84, - 0x1E002: 84, - 0x1E003: 84, - 0x1E004: 84, - 0x1E005: 84, - 0x1E006: 84, - 0x1E008: 84, - 0x1E009: 84, - 0x1E00A: 84, - 0x1E00B: 84, - 0x1E00C: 84, - 0x1E00D: 84, - 0x1E00E: 84, - 0x1E00F: 84, - 0x1E010: 84, - 0x1E011: 84, - 0x1E012: 84, - 0x1E013: 84, - 0x1E014: 84, - 0x1E015: 84, - 0x1E016: 84, - 0x1E017: 84, - 0x1E018: 84, - 0x1E01B: 84, - 0x1E01C: 84, - 0x1E01D: 84, - 0x1E01E: 84, - 0x1E01F: 84, - 0x1E020: 84, - 0x1E021: 84, - 0x1E023: 84, - 0x1E024: 84, - 0x1E026: 84, - 0x1E027: 84, - 0x1E028: 84, - 0x1E029: 84, - 0x1E02A: 84, - 0x1E08F: 84, - 0x1E130: 84, - 0x1E131: 84, - 0x1E132: 84, - 0x1E133: 84, - 0x1E134: 84, - 0x1E135: 84, - 0x1E136: 84, - 0x1E2AE: 84, - 0x1E2EC: 84, - 0x1E2ED: 84, - 0x1E2EE: 84, - 0x1E2EF: 84, - 0x1E4EC: 84, - 0x1E4ED: 84, - 0x1E4EE: 84, - 0x1E4EF: 84, - 0x1E5EE: 84, - 0x1E5EF: 84, - 0x1E6E3: 84, - 0x1E6E6: 84, - 0x1E6EE: 84, - 0x1E6EF: 84, - 0x1E6F5: 84, - 0x1E8D0: 84, - 0x1E8D1: 84, - 0x1E8D2: 84, - 0x1E8D3: 84, - 0x1E8D4: 84, - 0x1E8D5: 84, - 0x1E8D6: 84, - 0x1E900: 68, - 0x1E901: 68, - 0x1E902: 68, - 0x1E903: 68, - 0x1E904: 68, - 0x1E905: 68, - 0x1E906: 68, - 0x1E907: 68, - 0x1E908: 68, - 0x1E909: 68, - 0x1E90A: 68, - 0x1E90B: 68, - 0x1E90C: 68, - 0x1E90D: 68, - 0x1E90E: 68, - 0x1E90F: 68, - 0x1E910: 68, - 0x1E911: 68, - 0x1E912: 68, - 0x1E913: 68, - 0x1E914: 68, - 0x1E915: 68, - 0x1E916: 68, - 0x1E917: 68, - 0x1E918: 68, - 0x1E919: 68, - 0x1E91A: 68, - 0x1E91B: 68, - 0x1E91C: 68, - 0x1E91D: 68, - 0x1E91E: 68, - 0x1E91F: 68, - 0x1E920: 68, - 0x1E921: 68, - 0x1E922: 68, - 0x1E923: 68, - 0x1E924: 68, - 0x1E925: 68, - 0x1E926: 68, - 0x1E927: 68, - 0x1E928: 68, - 0x1E929: 68, - 0x1E92A: 68, - 0x1E92B: 68, - 0x1E92C: 68, - 0x1E92D: 68, - 0x1E92E: 68, - 0x1E92F: 68, - 0x1E930: 68, - 0x1E931: 68, - 0x1E932: 68, - 0x1E933: 68, - 0x1E934: 68, - 0x1E935: 68, - 0x1E936: 68, - 0x1E937: 68, - 0x1E938: 68, - 0x1E939: 68, - 0x1E93A: 68, - 0x1E93B: 68, - 0x1E93C: 68, - 0x1E93D: 68, - 0x1E93E: 68, - 0x1E93F: 68, - 0x1E940: 68, - 0x1E941: 68, - 0x1E942: 68, - 0x1E943: 68, - 0x1E944: 84, - 0x1E945: 84, - 0x1E946: 84, - 0x1E947: 84, - 0x1E948: 84, - 0x1E949: 84, - 0x1E94A: 84, - 0x1E94B: 84, - 0xE0001: 84, - 0xE0020: 84, - 0xE0021: 84, - 0xE0022: 84, - 0xE0023: 84, - 0xE0024: 84, - 0xE0025: 84, - 0xE0026: 84, - 0xE0027: 84, - 0xE0028: 84, - 0xE0029: 84, - 0xE002A: 84, - 0xE002B: 84, - 0xE002C: 84, - 0xE002D: 84, - 0xE002E: 84, - 0xE002F: 84, - 0xE0030: 84, - 0xE0031: 84, - 0xE0032: 84, - 0xE0033: 84, - 0xE0034: 84, - 0xE0035: 84, - 0xE0036: 84, - 0xE0037: 84, - 0xE0038: 84, - 0xE0039: 84, - 0xE003A: 84, - 0xE003B: 84, - 0xE003C: 84, - 0xE003D: 84, - 0xE003E: 84, - 0xE003F: 84, - 0xE0040: 84, - 0xE0041: 84, - 0xE0042: 84, - 0xE0043: 84, - 0xE0044: 84, - 0xE0045: 84, - 0xE0046: 84, - 0xE0047: 84, - 0xE0048: 84, - 0xE0049: 84, - 0xE004A: 84, - 0xE004B: 84, - 0xE004C: 84, - 0xE004D: 84, - 0xE004E: 84, - 0xE004F: 84, - 0xE0050: 84, - 0xE0051: 84, - 0xE0052: 84, - 0xE0053: 84, - 0xE0054: 84, - 0xE0055: 84, - 0xE0056: 84, - 0xE0057: 84, - 0xE0058: 84, - 0xE0059: 84, - 0xE005A: 84, - 0xE005B: 84, - 0xE005C: 84, - 0xE005D: 84, - 0xE005E: 84, - 0xE005F: 84, - 0xE0060: 84, - 0xE0061: 84, - 0xE0062: 84, - 0xE0063: 84, - 0xE0064: 84, - 0xE0065: 84, - 0xE0066: 84, - 0xE0067: 84, - 0xE0068: 84, - 0xE0069: 84, - 0xE006A: 84, - 0xE006B: 84, - 0xE006C: 84, - 0xE006D: 84, - 0xE006E: 84, - 0xE006F: 84, - 0xE0070: 84, - 0xE0071: 84, - 0xE0072: 84, - 0xE0073: 84, - 0xE0074: 84, - 0xE0075: 84, - 0xE0076: 84, - 0xE0077: 84, - 0xE0078: 84, - 0xE0079: 84, - 0xE007A: 84, - 0xE007B: 84, - 0xE007C: 84, - 0xE007D: 84, - 0xE007E: 84, - 0xE007F: 84, - 0xE0100: 84, - 0xE0101: 84, - 0xE0102: 84, - 0xE0103: 84, - 0xE0104: 84, - 0xE0105: 84, - 0xE0106: 84, - 0xE0107: 84, - 0xE0108: 84, - 0xE0109: 84, - 0xE010A: 84, - 0xE010B: 84, - 0xE010C: 84, - 0xE010D: 84, - 0xE010E: 84, - 0xE010F: 84, - 0xE0110: 84, - 0xE0111: 84, - 0xE0112: 84, - 0xE0113: 84, - 0xE0114: 84, - 0xE0115: 84, - 0xE0116: 84, - 0xE0117: 84, - 0xE0118: 84, - 0xE0119: 84, - 0xE011A: 84, - 0xE011B: 84, - 0xE011C: 84, - 0xE011D: 84, - 0xE011E: 84, - 0xE011F: 84, - 0xE0120: 84, - 0xE0121: 84, - 0xE0122: 84, - 0xE0123: 84, - 0xE0124: 84, - 0xE0125: 84, - 0xE0126: 84, - 0xE0127: 84, - 0xE0128: 84, - 0xE0129: 84, - 0xE012A: 84, - 0xE012B: 84, - 0xE012C: 84, - 0xE012D: 84, - 0xE012E: 84, - 0xE012F: 84, - 0xE0130: 84, - 0xE0131: 84, - 0xE0132: 84, - 0xE0133: 84, - 0xE0134: 84, - 0xE0135: 84, - 0xE0136: 84, - 0xE0137: 84, - 0xE0138: 84, - 0xE0139: 84, - 0xE013A: 84, - 0xE013B: 84, - 0xE013C: 84, - 0xE013D: 84, - 0xE013E: 84, - 0xE013F: 84, - 0xE0140: 84, - 0xE0141: 84, - 0xE0142: 84, - 0xE0143: 84, - 0xE0144: 84, - 0xE0145: 84, - 0xE0146: 84, - 0xE0147: 84, - 0xE0148: 84, - 0xE0149: 84, - 0xE014A: 84, - 0xE014B: 84, - 0xE014C: 84, - 0xE014D: 84, - 0xE014E: 84, - 0xE014F: 84, - 0xE0150: 84, - 0xE0151: 84, - 0xE0152: 84, - 0xE0153: 84, - 0xE0154: 84, - 0xE0155: 84, - 0xE0156: 84, - 0xE0157: 84, - 0xE0158: 84, - 0xE0159: 84, - 0xE015A: 84, - 0xE015B: 84, - 0xE015C: 84, - 0xE015D: 84, - 0xE015E: 84, - 0xE015F: 84, - 0xE0160: 84, - 0xE0161: 84, - 0xE0162: 84, - 0xE0163: 84, - 0xE0164: 84, - 0xE0165: 84, - 0xE0166: 84, - 0xE0167: 84, - 0xE0168: 84, - 0xE0169: 84, - 0xE016A: 84, - 0xE016B: 84, - 0xE016C: 84, - 0xE016D: 84, - 0xE016E: 84, - 0xE016F: 84, - 0xE0170: 84, - 0xE0171: 84, - 0xE0172: 84, - 0xE0173: 84, - 0xE0174: 84, - 0xE0175: 84, - 0xE0176: 84, - 0xE0177: 84, - 0xE0178: 84, - 0xE0179: 84, - 0xE017A: 84, - 0xE017B: 84, - 0xE017C: 84, - 0xE017D: 84, - 0xE017E: 84, - 0xE017F: 84, - 0xE0180: 84, - 0xE0181: 84, - 0xE0182: 84, - 0xE0183: 84, - 0xE0184: 84, - 0xE0185: 84, - 0xE0186: 84, - 0xE0187: 84, - 0xE0188: 84, - 0xE0189: 84, - 0xE018A: 84, - 0xE018B: 84, - 0xE018C: 84, - 0xE018D: 84, - 0xE018E: 84, - 0xE018F: 84, - 0xE0190: 84, - 0xE0191: 84, - 0xE0192: 84, - 0xE0193: 84, - 0xE0194: 84, - 0xE0195: 84, - 0xE0196: 84, - 0xE0197: 84, - 0xE0198: 84, - 0xE0199: 84, - 0xE019A: 84, - 0xE019B: 84, - 0xE019C: 84, - 0xE019D: 84, - 0xE019E: 84, - 0xE019F: 84, - 0xE01A0: 84, - 0xE01A1: 84, - 0xE01A2: 84, - 0xE01A3: 84, - 0xE01A4: 84, - 0xE01A5: 84, - 0xE01A6: 84, - 0xE01A7: 84, - 0xE01A8: 84, - 0xE01A9: 84, - 0xE01AA: 84, - 0xE01AB: 84, - 0xE01AC: 84, - 0xE01AD: 84, - 0xE01AE: 84, - 0xE01AF: 84, - 0xE01B0: 84, - 0xE01B1: 84, - 0xE01B2: 84, - 0xE01B3: 84, - 0xE01B4: 84, - 0xE01B5: 84, - 0xE01B6: 84, - 0xE01B7: 84, - 0xE01B8: 84, - 0xE01B9: 84, - 0xE01BA: 84, - 0xE01BB: 84, - 0xE01BC: 84, - 0xE01BD: 84, - 0xE01BE: 84, - 0xE01BF: 84, - 0xE01C0: 84, - 0xE01C1: 84, - 0xE01C2: 84, - 0xE01C3: 84, - 0xE01C4: 84, - 0xE01C5: 84, - 0xE01C6: 84, - 0xE01C7: 84, - 0xE01C8: 84, - 0xE01C9: 84, - 0xE01CA: 84, - 0xE01CB: 84, - 0xE01CC: 84, - 0xE01CD: 84, - 0xE01CE: 84, - 0xE01CF: 84, - 0xE01D0: 84, - 0xE01D1: 84, - 0xE01D2: 84, - 0xE01D3: 84, - 0xE01D4: 84, - 0xE01D5: 84, - 0xE01D6: 84, - 0xE01D7: 84, - 0xE01D8: 84, - 0xE01D9: 84, - 0xE01DA: 84, - 0xE01DB: 84, - 0xE01DC: 84, - 0xE01DD: 84, - 0xE01DE: 84, - 0xE01DF: 84, - 0xE01E0: 84, - 0xE01E1: 84, - 0xE01E2: 84, - 0xE01E3: 84, - 0xE01E4: 84, - 0xE01E5: 84, - 0xE01E6: 84, - 0xE01E7: 84, - 0xE01E8: 84, - 0xE01E9: 84, - 0xE01EA: 84, - 0xE01EB: 84, - 0xE01EC: 84, - 0xE01ED: 84, - 0xE01EE: 84, - 0xE01EF: 84, - } +joining_types = { + "C": ( + 0x64000000641, + 0x7FA000007FB, + 0x88300000886, + 0x180A0000180B, + 0x200D0000200E, + ), + "D": ( + 0x62000000621, + 0x62600000627, + 0x62800000629, + 0x62A0000062F, + 0x63300000640, + 0x64100000648, + 0x6490000064B, + 0x66E00000670, + 0x67800000688, + 0x69A000006C0, + 0x6C1000006C3, + 0x6CC000006CD, + 0x6CE000006CF, + 0x6D0000006D2, + 0x6FA000006FD, + 0x6FF00000700, + 0x71200000715, + 0x71A0000071E, + 0x71F00000728, + 0x7290000072A, + 0x72B0000072C, + 0x72D0000072F, + 0x74E00000759, + 0x75C0000076B, + 0x76D00000771, + 0x77200000773, + 0x77500000778, + 0x77A00000780, + 0x7CA000007EB, + 0x84100000846, + 0x84800000849, + 0x84A00000854, + 0x85500000856, + 0x86000000861, + 0x86200000866, + 0x86800000869, + 0x88600000887, + 0x8890000088E, + 0x88F00000890, + 0x8A0000008AA, + 0x8AF000008B1, + 0x8B3000008B9, + 0x8BA000008C9, + 0x180700001808, + 0x182000001879, + 0x1887000018A9, + 0x18AA000018AB, + 0xA8400000A872, + 0x10AC000010AC5, + 0x10AD300010AD7, + 0x10AD800010ADD, + 0x10ADE00010AE1, + 0x10AEB00010AEF, + 0x10B8000010B81, + 0x10B8200010B83, + 0x10B8600010B89, + 0x10B8A00010B8C, + 0x10B8D00010B8E, + 0x10B9000010B91, + 0x10BAD00010BAF, + 0x10D0100010D22, + 0x10D2300010D24, + 0x10EC300010EC5, + 0x10EC600010EC8, + 0x10F3000010F33, + 0x10F3400010F45, + 0x10F5100010F54, + 0x10F7000010F74, + 0x10F7600010F82, + 0x10FB000010FB1, + 0x10FB200010FB4, + 0x10FB800010FB9, + 0x10FBB00010FBD, + 0x10FBE00010FC0, + 0x10FC100010FC2, + 0x10FC400010FC5, + 0x10FCA00010FCB, + 0x1E9000001E944, + ), + "L": ( + 0xA8720000A873, + 0x10ACD00010ACE, + 0x10AD700010AD8, + 0x10D0000010D01, + 0x10FCB00010FCC, + ), + "R": ( + 0x62200000626, + 0x62700000628, + 0x6290000062A, + 0x62F00000633, + 0x64800000649, + 0x67100000674, + 0x67500000678, + 0x6880000069A, + 0x6C0000006C1, + 0x6C3000006CC, + 0x6CD000006CE, + 0x6CF000006D0, + 0x6D2000006D4, + 0x6D5000006D6, + 0x6EE000006F0, + 0x71000000711, + 0x7150000071A, + 0x71E0000071F, + 0x72800000729, + 0x72A0000072B, + 0x72C0000072D, + 0x72F00000730, + 0x74D0000074E, + 0x7590000075C, + 0x76B0000076D, + 0x77100000772, + 0x77300000775, + 0x7780000077A, + 0x84000000841, + 0x84600000848, + 0x8490000084A, + 0x85400000855, + 0x85600000859, + 0x86700000868, + 0x8690000086B, + 0x87000000883, + 0x88E0000088F, + 0x8AA000008AD, + 0x8AE000008AF, + 0x8B1000008B3, + 0x8B9000008BA, + 0x10AC500010AC6, + 0x10AC700010AC8, + 0x10AC900010ACB, + 0x10ACE00010AD3, + 0x10ADD00010ADE, + 0x10AE100010AE2, + 0x10AE400010AE5, + 0x10AEF00010AF0, + 0x10B8100010B82, + 0x10B8300010B86, + 0x10B8900010B8A, + 0x10B8C00010B8D, + 0x10B8E00010B90, + 0x10B9100010B92, + 0x10BA900010BAD, + 0x10D2200010D23, + 0x10EC200010EC3, + 0x10F3300010F34, + 0x10F5400010F55, + 0x10F7400010F76, + 0x10FB400010FB7, + 0x10FB900010FBB, + 0x10FBD00010FBE, + 0x10FC200010FC4, + 0x10FC900010FCA, + ), + "T": ( + 0xAD000000AE, + 0x30000000370, + 0x4830000048A, + 0x591000005BE, + 0x5BF000005C0, + 0x5C1000005C3, + 0x5C4000005C6, + 0x5C7000005C8, + 0x6100000061B, + 0x61C0000061D, + 0x64B00000660, + 0x67000000671, + 0x6D6000006DD, + 0x6DF000006E5, + 0x6E7000006E9, + 0x6EA000006EE, + 0x70F00000710, + 0x71100000712, + 0x7300000074B, + 0x7A6000007B1, + 0x7EB000007F4, + 0x7FD000007FE, + 0x8160000081A, + 0x81B00000824, + 0x82500000828, + 0x8290000082E, + 0x8590000085C, + 0x897000008A0, + 0x8CA000008E2, + 0x8E300000903, + 0x93A0000093B, + 0x93C0000093D, + 0x94100000949, + 0x94D0000094E, + 0x95100000958, + 0x96200000964, + 0x98100000982, + 0x9BC000009BD, + 0x9C1000009C5, + 0x9CD000009CE, + 0x9E2000009E4, + 0x9FE000009FF, + 0xA0100000A03, + 0xA3C00000A3D, + 0xA4100000A43, + 0xA4700000A49, + 0xA4B00000A4E, + 0xA5100000A52, + 0xA7000000A72, + 0xA7500000A76, + 0xA8100000A83, + 0xABC00000ABD, + 0xAC100000AC6, + 0xAC700000AC9, + 0xACD00000ACE, + 0xAE200000AE4, + 0xAFA00000B00, + 0xB0100000B02, + 0xB3C00000B3D, + 0xB3F00000B40, + 0xB4100000B45, + 0xB4D00000B4E, + 0xB5500000B57, + 0xB6200000B64, + 0xB8200000B83, + 0xBC000000BC1, + 0xBCD00000BCE, + 0xC0000000C01, + 0xC0400000C05, + 0xC3C00000C3D, + 0xC3E00000C41, + 0xC4600000C49, + 0xC4A00000C4E, + 0xC5500000C57, + 0xC6200000C64, + 0xC8100000C82, + 0xCBC00000CBD, + 0xCBF00000CC0, + 0xCC600000CC7, + 0xCCC00000CCE, + 0xCE200000CE4, + 0xD0000000D02, + 0xD3B00000D3D, + 0xD4100000D45, + 0xD4D00000D4E, + 0xD6200000D64, + 0xD8100000D82, + 0xDCA00000DCB, + 0xDD200000DD5, + 0xDD600000DD7, + 0xE3100000E32, + 0xE3400000E3B, + 0xE4700000E4F, + 0xEB100000EB2, + 0xEB400000EBD, + 0xEC800000ECF, + 0xF1800000F1A, + 0xF3500000F36, + 0xF3700000F38, + 0xF3900000F3A, + 0xF7100000F7F, + 0xF8000000F85, + 0xF8600000F88, + 0xF8D00000F98, + 0xF9900000FBD, + 0xFC600000FC7, + 0x102D00001031, + 0x103200001038, + 0x10390000103B, + 0x103D0000103F, + 0x10580000105A, + 0x105E00001061, + 0x107100001075, + 0x108200001083, + 0x108500001087, + 0x108D0000108E, + 0x109D0000109E, + 0x135D00001360, + 0x171200001715, + 0x173200001734, + 0x175200001754, + 0x177200001774, + 0x17B4000017B6, + 0x17B7000017BE, + 0x17C6000017C7, + 0x17C9000017D4, + 0x17DD000017DE, + 0x180B0000180E, + 0x180F00001810, + 0x188500001887, + 0x18A9000018AA, + 0x192000001923, + 0x192700001929, + 0x193200001933, + 0x19390000193C, + 0x1A1700001A19, + 0x1A1B00001A1C, + 0x1A5600001A57, + 0x1A5800001A5F, + 0x1A6000001A61, + 0x1A6200001A63, + 0x1A6500001A6D, + 0x1A7300001A7D, + 0x1A7F00001A80, + 0x1AB000001ADE, + 0x1AE000001AEC, + 0x1B0000001B04, + 0x1B3400001B35, + 0x1B3600001B3B, + 0x1B3C00001B3D, + 0x1B4200001B43, + 0x1B6B00001B74, + 0x1B8000001B82, + 0x1BA200001BA6, + 0x1BA800001BAA, + 0x1BAB00001BAE, + 0x1BE600001BE7, + 0x1BE800001BEA, + 0x1BED00001BEE, + 0x1BEF00001BF2, + 0x1C2C00001C34, + 0x1C3600001C38, + 0x1CD000001CD3, + 0x1CD400001CE1, + 0x1CE200001CE9, + 0x1CED00001CEE, + 0x1CF400001CF5, + 0x1CF800001CFA, + 0x1DC000001E00, + 0x200B0000200C, + 0x200E00002010, + 0x202A0000202F, + 0x206000002065, + 0x206A00002070, + 0x20D0000020F1, + 0x2CEF00002CF2, + 0x2D7F00002D80, + 0x2DE000002E00, + 0x302A0000302E, + 0x30990000309B, + 0xA66F0000A673, + 0xA6740000A67E, + 0xA69E0000A6A0, + 0xA6F00000A6F2, + 0xA8020000A803, + 0xA8060000A807, + 0xA80B0000A80C, + 0xA8250000A827, + 0xA82C0000A82D, + 0xA8C40000A8C6, + 0xA8E00000A8F2, + 0xA8FF0000A900, + 0xA9260000A92E, + 0xA9470000A952, + 0xA9800000A983, + 0xA9B30000A9B4, + 0xA9B60000A9BA, + 0xA9BC0000A9BE, + 0xA9E50000A9E6, + 0xAA290000AA2F, + 0xAA310000AA33, + 0xAA350000AA37, + 0xAA430000AA44, + 0xAA4C0000AA4D, + 0xAA7C0000AA7D, + 0xAAB00000AAB1, + 0xAAB20000AAB5, + 0xAAB70000AAB9, + 0xAABE0000AAC0, + 0xAAC10000AAC2, + 0xAAEC0000AAEE, + 0xAAF60000AAF7, + 0xABE50000ABE6, + 0xABE80000ABE9, + 0xABED0000ABEE, + 0xFB1E0000FB1F, + 0xFE000000FE10, + 0xFE200000FE30, + 0xFEFF0000FF00, + 0xFFF90000FFFC, + 0x101FD000101FE, + 0x102E0000102E1, + 0x103760001037B, + 0x10A0100010A04, + 0x10A0500010A07, + 0x10A0C00010A10, + 0x10A3800010A3B, + 0x10A3F00010A40, + 0x10AE500010AE7, + 0x10D2400010D28, + 0x10D6900010D6E, + 0x10EAB00010EAD, + 0x10EFA00010F00, + 0x10F4600010F51, + 0x10F8200010F86, + 0x1100100011002, + 0x1103800011047, + 0x1107000011071, + 0x1107300011075, + 0x1107F00011082, + 0x110B3000110B7, + 0x110B9000110BB, + 0x110C2000110C3, + 0x1110000011103, + 0x111270001112C, + 0x1112D00011135, + 0x1117300011174, + 0x1118000011182, + 0x111B6000111BF, + 0x111C9000111CD, + 0x111CF000111D0, + 0x1122F00011232, + 0x1123400011235, + 0x1123600011238, + 0x1123E0001123F, + 0x1124100011242, + 0x112DF000112E0, + 0x112E3000112EB, + 0x1130000011302, + 0x1133B0001133D, + 0x1134000011341, + 0x113660001136D, + 0x1137000011375, + 0x113BB000113C1, + 0x113CE000113CF, + 0x113D0000113D1, + 0x113D2000113D3, + 0x113E1000113E3, + 0x1143800011440, + 0x1144200011445, + 0x1144600011447, + 0x1145E0001145F, + 0x114B3000114B9, + 0x114BA000114BB, + 0x114BF000114C1, + 0x114C2000114C4, + 0x115B2000115B6, + 0x115BC000115BE, + 0x115BF000115C1, + 0x115DC000115DE, + 0x116330001163B, + 0x1163D0001163E, + 0x1163F00011641, + 0x116AB000116AC, + 0x116AD000116AE, + 0x116B0000116B6, + 0x116B7000116B8, + 0x1171D0001171E, + 0x1171F00011720, + 0x1172200011726, + 0x117270001172C, + 0x1182F00011838, + 0x118390001183B, + 0x1193B0001193D, + 0x1193E0001193F, + 0x1194300011944, + 0x119D4000119D8, + 0x119DA000119DC, + 0x119E0000119E1, + 0x11A0100011A0B, + 0x11A3300011A39, + 0x11A3B00011A3F, + 0x11A4700011A48, + 0x11A5100011A57, + 0x11A5900011A5C, + 0x11A8A00011A97, + 0x11A9800011A9A, + 0x11B6000011B61, + 0x11B6200011B65, + 0x11B6600011B67, + 0x11C3000011C37, + 0x11C3800011C3E, + 0x11C3F00011C40, + 0x11C9200011CA8, + 0x11CAA00011CB1, + 0x11CB200011CB4, + 0x11CB500011CB7, + 0x11D3100011D37, + 0x11D3A00011D3B, + 0x11D3C00011D3E, + 0x11D3F00011D46, + 0x11D4700011D48, + 0x11D9000011D92, + 0x11D9500011D96, + 0x11D9700011D98, + 0x11EF300011EF5, + 0x11F0000011F02, + 0x11F3600011F3B, + 0x11F4000011F41, + 0x11F4200011F43, + 0x11F5A00011F5B, + 0x1343000013441, + 0x1344700013456, + 0x1611E0001612A, + 0x1612D00016130, + 0x16AF000016AF5, + 0x16B3000016B37, + 0x16F4F00016F50, + 0x16F8F00016F93, + 0x16FE400016FE5, + 0x1BC9D0001BC9F, + 0x1BCA00001BCA4, + 0x1CF000001CF2E, + 0x1CF300001CF47, + 0x1D1670001D16A, + 0x1D1730001D183, + 0x1D1850001D18C, + 0x1D1AA0001D1AE, + 0x1D2420001D245, + 0x1DA000001DA37, + 0x1DA3B0001DA6D, + 0x1DA750001DA76, + 0x1DA840001DA85, + 0x1DA9B0001DAA0, + 0x1DAA10001DAB0, + 0x1E0000001E007, + 0x1E0080001E019, + 0x1E01B0001E022, + 0x1E0230001E025, + 0x1E0260001E02B, + 0x1E08F0001E090, + 0x1E1300001E137, + 0x1E2AE0001E2AF, + 0x1E2EC0001E2F0, + 0x1E4EC0001E4F0, + 0x1E5EE0001E5F0, + 0x1E6E30001E6E4, + 0x1E6E60001E6E7, + 0x1E6EE0001E6F0, + 0x1E6F50001E6F6, + 0x1E8D00001E8D7, + 0x1E9440001E94C, + 0xE0001000E0002, + 0xE0020000E0080, + 0xE0100000E01F0, + ), +} codepoint_classes = { diff --git a/contrib/python/idna/py3/idna/intranges.py b/contrib/python/idna/py3/idna/intranges.py index ea3455bb8da..19d77810caf 100644 --- a/contrib/python/idna/py3/idna/intranges.py +++ b/contrib/python/idna/py3/idna/intranges.py @@ -6,10 +6,9 @@ in the original list?" in time O(log(# runs)). """ import bisect -from typing import List, Tuple -def intranges_from_list(list_: List[int]) -> Tuple[int, ...]: +def intranges_from_list(list_: list[int]) -> tuple[int, ...]: """Represent a list of integers as a sequence of ranges: ((start_0, end_0), (start_1, end_1), ...), such that the original integers are exactly those x such that start_i <= x < end_i for some i. @@ -34,11 +33,11 @@ def _encode_range(start: int, end: int) -> int: return (start << 32) | end -def _decode_range(r: int) -> Tuple[int, int]: +def _decode_range(r: int) -> tuple[int, int]: return (r >> 32), (r & ((1 << 32) - 1)) -def intranges_contain(int_: int, ranges: Tuple[int, ...]) -> bool: +def intranges_contain(int_: int, ranges: tuple[int, ...]) -> bool: """Determine if `int_` falls into one of the ranges in `ranges`.""" tuple_ = _encode_range(int_, 0) pos = bisect.bisect_left(ranges, tuple_) diff --git a/contrib/python/idna/py3/idna/package_data.py b/contrib/python/idna/py3/idna/package_data.py index 75debb75d6f..94e40398cda 100644 --- a/contrib/python/idna/py3/idna/package_data.py +++ b/contrib/python/idna/py3/idna/package_data.py @@ -1 +1 @@ -__version__ = "3.15" +__version__ = "3.18" diff --git a/contrib/python/idna/py3/idna/uts46data.py b/contrib/python/idna/py3/idna/uts46data.py index e13aa90fb86..f2d931fe5d2 100644 --- a/contrib/python/idna/py3/idna/uts46data.py +++ b/contrib/python/idna/py3/idna/uts46data.py @@ -1,8383 +1,16896 @@ # This file is automatically generated by tools/idna-data -from typing import Tuple, Union +from array import array +from typing import Optional """IDNA Mapping Table from UTS46.""" __version__ = "17.0.0" -uts46data: Tuple[Union[Tuple[int, str], Tuple[int, str, str]], ...] = ( - (0x0, "V"), - (0x1, "V"), - (0x2, "V"), - (0x3, "V"), - (0x4, "V"), - (0x5, "V"), - (0x6, "V"), - (0x7, "V"), - (0x8, "V"), - (0x9, "V"), - (0xA, "V"), - (0xB, "V"), - (0xC, "V"), - (0xD, "V"), - (0xE, "V"), - (0xF, "V"), - (0x10, "V"), - (0x11, "V"), - (0x12, "V"), - (0x13, "V"), - (0x14, "V"), - (0x15, "V"), - (0x16, "V"), - (0x17, "V"), - (0x18, "V"), - (0x19, "V"), - (0x1A, "V"), - (0x1B, "V"), - (0x1C, "V"), - (0x1D, "V"), - (0x1E, "V"), - (0x1F, "V"), - (0x20, "V"), - (0x21, "V"), - (0x22, "V"), - (0x23, "V"), - (0x24, "V"), - (0x25, "V"), - (0x26, "V"), - (0x27, "V"), - (0x28, "V"), - (0x29, "V"), - (0x2A, "V"), - (0x2B, "V"), - (0x2C, "V"), - (0x2D, "V"), - (0x2E, "V"), - (0x2F, "V"), - (0x30, "V"), - (0x31, "V"), - (0x32, "V"), - (0x33, "V"), - (0x34, "V"), - (0x35, "V"), - (0x36, "V"), - (0x37, "V"), - (0x38, "V"), - (0x39, "V"), - (0x3A, "V"), - (0x3B, "V"), - (0x3C, "V"), - (0x3D, "V"), - (0x3E, "V"), - (0x3F, "V"), - (0x40, "V"), - (0x41, "M", "a"), - (0x42, "M", "b"), - (0x43, "M", "c"), - (0x44, "M", "d"), - (0x45, "M", "e"), - (0x46, "M", "f"), - (0x47, "M", "g"), - (0x48, "M", "h"), - (0x49, "M", "i"), - (0x4A, "M", "j"), - (0x4B, "M", "k"), - (0x4C, "M", "l"), - (0x4D, "M", "m"), - (0x4E, "M", "n"), - (0x4F, "M", "o"), - (0x50, "M", "p"), - (0x51, "M", "q"), - (0x52, "M", "r"), - (0x53, "M", "s"), - (0x54, "M", "t"), - (0x55, "M", "u"), - (0x56, "M", "v"), - (0x57, "M", "w"), - (0x58, "M", "x"), - (0x59, "M", "y"), - (0x5A, "M", "z"), - (0x5B, "V"), - (0x5C, "V"), - (0x5D, "V"), - (0x5E, "V"), - (0x5F, "V"), - (0x60, "V"), - (0x61, "V"), - (0x62, "V"), - (0x63, "V"), - (0x64, "V"), - (0x65, "V"), - (0x66, "V"), - (0x67, "V"), - (0x68, "V"), - (0x69, "V"), - (0x6A, "V"), - (0x6B, "V"), - (0x6C, "V"), - (0x6D, "V"), - (0x6E, "V"), - (0x6F, "V"), - (0x70, "V"), - (0x71, "V"), - (0x72, "V"), - (0x73, "V"), - (0x74, "V"), - (0x75, "V"), - (0x76, "V"), - (0x77, "V"), - (0x78, "V"), - (0x79, "V"), - (0x7A, "V"), - (0x7B, "V"), - (0x7C, "V"), - (0x7D, "V"), - (0x7E, "V"), - (0x7F, "V"), - (0x80, "X"), - (0x81, "X"), - (0x82, "X"), - (0x83, "X"), - (0x84, "X"), - (0x85, "X"), - (0x86, "X"), - (0x87, "X"), - (0x88, "X"), - (0x89, "X"), - (0x8A, "X"), - (0x8B, "X"), - (0x8C, "X"), - (0x8D, "X"), - (0x8E, "X"), - (0x8F, "X"), - (0x90, "X"), - (0x91, "X"), - (0x92, "X"), - (0x93, "X"), - (0x94, "X"), - (0x95, "X"), - (0x96, "X"), - (0x97, "X"), - (0x98, "X"), - (0x99, "X"), - (0x9A, "X"), - (0x9B, "X"), - (0x9C, "X"), - (0x9D, "X"), - (0x9E, "X"), - (0x9F, "X"), - (0xA0, "M", " "), - (0xA1, "V"), - (0xA2, "V"), - (0xA3, "V"), - (0xA4, "V"), - (0xA5, "V"), - (0xA6, "V"), - (0xA7, "V"), - (0xA8, "M", " ̈"), - (0xA9, "V"), - (0xAA, "M", "a"), - (0xAB, "V"), - (0xAC, "V"), - (0xAD, "I"), - (0xAE, "V"), - (0xAF, "M", " ̄"), - (0xB0, "V"), - (0xB1, "V"), - (0xB2, "M", "2"), - (0xB3, "M", "3"), - (0xB4, "M", " ́"), - (0xB5, "M", "μ"), - (0xB6, "V"), - (0xB7, "V"), - (0xB8, "M", " ̧"), - (0xB9, "M", "1"), - (0xBA, "M", "o"), - (0xBB, "V"), - (0xBC, "M", "1⁄4"), - (0xBD, "M", "1⁄2"), - (0xBE, "M", "3⁄4"), - (0xBF, "V"), - (0xC0, "M", "à"), - (0xC1, "M", "á"), - (0xC2, "M", "â"), - (0xC3, "M", "ã"), - (0xC4, "M", "ä"), - (0xC5, "M", "å"), - (0xC6, "M", "æ"), - (0xC7, "M", "ç"), - (0xC8, "M", "è"), - (0xC9, "M", "é"), - (0xCA, "M", "ê"), - (0xCB, "M", "ë"), - (0xCC, "M", "ì"), - (0xCD, "M", "í"), - (0xCE, "M", "î"), - (0xCF, "M", "ï"), - (0xD0, "M", "ð"), - (0xD1, "M", "ñ"), - (0xD2, "M", "ò"), - (0xD3, "M", "ó"), - (0xD4, "M", "ô"), - (0xD5, "M", "õ"), - (0xD6, "M", "ö"), - (0xD7, "V"), - (0xD8, "M", "ø"), - (0xD9, "M", "ù"), - (0xDA, "M", "ú"), - (0xDB, "M", "û"), - (0xDC, "M", "ü"), - (0xDD, "M", "ý"), - (0xDE, "M", "þ"), - (0xDF, "D", "ss"), - (0xE0, "V"), - (0xE1, "V"), - (0xE2, "V"), - (0xE3, "V"), - (0xE4, "V"), - (0xE5, "V"), - (0xE6, "V"), - (0xE7, "V"), - (0xE8, "V"), - (0xE9, "V"), - (0xEA, "V"), - (0xEB, "V"), - (0xEC, "V"), - (0xED, "V"), - (0xEE, "V"), - (0xEF, "V"), - (0xF0, "V"), - (0xF1, "V"), - (0xF2, "V"), - (0xF3, "V"), - (0xF4, "V"), - (0xF5, "V"), - (0xF6, "V"), - (0xF7, "V"), - (0xF8, "V"), - (0xF9, "V"), - (0xFA, "V"), - (0xFB, "V"), - (0xFC, "V"), - (0xFD, "V"), - (0xFE, "V"), - (0xFF, "V"), - (0x100, "M", "ā"), - (0x101, "V"), - (0x102, "M", "ă"), - (0x103, "V"), - (0x104, "M", "ą"), - (0x105, "V"), - (0x106, "M", "ć"), - (0x107, "V"), - (0x108, "M", "ĉ"), - (0x109, "V"), - (0x10A, "M", "ċ"), - (0x10B, "V"), - (0x10C, "M", "č"), - (0x10D, "V"), - (0x10E, "M", "ď"), - (0x10F, "V"), - (0x110, "M", "đ"), - (0x111, "V"), - (0x112, "M", "ē"), - (0x113, "V"), - (0x114, "M", "ĕ"), - (0x115, "V"), - (0x116, "M", "ė"), - (0x117, "V"), - (0x118, "M", "ę"), - (0x119, "V"), - (0x11A, "M", "ě"), - (0x11B, "V"), - (0x11C, "M", "ĝ"), - (0x11D, "V"), - (0x11E, "M", "ğ"), - (0x11F, "V"), - (0x120, "M", "ġ"), - (0x121, "V"), - (0x122, "M", "ģ"), - (0x123, "V"), - (0x124, "M", "ĥ"), - (0x125, "V"), - (0x126, "M", "ħ"), - (0x127, "V"), - (0x128, "M", "ĩ"), - (0x129, "V"), - (0x12A, "M", "ī"), - (0x12B, "V"), - (0x12C, "M", "ĭ"), - (0x12D, "V"), - (0x12E, "M", "į"), - (0x12F, "V"), - (0x130, "M", "i̇"), - (0x131, "V"), - (0x132, "M", "ij"), - (0x134, "M", "ĵ"), - (0x135, "V"), - (0x136, "M", "ķ"), - (0x137, "V"), - (0x139, "M", "ĺ"), - (0x13A, "V"), - (0x13B, "M", "ļ"), - (0x13C, "V"), - (0x13D, "M", "ľ"), - (0x13E, "V"), - (0x13F, "M", "l·"), - (0x141, "M", "ł"), - (0x142, "V"), - (0x143, "M", "ń"), - (0x144, "V"), - (0x145, "M", "ņ"), - (0x146, "V"), - (0x147, "M", "ň"), - (0x148, "V"), - (0x149, "M", "ʼn"), - (0x14A, "M", "ŋ"), - (0x14B, "V"), - (0x14C, "M", "ō"), - (0x14D, "V"), - (0x14E, "M", "ŏ"), - (0x14F, "V"), - (0x150, "M", "ő"), - (0x151, "V"), - (0x152, "M", "œ"), - (0x153, "V"), - (0x154, "M", "ŕ"), - (0x155, "V"), - (0x156, "M", "ŗ"), - (0x157, "V"), - (0x158, "M", "ř"), - (0x159, "V"), - (0x15A, "M", "ś"), - (0x15B, "V"), - (0x15C, "M", "ŝ"), - (0x15D, "V"), - (0x15E, "M", "ş"), - (0x15F, "V"), - (0x160, "M", "š"), - (0x161, "V"), - (0x162, "M", "ţ"), - (0x163, "V"), - (0x164, "M", "ť"), - (0x165, "V"), - (0x166, "M", "ŧ"), - (0x167, "V"), - (0x168, "M", "ũ"), - (0x169, "V"), - (0x16A, "M", "ū"), - (0x16B, "V"), - (0x16C, "M", "ŭ"), - (0x16D, "V"), - (0x16E, "M", "ů"), - (0x16F, "V"), - (0x170, "M", "ű"), - (0x171, "V"), - (0x172, "M", "ų"), - (0x173, "V"), - (0x174, "M", "ŵ"), - (0x175, "V"), - (0x176, "M", "ŷ"), - (0x177, "V"), - (0x178, "M", "ÿ"), - (0x179, "M", "ź"), - (0x17A, "V"), - (0x17B, "M", "ż"), - (0x17C, "V"), - (0x17D, "M", "ž"), - (0x17E, "V"), - (0x17F, "M", "s"), - (0x180, "V"), - (0x181, "M", "ɓ"), - (0x182, "M", "ƃ"), - (0x183, "V"), - (0x184, "M", "ƅ"), - (0x185, "V"), - (0x186, "M", "ɔ"), - (0x187, "M", "ƈ"), - (0x188, "V"), - (0x189, "M", "ɖ"), - (0x18A, "M", "ɗ"), - (0x18B, "M", "ƌ"), - (0x18C, "V"), - (0x18E, "M", "ǝ"), - (0x18F, "M", "ə"), - (0x190, "M", "ɛ"), - (0x191, "M", "ƒ"), - (0x192, "V"), - (0x193, "M", "ɠ"), - (0x194, "M", "ɣ"), - (0x195, "V"), - (0x196, "M", "ɩ"), - (0x197, "M", "ɨ"), - (0x198, "M", "ƙ"), - (0x199, "V"), - (0x19C, "M", "ɯ"), - (0x19D, "M", "ɲ"), - (0x19E, "V"), - (0x19F, "M", "ɵ"), - (0x1A0, "M", "ơ"), - (0x1A1, "V"), - (0x1A2, "M", "ƣ"), - (0x1A3, "V"), - (0x1A4, "M", "ƥ"), - (0x1A5, "V"), - (0x1A6, "M", "ʀ"), - (0x1A7, "M", "ƨ"), - (0x1A8, "V"), - (0x1A9, "M", "ʃ"), - (0x1AA, "V"), - (0x1AC, "M", "ƭ"), - (0x1AD, "V"), - (0x1AE, "M", "ʈ"), - (0x1AF, "M", "ư"), - (0x1B0, "V"), - (0x1B1, "M", "ʊ"), - (0x1B2, "M", "ʋ"), - (0x1B3, "M", "ƴ"), - (0x1B4, "V"), - (0x1B5, "M", "ƶ"), - (0x1B6, "V"), - (0x1B7, "M", "ʒ"), - (0x1B8, "M", "ƹ"), - (0x1B9, "V"), - (0x1BC, "M", "ƽ"), - (0x1BD, "V"), - (0x1C4, "M", "dž"), - (0x1C7, "M", "lj"), - (0x1CA, "M", "nj"), - (0x1CD, "M", "ǎ"), - (0x1CE, "V"), - (0x1CF, "M", "ǐ"), - (0x1D0, "V"), - (0x1D1, "M", "ǒ"), - (0x1D2, "V"), - (0x1D3, "M", "ǔ"), - (0x1D4, "V"), - (0x1D5, "M", "ǖ"), - (0x1D6, "V"), - (0x1D7, "M", "ǘ"), - (0x1D8, "V"), - (0x1D9, "M", "ǚ"), - (0x1DA, "V"), - (0x1DB, "M", "ǜ"), - (0x1DC, "V"), - (0x1DE, "M", "ǟ"), - (0x1DF, "V"), - (0x1E0, "M", "ǡ"), - (0x1E1, "V"), - (0x1E2, "M", "ǣ"), - (0x1E3, "V"), - (0x1E4, "M", "ǥ"), - (0x1E5, "V"), - (0x1E6, "M", "ǧ"), - (0x1E7, "V"), - (0x1E8, "M", "ǩ"), - (0x1E9, "V"), - (0x1EA, "M", "ǫ"), - (0x1EB, "V"), - (0x1EC, "M", "ǭ"), - (0x1ED, "V"), - (0x1EE, "M", "ǯ"), - (0x1EF, "V"), - (0x1F1, "M", "dz"), - (0x1F4, "M", "ǵ"), - (0x1F5, "V"), - (0x1F6, "M", "ƕ"), - (0x1F7, "M", "ƿ"), - (0x1F8, "M", "ǹ"), - (0x1F9, "V"), - (0x1FA, "M", "ǻ"), - (0x1FB, "V"), - (0x1FC, "M", "ǽ"), - (0x1FD, "V"), - (0x1FE, "M", "ǿ"), - (0x1FF, "V"), - (0x200, "M", "ȁ"), - (0x201, "V"), - (0x202, "M", "ȃ"), - (0x203, "V"), - (0x204, "M", "ȅ"), - (0x205, "V"), - (0x206, "M", "ȇ"), - (0x207, "V"), - (0x208, "M", "ȉ"), - (0x209, "V"), - (0x20A, "M", "ȋ"), - (0x20B, "V"), - (0x20C, "M", "ȍ"), - (0x20D, "V"), - (0x20E, "M", "ȏ"), - (0x20F, "V"), - (0x210, "M", "ȑ"), - (0x211, "V"), - (0x212, "M", "ȓ"), - (0x213, "V"), - (0x214, "M", "ȕ"), - (0x215, "V"), - (0x216, "M", "ȗ"), - (0x217, "V"), - (0x218, "M", "ș"), - (0x219, "V"), - (0x21A, "M", "ț"), - (0x21B, "V"), - (0x21C, "M", "ȝ"), - (0x21D, "V"), - (0x21E, "M", "ȟ"), - (0x21F, "V"), - (0x220, "M", "ƞ"), - (0x221, "V"), - (0x222, "M", "ȣ"), - (0x223, "V"), - (0x224, "M", "ȥ"), - (0x225, "V"), - (0x226, "M", "ȧ"), - (0x227, "V"), - (0x228, "M", "ȩ"), - (0x229, "V"), - (0x22A, "M", "ȫ"), - (0x22B, "V"), - (0x22C, "M", "ȭ"), - (0x22D, "V"), - (0x22E, "M", "ȯ"), - (0x22F, "V"), - (0x230, "M", "ȱ"), - (0x231, "V"), - (0x232, "M", "ȳ"), - (0x233, "V"), - (0x23A, "M", "ⱥ"), - (0x23B, "M", "ȼ"), - (0x23C, "V"), - (0x23D, "M", "ƚ"), - (0x23E, "M", "ⱦ"), - (0x23F, "V"), - (0x241, "M", "ɂ"), - (0x242, "V"), - (0x243, "M", "ƀ"), - (0x244, "M", "ʉ"), - (0x245, "M", "ʌ"), - (0x246, "M", "ɇ"), - (0x247, "V"), - (0x248, "M", "ɉ"), - (0x249, "V"), - (0x24A, "M", "ɋ"), - (0x24B, "V"), - (0x24C, "M", "ɍ"), - (0x24D, "V"), - (0x24E, "M", "ɏ"), - (0x24F, "V"), - (0x2B0, "M", "h"), - (0x2B1, "M", "ɦ"), - (0x2B2, "M", "j"), - (0x2B3, "M", "r"), - (0x2B4, "M", "ɹ"), - (0x2B5, "M", "ɻ"), - (0x2B6, "M", "ʁ"), - (0x2B7, "M", "w"), - (0x2B8, "M", "y"), - (0x2B9, "V"), - (0x2D8, "M", " ̆"), - (0x2D9, "M", " ̇"), - (0x2DA, "M", " ̊"), - (0x2DB, "M", " ̨"), - (0x2DC, "M", " ̃"), - (0x2DD, "M", " ̋"), - (0x2DE, "V"), - (0x2E0, "M", "ɣ"), - (0x2E1, "M", "l"), - (0x2E2, "M", "s"), - (0x2E3, "M", "x"), - (0x2E4, "M", "ʕ"), - (0x2E5, "V"), - (0x340, "M", "̀"), - (0x341, "M", "́"), - (0x342, "V"), - (0x343, "M", "̓"), - (0x344, "M", "̈́"), - (0x345, "M", "ι"), - (0x346, "V"), - (0x34F, "I"), - (0x350, "V"), - (0x370, "M", "ͱ"), - (0x371, "V"), - (0x372, "M", "ͳ"), - (0x373, "V"), - (0x374, "M", "ʹ"), - (0x375, "V"), - (0x376, "M", "ͷ"), - (0x377, "V"), - (0x378, "X"), - (0x37A, "M", " ι"), - (0x37B, "V"), - (0x37E, "M", ";"), - (0x37F, "M", "ϳ"), - (0x380, "X"), - (0x384, "M", " ́"), - (0x385, "M", " ̈́"), - (0x386, "M", "ά"), - (0x387, "M", "·"), - (0x388, "M", "έ"), - (0x389, "M", "ή"), - (0x38A, "M", "ί"), - (0x38B, "X"), - (0x38C, "M", "ό"), - (0x38D, "X"), - (0x38E, "M", "ύ"), - (0x38F, "M", "ώ"), - (0x390, "V"), - (0x391, "M", "α"), - (0x392, "M", "β"), - (0x393, "M", "γ"), - (0x394, "M", "δ"), - (0x395, "M", "ε"), - (0x396, "M", "ζ"), - (0x397, "M", "η"), - (0x398, "M", "θ"), - (0x399, "M", "ι"), - (0x39A, "M", "κ"), - (0x39B, "M", "λ"), - (0x39C, "M", "μ"), - (0x39D, "M", "ν"), - (0x39E, "M", "ξ"), - (0x39F, "M", "ο"), - (0x3A0, "M", "π"), - (0x3A1, "M", "ρ"), - (0x3A2, "X"), - (0x3A3, "M", "σ"), - (0x3A4, "M", "τ"), - (0x3A5, "M", "υ"), - (0x3A6, "M", "φ"), - (0x3A7, "M", "χ"), - (0x3A8, "M", "ψ"), - (0x3A9, "M", "ω"), - (0x3AA, "M", "ϊ"), - (0x3AB, "M", "ϋ"), - (0x3AC, "V"), - (0x3C2, "D", "σ"), - (0x3C3, "V"), - (0x3CF, "M", "ϗ"), - (0x3D0, "M", "β"), - (0x3D1, "M", "θ"), - (0x3D2, "M", "υ"), - (0x3D3, "M", "ύ"), - (0x3D4, "M", "ϋ"), - (0x3D5, "M", "φ"), - (0x3D6, "M", "π"), - (0x3D7, "V"), - (0x3D8, "M", "ϙ"), - (0x3D9, "V"), - (0x3DA, "M", "ϛ"), - (0x3DB, "V"), - (0x3DC, "M", "ϝ"), - (0x3DD, "V"), - (0x3DE, "M", "ϟ"), - (0x3DF, "V"), - (0x3E0, "M", "ϡ"), - (0x3E1, "V"), - (0x3E2, "M", "ϣ"), - (0x3E3, "V"), - (0x3E4, "M", "ϥ"), - (0x3E5, "V"), - (0x3E6, "M", "ϧ"), - (0x3E7, "V"), - (0x3E8, "M", "ϩ"), - (0x3E9, "V"), - (0x3EA, "M", "ϫ"), - (0x3EB, "V"), - (0x3EC, "M", "ϭ"), - (0x3ED, "V"), - (0x3EE, "M", "ϯ"), - (0x3EF, "V"), - (0x3F0, "M", "κ"), - (0x3F1, "M", "ρ"), - (0x3F2, "M", "σ"), - (0x3F3, "V"), - (0x3F4, "M", "θ"), - (0x3F5, "M", "ε"), - (0x3F6, "V"), - (0x3F7, "M", "ϸ"), - (0x3F8, "V"), - (0x3F9, "M", "σ"), - (0x3FA, "M", "ϻ"), - (0x3FB, "V"), - (0x3FD, "M", "ͻ"), - (0x3FE, "M", "ͼ"), - (0x3FF, "M", "ͽ"), - (0x400, "M", "ѐ"), - (0x401, "M", "ё"), - (0x402, "M", "ђ"), - (0x403, "M", "ѓ"), - (0x404, "M", "є"), - (0x405, "M", "ѕ"), - (0x406, "M", "і"), - (0x407, "M", "ї"), - (0x408, "M", "ј"), - (0x409, "M", "љ"), - (0x40A, "M", "њ"), - (0x40B, "M", "ћ"), - (0x40C, "M", "ќ"), - (0x40D, "M", "ѝ"), - (0x40E, "M", "ў"), - (0x40F, "M", "џ"), - (0x410, "M", "а"), - (0x411, "M", "б"), - (0x412, "M", "в"), - (0x413, "M", "г"), - (0x414, "M", "д"), - (0x415, "M", "е"), - (0x416, "M", "ж"), - (0x417, "M", "з"), - (0x418, "M", "и"), - (0x419, "M", "й"), - (0x41A, "M", "к"), - (0x41B, "M", "л"), - (0x41C, "M", "м"), - (0x41D, "M", "н"), - (0x41E, "M", "о"), - (0x41F, "M", "п"), - (0x420, "M", "р"), - (0x421, "M", "с"), - (0x422, "M", "т"), - (0x423, "M", "у"), - (0x424, "M", "ф"), - (0x425, "M", "х"), - (0x426, "M", "ц"), - (0x427, "M", "ч"), - (0x428, "M", "ш"), - (0x429, "M", "щ"), - (0x42A, "M", "ъ"), - (0x42B, "M", "ы"), - (0x42C, "M", "ь"), - (0x42D, "M", "э"), - (0x42E, "M", "ю"), - (0x42F, "M", "я"), - (0x430, "V"), - (0x460, "M", "ѡ"), - (0x461, "V"), - (0x462, "M", "ѣ"), - (0x463, "V"), - (0x464, "M", "ѥ"), - (0x465, "V"), - (0x466, "M", "ѧ"), - (0x467, "V"), - (0x468, "M", "ѩ"), - (0x469, "V"), - (0x46A, "M", "ѫ"), - (0x46B, "V"), - (0x46C, "M", "ѭ"), - (0x46D, "V"), - (0x46E, "M", "ѯ"), - (0x46F, "V"), - (0x470, "M", "ѱ"), - (0x471, "V"), - (0x472, "M", "ѳ"), - (0x473, "V"), - (0x474, "M", "ѵ"), - (0x475, "V"), - (0x476, "M", "ѷ"), - (0x477, "V"), - (0x478, "M", "ѹ"), - (0x479, "V"), - (0x47A, "M", "ѻ"), - (0x47B, "V"), - (0x47C, "M", "ѽ"), - (0x47D, "V"), - (0x47E, "M", "ѿ"), - (0x47F, "V"), - (0x480, "M", "ҁ"), - (0x481, "V"), - (0x48A, "M", "ҋ"), - (0x48B, "V"), - (0x48C, "M", "ҍ"), - (0x48D, "V"), - (0x48E, "M", "ҏ"), - (0x48F, "V"), - (0x490, "M", "ґ"), - (0x491, "V"), - (0x492, "M", "ғ"), - (0x493, "V"), - (0x494, "M", "ҕ"), - (0x495, "V"), - (0x496, "M", "җ"), - (0x497, "V"), - (0x498, "M", "ҙ"), - (0x499, "V"), - (0x49A, "M", "қ"), - (0x49B, "V"), - (0x49C, "M", "ҝ"), - (0x49D, "V"), - (0x49E, "M", "ҟ"), - (0x49F, "V"), - (0x4A0, "M", "ҡ"), - (0x4A1, "V"), - (0x4A2, "M", "ң"), - (0x4A3, "V"), - (0x4A4, "M", "ҥ"), - (0x4A5, "V"), - (0x4A6, "M", "ҧ"), - (0x4A7, "V"), - (0x4A8, "M", "ҩ"), - (0x4A9, "V"), - (0x4AA, "M", "ҫ"), - (0x4AB, "V"), - (0x4AC, "M", "ҭ"), - (0x4AD, "V"), - (0x4AE, "M", "ү"), - (0x4AF, "V"), - (0x4B0, "M", "ұ"), - (0x4B1, "V"), - (0x4B2, "M", "ҳ"), - (0x4B3, "V"), - (0x4B4, "M", "ҵ"), - (0x4B5, "V"), - (0x4B6, "M", "ҷ"), - (0x4B7, "V"), - (0x4B8, "M", "ҹ"), - (0x4B9, "V"), - (0x4BA, "M", "һ"), - (0x4BB, "V"), - (0x4BC, "M", "ҽ"), - (0x4BD, "V"), - (0x4BE, "M", "ҿ"), - (0x4BF, "V"), - (0x4C0, "M", "ӏ"), - (0x4C1, "M", "ӂ"), - (0x4C2, "V"), - (0x4C3, "M", "ӄ"), - (0x4C4, "V"), - (0x4C5, "M", "ӆ"), - (0x4C6, "V"), - (0x4C7, "M", "ӈ"), - (0x4C8, "V"), - (0x4C9, "M", "ӊ"), - (0x4CA, "V"), - (0x4CB, "M", "ӌ"), - (0x4CC, "V"), - (0x4CD, "M", "ӎ"), - (0x4CE, "V"), - (0x4D0, "M", "ӑ"), - (0x4D1, "V"), - (0x4D2, "M", "ӓ"), - (0x4D3, "V"), - (0x4D4, "M", "ӕ"), - (0x4D5, "V"), - (0x4D6, "M", "ӗ"), - (0x4D7, "V"), - (0x4D8, "M", "ә"), - (0x4D9, "V"), - (0x4DA, "M", "ӛ"), - (0x4DB, "V"), - (0x4DC, "M", "ӝ"), - (0x4DD, "V"), - (0x4DE, "M", "ӟ"), - (0x4DF, "V"), - (0x4E0, "M", "ӡ"), - (0x4E1, "V"), - (0x4E2, "M", "ӣ"), - (0x4E3, "V"), - (0x4E4, "M", "ӥ"), - (0x4E5, "V"), - (0x4E6, "M", "ӧ"), - (0x4E7, "V"), - (0x4E8, "M", "ө"), - (0x4E9, "V"), - (0x4EA, "M", "ӫ"), - (0x4EB, "V"), - (0x4EC, "M", "ӭ"), - (0x4ED, "V"), - (0x4EE, "M", "ӯ"), - (0x4EF, "V"), - (0x4F0, "M", "ӱ"), - (0x4F1, "V"), - (0x4F2, "M", "ӳ"), - (0x4F3, "V"), - (0x4F4, "M", "ӵ"), - (0x4F5, "V"), - (0x4F6, "M", "ӷ"), - (0x4F7, "V"), - (0x4F8, "M", "ӹ"), - (0x4F9, "V"), - (0x4FA, "M", "ӻ"), - (0x4FB, "V"), - (0x4FC, "M", "ӽ"), - (0x4FD, "V"), - (0x4FE, "M", "ӿ"), - (0x4FF, "V"), - (0x500, "M", "ԁ"), - (0x501, "V"), - (0x502, "M", "ԃ"), - (0x503, "V"), - (0x504, "M", "ԅ"), - (0x505, "V"), - (0x506, "M", "ԇ"), - (0x507, "V"), - (0x508, "M", "ԉ"), - (0x509, "V"), - (0x50A, "M", "ԋ"), - (0x50B, "V"), - (0x50C, "M", "ԍ"), - (0x50D, "V"), - (0x50E, "M", "ԏ"), - (0x50F, "V"), - (0x510, "M", "ԑ"), - (0x511, "V"), - (0x512, "M", "ԓ"), - (0x513, "V"), - (0x514, "M", "ԕ"), - (0x515, "V"), - (0x516, "M", "ԗ"), - (0x517, "V"), - (0x518, "M", "ԙ"), - (0x519, "V"), - (0x51A, "M", "ԛ"), - (0x51B, "V"), - (0x51C, "M", "ԝ"), - (0x51D, "V"), - (0x51E, "M", "ԟ"), - (0x51F, "V"), - (0x520, "M", "ԡ"), - (0x521, "V"), - (0x522, "M", "ԣ"), - (0x523, "V"), - (0x524, "M", "ԥ"), - (0x525, "V"), - (0x526, "M", "ԧ"), - (0x527, "V"), - (0x528, "M", "ԩ"), - (0x529, "V"), - (0x52A, "M", "ԫ"), - (0x52B, "V"), - (0x52C, "M", "ԭ"), - (0x52D, "V"), - (0x52E, "M", "ԯ"), - (0x52F, "V"), - (0x530, "X"), - (0x531, "M", "ա"), - (0x532, "M", "բ"), - (0x533, "M", "գ"), - (0x534, "M", "դ"), - (0x535, "M", "ե"), - (0x536, "M", "զ"), - (0x537, "M", "է"), - (0x538, "M", "ը"), - (0x539, "M", "թ"), - (0x53A, "M", "ժ"), - (0x53B, "M", "ի"), - (0x53C, "M", "լ"), - (0x53D, "M", "խ"), - (0x53E, "M", "ծ"), - (0x53F, "M", "կ"), - (0x540, "M", "հ"), - (0x541, "M", "ձ"), - (0x542, "M", "ղ"), - (0x543, "M", "ճ"), - (0x544, "M", "մ"), - (0x545, "M", "յ"), - (0x546, "M", "ն"), - (0x547, "M", "շ"), - (0x548, "M", "ո"), - (0x549, "M", "չ"), - (0x54A, "M", "պ"), - (0x54B, "M", "ջ"), - (0x54C, "M", "ռ"), - (0x54D, "M", "ս"), - (0x54E, "M", "վ"), - (0x54F, "M", "տ"), - (0x550, "M", "ր"), - (0x551, "M", "ց"), - (0x552, "M", "ւ"), - (0x553, "M", "փ"), - (0x554, "M", "ք"), - (0x555, "M", "օ"), - (0x556, "M", "ֆ"), - (0x557, "X"), - (0x559, "V"), - (0x587, "M", "եւ"), - (0x588, "V"), - (0x58B, "X"), - (0x58D, "V"), - (0x590, "X"), - (0x591, "V"), - (0x5C8, "X"), - (0x5D0, "V"), - (0x5EB, "X"), - (0x5EF, "V"), - (0x5F5, "X"), - (0x606, "V"), - (0x61C, "X"), - (0x61D, "V"), - (0x675, "M", "اٴ"), - (0x676, "M", "وٴ"), - (0x677, "M", "ۇٴ"), - (0x678, "M", "يٴ"), - (0x679, "V"), - (0x6DD, "X"), - (0x6DE, "V"), - (0x70E, "X"), - (0x710, "V"), - (0x74B, "X"), - (0x74D, "V"), - (0x7B2, "X"), - (0x7C0, "V"), - (0x7FB, "X"), - (0x7FD, "V"), - (0x82E, "X"), - (0x830, "V"), - (0x83F, "X"), - (0x840, "V"), - (0x85C, "X"), - (0x85E, "V"), - (0x85F, "X"), - (0x860, "V"), - (0x86B, "X"), - (0x870, "V"), - (0x890, "X"), - (0x897, "V"), - (0x8E2, "X"), - (0x8E3, "V"), - (0x958, "M", "क़"), - (0x959, "M", "ख़"), - (0x95A, "M", "ग़"), - (0x95B, "M", "ज़"), - (0x95C, "M", "ड़"), - (0x95D, "M", "ढ़"), - (0x95E, "M", "फ़"), - (0x95F, "M", "य़"), - (0x960, "V"), - (0x984, "X"), - (0x985, "V"), - (0x98D, "X"), - (0x98F, "V"), - (0x991, "X"), - (0x993, "V"), - (0x9A9, "X"), - (0x9AA, "V"), - (0x9B1, "X"), - (0x9B2, "V"), - (0x9B3, "X"), - (0x9B6, "V"), - (0x9BA, "X"), - (0x9BC, "V"), - (0x9C5, "X"), - (0x9C7, "V"), - (0x9C9, "X"), - (0x9CB, "V"), - (0x9CF, "X"), - (0x9D7, "V"), - (0x9D8, "X"), - (0x9DC, "M", "ড়"), - (0x9DD, "M", "ঢ়"), - (0x9DE, "X"), - (0x9DF, "M", "য়"), - (0x9E0, "V"), - (0x9E4, "X"), - (0x9E6, "V"), - (0x9FF, "X"), - (0xA01, "V"), - (0xA04, "X"), - (0xA05, "V"), - (0xA0B, "X"), - (0xA0F, "V"), - (0xA11, "X"), - (0xA13, "V"), - (0xA29, "X"), - (0xA2A, "V"), - (0xA31, "X"), - (0xA32, "V"), - (0xA33, "M", "ਲ਼"), - (0xA34, "X"), - (0xA35, "V"), - (0xA36, "M", "ਸ਼"), - (0xA37, "X"), - (0xA38, "V"), - (0xA3A, "X"), - (0xA3C, "V"), - (0xA3D, "X"), - (0xA3E, "V"), - (0xA43, "X"), - (0xA47, "V"), - (0xA49, "X"), - (0xA4B, "V"), - (0xA4E, "X"), - (0xA51, "V"), - (0xA52, "X"), - (0xA59, "M", "ਖ਼"), - (0xA5A, "M", "ਗ਼"), - (0xA5B, "M", "ਜ਼"), - (0xA5C, "V"), - (0xA5D, "X"), - (0xA5E, "M", "ਫ਼"), - (0xA5F, "X"), - (0xA66, "V"), - (0xA77, "X"), - (0xA81, "V"), - (0xA84, "X"), - (0xA85, "V"), - (0xA8E, "X"), - (0xA8F, "V"), - (0xA92, "X"), - (0xA93, "V"), - (0xAA9, "X"), - (0xAAA, "V"), - (0xAB1, "X"), - (0xAB2, "V"), - (0xAB4, "X"), - (0xAB5, "V"), - (0xABA, "X"), - (0xABC, "V"), - (0xAC6, "X"), - (0xAC7, "V"), - (0xACA, "X"), - (0xACB, "V"), - (0xACE, "X"), - (0xAD0, "V"), - (0xAD1, "X"), - (0xAE0, "V"), - (0xAE4, "X"), - (0xAE6, "V"), - (0xAF2, "X"), - (0xAF9, "V"), - (0xB00, "X"), - (0xB01, "V"), - (0xB04, "X"), - (0xB05, "V"), - (0xB0D, "X"), - (0xB0F, "V"), - (0xB11, "X"), - (0xB13, "V"), - (0xB29, "X"), - (0xB2A, "V"), - (0xB31, "X"), - (0xB32, "V"), - (0xB34, "X"), - (0xB35, "V"), - (0xB3A, "X"), - (0xB3C, "V"), - (0xB45, "X"), - (0xB47, "V"), - (0xB49, "X"), - (0xB4B, "V"), - (0xB4E, "X"), - (0xB55, "V"), - (0xB58, "X"), - (0xB5C, "M", "ଡ଼"), - (0xB5D, "M", "ଢ଼"), - (0xB5E, "X"), - (0xB5F, "V"), - (0xB64, "X"), - (0xB66, "V"), - (0xB78, "X"), - (0xB82, "V"), - (0xB84, "X"), - (0xB85, "V"), - (0xB8B, "X"), - (0xB8E, "V"), - (0xB91, "X"), - (0xB92, "V"), - (0xB96, "X"), - (0xB99, "V"), - (0xB9B, "X"), - (0xB9C, "V"), - (0xB9D, "X"), - (0xB9E, "V"), - (0xBA0, "X"), - (0xBA3, "V"), - (0xBA5, "X"), - (0xBA8, "V"), - (0xBAB, "X"), - (0xBAE, "V"), - (0xBBA, "X"), - (0xBBE, "V"), - (0xBC3, "X"), - (0xBC6, "V"), - (0xBC9, "X"), - (0xBCA, "V"), - (0xBCE, "X"), - (0xBD0, "V"), - (0xBD1, "X"), - (0xBD7, "V"), - (0xBD8, "X"), - (0xBE6, "V"), - (0xBFB, "X"), - (0xC00, "V"), - (0xC0D, "X"), - (0xC0E, "V"), - (0xC11, "X"), - (0xC12, "V"), - (0xC29, "X"), - (0xC2A, "V"), - (0xC3A, "X"), - (0xC3C, "V"), - (0xC45, "X"), - (0xC46, "V"), - (0xC49, "X"), - (0xC4A, "V"), - (0xC4E, "X"), - (0xC55, "V"), - (0xC57, "X"), - (0xC58, "V"), - (0xC5B, "X"), - (0xC5C, "V"), - (0xC5E, "X"), - (0xC60, "V"), - (0xC64, "X"), - (0xC66, "V"), - (0xC70, "X"), - (0xC77, "V"), - (0xC8D, "X"), - (0xC8E, "V"), - (0xC91, "X"), - (0xC92, "V"), - (0xCA9, "X"), - (0xCAA, "V"), - (0xCB4, "X"), - (0xCB5, "V"), - (0xCBA, "X"), - (0xCBC, "V"), - (0xCC5, "X"), - (0xCC6, "V"), - (0xCC9, "X"), - (0xCCA, "V"), - (0xCCE, "X"), - (0xCD5, "V"), - (0xCD7, "X"), - (0xCDC, "V"), - (0xCDF, "X"), - (0xCE0, "V"), - (0xCE4, "X"), - (0xCE6, "V"), - (0xCF0, "X"), - (0xCF1, "V"), - (0xCF4, "X"), - (0xD00, "V"), - (0xD0D, "X"), - (0xD0E, "V"), - (0xD11, "X"), - (0xD12, "V"), - (0xD45, "X"), - (0xD46, "V"), - (0xD49, "X"), - (0xD4A, "V"), - (0xD50, "X"), - (0xD54, "V"), - (0xD64, "X"), - (0xD66, "V"), - (0xD80, "X"), - (0xD81, "V"), - (0xD84, "X"), - (0xD85, "V"), - (0xD97, "X"), - (0xD9A, "V"), - (0xDB2, "X"), - (0xDB3, "V"), - (0xDBC, "X"), - (0xDBD, "V"), - (0xDBE, "X"), - (0xDC0, "V"), - (0xDC7, "X"), - (0xDCA, "V"), - (0xDCB, "X"), - (0xDCF, "V"), - (0xDD5, "X"), - (0xDD6, "V"), - (0xDD7, "X"), - (0xDD8, "V"), - (0xDE0, "X"), - (0xDE6, "V"), - (0xDF0, "X"), - (0xDF2, "V"), - (0xDF5, "X"), - (0xE01, "V"), - (0xE33, "M", "ํา"), - (0xE34, "V"), - (0xE3B, "X"), - (0xE3F, "V"), - (0xE5C, "X"), - (0xE81, "V"), - (0xE83, "X"), - (0xE84, "V"), - (0xE85, "X"), - (0xE86, "V"), - (0xE8B, "X"), - (0xE8C, "V"), - (0xEA4, "X"), - (0xEA5, "V"), - (0xEA6, "X"), - (0xEA7, "V"), - (0xEB3, "M", "ໍາ"), - (0xEB4, "V"), - (0xEBE, "X"), - (0xEC0, "V"), - (0xEC5, "X"), - (0xEC6, "V"), - (0xEC7, "X"), - (0xEC8, "V"), - (0xECF, "X"), - (0xED0, "V"), - (0xEDA, "X"), - (0xEDC, "M", "ຫນ"), - (0xEDD, "M", "ຫມ"), - (0xEDE, "V"), - (0xEE0, "X"), - (0xF00, "V"), - (0xF0C, "M", "་"), - (0xF0D, "V"), - (0xF43, "M", "གྷ"), - (0xF44, "V"), - (0xF48, "X"), - (0xF49, "V"), - (0xF4D, "M", "ཌྷ"), - (0xF4E, "V"), - (0xF52, "M", "དྷ"), - (0xF53, "V"), - (0xF57, "M", "བྷ"), - (0xF58, "V"), - (0xF5C, "M", "ཛྷ"), - (0xF5D, "V"), - (0xF69, "M", "ཀྵ"), - (0xF6A, "V"), - (0xF6D, "X"), - (0xF71, "V"), - (0xF73, "M", "ཱི"), - (0xF74, "V"), - (0xF75, "M", "ཱུ"), - (0xF76, "M", "ྲྀ"), - (0xF77, "M", "ྲཱྀ"), - (0xF78, "M", "ླྀ"), - (0xF79, "M", "ླཱྀ"), - (0xF7A, "V"), - (0xF81, "M", "ཱྀ"), - (0xF82, "V"), - (0xF93, "M", "ྒྷ"), - (0xF94, "V"), - (0xF98, "X"), - (0xF99, "V"), - (0xF9D, "M", "ྜྷ"), - (0xF9E, "V"), - (0xFA2, "M", "ྡྷ"), - (0xFA3, "V"), - (0xFA7, "M", "ྦྷ"), - (0xFA8, "V"), - (0xFAC, "M", "ྫྷ"), - (0xFAD, "V"), - (0xFB9, "M", "ྐྵ"), - (0xFBA, "V"), - (0xFBD, "X"), - (0xFBE, "V"), - (0xFCD, "X"), - (0xFCE, "V"), - (0xFDB, "X"), - (0x1000, "V"), - (0x10A0, "M", "ⴀ"), - (0x10A1, "M", "ⴁ"), - (0x10A2, "M", "ⴂ"), - (0x10A3, "M", "ⴃ"), - (0x10A4, "M", "ⴄ"), - (0x10A5, "M", "ⴅ"), - (0x10A6, "M", "ⴆ"), - (0x10A7, "M", "ⴇ"), - (0x10A8, "M", "ⴈ"), - (0x10A9, "M", "ⴉ"), - (0x10AA, "M", "ⴊ"), - (0x10AB, "M", "ⴋ"), - (0x10AC, "M", "ⴌ"), - (0x10AD, "M", "ⴍ"), - (0x10AE, "M", "ⴎ"), - (0x10AF, "M", "ⴏ"), - (0x10B0, "M", "ⴐ"), - (0x10B1, "M", "ⴑ"), - (0x10B2, "M", "ⴒ"), - (0x10B3, "M", "ⴓ"), - (0x10B4, "M", "ⴔ"), - (0x10B5, "M", "ⴕ"), - (0x10B6, "M", "ⴖ"), - (0x10B7, "M", "ⴗ"), - (0x10B8, "M", "ⴘ"), - (0x10B9, "M", "ⴙ"), - (0x10BA, "M", "ⴚ"), - (0x10BB, "M", "ⴛ"), - (0x10BC, "M", "ⴜ"), - (0x10BD, "M", "ⴝ"), - (0x10BE, "M", "ⴞ"), - (0x10BF, "M", "ⴟ"), - (0x10C0, "M", "ⴠ"), - (0x10C1, "M", "ⴡ"), - (0x10C2, "M", "ⴢ"), - (0x10C3, "M", "ⴣ"), - (0x10C4, "M", "ⴤ"), - (0x10C5, "M", "ⴥ"), - (0x10C6, "X"), - (0x10C7, "M", "ⴧ"), - (0x10C8, "X"), - (0x10CD, "M", "ⴭ"), - (0x10CE, "X"), - (0x10D0, "V"), - (0x10FC, "M", "ნ"), - (0x10FD, "V"), - (0x115F, "I"), - (0x1161, "V"), - (0x1249, "X"), - (0x124A, "V"), - (0x124E, "X"), - (0x1250, "V"), - (0x1257, "X"), - (0x1258, "V"), - (0x1259, "X"), - (0x125A, "V"), - (0x125E, "X"), - (0x1260, "V"), - (0x1289, "X"), - (0x128A, "V"), - (0x128E, "X"), - (0x1290, "V"), - (0x12B1, "X"), - (0x12B2, "V"), - (0x12B6, "X"), - (0x12B8, "V"), - (0x12BF, "X"), - (0x12C0, "V"), - (0x12C1, "X"), - (0x12C2, "V"), - (0x12C6, "X"), - (0x12C8, "V"), - (0x12D7, "X"), - (0x12D8, "V"), - (0x1311, "X"), - (0x1312, "V"), - (0x1316, "X"), - (0x1318, "V"), - (0x135B, "X"), - (0x135D, "V"), - (0x137D, "X"), - (0x1380, "V"), - (0x139A, "X"), - (0x13A0, "V"), - (0x13F6, "X"), - (0x13F8, "M", "Ᏸ"), - (0x13F9, "M", "Ᏹ"), - (0x13FA, "M", "Ᏺ"), - (0x13FB, "M", "Ᏻ"), - (0x13FC, "M", "Ᏼ"), - (0x13FD, "M", "Ᏽ"), - (0x13FE, "X"), - (0x1400, "V"), - (0x1680, "X"), - (0x1681, "V"), - (0x169D, "X"), - (0x16A0, "V"), - (0x16F9, "X"), - (0x1700, "V"), - (0x1716, "X"), - (0x171F, "V"), - (0x1737, "X"), - (0x1740, "V"), - (0x1754, "X"), - (0x1760, "V"), - (0x176D, "X"), - (0x176E, "V"), - (0x1771, "X"), - (0x1772, "V"), - (0x1774, "X"), - (0x1780, "V"), - (0x17B4, "I"), - (0x17B6, "V"), - (0x17DE, "X"), - (0x17E0, "V"), - (0x17EA, "X"), - (0x17F0, "V"), - (0x17FA, "X"), - (0x1800, "V"), - (0x180B, "I"), - (0x1810, "V"), - (0x181A, "X"), - (0x1820, "V"), - (0x1879, "X"), - (0x1880, "V"), - (0x18AB, "X"), - (0x18B0, "V"), - (0x18F6, "X"), - (0x1900, "V"), - (0x191F, "X"), - (0x1920, "V"), - (0x192C, "X"), - (0x1930, "V"), - (0x193C, "X"), - (0x1940, "V"), - (0x1941, "X"), - (0x1944, "V"), - (0x196E, "X"), - (0x1970, "V"), - (0x1975, "X"), - (0x1980, "V"), - (0x19AC, "X"), - (0x19B0, "V"), - (0x19CA, "X"), - (0x19D0, "V"), - (0x19DB, "X"), - (0x19DE, "V"), - (0x1A1C, "X"), - (0x1A1E, "V"), - (0x1A5F, "X"), - (0x1A60, "V"), - (0x1A7D, "X"), - (0x1A7F, "V"), - (0x1A8A, "X"), - (0x1A90, "V"), - (0x1A9A, "X"), - (0x1AA0, "V"), - (0x1AAE, "X"), - (0x1AB0, "V"), - (0x1ADE, "X"), - (0x1AE0, "V"), - (0x1AEC, "X"), - (0x1B00, "V"), - (0x1B4D, "X"), - (0x1B4E, "V"), - (0x1BF4, "X"), - (0x1BFC, "V"), - (0x1C38, "X"), - (0x1C3B, "V"), - (0x1C4A, "X"), - (0x1C4D, "V"), - (0x1C80, "M", "в"), - (0x1C81, "M", "д"), - (0x1C82, "M", "о"), - (0x1C83, "M", "с"), - (0x1C84, "M", "т"), - (0x1C86, "M", "ъ"), - (0x1C87, "M", "ѣ"), - (0x1C88, "M", "ꙋ"), - (0x1C89, "M", ""), - (0x1C8A, "V"), - (0x1C8B, "X"), - (0x1C90, "M", "ა"), - (0x1C91, "M", "ბ"), - (0x1C92, "M", "გ"), - (0x1C93, "M", "დ"), - (0x1C94, "M", "ე"), - (0x1C95, "M", "ვ"), - (0x1C96, "M", "ზ"), - (0x1C97, "M", "თ"), - (0x1C98, "M", "ი"), - (0x1C99, "M", "კ"), - (0x1C9A, "M", "ლ"), - (0x1C9B, "M", "მ"), - (0x1C9C, "M", "ნ"), - (0x1C9D, "M", "ო"), - (0x1C9E, "M", "პ"), - (0x1C9F, "M", "ჟ"), - (0x1CA0, "M", "რ"), - (0x1CA1, "M", "ს"), - (0x1CA2, "M", "ტ"), - (0x1CA3, "M", "უ"), - (0x1CA4, "M", "ფ"), - (0x1CA5, "M", "ქ"), - (0x1CA6, "M", "ღ"), - (0x1CA7, "M", "ყ"), - (0x1CA8, "M", "შ"), - (0x1CA9, "M", "ჩ"), - (0x1CAA, "M", "ც"), - (0x1CAB, "M", "ძ"), - (0x1CAC, "M", "წ"), - (0x1CAD, "M", "ჭ"), - (0x1CAE, "M", "ხ"), - (0x1CAF, "M", "ჯ"), - (0x1CB0, "M", "ჰ"), - (0x1CB1, "M", "ჱ"), - (0x1CB2, "M", "ჲ"), - (0x1CB3, "M", "ჳ"), - (0x1CB4, "M", "ჴ"), - (0x1CB5, "M", "ჵ"), - (0x1CB6, "M", "ჶ"), - (0x1CB7, "M", "ჷ"), - (0x1CB8, "M", "ჸ"), - (0x1CB9, "M", "ჹ"), - (0x1CBA, "M", "ჺ"), - (0x1CBB, "X"), - (0x1CBD, "M", "ჽ"), - (0x1CBE, "M", "ჾ"), - (0x1CBF, "M", "ჿ"), - (0x1CC0, "V"), - (0x1CC8, "X"), - (0x1CD0, "V"), - (0x1CFB, "X"), - (0x1D00, "V"), - (0x1D2C, "M", "a"), - (0x1D2D, "M", "æ"), - (0x1D2E, "M", "b"), - (0x1D2F, "V"), - (0x1D30, "M", "d"), - (0x1D31, "M", "e"), - (0x1D32, "M", "ǝ"), - (0x1D33, "M", "g"), - (0x1D34, "M", "h"), - (0x1D35, "M", "i"), - (0x1D36, "M", "j"), - (0x1D37, "M", "k"), - (0x1D38, "M", "l"), - (0x1D39, "M", "m"), - (0x1D3A, "M", "n"), - (0x1D3B, "V"), - (0x1D3C, "M", "o"), - (0x1D3D, "M", "ȣ"), - (0x1D3E, "M", "p"), - (0x1D3F, "M", "r"), - (0x1D40, "M", "t"), - (0x1D41, "M", "u"), - (0x1D42, "M", "w"), - (0x1D43, "M", "a"), - (0x1D44, "M", "ɐ"), - (0x1D45, "M", "ɑ"), - (0x1D46, "M", "ᴂ"), - (0x1D47, "M", "b"), - (0x1D48, "M", "d"), - (0x1D49, "M", "e"), - (0x1D4A, "M", "ə"), - (0x1D4B, "M", "ɛ"), - (0x1D4C, "M", "ɜ"), - (0x1D4D, "M", "g"), - (0x1D4E, "V"), - (0x1D4F, "M", "k"), - (0x1D50, "M", "m"), - (0x1D51, "M", "ŋ"), - (0x1D52, "M", "o"), - (0x1D53, "M", "ɔ"), - (0x1D54, "M", "ᴖ"), - (0x1D55, "M", "ᴗ"), - (0x1D56, "M", "p"), - (0x1D57, "M", "t"), - (0x1D58, "M", "u"), - (0x1D59, "M", "ᴝ"), - (0x1D5A, "M", "ɯ"), - (0x1D5B, "M", "v"), - (0x1D5C, "M", "ᴥ"), - (0x1D5D, "M", "β"), - (0x1D5E, "M", "γ"), - (0x1D5F, "M", "δ"), - (0x1D60, "M", "φ"), - (0x1D61, "M", "χ"), - (0x1D62, "M", "i"), - (0x1D63, "M", "r"), - (0x1D64, "M", "u"), - (0x1D65, "M", "v"), - (0x1D66, "M", "β"), - (0x1D67, "M", "γ"), - (0x1D68, "M", "ρ"), - (0x1D69, "M", "φ"), - (0x1D6A, "M", "χ"), - (0x1D6B, "V"), - (0x1D78, "M", "н"), - (0x1D79, "V"), - (0x1D9B, "M", "ɒ"), - (0x1D9C, "M", "c"), - (0x1D9D, "M", "ɕ"), - (0x1D9E, "M", "ð"), - (0x1D9F, "M", "ɜ"), - (0x1DA0, "M", "f"), - (0x1DA1, "M", "ɟ"), - (0x1DA2, "M", "ɡ"), - (0x1DA3, "M", "ɥ"), - (0x1DA4, "M", "ɨ"), - (0x1DA5, "M", "ɩ"), - (0x1DA6, "M", "ɪ"), - (0x1DA7, "M", "ᵻ"), - (0x1DA8, "M", "ʝ"), - (0x1DA9, "M", "ɭ"), - (0x1DAA, "M", "ᶅ"), - (0x1DAB, "M", "ʟ"), - (0x1DAC, "M", "ɱ"), - (0x1DAD, "M", "ɰ"), - (0x1DAE, "M", "ɲ"), - (0x1DAF, "M", "ɳ"), - (0x1DB0, "M", "ɴ"), - (0x1DB1, "M", "ɵ"), - (0x1DB2, "M", "ɸ"), - (0x1DB3, "M", "ʂ"), - (0x1DB4, "M", "ʃ"), - (0x1DB5, "M", "ƫ"), - (0x1DB6, "M", "ʉ"), - (0x1DB7, "M", "ʊ"), - (0x1DB8, "M", "ᴜ"), - (0x1DB9, "M", "ʋ"), - (0x1DBA, "M", "ʌ"), - (0x1DBB, "M", "z"), - (0x1DBC, "M", "ʐ"), - (0x1DBD, "M", "ʑ"), - (0x1DBE, "M", "ʒ"), - (0x1DBF, "M", "θ"), - (0x1DC0, "V"), - (0x1E00, "M", "ḁ"), - (0x1E01, "V"), - (0x1E02, "M", "ḃ"), - (0x1E03, "V"), - (0x1E04, "M", "ḅ"), - (0x1E05, "V"), - (0x1E06, "M", "ḇ"), - (0x1E07, "V"), - (0x1E08, "M", "ḉ"), - (0x1E09, "V"), - (0x1E0A, "M", "ḋ"), - (0x1E0B, "V"), - (0x1E0C, "M", "ḍ"), - (0x1E0D, "V"), - (0x1E0E, "M", "ḏ"), - (0x1E0F, "V"), - (0x1E10, "M", "ḑ"), - (0x1E11, "V"), - (0x1E12, "M", "ḓ"), - (0x1E13, "V"), - (0x1E14, "M", "ḕ"), - (0x1E15, "V"), - (0x1E16, "M", "ḗ"), - (0x1E17, "V"), - (0x1E18, "M", "ḙ"), - (0x1E19, "V"), - (0x1E1A, "M", "ḛ"), - (0x1E1B, "V"), - (0x1E1C, "M", "ḝ"), - (0x1E1D, "V"), - (0x1E1E, "M", "ḟ"), - (0x1E1F, "V"), - (0x1E20, "M", "ḡ"), - (0x1E21, "V"), - (0x1E22, "M", "ḣ"), - (0x1E23, "V"), - (0x1E24, "M", "ḥ"), - (0x1E25, "V"), - (0x1E26, "M", "ḧ"), - (0x1E27, "V"), - (0x1E28, "M", "ḩ"), - (0x1E29, "V"), - (0x1E2A, "M", "ḫ"), - (0x1E2B, "V"), - (0x1E2C, "M", "ḭ"), - (0x1E2D, "V"), - (0x1E2E, "M", "ḯ"), - (0x1E2F, "V"), - (0x1E30, "M", "ḱ"), - (0x1E31, "V"), - (0x1E32, "M", "ḳ"), - (0x1E33, "V"), - (0x1E34, "M", "ḵ"), - (0x1E35, "V"), - (0x1E36, "M", "ḷ"), - (0x1E37, "V"), - (0x1E38, "M", "ḹ"), - (0x1E39, "V"), - (0x1E3A, "M", "ḻ"), - (0x1E3B, "V"), - (0x1E3C, "M", "ḽ"), - (0x1E3D, "V"), - (0x1E3E, "M", "ḿ"), - (0x1E3F, "V"), - (0x1E40, "M", "ṁ"), - (0x1E41, "V"), - (0x1E42, "M", "ṃ"), - (0x1E43, "V"), - (0x1E44, "M", "ṅ"), - (0x1E45, "V"), - (0x1E46, "M", "ṇ"), - (0x1E47, "V"), - (0x1E48, "M", "ṉ"), - (0x1E49, "V"), - (0x1E4A, "M", "ṋ"), - (0x1E4B, "V"), - (0x1E4C, "M", "ṍ"), - (0x1E4D, "V"), - (0x1E4E, "M", "ṏ"), - (0x1E4F, "V"), - (0x1E50, "M", "ṑ"), - (0x1E51, "V"), - (0x1E52, "M", "ṓ"), - (0x1E53, "V"), - (0x1E54, "M", "ṕ"), - (0x1E55, "V"), - (0x1E56, "M", "ṗ"), - (0x1E57, "V"), - (0x1E58, "M", "ṙ"), - (0x1E59, "V"), - (0x1E5A, "M", "ṛ"), - (0x1E5B, "V"), - (0x1E5C, "M", "ṝ"), - (0x1E5D, "V"), - (0x1E5E, "M", "ṟ"), - (0x1E5F, "V"), - (0x1E60, "M", "ṡ"), - (0x1E61, "V"), - (0x1E62, "M", "ṣ"), - (0x1E63, "V"), - (0x1E64, "M", "ṥ"), - (0x1E65, "V"), - (0x1E66, "M", "ṧ"), - (0x1E67, "V"), - (0x1E68, "M", "ṩ"), - (0x1E69, "V"), - (0x1E6A, "M", "ṫ"), - (0x1E6B, "V"), - (0x1E6C, "M", "ṭ"), - (0x1E6D, "V"), - (0x1E6E, "M", "ṯ"), - (0x1E6F, "V"), - (0x1E70, "M", "ṱ"), - (0x1E71, "V"), - (0x1E72, "M", "ṳ"), - (0x1E73, "V"), - (0x1E74, "M", "ṵ"), - (0x1E75, "V"), - (0x1E76, "M", "ṷ"), - (0x1E77, "V"), - (0x1E78, "M", "ṹ"), - (0x1E79, "V"), - (0x1E7A, "M", "ṻ"), - (0x1E7B, "V"), - (0x1E7C, "M", "ṽ"), - (0x1E7D, "V"), - (0x1E7E, "M", "ṿ"), - (0x1E7F, "V"), - (0x1E80, "M", "ẁ"), - (0x1E81, "V"), - (0x1E82, "M", "ẃ"), - (0x1E83, "V"), - (0x1E84, "M", "ẅ"), - (0x1E85, "V"), - (0x1E86, "M", "ẇ"), - (0x1E87, "V"), - (0x1E88, "M", "ẉ"), - (0x1E89, "V"), - (0x1E8A, "M", "ẋ"), - (0x1E8B, "V"), - (0x1E8C, "M", "ẍ"), - (0x1E8D, "V"), - (0x1E8E, "M", "ẏ"), - (0x1E8F, "V"), - (0x1E90, "M", "ẑ"), - (0x1E91, "V"), - (0x1E92, "M", "ẓ"), - (0x1E93, "V"), - (0x1E94, "M", "ẕ"), - (0x1E95, "V"), - (0x1E9A, "M", "aʾ"), - (0x1E9B, "M", "ṡ"), - (0x1E9C, "V"), - (0x1E9E, "M", "ß"), - (0x1E9F, "V"), - (0x1EA0, "M", "ạ"), - (0x1EA1, "V"), - (0x1EA2, "M", "ả"), - (0x1EA3, "V"), - (0x1EA4, "M", "ấ"), - (0x1EA5, "V"), - (0x1EA6, "M", "ầ"), - (0x1EA7, "V"), - (0x1EA8, "M", "ẩ"), - (0x1EA9, "V"), - (0x1EAA, "M", "ẫ"), - (0x1EAB, "V"), - (0x1EAC, "M", "ậ"), - (0x1EAD, "V"), - (0x1EAE, "M", "ắ"), - (0x1EAF, "V"), - (0x1EB0, "M", "ằ"), - (0x1EB1, "V"), - (0x1EB2, "M", "ẳ"), - (0x1EB3, "V"), - (0x1EB4, "M", "ẵ"), - (0x1EB5, "V"), - (0x1EB6, "M", "ặ"), - (0x1EB7, "V"), - (0x1EB8, "M", "ẹ"), - (0x1EB9, "V"), - (0x1EBA, "M", "ẻ"), - (0x1EBB, "V"), - (0x1EBC, "M", "ẽ"), - (0x1EBD, "V"), - (0x1EBE, "M", "ế"), - (0x1EBF, "V"), - (0x1EC0, "M", "ề"), - (0x1EC1, "V"), - (0x1EC2, "M", "ể"), - (0x1EC3, "V"), - (0x1EC4, "M", "ễ"), - (0x1EC5, "V"), - (0x1EC6, "M", "ệ"), - (0x1EC7, "V"), - (0x1EC8, "M", "ỉ"), - (0x1EC9, "V"), - (0x1ECA, "M", "ị"), - (0x1ECB, "V"), - (0x1ECC, "M", "ọ"), - (0x1ECD, "V"), - (0x1ECE, "M", "ỏ"), - (0x1ECF, "V"), - (0x1ED0, "M", "ố"), - (0x1ED1, "V"), - (0x1ED2, "M", "ồ"), - (0x1ED3, "V"), - (0x1ED4, "M", "ổ"), - (0x1ED5, "V"), - (0x1ED6, "M", "ỗ"), - (0x1ED7, "V"), - (0x1ED8, "M", "ộ"), - (0x1ED9, "V"), - (0x1EDA, "M", "ớ"), - (0x1EDB, "V"), - (0x1EDC, "M", "ờ"), - (0x1EDD, "V"), - (0x1EDE, "M", "ở"), - (0x1EDF, "V"), - (0x1EE0, "M", "ỡ"), - (0x1EE1, "V"), - (0x1EE2, "M", "ợ"), - (0x1EE3, "V"), - (0x1EE4, "M", "ụ"), - (0x1EE5, "V"), - (0x1EE6, "M", "ủ"), - (0x1EE7, "V"), - (0x1EE8, "M", "ứ"), - (0x1EE9, "V"), - (0x1EEA, "M", "ừ"), - (0x1EEB, "V"), - (0x1EEC, "M", "ử"), - (0x1EED, "V"), - (0x1EEE, "M", "ữ"), - (0x1EEF, "V"), - (0x1EF0, "M", "ự"), - (0x1EF1, "V"), - (0x1EF2, "M", "ỳ"), - (0x1EF3, "V"), - (0x1EF4, "M", "ỵ"), - (0x1EF5, "V"), - (0x1EF6, "M", "ỷ"), - (0x1EF7, "V"), - (0x1EF8, "M", "ỹ"), - (0x1EF9, "V"), - (0x1EFA, "M", "ỻ"), - (0x1EFB, "V"), - (0x1EFC, "M", "ỽ"), - (0x1EFD, "V"), - (0x1EFE, "M", "ỿ"), - (0x1EFF, "V"), - (0x1F08, "M", "ἀ"), - (0x1F09, "M", "ἁ"), - (0x1F0A, "M", "ἂ"), - (0x1F0B, "M", "ἃ"), - (0x1F0C, "M", "ἄ"), - (0x1F0D, "M", "ἅ"), - (0x1F0E, "M", "ἆ"), - (0x1F0F, "M", "ἇ"), - (0x1F10, "V"), - (0x1F16, "X"), - (0x1F18, "M", "ἐ"), - (0x1F19, "M", "ἑ"), - (0x1F1A, "M", "ἒ"), - (0x1F1B, "M", "ἓ"), - (0x1F1C, "M", "ἔ"), - (0x1F1D, "M", "ἕ"), - (0x1F1E, "X"), - (0x1F20, "V"), - (0x1F28, "M", "ἠ"), - (0x1F29, "M", "ἡ"), - (0x1F2A, "M", "ἢ"), - (0x1F2B, "M", "ἣ"), - (0x1F2C, "M", "ἤ"), - (0x1F2D, "M", "ἥ"), - (0x1F2E, "M", "ἦ"), - (0x1F2F, "M", "ἧ"), - (0x1F30, "V"), - (0x1F38, "M", "ἰ"), - (0x1F39, "M", "ἱ"), - (0x1F3A, "M", "ἲ"), - (0x1F3B, "M", "ἳ"), - (0x1F3C, "M", "ἴ"), - (0x1F3D, "M", "ἵ"), - (0x1F3E, "M", "ἶ"), - (0x1F3F, "M", "ἷ"), - (0x1F40, "V"), - (0x1F46, "X"), - (0x1F48, "M", "ὀ"), - (0x1F49, "M", "ὁ"), - (0x1F4A, "M", "ὂ"), - (0x1F4B, "M", "ὃ"), - (0x1F4C, "M", "ὄ"), - (0x1F4D, "M", "ὅ"), - (0x1F4E, "X"), - (0x1F50, "V"), - (0x1F58, "X"), - (0x1F59, "M", "ὑ"), - (0x1F5A, "X"), - (0x1F5B, "M", "ὓ"), - (0x1F5C, "X"), - (0x1F5D, "M", "ὕ"), - (0x1F5E, "X"), - (0x1F5F, "M", "ὗ"), - (0x1F60, "V"), - (0x1F68, "M", "ὠ"), - (0x1F69, "M", "ὡ"), - (0x1F6A, "M", "ὢ"), - (0x1F6B, "M", "ὣ"), - (0x1F6C, "M", "ὤ"), - (0x1F6D, "M", "ὥ"), - (0x1F6E, "M", "ὦ"), - (0x1F6F, "M", "ὧ"), - (0x1F70, "V"), - (0x1F71, "M", "ά"), - (0x1F72, "V"), - (0x1F73, "M", "έ"), - (0x1F74, "V"), - (0x1F75, "M", "ή"), - (0x1F76, "V"), - (0x1F77, "M", "ί"), - (0x1F78, "V"), - (0x1F79, "M", "ό"), - (0x1F7A, "V"), - (0x1F7B, "M", "ύ"), - (0x1F7C, "V"), - (0x1F7D, "M", "ώ"), - (0x1F7E, "X"), - (0x1F80, "M", "ἀι"), - (0x1F81, "M", "ἁι"), - (0x1F82, "M", "ἂι"), - (0x1F83, "M", "ἃι"), - (0x1F84, "M", "ἄι"), - (0x1F85, "M", "ἅι"), - (0x1F86, "M", "ἆι"), - (0x1F87, "M", "ἇι"), - (0x1F88, "M", "ἀι"), - (0x1F89, "M", "ἁι"), - (0x1F8A, "M", "ἂι"), - (0x1F8B, "M", "ἃι"), - (0x1F8C, "M", "ἄι"), - (0x1F8D, "M", "ἅι"), - (0x1F8E, "M", "ἆι"), - (0x1F8F, "M", "ἇι"), - (0x1F90, "M", "ἠι"), - (0x1F91, "M", "ἡι"), - (0x1F92, "M", "ἢι"), - (0x1F93, "M", "ἣι"), - (0x1F94, "M", "ἤι"), - (0x1F95, "M", "ἥι"), - (0x1F96, "M", "ἦι"), - (0x1F97, "M", "ἧι"), - (0x1F98, "M", "ἠι"), - (0x1F99, "M", "ἡι"), - (0x1F9A, "M", "ἢι"), - (0x1F9B, "M", "ἣι"), - (0x1F9C, "M", "ἤι"), - (0x1F9D, "M", "ἥι"), - (0x1F9E, "M", "ἦι"), - (0x1F9F, "M", "ἧι"), - (0x1FA0, "M", "ὠι"), - (0x1FA1, "M", "ὡι"), - (0x1FA2, "M", "ὢι"), - (0x1FA3, "M", "ὣι"), - (0x1FA4, "M", "ὤι"), - (0x1FA5, "M", "ὥι"), - (0x1FA6, "M", "ὦι"), - (0x1FA7, "M", "ὧι"), - (0x1FA8, "M", "ὠι"), - (0x1FA9, "M", "ὡι"), - (0x1FAA, "M", "ὢι"), - (0x1FAB, "M", "ὣι"), - (0x1FAC, "M", "ὤι"), - (0x1FAD, "M", "ὥι"), - (0x1FAE, "M", "ὦι"), - (0x1FAF, "M", "ὧι"), - (0x1FB0, "V"), - (0x1FB2, "M", "ὰι"), - (0x1FB3, "M", "αι"), - (0x1FB4, "M", "άι"), - (0x1FB5, "X"), - (0x1FB6, "V"), - (0x1FB7, "M", "ᾶι"), - (0x1FB8, "M", "ᾰ"), - (0x1FB9, "M", "ᾱ"), - (0x1FBA, "M", "ὰ"), - (0x1FBB, "M", "ά"), - (0x1FBC, "M", "αι"), - (0x1FBD, "M", " ̓"), - (0x1FBE, "M", "ι"), - (0x1FBF, "M", " ̓"), - (0x1FC0, "M", " ͂"), - (0x1FC1, "M", " ̈͂"), - (0x1FC2, "M", "ὴι"), - (0x1FC3, "M", "ηι"), - (0x1FC4, "M", "ήι"), - (0x1FC5, "X"), - (0x1FC6, "V"), - (0x1FC7, "M", "ῆι"), - (0x1FC8, "M", "ὲ"), - (0x1FC9, "M", "έ"), - (0x1FCA, "M", "ὴ"), - (0x1FCB, "M", "ή"), - (0x1FCC, "M", "ηι"), - (0x1FCD, "M", " ̓̀"), - (0x1FCE, "M", " ̓́"), - (0x1FCF, "M", " ̓͂"), - (0x1FD0, "V"), - (0x1FD3, "M", "ΐ"), - (0x1FD4, "X"), - (0x1FD6, "V"), - (0x1FD8, "M", "ῐ"), - (0x1FD9, "M", "ῑ"), - (0x1FDA, "M", "ὶ"), - (0x1FDB, "M", "ί"), - (0x1FDC, "X"), - (0x1FDD, "M", " ̔̀"), - (0x1FDE, "M", " ̔́"), - (0x1FDF, "M", " ̔͂"), - (0x1FE0, "V"), - (0x1FE3, "M", "ΰ"), - (0x1FE4, "V"), - (0x1FE8, "M", "ῠ"), - (0x1FE9, "M", "ῡ"), - (0x1FEA, "M", "ὺ"), - (0x1FEB, "M", "ύ"), - (0x1FEC, "M", "ῥ"), - (0x1FED, "M", " ̈̀"), - (0x1FEE, "M", " ̈́"), - (0x1FEF, "M", "`"), - (0x1FF0, "X"), - (0x1FF2, "M", "ὼι"), - (0x1FF3, "M", "ωι"), - (0x1FF4, "M", "ώι"), - (0x1FF5, "X"), - (0x1FF6, "V"), - (0x1FF7, "M", "ῶι"), - (0x1FF8, "M", "ὸ"), - (0x1FF9, "M", "ό"), - (0x1FFA, "M", "ὼ"), - (0x1FFB, "M", "ώ"), - (0x1FFC, "M", "ωι"), - (0x1FFD, "M", " ́"), - (0x1FFE, "M", " ̔"), - (0x1FFF, "X"), - (0x2000, "M", " "), - (0x200B, "I"), - (0x200C, "D", ""), - (0x200E, "X"), - (0x2010, "V"), - (0x2011, "M", "‐"), - (0x2012, "V"), - (0x2017, "M", " ̳"), - (0x2018, "V"), - (0x2024, "X"), - (0x2027, "V"), - (0x2028, "X"), - (0x202F, "M", " "), - (0x2030, "V"), - (0x2033, "M", "′′"), - (0x2034, "M", "′′′"), - (0x2035, "V"), - (0x2036, "M", "‵‵"), - (0x2037, "M", "‵‵‵"), - (0x2038, "V"), - (0x203C, "M", "!!"), - (0x203D, "V"), - (0x203E, "M", " ̅"), - (0x203F, "V"), - (0x2047, "M", "??"), - (0x2048, "M", "?!"), - (0x2049, "M", "!?"), - (0x204A, "V"), - (0x2057, "M", "′′′′"), - (0x2058, "V"), - (0x205F, "M", " "), - (0x2060, "I"), - (0x2065, "X"), - (0x206A, "I"), - (0x2070, "M", "0"), - (0x2071, "M", "i"), - (0x2072, "X"), - (0x2074, "M", "4"), - (0x2075, "M", "5"), - (0x2076, "M", "6"), - (0x2077, "M", "7"), - (0x2078, "M", "8"), - (0x2079, "M", "9"), - (0x207A, "M", "+"), - (0x207B, "M", "−"), - (0x207C, "M", "="), - (0x207D, "M", "("), - (0x207E, "M", ")"), - (0x207F, "M", "n"), - (0x2080, "M", "0"), - (0x2081, "M", "1"), - (0x2082, "M", "2"), - (0x2083, "M", "3"), - (0x2084, "M", "4"), - (0x2085, "M", "5"), - (0x2086, "M", "6"), - (0x2087, "M", "7"), - (0x2088, "M", "8"), - (0x2089, "M", "9"), - (0x208A, "M", "+"), - (0x208B, "M", "−"), - (0x208C, "M", "="), - (0x208D, "M", "("), - (0x208E, "M", ")"), - (0x208F, "X"), - (0x2090, "M", "a"), - (0x2091, "M", "e"), - (0x2092, "M", "o"), - (0x2093, "M", "x"), - (0x2094, "M", "ə"), - (0x2095, "M", "h"), - (0x2096, "M", "k"), - (0x2097, "M", "l"), - (0x2098, "M", "m"), - (0x2099, "M", "n"), - (0x209A, "M", "p"), - (0x209B, "M", "s"), - (0x209C, "M", "t"), - (0x209D, "X"), - (0x20A0, "V"), - (0x20A8, "M", "rs"), - (0x20A9, "V"), - (0x20C2, "X"), - (0x20D0, "V"), - (0x20F1, "X"), - (0x2100, "M", "a/c"), - (0x2101, "M", "a/s"), - (0x2102, "M", "c"), - (0x2103, "M", "°c"), - (0x2104, "V"), - (0x2105, "M", "c/o"), - (0x2106, "M", "c/u"), - (0x2107, "M", "ɛ"), - (0x2108, "V"), - (0x2109, "M", "°f"), - (0x210A, "M", "g"), - (0x210B, "M", "h"), - (0x210F, "M", "ħ"), - (0x2110, "M", "i"), - (0x2112, "M", "l"), - (0x2114, "V"), - (0x2115, "M", "n"), - (0x2116, "M", "no"), - (0x2117, "V"), - (0x2119, "M", "p"), - (0x211A, "M", "q"), - (0x211B, "M", "r"), - (0x211E, "V"), - (0x2120, "M", "sm"), - (0x2121, "M", "tel"), - (0x2122, "M", "tm"), - (0x2123, "V"), - (0x2124, "M", "z"), - (0x2125, "V"), - (0x2126, "M", "ω"), - (0x2127, "V"), - (0x2128, "M", "z"), - (0x2129, "V"), - (0x212A, "M", "k"), - (0x212B, "M", "å"), - (0x212C, "M", "b"), - (0x212D, "M", "c"), - (0x212E, "V"), - (0x212F, "M", "e"), - (0x2131, "M", "f"), - (0x2132, "M", "ⅎ"), - (0x2133, "M", "m"), - (0x2134, "M", "o"), - (0x2135, "M", "א"), - (0x2136, "M", "ב"), - (0x2137, "M", "ג"), - (0x2138, "M", "ד"), - (0x2139, "M", "i"), - (0x213A, "V"), - (0x213B, "M", "fax"), - (0x213C, "M", "π"), - (0x213D, "M", "γ"), - (0x213F, "M", "π"), - (0x2140, "M", "∑"), - (0x2141, "V"), - (0x2145, "M", "d"), - (0x2147, "M", "e"), - (0x2148, "M", "i"), - (0x2149, "M", "j"), - (0x214A, "V"), - (0x2150, "M", "1⁄7"), - (0x2151, "M", "1⁄9"), - (0x2152, "M", "1⁄10"), - (0x2153, "M", "1⁄3"), - (0x2154, "M", "2⁄3"), - (0x2155, "M", "1⁄5"), - (0x2156, "M", "2⁄5"), - (0x2157, "M", "3⁄5"), - (0x2158, "M", "4⁄5"), - (0x2159, "M", "1⁄6"), - (0x215A, "M", "5⁄6"), - (0x215B, "M", "1⁄8"), - (0x215C, "M", "3⁄8"), - (0x215D, "M", "5⁄8"), - (0x215E, "M", "7⁄8"), - (0x215F, "M", "1⁄"), - (0x2160, "M", "i"), - (0x2161, "M", "ii"), - (0x2162, "M", "iii"), - (0x2163, "M", "iv"), - (0x2164, "M", "v"), - (0x2165, "M", "vi"), - (0x2166, "M", "vii"), - (0x2167, "M", "viii"), - (0x2168, "M", "ix"), - (0x2169, "M", "x"), - (0x216A, "M", "xi"), - (0x216B, "M", "xii"), - (0x216C, "M", "l"), - (0x216D, "M", "c"), - (0x216E, "M", "d"), - (0x216F, "M", "m"), - (0x2170, "M", "i"), - (0x2171, "M", "ii"), - (0x2172, "M", "iii"), - (0x2173, "M", "iv"), - (0x2174, "M", "v"), - (0x2175, "M", "vi"), - (0x2176, "M", "vii"), - (0x2177, "M", "viii"), - (0x2178, "M", "ix"), - (0x2179, "M", "x"), - (0x217A, "M", "xi"), - (0x217B, "M", "xii"), - (0x217C, "M", "l"), - (0x217D, "M", "c"), - (0x217E, "M", "d"), - (0x217F, "M", "m"), - (0x2180, "V"), - (0x2183, "M", "ↄ"), - (0x2184, "V"), - (0x2189, "M", "0⁄3"), - (0x218A, "V"), - (0x218C, "X"), - (0x2190, "V"), - (0x222C, "M", "∫∫"), - (0x222D, "M", "∫∫∫"), - (0x222E, "V"), - (0x222F, "M", "∮∮"), - (0x2230, "M", "∮∮∮"), - (0x2231, "V"), - (0x2329, "M", "〈"), - (0x232A, "M", "〉"), - (0x232B, "V"), - (0x242A, "X"), - (0x2440, "V"), - (0x244B, "X"), - (0x2460, "M", "1"), - (0x2461, "M", "2"), - (0x2462, "M", "3"), - (0x2463, "M", "4"), - (0x2464, "M", "5"), - (0x2465, "M", "6"), - (0x2466, "M", "7"), - (0x2467, "M", "8"), - (0x2468, "M", "9"), - (0x2469, "M", "10"), - (0x246A, "M", "11"), - (0x246B, "M", "12"), - (0x246C, "M", "13"), - (0x246D, "M", "14"), - (0x246E, "M", "15"), - (0x246F, "M", "16"), - (0x2470, "M", "17"), - (0x2471, "M", "18"), - (0x2472, "M", "19"), - (0x2473, "M", "20"), - (0x2474, "M", "(1)"), - (0x2475, "M", "(2)"), - (0x2476, "M", "(3)"), - (0x2477, "M", "(4)"), - (0x2478, "M", "(5)"), - (0x2479, "M", "(6)"), - (0x247A, "M", "(7)"), - (0x247B, "M", "(8)"), - (0x247C, "M", "(9)"), - (0x247D, "M", "(10)"), - (0x247E, "M", "(11)"), - (0x247F, "M", "(12)"), - (0x2480, "M", "(13)"), - (0x2481, "M", "(14)"), - (0x2482, "M", "(15)"), - (0x2483, "M", "(16)"), - (0x2484, "M", "(17)"), - (0x2485, "M", "(18)"), - (0x2486, "M", "(19)"), - (0x2487, "M", "(20)"), - (0x2488, "X"), - (0x249C, "M", "(a)"), - (0x249D, "M", "(b)"), - (0x249E, "M", "(c)"), - (0x249F, "M", "(d)"), - (0x24A0, "M", "(e)"), - (0x24A1, "M", "(f)"), - (0x24A2, "M", "(g)"), - (0x24A3, "M", "(h)"), - (0x24A4, "M", "(i)"), - (0x24A5, "M", "(j)"), - (0x24A6, "M", "(k)"), - (0x24A7, "M", "(l)"), - (0x24A8, "M", "(m)"), - (0x24A9, "M", "(n)"), - (0x24AA, "M", "(o)"), - (0x24AB, "M", "(p)"), - (0x24AC, "M", "(q)"), - (0x24AD, "M", "(r)"), - (0x24AE, "M", "(s)"), - (0x24AF, "M", "(t)"), - (0x24B0, "M", "(u)"), - (0x24B1, "M", "(v)"), - (0x24B2, "M", "(w)"), - (0x24B3, "M", "(x)"), - (0x24B4, "M", "(y)"), - (0x24B5, "M", "(z)"), - (0x24B6, "M", "a"), - (0x24B7, "M", "b"), - (0x24B8, "M", "c"), - (0x24B9, "M", "d"), - (0x24BA, "M", "e"), - (0x24BB, "M", "f"), - (0x24BC, "M", "g"), - (0x24BD, "M", "h"), - (0x24BE, "M", "i"), - (0x24BF, "M", "j"), - (0x24C0, "M", "k"), - (0x24C1, "M", "l"), - (0x24C2, "M", "m"), - (0x24C3, "M", "n"), - (0x24C4, "M", "o"), - (0x24C5, "M", "p"), - (0x24C6, "M", "q"), - (0x24C7, "M", "r"), - (0x24C8, "M", "s"), - (0x24C9, "M", "t"), - (0x24CA, "M", "u"), - (0x24CB, "M", "v"), - (0x24CC, "M", "w"), - (0x24CD, "M", "x"), - (0x24CE, "M", "y"), - (0x24CF, "M", "z"), - (0x24D0, "M", "a"), - (0x24D1, "M", "b"), - (0x24D2, "M", "c"), - (0x24D3, "M", "d"), - (0x24D4, "M", "e"), - (0x24D5, "M", "f"), - (0x24D6, "M", "g"), - (0x24D7, "M", "h"), - (0x24D8, "M", "i"), - (0x24D9, "M", "j"), - (0x24DA, "M", "k"), - (0x24DB, "M", "l"), - (0x24DC, "M", "m"), - (0x24DD, "M", "n"), - (0x24DE, "M", "o"), - (0x24DF, "M", "p"), - (0x24E0, "M", "q"), - (0x24E1, "M", "r"), - (0x24E2, "M", "s"), - (0x24E3, "M", "t"), - (0x24E4, "M", "u"), - (0x24E5, "M", "v"), - (0x24E6, "M", "w"), - (0x24E7, "M", "x"), - (0x24E8, "M", "y"), - (0x24E9, "M", "z"), - (0x24EA, "M", "0"), - (0x24EB, "V"), - (0x2A0C, "M", "∫∫∫∫"), - (0x2A0D, "V"), - (0x2A74, "M", "::="), - (0x2A75, "M", "=="), - (0x2A76, "M", "==="), - (0x2A77, "V"), - (0x2ADC, "M", "⫝̸"), - (0x2ADD, "V"), - (0x2B74, "X"), - (0x2B76, "V"), - (0x2C00, "M", "ⰰ"), - (0x2C01, "M", "ⰱ"), - (0x2C02, "M", "ⰲ"), - (0x2C03, "M", "ⰳ"), - (0x2C04, "M", "ⰴ"), - (0x2C05, "M", "ⰵ"), - (0x2C06, "M", "ⰶ"), - (0x2C07, "M", "ⰷ"), - (0x2C08, "M", "ⰸ"), - (0x2C09, "M", "ⰹ"), - (0x2C0A, "M", "ⰺ"), - (0x2C0B, "M", "ⰻ"), - (0x2C0C, "M", "ⰼ"), - (0x2C0D, "M", "ⰽ"), - (0x2C0E, "M", "ⰾ"), - (0x2C0F, "M", "ⰿ"), - (0x2C10, "M", "ⱀ"), - (0x2C11, "M", "ⱁ"), - (0x2C12, "M", "ⱂ"), - (0x2C13, "M", "ⱃ"), - (0x2C14, "M", "ⱄ"), - (0x2C15, "M", "ⱅ"), - (0x2C16, "M", "ⱆ"), - (0x2C17, "M", "ⱇ"), - (0x2C18, "M", "ⱈ"), - (0x2C19, "M", "ⱉ"), - (0x2C1A, "M", "ⱊ"), - (0x2C1B, "M", "ⱋ"), - (0x2C1C, "M", "ⱌ"), - (0x2C1D, "M", "ⱍ"), - (0x2C1E, "M", "ⱎ"), - (0x2C1F, "M", "ⱏ"), - (0x2C20, "M", "ⱐ"), - (0x2C21, "M", "ⱑ"), - (0x2C22, "M", "ⱒ"), - (0x2C23, "M", "ⱓ"), - (0x2C24, "M", "ⱔ"), - (0x2C25, "M", "ⱕ"), - (0x2C26, "M", "ⱖ"), - (0x2C27, "M", "ⱗ"), - (0x2C28, "M", "ⱘ"), - (0x2C29, "M", "ⱙ"), - (0x2C2A, "M", "ⱚ"), - (0x2C2B, "M", "ⱛ"), - (0x2C2C, "M", "ⱜ"), - (0x2C2D, "M", "ⱝ"), - (0x2C2E, "M", "ⱞ"), - (0x2C2F, "M", "ⱟ"), - (0x2C30, "V"), - (0x2C60, "M", "ⱡ"), - (0x2C61, "V"), - (0x2C62, "M", "ɫ"), - (0x2C63, "M", "ᵽ"), - (0x2C64, "M", "ɽ"), - (0x2C65, "V"), - (0x2C67, "M", "ⱨ"), - (0x2C68, "V"), - (0x2C69, "M", "ⱪ"), - (0x2C6A, "V"), - (0x2C6B, "M", "ⱬ"), - (0x2C6C, "V"), - (0x2C6D, "M", "ɑ"), - (0x2C6E, "M", "ɱ"), - (0x2C6F, "M", "ɐ"), - (0x2C70, "M", "ɒ"), - (0x2C71, "V"), - (0x2C72, "M", "ⱳ"), - (0x2C73, "V"), - (0x2C75, "M", "ⱶ"), - (0x2C76, "V"), - (0x2C7C, "M", "j"), - (0x2C7D, "M", "v"), - (0x2C7E, "M", "ȿ"), - (0x2C7F, "M", "ɀ"), - (0x2C80, "M", "ⲁ"), - (0x2C81, "V"), - (0x2C82, "M", "ⲃ"), - (0x2C83, "V"), - (0x2C84, "M", "ⲅ"), - (0x2C85, "V"), - (0x2C86, "M", "ⲇ"), - (0x2C87, "V"), - (0x2C88, "M", "ⲉ"), - (0x2C89, "V"), - (0x2C8A, "M", "ⲋ"), - (0x2C8B, "V"), - (0x2C8C, "M", "ⲍ"), - (0x2C8D, "V"), - (0x2C8E, "M", "ⲏ"), - (0x2C8F, "V"), - (0x2C90, "M", "ⲑ"), - (0x2C91, "V"), - (0x2C92, "M", "ⲓ"), - (0x2C93, "V"), - (0x2C94, "M", "ⲕ"), - (0x2C95, "V"), - (0x2C96, "M", "ⲗ"), - (0x2C97, "V"), - (0x2C98, "M", "ⲙ"), - (0x2C99, "V"), - (0x2C9A, "M", "ⲛ"), - (0x2C9B, "V"), - (0x2C9C, "M", "ⲝ"), - (0x2C9D, "V"), - (0x2C9E, "M", "ⲟ"), - (0x2C9F, "V"), - (0x2CA0, "M", "ⲡ"), - (0x2CA1, "V"), - (0x2CA2, "M", "ⲣ"), - (0x2CA3, "V"), - (0x2CA4, "M", "ⲥ"), - (0x2CA5, "V"), - (0x2CA6, "M", "ⲧ"), - (0x2CA7, "V"), - (0x2CA8, "M", "ⲩ"), - (0x2CA9, "V"), - (0x2CAA, "M", "ⲫ"), - (0x2CAB, "V"), - (0x2CAC, "M", "ⲭ"), - (0x2CAD, "V"), - (0x2CAE, "M", "ⲯ"), - (0x2CAF, "V"), - (0x2CB0, "M", "ⲱ"), - (0x2CB1, "V"), - (0x2CB2, "M", "ⲳ"), - (0x2CB3, "V"), - (0x2CB4, "M", "ⲵ"), - (0x2CB5, "V"), - (0x2CB6, "M", "ⲷ"), - (0x2CB7, "V"), - (0x2CB8, "M", "ⲹ"), - (0x2CB9, "V"), - (0x2CBA, "M", "ⲻ"), - (0x2CBB, "V"), - (0x2CBC, "M", "ⲽ"), - (0x2CBD, "V"), - (0x2CBE, "M", "ⲿ"), - (0x2CBF, "V"), - (0x2CC0, "M", "ⳁ"), - (0x2CC1, "V"), - (0x2CC2, "M", "ⳃ"), - (0x2CC3, "V"), - (0x2CC4, "M", "ⳅ"), - (0x2CC5, "V"), - (0x2CC6, "M", "ⳇ"), - (0x2CC7, "V"), - (0x2CC8, "M", "ⳉ"), - (0x2CC9, "V"), - (0x2CCA, "M", "ⳋ"), - (0x2CCB, "V"), - (0x2CCC, "M", "ⳍ"), - (0x2CCD, "V"), - (0x2CCE, "M", "ⳏ"), - (0x2CCF, "V"), - (0x2CD0, "M", "ⳑ"), - (0x2CD1, "V"), - (0x2CD2, "M", "ⳓ"), - (0x2CD3, "V"), - (0x2CD4, "M", "ⳕ"), - (0x2CD5, "V"), - (0x2CD6, "M", "ⳗ"), - (0x2CD7, "V"), - (0x2CD8, "M", "ⳙ"), - (0x2CD9, "V"), - (0x2CDA, "M", "ⳛ"), - (0x2CDB, "V"), - (0x2CDC, "M", "ⳝ"), - (0x2CDD, "V"), - (0x2CDE, "M", "ⳟ"), - (0x2CDF, "V"), - (0x2CE0, "M", "ⳡ"), - (0x2CE1, "V"), - (0x2CE2, "M", "ⳣ"), - (0x2CE3, "V"), - (0x2CEB, "M", "ⳬ"), - (0x2CEC, "V"), - (0x2CED, "M", "ⳮ"), - (0x2CEE, "V"), - (0x2CF2, "M", "ⳳ"), - (0x2CF3, "V"), - (0x2CF4, "X"), - (0x2CF9, "V"), - (0x2D26, "X"), - (0x2D27, "V"), - (0x2D28, "X"), - (0x2D2D, "V"), - (0x2D2E, "X"), - (0x2D30, "V"), - (0x2D68, "X"), - (0x2D6F, "M", "ⵡ"), - (0x2D70, "V"), - (0x2D71, "X"), - (0x2D7F, "V"), - (0x2D97, "X"), - (0x2DA0, "V"), - (0x2DA7, "X"), - (0x2DA8, "V"), - (0x2DAF, "X"), - (0x2DB0, "V"), - (0x2DB7, "X"), - (0x2DB8, "V"), - (0x2DBF, "X"), - (0x2DC0, "V"), - (0x2DC7, "X"), - (0x2DC8, "V"), - (0x2DCF, "X"), - (0x2DD0, "V"), - (0x2DD7, "X"), - (0x2DD8, "V"), - (0x2DDF, "X"), - (0x2DE0, "V"), - (0x2E5E, "X"), - (0x2E80, "V"), - (0x2E9A, "X"), - (0x2E9B, "V"), - (0x2E9F, "M", "母"), - (0x2EA0, "V"), - (0x2EF3, "M", "龟"), - (0x2EF4, "X"), - (0x2F00, "M", "一"), - (0x2F01, "M", "丨"), - (0x2F02, "M", "丶"), - (0x2F03, "M", "丿"), - (0x2F04, "M", "乙"), - (0x2F05, "M", "亅"), - (0x2F06, "M", "二"), - (0x2F07, "M", "亠"), - (0x2F08, "M", "人"), - (0x2F09, "M", "儿"), - (0x2F0A, "M", "入"), - (0x2F0B, "M", "八"), - (0x2F0C, "M", "冂"), - (0x2F0D, "M", "冖"), - (0x2F0E, "M", "冫"), - (0x2F0F, "M", "几"), - (0x2F10, "M", "凵"), - (0x2F11, "M", "刀"), - (0x2F12, "M", "力"), - (0x2F13, "M", "勹"), - (0x2F14, "M", "匕"), - (0x2F15, "M", "匚"), - (0x2F16, "M", "匸"), - (0x2F17, "M", "十"), - (0x2F18, "M", "卜"), - (0x2F19, "M", "卩"), - (0x2F1A, "M", "厂"), - (0x2F1B, "M", "厶"), - (0x2F1C, "M", "又"), - (0x2F1D, "M", "口"), - (0x2F1E, "M", "囗"), - (0x2F1F, "M", "土"), - (0x2F20, "M", "士"), - (0x2F21, "M", "夂"), - (0x2F22, "M", "夊"), - (0x2F23, "M", "夕"), - (0x2F24, "M", "大"), - (0x2F25, "M", "女"), - (0x2F26, "M", "子"), - (0x2F27, "M", "宀"), - (0x2F28, "M", "寸"), - (0x2F29, "M", "小"), - (0x2F2A, "M", "尢"), - (0x2F2B, "M", "尸"), - (0x2F2C, "M", "屮"), - (0x2F2D, "M", "山"), - (0x2F2E, "M", "巛"), - (0x2F2F, "M", "工"), - (0x2F30, "M", "己"), - (0x2F31, "M", "巾"), - (0x2F32, "M", "干"), - (0x2F33, "M", "幺"), - (0x2F34, "M", "广"), - (0x2F35, "M", "廴"), - (0x2F36, "M", "廾"), - (0x2F37, "M", "弋"), - (0x2F38, "M", "弓"), - (0x2F39, "M", "彐"), - (0x2F3A, "M", "彡"), - (0x2F3B, "M", "彳"), - (0x2F3C, "M", "心"), - (0x2F3D, "M", "戈"), - (0x2F3E, "M", "戶"), - (0x2F3F, "M", "手"), - (0x2F40, "M", "支"), - (0x2F41, "M", "攴"), - (0x2F42, "M", "文"), - (0x2F43, "M", "斗"), - (0x2F44, "M", "斤"), - (0x2F45, "M", "方"), - (0x2F46, "M", "无"), - (0x2F47, "M", "日"), - (0x2F48, "M", "曰"), - (0x2F49, "M", "月"), - (0x2F4A, "M", "木"), - (0x2F4B, "M", "欠"), - (0x2F4C, "M", "止"), - (0x2F4D, "M", "歹"), - (0x2F4E, "M", "殳"), - (0x2F4F, "M", "毋"), - (0x2F50, "M", "比"), - (0x2F51, "M", "毛"), - (0x2F52, "M", "氏"), - (0x2F53, "M", "气"), - (0x2F54, "M", "水"), - (0x2F55, "M", "火"), - (0x2F56, "M", "爪"), - (0x2F57, "M", "父"), - (0x2F58, "M", "爻"), - (0x2F59, "M", "爿"), - (0x2F5A, "M", "片"), - (0x2F5B, "M", "牙"), - (0x2F5C, "M", "牛"), - (0x2F5D, "M", "犬"), - (0x2F5E, "M", "玄"), - (0x2F5F, "M", "玉"), - (0x2F60, "M", "瓜"), - (0x2F61, "M", "瓦"), - (0x2F62, "M", "甘"), - (0x2F63, "M", "生"), - (0x2F64, "M", "用"), - (0x2F65, "M", "田"), - (0x2F66, "M", "疋"), - (0x2F67, "M", "疒"), - (0x2F68, "M", "癶"), - (0x2F69, "M", "白"), - (0x2F6A, "M", "皮"), - (0x2F6B, "M", "皿"), - (0x2F6C, "M", "目"), - (0x2F6D, "M", "矛"), - (0x2F6E, "M", "矢"), - (0x2F6F, "M", "石"), - (0x2F70, "M", "示"), - (0x2F71, "M", "禸"), - (0x2F72, "M", "禾"), - (0x2F73, "M", "穴"), - (0x2F74, "M", "立"), - (0x2F75, "M", "竹"), - (0x2F76, "M", "米"), - (0x2F77, "M", "糸"), - (0x2F78, "M", "缶"), - (0x2F79, "M", "网"), - (0x2F7A, "M", "羊"), - (0x2F7B, "M", "羽"), - (0x2F7C, "M", "老"), - (0x2F7D, "M", "而"), - (0x2F7E, "M", "耒"), - (0x2F7F, "M", "耳"), - (0x2F80, "M", "聿"), - (0x2F81, "M", "肉"), - (0x2F82, "M", "臣"), - (0x2F83, "M", "自"), - (0x2F84, "M", "至"), - (0x2F85, "M", "臼"), - (0x2F86, "M", "舌"), - (0x2F87, "M", "舛"), - (0x2F88, "M", "舟"), - (0x2F89, "M", "艮"), - (0x2F8A, "M", "色"), - (0x2F8B, "M", "艸"), - (0x2F8C, "M", "虍"), - (0x2F8D, "M", "虫"), - (0x2F8E, "M", "血"), - (0x2F8F, "M", "行"), - (0x2F90, "M", "衣"), - (0x2F91, "M", "襾"), - (0x2F92, "M", "見"), - (0x2F93, "M", "角"), - (0x2F94, "M", "言"), - (0x2F95, "M", "谷"), - (0x2F96, "M", "豆"), - (0x2F97, "M", "豕"), - (0x2F98, "M", "豸"), - (0x2F99, "M", "貝"), - (0x2F9A, "M", "赤"), - (0x2F9B, "M", "走"), - (0x2F9C, "M", "足"), - (0x2F9D, "M", "身"), - (0x2F9E, "M", "車"), - (0x2F9F, "M", "辛"), - (0x2FA0, "M", "辰"), - (0x2FA1, "M", "辵"), - (0x2FA2, "M", "邑"), - (0x2FA3, "M", "酉"), - (0x2FA4, "M", "釆"), - (0x2FA5, "M", "里"), - (0x2FA6, "M", "金"), - (0x2FA7, "M", "長"), - (0x2FA8, "M", "門"), - (0x2FA9, "M", "阜"), - (0x2FAA, "M", "隶"), - (0x2FAB, "M", "隹"), - (0x2FAC, "M", "雨"), - (0x2FAD, "M", "靑"), - (0x2FAE, "M", "非"), - (0x2FAF, "M", "面"), - (0x2FB0, "M", "革"), - (0x2FB1, "M", "韋"), - (0x2FB2, "M", "韭"), - (0x2FB3, "M", "音"), - (0x2FB4, "M", "頁"), - (0x2FB5, "M", "風"), - (0x2FB6, "M", "飛"), - (0x2FB7, "M", "食"), - (0x2FB8, "M", "首"), - (0x2FB9, "M", "香"), - (0x2FBA, "M", "馬"), - (0x2FBB, "M", "骨"), - (0x2FBC, "M", "高"), - (0x2FBD, "M", "髟"), - (0x2FBE, "M", "鬥"), - (0x2FBF, "M", "鬯"), - (0x2FC0, "M", "鬲"), - (0x2FC1, "M", "鬼"), - (0x2FC2, "M", "魚"), - (0x2FC3, "M", "鳥"), - (0x2FC4, "M", "鹵"), - (0x2FC5, "M", "鹿"), - (0x2FC6, "M", "麥"), - (0x2FC7, "M", "麻"), - (0x2FC8, "M", "黃"), - (0x2FC9, "M", "黍"), - (0x2FCA, "M", "黑"), - (0x2FCB, "M", "黹"), - (0x2FCC, "M", "黽"), - (0x2FCD, "M", "鼎"), - (0x2FCE, "M", "鼓"), - (0x2FCF, "M", "鼠"), - (0x2FD0, "M", "鼻"), - (0x2FD1, "M", "齊"), - (0x2FD2, "M", "齒"), - (0x2FD3, "M", "龍"), - (0x2FD4, "M", "龜"), - (0x2FD5, "M", "龠"), - (0x2FD6, "X"), - (0x3000, "M", " "), - (0x3001, "V"), - (0x3002, "M", "."), - (0x3003, "V"), - (0x3036, "M", "〒"), - (0x3037, "V"), - (0x3038, "M", "十"), - (0x3039, "M", "卄"), - (0x303A, "M", "卅"), - (0x303B, "V"), - (0x3040, "X"), - (0x3041, "V"), - (0x3097, "X"), - (0x3099, "V"), - (0x309B, "M", " ゙"), - (0x309C, "M", " ゚"), - (0x309D, "V"), - (0x309F, "M", "より"), - (0x30A0, "V"), - (0x30FF, "M", "コト"), - (0x3100, "X"), - (0x3105, "V"), - (0x3130, "X"), - (0x3131, "M", "ᄀ"), - (0x3132, "M", "ᄁ"), - (0x3133, "M", "ᆪ"), - (0x3134, "M", "ᄂ"), - (0x3135, "M", "ᆬ"), - (0x3136, "M", "ᆭ"), - (0x3137, "M", "ᄃ"), - (0x3138, "M", "ᄄ"), - (0x3139, "M", "ᄅ"), - (0x313A, "M", "ᆰ"), - (0x313B, "M", "ᆱ"), - (0x313C, "M", "ᆲ"), - (0x313D, "M", "ᆳ"), - (0x313E, "M", "ᆴ"), - (0x313F, "M", "ᆵ"), - (0x3140, "M", "ᄚ"), - (0x3141, "M", "ᄆ"), - (0x3142, "M", "ᄇ"), - (0x3143, "M", "ᄈ"), - (0x3144, "M", "ᄡ"), - (0x3145, "M", "ᄉ"), - (0x3146, "M", "ᄊ"), - (0x3147, "M", "ᄋ"), - (0x3148, "M", "ᄌ"), - (0x3149, "M", "ᄍ"), - (0x314A, "M", "ᄎ"), - (0x314B, "M", "ᄏ"), - (0x314C, "M", "ᄐ"), - (0x314D, "M", "ᄑ"), - (0x314E, "M", "ᄒ"), - (0x314F, "M", "ᅡ"), - (0x3150, "M", "ᅢ"), - (0x3151, "M", "ᅣ"), - (0x3152, "M", "ᅤ"), - (0x3153, "M", "ᅥ"), - (0x3154, "M", "ᅦ"), - (0x3155, "M", "ᅧ"), - (0x3156, "M", "ᅨ"), - (0x3157, "M", "ᅩ"), - (0x3158, "M", "ᅪ"), - (0x3159, "M", "ᅫ"), - (0x315A, "M", "ᅬ"), - (0x315B, "M", "ᅭ"), - (0x315C, "M", "ᅮ"), - (0x315D, "M", "ᅯ"), - (0x315E, "M", "ᅰ"), - (0x315F, "M", "ᅱ"), - (0x3160, "M", "ᅲ"), - (0x3161, "M", "ᅳ"), - (0x3162, "M", "ᅴ"), - (0x3163, "M", "ᅵ"), - (0x3164, "I"), - (0x3165, "M", "ᄔ"), - (0x3166, "M", "ᄕ"), - (0x3167, "M", "ᇇ"), - (0x3168, "M", "ᇈ"), - (0x3169, "M", "ᇌ"), - (0x316A, "M", "ᇎ"), - (0x316B, "M", "ᇓ"), - (0x316C, "M", "ᇗ"), - (0x316D, "M", "ᇙ"), - (0x316E, "M", "ᄜ"), - (0x316F, "M", "ᇝ"), - (0x3170, "M", "ᇟ"), - (0x3171, "M", "ᄝ"), - (0x3172, "M", "ᄞ"), - (0x3173, "M", "ᄠ"), - (0x3174, "M", "ᄢ"), - (0x3175, "M", "ᄣ"), - (0x3176, "M", "ᄧ"), - (0x3177, "M", "ᄩ"), - (0x3178, "M", "ᄫ"), - (0x3179, "M", "ᄬ"), - (0x317A, "M", "ᄭ"), - (0x317B, "M", "ᄮ"), - (0x317C, "M", "ᄯ"), - (0x317D, "M", "ᄲ"), - (0x317E, "M", "ᄶ"), - (0x317F, "M", "ᅀ"), - (0x3180, "M", "ᅇ"), - (0x3181, "M", "ᅌ"), - (0x3182, "M", "ᇱ"), - (0x3183, "M", "ᇲ"), - (0x3184, "M", "ᅗ"), - (0x3185, "M", "ᅘ"), - (0x3186, "M", "ᅙ"), - (0x3187, "M", "ᆄ"), - (0x3188, "M", "ᆅ"), - (0x3189, "M", "ᆈ"), - (0x318A, "M", "ᆑ"), - (0x318B, "M", "ᆒ"), - (0x318C, "M", "ᆔ"), - (0x318D, "M", "ᆞ"), - (0x318E, "M", "ᆡ"), - (0x318F, "X"), - (0x3190, "V"), - (0x3192, "M", "一"), - (0x3193, "M", "二"), - (0x3194, "M", "三"), - (0x3195, "M", "四"), - (0x3196, "M", "上"), - (0x3197, "M", "中"), - (0x3198, "M", "下"), - (0x3199, "M", "甲"), - (0x319A, "M", "乙"), - (0x319B, "M", "丙"), - (0x319C, "M", "丁"), - (0x319D, "M", "天"), - (0x319E, "M", "地"), - (0x319F, "M", "人"), - (0x31A0, "V"), - (0x31E6, "X"), - (0x31F0, "V"), - (0x3200, "M", "(ᄀ)"), - (0x3201, "M", "(ᄂ)"), - (0x3202, "M", "(ᄃ)"), - (0x3203, "M", "(ᄅ)"), - (0x3204, "M", "(ᄆ)"), - (0x3205, "M", "(ᄇ)"), - (0x3206, "M", "(ᄉ)"), - (0x3207, "M", "(ᄋ)"), - (0x3208, "M", "(ᄌ)"), - (0x3209, "M", "(ᄎ)"), - (0x320A, "M", "(ᄏ)"), - (0x320B, "M", "(ᄐ)"), - (0x320C, "M", "(ᄑ)"), - (0x320D, "M", "(ᄒ)"), - (0x320E, "M", "(가)"), - (0x320F, "M", "(나)"), - (0x3210, "M", "(다)"), - (0x3211, "M", "(라)"), - (0x3212, "M", "(마)"), - (0x3213, "M", "(바)"), - (0x3214, "M", "(사)"), - (0x3215, "M", "(아)"), - (0x3216, "M", "(자)"), - (0x3217, "M", "(차)"), - (0x3218, "M", "(카)"), - (0x3219, "M", "(타)"), - (0x321A, "M", "(파)"), - (0x321B, "M", "(하)"), - (0x321C, "M", "(주)"), - (0x321D, "M", "(오전)"), - (0x321E, "M", "(오후)"), - (0x321F, "X"), - (0x3220, "M", "(一)"), - (0x3221, "M", "(二)"), - (0x3222, "M", "(三)"), - (0x3223, "M", "(四)"), - (0x3224, "M", "(五)"), - (0x3225, "M", "(六)"), - (0x3226, "M", "(七)"), - (0x3227, "M", "(八)"), - (0x3228, "M", "(九)"), - (0x3229, "M", "(十)"), - (0x322A, "M", "(月)"), - (0x322B, "M", "(火)"), - (0x322C, "M", "(水)"), - (0x322D, "M", "(木)"), - (0x322E, "M", "(金)"), - (0x322F, "M", "(土)"), - (0x3230, "M", "(日)"), - (0x3231, "M", "(株)"), - (0x3232, "M", "(有)"), - (0x3233, "M", "(社)"), - (0x3234, "M", "(名)"), - (0x3235, "M", "(特)"), - (0x3236, "M", "(財)"), - (0x3237, "M", "(祝)"), - (0x3238, "M", "(労)"), - (0x3239, "M", "(代)"), - (0x323A, "M", "(呼)"), - (0x323B, "M", "(学)"), - (0x323C, "M", "(監)"), - (0x323D, "M", "(企)"), - (0x323E, "M", "(資)"), - (0x323F, "M", "(協)"), - (0x3240, "M", "(祭)"), - (0x3241, "M", "(休)"), - (0x3242, "M", "(自)"), - (0x3243, "M", "(至)"), - (0x3244, "M", "問"), - (0x3245, "M", "幼"), - (0x3246, "M", "文"), - (0x3247, "M", "箏"), - (0x3248, "V"), - (0x3250, "M", "pte"), - (0x3251, "M", "21"), - (0x3252, "M", "22"), - (0x3253, "M", "23"), - (0x3254, "M", "24"), - (0x3255, "M", "25"), - (0x3256, "M", "26"), - (0x3257, "M", "27"), - (0x3258, "M", "28"), - (0x3259, "M", "29"), - (0x325A, "M", "30"), - (0x325B, "M", "31"), - (0x325C, "M", "32"), - (0x325D, "M", "33"), - (0x325E, "M", "34"), - (0x325F, "M", "35"), - (0x3260, "M", "ᄀ"), - (0x3261, "M", "ᄂ"), - (0x3262, "M", "ᄃ"), - (0x3263, "M", "ᄅ"), - (0x3264, "M", "ᄆ"), - (0x3265, "M", "ᄇ"), - (0x3266, "M", "ᄉ"), - (0x3267, "M", "ᄋ"), - (0x3268, "M", "ᄌ"), - (0x3269, "M", "ᄎ"), - (0x326A, "M", "ᄏ"), - (0x326B, "M", "ᄐ"), - (0x326C, "M", "ᄑ"), - (0x326D, "M", "ᄒ"), - (0x326E, "M", "가"), - (0x326F, "M", "나"), - (0x3270, "M", "다"), - (0x3271, "M", "라"), - (0x3272, "M", "마"), - (0x3273, "M", "바"), - (0x3274, "M", "사"), - (0x3275, "M", "아"), - (0x3276, "M", "자"), - (0x3277, "M", "차"), - (0x3278, "M", "카"), - (0x3279, "M", "타"), - (0x327A, "M", "파"), - (0x327B, "M", "하"), - (0x327C, "M", "참고"), - (0x327D, "M", "주의"), - (0x327E, "M", "우"), - (0x327F, "V"), - (0x3280, "M", "一"), - (0x3281, "M", "二"), - (0x3282, "M", "三"), - (0x3283, "M", "四"), - (0x3284, "M", "五"), - (0x3285, "M", "六"), - (0x3286, "M", "七"), - (0x3287, "M", "八"), - (0x3288, "M", "九"), - (0x3289, "M", "十"), - (0x328A, "M", "月"), - (0x328B, "M", "火"), - (0x328C, "M", "水"), - (0x328D, "M", "木"), - (0x328E, "M", "金"), - (0x328F, "M", "土"), - (0x3290, "M", "日"), - (0x3291, "M", "株"), - (0x3292, "M", "有"), - (0x3293, "M", "社"), - (0x3294, "M", "名"), - (0x3295, "M", "特"), - (0x3296, "M", "財"), - (0x3297, "M", "祝"), - (0x3298, "M", "労"), - (0x3299, "M", "秘"), - (0x329A, "M", "男"), - (0x329B, "M", "女"), - (0x329C, "M", "適"), - (0x329D, "M", "優"), - (0x329E, "M", "印"), - (0x329F, "M", "注"), - (0x32A0, "M", "項"), - (0x32A1, "M", "休"), - (0x32A2, "M", "写"), - (0x32A3, "M", "正"), - (0x32A4, "M", "上"), - (0x32A5, "M", "中"), - (0x32A6, "M", "下"), - (0x32A7, "M", "左"), - (0x32A8, "M", "右"), - (0x32A9, "M", "医"), - (0x32AA, "M", "宗"), - (0x32AB, "M", "学"), - (0x32AC, "M", "監"), - (0x32AD, "M", "企"), - (0x32AE, "M", "資"), - (0x32AF, "M", "協"), - (0x32B0, "M", "夜"), - (0x32B1, "M", "36"), - (0x32B2, "M", "37"), - (0x32B3, "M", "38"), - (0x32B4, "M", "39"), - (0x32B5, "M", "40"), - (0x32B6, "M", "41"), - (0x32B7, "M", "42"), - (0x32B8, "M", "43"), - (0x32B9, "M", "44"), - (0x32BA, "M", "45"), - (0x32BB, "M", "46"), - (0x32BC, "M", "47"), - (0x32BD, "M", "48"), - (0x32BE, "M", "49"), - (0x32BF, "M", "50"), - (0x32C0, "M", "1月"), - (0x32C1, "M", "2月"), - (0x32C2, "M", "3月"), - (0x32C3, "M", "4月"), - (0x32C4, "M", "5月"), - (0x32C5, "M", "6月"), - (0x32C6, "M", "7月"), - (0x32C7, "M", "8月"), - (0x32C8, "M", "9月"), - (0x32C9, "M", "10月"), - (0x32CA, "M", "11月"), - (0x32CB, "M", "12月"), - (0x32CC, "M", "hg"), - (0x32CD, "M", "erg"), - (0x32CE, "M", "ev"), - (0x32CF, "M", "ltd"), - (0x32D0, "M", "ア"), - (0x32D1, "M", "イ"), - (0x32D2, "M", "ウ"), - (0x32D3, "M", "エ"), - (0x32D4, "M", "オ"), - (0x32D5, "M", "カ"), - (0x32D6, "M", "キ"), - (0x32D7, "M", "ク"), - (0x32D8, "M", "ケ"), - (0x32D9, "M", "コ"), - (0x32DA, "M", "サ"), - (0x32DB, "M", "シ"), - (0x32DC, "M", "ス"), - (0x32DD, "M", "セ"), - (0x32DE, "M", "ソ"), - (0x32DF, "M", "タ"), - (0x32E0, "M", "チ"), - (0x32E1, "M", "ツ"), - (0x32E2, "M", "テ"), - (0x32E3, "M", "ト"), - (0x32E4, "M", "ナ"), - (0x32E5, "M", "ニ"), - (0x32E6, "M", "ヌ"), - (0x32E7, "M", "ネ"), - (0x32E8, "M", "ノ"), - (0x32E9, "M", "ハ"), - (0x32EA, "M", "ヒ"), - (0x32EB, "M", "フ"), - (0x32EC, "M", "ヘ"), - (0x32ED, "M", "ホ"), - (0x32EE, "M", "マ"), - (0x32EF, "M", "ミ"), - (0x32F0, "M", "ム"), - (0x32F1, "M", "メ"), - (0x32F2, "M", "モ"), - (0x32F3, "M", "ヤ"), - (0x32F4, "M", "ユ"), - (0x32F5, "M", "ヨ"), - (0x32F6, "M", "ラ"), - (0x32F7, "M", "リ"), - (0x32F8, "M", "ル"), - (0x32F9, "M", "レ"), - (0x32FA, "M", "ロ"), - (0x32FB, "M", "ワ"), - (0x32FC, "M", "ヰ"), - (0x32FD, "M", "ヱ"), - (0x32FE, "M", "ヲ"), - (0x32FF, "M", "令和"), - (0x3300, "M", "アパート"), - (0x3301, "M", "アルファ"), - (0x3302, "M", "アンペア"), - (0x3303, "M", "アール"), - (0x3304, "M", "イニング"), - (0x3305, "M", "インチ"), - (0x3306, "M", "ウォン"), - (0x3307, "M", "エスクード"), - (0x3308, "M", "エーカー"), - (0x3309, "M", "オンス"), - (0x330A, "M", "オーム"), - (0x330B, "M", "カイリ"), - (0x330C, "M", "カラット"), - (0x330D, "M", "カロリー"), - (0x330E, "M", "ガロン"), - (0x330F, "M", "ガンマ"), - (0x3310, "M", "ギガ"), - (0x3311, "M", "ギニー"), - (0x3312, "M", "キュリー"), - (0x3313, "M", "ギルダー"), - (0x3314, "M", "キロ"), - (0x3315, "M", "キログラム"), - (0x3316, "M", "キロメートル"), - (0x3317, "M", "キロワット"), - (0x3318, "M", "グラム"), - (0x3319, "M", "グラムトン"), - (0x331A, "M", "クルゼイロ"), - (0x331B, "M", "クローネ"), - (0x331C, "M", "ケース"), - (0x331D, "M", "コルナ"), - (0x331E, "M", "コーポ"), - (0x331F, "M", "サイクル"), - (0x3320, "M", "サンチーム"), - (0x3321, "M", "シリング"), - (0x3322, "M", "センチ"), - (0x3323, "M", "セント"), - (0x3324, "M", "ダース"), - (0x3325, "M", "デシ"), - (0x3326, "M", "ドル"), - (0x3327, "M", "トン"), - (0x3328, "M", "ナノ"), - (0x3329, "M", "ノット"), - (0x332A, "M", "ハイツ"), - (0x332B, "M", "パーセント"), - (0x332C, "M", "パーツ"), - (0x332D, "M", "バーレル"), - (0x332E, "M", "ピアストル"), - (0x332F, "M", "ピクル"), - (0x3330, "M", "ピコ"), - (0x3331, "M", "ビル"), - (0x3332, "M", "ファラッド"), - (0x3333, "M", "フィート"), - (0x3334, "M", "ブッシェル"), - (0x3335, "M", "フラン"), - (0x3336, "M", "ヘクタール"), - (0x3337, "M", "ペソ"), - (0x3338, "M", "ペニヒ"), - (0x3339, "M", "ヘルツ"), - (0x333A, "M", "ペンス"), - (0x333B, "M", "ページ"), - (0x333C, "M", "ベータ"), - (0x333D, "M", "ポイント"), - (0x333E, "M", "ボルト"), - (0x333F, "M", "ホン"), - (0x3340, "M", "ポンド"), - (0x3341, "M", "ホール"), - (0x3342, "M", "ホーン"), - (0x3343, "M", "マイクロ"), - (0x3344, "M", "マイル"), - (0x3345, "M", "マッハ"), - (0x3346, "M", "マルク"), - (0x3347, "M", "マンション"), - (0x3348, "M", "ミクロン"), - (0x3349, "M", "ミリ"), - (0x334A, "M", "ミリバール"), - (0x334B, "M", "メガ"), - (0x334C, "M", "メガトン"), - (0x334D, "M", "メートル"), - (0x334E, "M", "ヤード"), - (0x334F, "M", "ヤール"), - (0x3350, "M", "ユアン"), - (0x3351, "M", "リットル"), - (0x3352, "M", "リラ"), - (0x3353, "M", "ルピー"), - (0x3354, "M", "ルーブル"), - (0x3355, "M", "レム"), - (0x3356, "M", "レントゲン"), - (0x3357, "M", "ワット"), - (0x3358, "M", "0点"), - (0x3359, "M", "1点"), - (0x335A, "M", "2点"), - (0x335B, "M", "3点"), - (0x335C, "M", "4点"), - (0x335D, "M", "5点"), - (0x335E, "M", "6点"), - (0x335F, "M", "7点"), - (0x3360, "M", "8点"), - (0x3361, "M", "9点"), - (0x3362, "M", "10点"), - (0x3363, "M", "11点"), - (0x3364, "M", "12点"), - (0x3365, "M", "13点"), - (0x3366, "M", "14点"), - (0x3367, "M", "15点"), - (0x3368, "M", "16点"), - (0x3369, "M", "17点"), - (0x336A, "M", "18点"), - (0x336B, "M", "19点"), - (0x336C, "M", "20点"), - (0x336D, "M", "21点"), - (0x336E, "M", "22点"), - (0x336F, "M", "23点"), - (0x3370, "M", "24点"), - (0x3371, "M", "hpa"), - (0x3372, "M", "da"), - (0x3373, "M", "au"), - (0x3374, "M", "bar"), - (0x3375, "M", "ov"), - (0x3376, "M", "pc"), - (0x3377, "M", "dm"), - (0x3378, "M", "dm2"), - (0x3379, "M", "dm3"), - (0x337A, "M", "iu"), - (0x337B, "M", "平成"), - (0x337C, "M", "昭和"), - (0x337D, "M", "大正"), - (0x337E, "M", "明治"), - (0x337F, "M", "株式会社"), - (0x3380, "M", "pa"), - (0x3381, "M", "na"), - (0x3382, "M", "μa"), - (0x3383, "M", "ma"), - (0x3384, "M", "ka"), - (0x3385, "M", "kb"), - (0x3386, "M", "mb"), - (0x3387, "M", "gb"), - (0x3388, "M", "cal"), - (0x3389, "M", "kcal"), - (0x338A, "M", "pf"), - (0x338B, "M", "nf"), - (0x338C, "M", "μf"), - (0x338D, "M", "μg"), - (0x338E, "M", "mg"), - (0x338F, "M", "kg"), - (0x3390, "M", "hz"), - (0x3391, "M", "khz"), - (0x3392, "M", "mhz"), - (0x3393, "M", "ghz"), - (0x3394, "M", "thz"), - (0x3395, "M", "μl"), - (0x3396, "M", "ml"), - (0x3397, "M", "dl"), - (0x3398, "M", "kl"), - (0x3399, "M", "fm"), - (0x339A, "M", "nm"), - (0x339B, "M", "μm"), - (0x339C, "M", "mm"), - (0x339D, "M", "cm"), - (0x339E, "M", "km"), - (0x339F, "M", "mm2"), - (0x33A0, "M", "cm2"), - (0x33A1, "M", "m2"), - (0x33A2, "M", "km2"), - (0x33A3, "M", "mm3"), - (0x33A4, "M", "cm3"), - (0x33A5, "M", "m3"), - (0x33A6, "M", "km3"), - (0x33A7, "M", "m∕s"), - (0x33A8, "M", "m∕s2"), - (0x33A9, "M", "pa"), - (0x33AA, "M", "kpa"), - (0x33AB, "M", "mpa"), - (0x33AC, "M", "gpa"), - (0x33AD, "M", "rad"), - (0x33AE, "M", "rad∕s"), - (0x33AF, "M", "rad∕s2"), - (0x33B0, "M", "ps"), - (0x33B1, "M", "ns"), - (0x33B2, "M", "μs"), - (0x33B3, "M", "ms"), - (0x33B4, "M", "pv"), - (0x33B5, "M", "nv"), - (0x33B6, "M", "μv"), - (0x33B7, "M", "mv"), - (0x33B8, "M", "kv"), - (0x33B9, "M", "mv"), - (0x33BA, "M", "pw"), - (0x33BB, "M", "nw"), - (0x33BC, "M", "μw"), - (0x33BD, "M", "mw"), - (0x33BE, "M", "kw"), - (0x33BF, "M", "mw"), - (0x33C0, "M", "kω"), - (0x33C1, "M", "mω"), - (0x33C2, "X"), - (0x33C3, "M", "bq"), - (0x33C4, "M", "cc"), - (0x33C5, "M", "cd"), - (0x33C6, "M", "c∕kg"), - (0x33C7, "X"), - (0x33C8, "M", "db"), - (0x33C9, "M", "gy"), - (0x33CA, "M", "ha"), - (0x33CB, "M", "hp"), - (0x33CC, "M", "in"), - (0x33CD, "M", "kk"), - (0x33CE, "M", "km"), - (0x33CF, "M", "kt"), - (0x33D0, "M", "lm"), - (0x33D1, "M", "ln"), - (0x33D2, "M", "log"), - (0x33D3, "M", "lx"), - (0x33D4, "M", "mb"), - (0x33D5, "M", "mil"), - (0x33D6, "M", "mol"), - (0x33D7, "M", "ph"), - (0x33D8, "X"), - (0x33D9, "M", "ppm"), - (0x33DA, "M", "pr"), - (0x33DB, "M", "sr"), - (0x33DC, "M", "sv"), - (0x33DD, "M", "wb"), - (0x33DE, "M", "v∕m"), - (0x33DF, "M", "a∕m"), - (0x33E0, "M", "1日"), - (0x33E1, "M", "2日"), - (0x33E2, "M", "3日"), - (0x33E3, "M", "4日"), - (0x33E4, "M", "5日"), - (0x33E5, "M", "6日"), - (0x33E6, "M", "7日"), - (0x33E7, "M", "8日"), - (0x33E8, "M", "9日"), - (0x33E9, "M", "10日"), - (0x33EA, "M", "11日"), - (0x33EB, "M", "12日"), - (0x33EC, "M", "13日"), - (0x33ED, "M", "14日"), - (0x33EE, "M", "15日"), - (0x33EF, "M", "16日"), - (0x33F0, "M", "17日"), - (0x33F1, "M", "18日"), - (0x33F2, "M", "19日"), - (0x33F3, "M", "20日"), - (0x33F4, "M", "21日"), - (0x33F5, "M", "22日"), - (0x33F6, "M", "23日"), - (0x33F7, "M", "24日"), - (0x33F8, "M", "25日"), - (0x33F9, "M", "26日"), - (0x33FA, "M", "27日"), - (0x33FB, "M", "28日"), - (0x33FC, "M", "29日"), - (0x33FD, "M", "30日"), - (0x33FE, "M", "31日"), - (0x33FF, "M", "gal"), - (0x3400, "V"), - (0xA48D, "X"), - (0xA490, "V"), - (0xA4C7, "X"), - (0xA4D0, "V"), - (0xA62C, "X"), - (0xA640, "M", "ꙁ"), - (0xA641, "V"), - (0xA642, "M", "ꙃ"), - (0xA643, "V"), - (0xA644, "M", "ꙅ"), - (0xA645, "V"), - (0xA646, "M", "ꙇ"), - (0xA647, "V"), - (0xA648, "M", "ꙉ"), - (0xA649, "V"), - (0xA64A, "M", "ꙋ"), - (0xA64B, "V"), - (0xA64C, "M", "ꙍ"), - (0xA64D, "V"), - (0xA64E, "M", "ꙏ"), - (0xA64F, "V"), - (0xA650, "M", "ꙑ"), - (0xA651, "V"), - (0xA652, "M", "ꙓ"), - (0xA653, "V"), - (0xA654, "M", "ꙕ"), - (0xA655, "V"), - (0xA656, "M", "ꙗ"), - (0xA657, "V"), - (0xA658, "M", "ꙙ"), - (0xA659, "V"), - (0xA65A, "M", "ꙛ"), - (0xA65B, "V"), - (0xA65C, "M", "ꙝ"), - (0xA65D, "V"), - (0xA65E, "M", "ꙟ"), - (0xA65F, "V"), - (0xA660, "M", "ꙡ"), - (0xA661, "V"), - (0xA662, "M", "ꙣ"), - (0xA663, "V"), - (0xA664, "M", "ꙥ"), - (0xA665, "V"), - (0xA666, "M", "ꙧ"), - (0xA667, "V"), - (0xA668, "M", "ꙩ"), - (0xA669, "V"), - (0xA66A, "M", "ꙫ"), - (0xA66B, "V"), - (0xA66C, "M", "ꙭ"), - (0xA66D, "V"), - (0xA680, "M", "ꚁ"), - (0xA681, "V"), - (0xA682, "M", "ꚃ"), - (0xA683, "V"), - (0xA684, "M", "ꚅ"), - (0xA685, "V"), - (0xA686, "M", "ꚇ"), - (0xA687, "V"), - (0xA688, "M", "ꚉ"), - (0xA689, "V"), - (0xA68A, "M", "ꚋ"), - (0xA68B, "V"), - (0xA68C, "M", "ꚍ"), - (0xA68D, "V"), - (0xA68E, "M", "ꚏ"), - (0xA68F, "V"), - (0xA690, "M", "ꚑ"), - (0xA691, "V"), - (0xA692, "M", "ꚓ"), - (0xA693, "V"), - (0xA694, "M", "ꚕ"), - (0xA695, "V"), - (0xA696, "M", "ꚗ"), - (0xA697, "V"), - (0xA698, "M", "ꚙ"), - (0xA699, "V"), - (0xA69A, "M", "ꚛ"), - (0xA69B, "V"), - (0xA69C, "M", "ъ"), - (0xA69D, "M", "ь"), - (0xA69E, "V"), - (0xA6F8, "X"), - (0xA700, "V"), - (0xA722, "M", "ꜣ"), - (0xA723, "V"), - (0xA724, "M", "ꜥ"), - (0xA725, "V"), - (0xA726, "M", "ꜧ"), - (0xA727, "V"), - (0xA728, "M", "ꜩ"), - (0xA729, "V"), - (0xA72A, "M", "ꜫ"), - (0xA72B, "V"), - (0xA72C, "M", "ꜭ"), - (0xA72D, "V"), - (0xA72E, "M", "ꜯ"), - (0xA72F, "V"), - (0xA732, "M", "ꜳ"), - (0xA733, "V"), - (0xA734, "M", "ꜵ"), - (0xA735, "V"), - (0xA736, "M", "ꜷ"), - (0xA737, "V"), - (0xA738, "M", "ꜹ"), - (0xA739, "V"), - (0xA73A, "M", "ꜻ"), - (0xA73B, "V"), - (0xA73C, "M", "ꜽ"), - (0xA73D, "V"), - (0xA73E, "M", "ꜿ"), - (0xA73F, "V"), - (0xA740, "M", "ꝁ"), - (0xA741, "V"), - (0xA742, "M", "ꝃ"), - (0xA743, "V"), - (0xA744, "M", "ꝅ"), - (0xA745, "V"), - (0xA746, "M", "ꝇ"), - (0xA747, "V"), - (0xA748, "M", "ꝉ"), - (0xA749, "V"), - (0xA74A, "M", "ꝋ"), - (0xA74B, "V"), - (0xA74C, "M", "ꝍ"), - (0xA74D, "V"), - (0xA74E, "M", "ꝏ"), - (0xA74F, "V"), - (0xA750, "M", "ꝑ"), - (0xA751, "V"), - (0xA752, "M", "ꝓ"), - (0xA753, "V"), - (0xA754, "M", "ꝕ"), - (0xA755, "V"), - (0xA756, "M", "ꝗ"), - (0xA757, "V"), - (0xA758, "M", "ꝙ"), - (0xA759, "V"), - (0xA75A, "M", "ꝛ"), - (0xA75B, "V"), - (0xA75C, "M", "ꝝ"), - (0xA75D, "V"), - (0xA75E, "M", "ꝟ"), - (0xA75F, "V"), - (0xA760, "M", "ꝡ"), - (0xA761, "V"), - (0xA762, "M", "ꝣ"), - (0xA763, "V"), - (0xA764, "M", "ꝥ"), - (0xA765, "V"), - (0xA766, "M", "ꝧ"), - (0xA767, "V"), - (0xA768, "M", "ꝩ"), - (0xA769, "V"), - (0xA76A, "M", "ꝫ"), - (0xA76B, "V"), - (0xA76C, "M", "ꝭ"), - (0xA76D, "V"), - (0xA76E, "M", "ꝯ"), - (0xA76F, "V"), - (0xA770, "M", "ꝯ"), - (0xA771, "V"), - (0xA779, "M", "ꝺ"), - (0xA77A, "V"), - (0xA77B, "M", "ꝼ"), - (0xA77C, "V"), - (0xA77D, "M", "ᵹ"), - (0xA77E, "M", "ꝿ"), - (0xA77F, "V"), - (0xA780, "M", "ꞁ"), - (0xA781, "V"), - (0xA782, "M", "ꞃ"), - (0xA783, "V"), - (0xA784, "M", "ꞅ"), - (0xA785, "V"), - (0xA786, "M", "ꞇ"), - (0xA787, "V"), - (0xA78B, "M", "ꞌ"), - (0xA78C, "V"), - (0xA78D, "M", "ɥ"), - (0xA78E, "V"), - (0xA790, "M", "ꞑ"), - (0xA791, "V"), - (0xA792, "M", "ꞓ"), - (0xA793, "V"), - (0xA796, "M", "ꞗ"), - (0xA797, "V"), - (0xA798, "M", "ꞙ"), - (0xA799, "V"), - (0xA79A, "M", "ꞛ"), - (0xA79B, "V"), - (0xA79C, "M", "ꞝ"), - (0xA79D, "V"), - (0xA79E, "M", "ꞟ"), - (0xA79F, "V"), - (0xA7A0, "M", "ꞡ"), - (0xA7A1, "V"), - (0xA7A2, "M", "ꞣ"), - (0xA7A3, "V"), - (0xA7A4, "M", "ꞥ"), - (0xA7A5, "V"), - (0xA7A6, "M", "ꞧ"), - (0xA7A7, "V"), - (0xA7A8, "M", "ꞩ"), - (0xA7A9, "V"), - (0xA7AA, "M", "ɦ"), - (0xA7AB, "M", "ɜ"), - (0xA7AC, "M", "ɡ"), - (0xA7AD, "M", "ɬ"), - (0xA7AE, "M", "ɪ"), - (0xA7AF, "V"), - (0xA7B0, "M", "ʞ"), - (0xA7B1, "M", "ʇ"), - (0xA7B2, "M", "ʝ"), - (0xA7B3, "M", "ꭓ"), - (0xA7B4, "M", "ꞵ"), - (0xA7B5, "V"), - (0xA7B6, "M", "ꞷ"), - (0xA7B7, "V"), - (0xA7B8, "M", "ꞹ"), - (0xA7B9, "V"), - (0xA7BA, "M", "ꞻ"), - (0xA7BB, "V"), - (0xA7BC, "M", "ꞽ"), - (0xA7BD, "V"), - (0xA7BE, "M", "ꞿ"), - (0xA7BF, "V"), - (0xA7C0, "M", "ꟁ"), - (0xA7C1, "V"), - (0xA7C2, "M", "ꟃ"), - (0xA7C3, "V"), - (0xA7C4, "M", "ꞔ"), - (0xA7C5, "M", "ʂ"), - (0xA7C6, "M", "ᶎ"), - (0xA7C7, "M", "ꟈ"), - (0xA7C8, "V"), - (0xA7C9, "M", "ꟊ"), - (0xA7CA, "V"), - (0xA7CB, "M", "ɤ"), - (0xA7CC, "M", ""), - (0xA7CD, "V"), - (0xA7CE, "M", ""), - (0xA7CF, "V"), - (0xA7D0, "M", "ꟑ"), - (0xA7D1, "V"), - (0xA7D2, "M", "ꟓ"), - (0xA7D3, "V"), - (0xA7D4, "M", "ꟕ"), - (0xA7D5, "V"), - (0xA7D6, "M", "ꟗ"), - (0xA7D7, "V"), - (0xA7D8, "M", "ꟙ"), - (0xA7D9, "V"), - (0xA7DA, "M", ""), - (0xA7DB, "V"), - (0xA7DC, "M", "ƛ"), - (0xA7DD, "X"), - (0xA7F1, "M", "s"), - (0xA7F2, "M", "c"), - (0xA7F3, "M", "f"), - (0xA7F4, "M", "q"), - (0xA7F5, "M", "ꟶ"), - (0xA7F6, "V"), - (0xA7F8, "M", "ħ"), - (0xA7F9, "M", "œ"), - (0xA7FA, "V"), - (0xA82D, "X"), - (0xA830, "V"), - (0xA83A, "X"), - (0xA840, "V"), - (0xA878, "X"), - (0xA880, "V"), - (0xA8C6, "X"), - (0xA8CE, "V"), - (0xA8DA, "X"), - (0xA8E0, "V"), - (0xA954, "X"), - (0xA95F, "V"), - (0xA97D, "X"), - (0xA980, "V"), - (0xA9CE, "X"), - (0xA9CF, "V"), - (0xA9DA, "X"), - (0xA9DE, "V"), - (0xA9FF, "X"), - (0xAA00, "V"), - (0xAA37, "X"), - (0xAA40, "V"), - (0xAA4E, "X"), - (0xAA50, "V"), - (0xAA5A, "X"), - (0xAA5C, "V"), - (0xAAC3, "X"), - (0xAADB, "V"), - (0xAAF7, "X"), - (0xAB01, "V"), - (0xAB07, "X"), - (0xAB09, "V"), - (0xAB0F, "X"), - (0xAB11, "V"), - (0xAB17, "X"), - (0xAB20, "V"), - (0xAB27, "X"), - (0xAB28, "V"), - (0xAB2F, "X"), - (0xAB30, "V"), - (0xAB5C, "M", "ꜧ"), - (0xAB5D, "M", "ꬷ"), - (0xAB5E, "M", "ɫ"), - (0xAB5F, "M", "ꭒ"), - (0xAB60, "V"), - (0xAB69, "M", "ʍ"), - (0xAB6A, "V"), - (0xAB6C, "X"), - (0xAB70, "M", "Ꭰ"), - (0xAB71, "M", "Ꭱ"), - (0xAB72, "M", "Ꭲ"), - (0xAB73, "M", "Ꭳ"), - (0xAB74, "M", "Ꭴ"), - (0xAB75, "M", "Ꭵ"), - (0xAB76, "M", "Ꭶ"), - (0xAB77, "M", "Ꭷ"), - (0xAB78, "M", "Ꭸ"), - (0xAB79, "M", "Ꭹ"), - (0xAB7A, "M", "Ꭺ"), - (0xAB7B, "M", "Ꭻ"), - (0xAB7C, "M", "Ꭼ"), - (0xAB7D, "M", "Ꭽ"), - (0xAB7E, "M", "Ꭾ"), - (0xAB7F, "M", "Ꭿ"), - (0xAB80, "M", "Ꮀ"), - (0xAB81, "M", "Ꮁ"), - (0xAB82, "M", "Ꮂ"), - (0xAB83, "M", "Ꮃ"), - (0xAB84, "M", "Ꮄ"), - (0xAB85, "M", "Ꮅ"), - (0xAB86, "M", "Ꮆ"), - (0xAB87, "M", "Ꮇ"), - (0xAB88, "M", "Ꮈ"), - (0xAB89, "M", "Ꮉ"), - (0xAB8A, "M", "Ꮊ"), - (0xAB8B, "M", "Ꮋ"), - (0xAB8C, "M", "Ꮌ"), - (0xAB8D, "M", "Ꮍ"), - (0xAB8E, "M", "Ꮎ"), - (0xAB8F, "M", "Ꮏ"), - (0xAB90, "M", "Ꮐ"), - (0xAB91, "M", "Ꮑ"), - (0xAB92, "M", "Ꮒ"), - (0xAB93, "M", "Ꮓ"), - (0xAB94, "M", "Ꮔ"), - (0xAB95, "M", "Ꮕ"), - (0xAB96, "M", "Ꮖ"), - (0xAB97, "M", "Ꮗ"), - (0xAB98, "M", "Ꮘ"), - (0xAB99, "M", "Ꮙ"), - (0xAB9A, "M", "Ꮚ"), - (0xAB9B, "M", "Ꮛ"), - (0xAB9C, "M", "Ꮜ"), - (0xAB9D, "M", "Ꮝ"), - (0xAB9E, "M", "Ꮞ"), - (0xAB9F, "M", "Ꮟ"), - (0xABA0, "M", "Ꮠ"), - (0xABA1, "M", "Ꮡ"), - (0xABA2, "M", "Ꮢ"), - (0xABA3, "M", "Ꮣ"), - (0xABA4, "M", "Ꮤ"), - (0xABA5, "M", "Ꮥ"), - (0xABA6, "M", "Ꮦ"), - (0xABA7, "M", "Ꮧ"), - (0xABA8, "M", "Ꮨ"), - (0xABA9, "M", "Ꮩ"), - (0xABAA, "M", "Ꮪ"), - (0xABAB, "M", "Ꮫ"), - (0xABAC, "M", "Ꮬ"), - (0xABAD, "M", "Ꮭ"), - (0xABAE, "M", "Ꮮ"), - (0xABAF, "M", "Ꮯ"), - (0xABB0, "M", "Ꮰ"), - (0xABB1, "M", "Ꮱ"), - (0xABB2, "M", "Ꮲ"), - (0xABB3, "M", "Ꮳ"), - (0xABB4, "M", "Ꮴ"), - (0xABB5, "M", "Ꮵ"), - (0xABB6, "M", "Ꮶ"), - (0xABB7, "M", "Ꮷ"), - (0xABB8, "M", "Ꮸ"), - (0xABB9, "M", "Ꮹ"), - (0xABBA, "M", "Ꮺ"), - (0xABBB, "M", "Ꮻ"), - (0xABBC, "M", "Ꮼ"), - (0xABBD, "M", "Ꮽ"), - (0xABBE, "M", "Ꮾ"), - (0xABBF, "M", "Ꮿ"), - (0xABC0, "V"), - (0xABEE, "X"), - (0xABF0, "V"), - (0xABFA, "X"), - (0xAC00, "V"), - (0xD7A4, "X"), - (0xD7B0, "V"), - (0xD7C7, "X"), - (0xD7CB, "V"), - (0xD7FC, "X"), - (0xF900, "M", "豈"), - (0xF901, "M", "更"), - (0xF902, "M", "車"), - (0xF903, "M", "賈"), - (0xF904, "M", "滑"), - (0xF905, "M", "串"), - (0xF906, "M", "句"), - (0xF907, "M", "龜"), - (0xF909, "M", "契"), - (0xF90A, "M", "金"), - (0xF90B, "M", "喇"), - (0xF90C, "M", "奈"), - (0xF90D, "M", "懶"), - (0xF90E, "M", "癩"), - (0xF90F, "M", "羅"), - (0xF910, "M", "蘿"), - (0xF911, "M", "螺"), - (0xF912, "M", "裸"), - (0xF913, "M", "邏"), - (0xF914, "M", "樂"), - (0xF915, "M", "洛"), - (0xF916, "M", "烙"), - (0xF917, "M", "珞"), - (0xF918, "M", "落"), - (0xF919, "M", "酪"), - (0xF91A, "M", "駱"), - (0xF91B, "M", "亂"), - (0xF91C, "M", "卵"), - (0xF91D, "M", "欄"), - (0xF91E, "M", "爛"), - (0xF91F, "M", "蘭"), - (0xF920, "M", "鸞"), - (0xF921, "M", "嵐"), - (0xF922, "M", "濫"), - (0xF923, "M", "藍"), - (0xF924, "M", "襤"), - (0xF925, "M", "拉"), - (0xF926, "M", "臘"), - (0xF927, "M", "蠟"), - (0xF928, "M", "廊"), - (0xF929, "M", "朗"), - (0xF92A, "M", "浪"), - (0xF92B, "M", "狼"), - (0xF92C, "M", "郎"), - (0xF92D, "M", "來"), - (0xF92E, "M", "冷"), - (0xF92F, "M", "勞"), - (0xF930, "M", "擄"), - (0xF931, "M", "櫓"), - (0xF932, "M", "爐"), - (0xF933, "M", "盧"), - (0xF934, "M", "老"), - (0xF935, "M", "蘆"), - (0xF936, "M", "虜"), - (0xF937, "M", "路"), - (0xF938, "M", "露"), - (0xF939, "M", "魯"), - (0xF93A, "M", "鷺"), - (0xF93B, "M", "碌"), - (0xF93C, "M", "祿"), - (0xF93D, "M", "綠"), - (0xF93E, "M", "菉"), - (0xF93F, "M", "錄"), - (0xF940, "M", "鹿"), - (0xF941, "M", "論"), - (0xF942, "M", "壟"), - (0xF943, "M", "弄"), - (0xF944, "M", "籠"), - (0xF945, "M", "聾"), - (0xF946, "M", "牢"), - (0xF947, "M", "磊"), - (0xF948, "M", "賂"), - (0xF949, "M", "雷"), - (0xF94A, "M", "壘"), - (0xF94B, "M", "屢"), - (0xF94C, "M", "樓"), - (0xF94D, "M", "淚"), - (0xF94E, "M", "漏"), - (0xF94F, "M", "累"), - (0xF950, "M", "縷"), - (0xF951, "M", "陋"), - (0xF952, "M", "勒"), - (0xF953, "M", "肋"), - (0xF954, "M", "凜"), - (0xF955, "M", "凌"), - (0xF956, "M", "稜"), - (0xF957, "M", "綾"), - (0xF958, "M", "菱"), - (0xF959, "M", "陵"), - (0xF95A, "M", "讀"), - (0xF95B, "M", "拏"), - (0xF95C, "M", "樂"), - (0xF95D, "M", "諾"), - (0xF95E, "M", "丹"), - (0xF95F, "M", "寧"), - (0xF960, "M", "怒"), - (0xF961, "M", "率"), - (0xF962, "M", "異"), - (0xF963, "M", "北"), - (0xF964, "M", "磻"), - (0xF965, "M", "便"), - (0xF966, "M", "復"), - (0xF967, "M", "不"), - (0xF968, "M", "泌"), - (0xF969, "M", "數"), - (0xF96A, "M", "索"), - (0xF96B, "M", "參"), - (0xF96C, "M", "塞"), - (0xF96D, "M", "省"), - (0xF96E, "M", "葉"), - (0xF96F, "M", "說"), - (0xF970, "M", "殺"), - (0xF971, "M", "辰"), - (0xF972, "M", "沈"), - (0xF973, "M", "拾"), - (0xF974, "M", "若"), - (0xF975, "M", "掠"), - (0xF976, "M", "略"), - (0xF977, "M", "亮"), - (0xF978, "M", "兩"), - (0xF979, "M", "凉"), - (0xF97A, "M", "梁"), - (0xF97B, "M", "糧"), - (0xF97C, "M", "良"), - (0xF97D, "M", "諒"), - (0xF97E, "M", "量"), - (0xF97F, "M", "勵"), - (0xF980, "M", "呂"), - (0xF981, "M", "女"), - (0xF982, "M", "廬"), - (0xF983, "M", "旅"), - (0xF984, "M", "濾"), - (0xF985, "M", "礪"), - (0xF986, "M", "閭"), - (0xF987, "M", "驪"), - (0xF988, "M", "麗"), - (0xF989, "M", "黎"), - (0xF98A, "M", "力"), - (0xF98B, "M", "曆"), - (0xF98C, "M", "歷"), - (0xF98D, "M", "轢"), - (0xF98E, "M", "年"), - (0xF98F, "M", "憐"), - (0xF990, "M", "戀"), - (0xF991, "M", "撚"), - (0xF992, "M", "漣"), - (0xF993, "M", "煉"), - (0xF994, "M", "璉"), - (0xF995, "M", "秊"), - (0xF996, "M", "練"), - (0xF997, "M", "聯"), - (0xF998, "M", "輦"), - (0xF999, "M", "蓮"), - (0xF99A, "M", "連"), - (0xF99B, "M", "鍊"), - (0xF99C, "M", "列"), - (0xF99D, "M", "劣"), - (0xF99E, "M", "咽"), - (0xF99F, "M", "烈"), - (0xF9A0, "M", "裂"), - (0xF9A1, "M", "說"), - (0xF9A2, "M", "廉"), - (0xF9A3, "M", "念"), - (0xF9A4, "M", "捻"), - (0xF9A5, "M", "殮"), - (0xF9A6, "M", "簾"), - (0xF9A7, "M", "獵"), - (0xF9A8, "M", "令"), - (0xF9A9, "M", "囹"), - (0xF9AA, "M", "寧"), - (0xF9AB, "M", "嶺"), - (0xF9AC, "M", "怜"), - (0xF9AD, "M", "玲"), - (0xF9AE, "M", "瑩"), - (0xF9AF, "M", "羚"), - (0xF9B0, "M", "聆"), - (0xF9B1, "M", "鈴"), - (0xF9B2, "M", "零"), - (0xF9B3, "M", "靈"), - (0xF9B4, "M", "領"), - (0xF9B5, "M", "例"), - (0xF9B6, "M", "禮"), - (0xF9B7, "M", "醴"), - (0xF9B8, "M", "隸"), - (0xF9B9, "M", "惡"), - (0xF9BA, "M", "了"), - (0xF9BB, "M", "僚"), - (0xF9BC, "M", "寮"), - (0xF9BD, "M", "尿"), - (0xF9BE, "M", "料"), - (0xF9BF, "M", "樂"), - (0xF9C0, "M", "燎"), - (0xF9C1, "M", "療"), - (0xF9C2, "M", "蓼"), - (0xF9C3, "M", "遼"), - (0xF9C4, "M", "龍"), - (0xF9C5, "M", "暈"), - (0xF9C6, "M", "阮"), - (0xF9C7, "M", "劉"), - (0xF9C8, "M", "杻"), - (0xF9C9, "M", "柳"), - (0xF9CA, "M", "流"), - (0xF9CB, "M", "溜"), - (0xF9CC, "M", "琉"), - (0xF9CD, "M", "留"), - (0xF9CE, "M", "硫"), - (0xF9CF, "M", "紐"), - (0xF9D0, "M", "類"), - (0xF9D1, "M", "六"), - (0xF9D2, "M", "戮"), - (0xF9D3, "M", "陸"), - (0xF9D4, "M", "倫"), - (0xF9D5, "M", "崙"), - (0xF9D6, "M", "淪"), - (0xF9D7, "M", "輪"), - (0xF9D8, "M", "律"), - (0xF9D9, "M", "慄"), - (0xF9DA, "M", "栗"), - (0xF9DB, "M", "率"), - (0xF9DC, "M", "隆"), - (0xF9DD, "M", "利"), - (0xF9DE, "M", "吏"), - (0xF9DF, "M", "履"), - (0xF9E0, "M", "易"), - (0xF9E1, "M", "李"), - (0xF9E2, "M", "梨"), - (0xF9E3, "M", "泥"), - (0xF9E4, "M", "理"), - (0xF9E5, "M", "痢"), - (0xF9E6, "M", "罹"), - (0xF9E7, "M", "裏"), - (0xF9E8, "M", "裡"), - (0xF9E9, "M", "里"), - (0xF9EA, "M", "離"), - (0xF9EB, "M", "匿"), - (0xF9EC, "M", "溺"), - (0xF9ED, "M", "吝"), - (0xF9EE, "M", "燐"), - (0xF9EF, "M", "璘"), - (0xF9F0, "M", "藺"), - (0xF9F1, "M", "隣"), - (0xF9F2, "M", "鱗"), - (0xF9F3, "M", "麟"), - (0xF9F4, "M", "林"), - (0xF9F5, "M", "淋"), - (0xF9F6, "M", "臨"), - (0xF9F7, "M", "立"), - (0xF9F8, "M", "笠"), - (0xF9F9, "M", "粒"), - (0xF9FA, "M", "狀"), - (0xF9FB, "M", "炙"), - (0xF9FC, "M", "識"), - (0xF9FD, "M", "什"), - (0xF9FE, "M", "茶"), - (0xF9FF, "M", "刺"), - (0xFA00, "M", "切"), - (0xFA01, "M", "度"), - (0xFA02, "M", "拓"), - (0xFA03, "M", "糖"), - (0xFA04, "M", "宅"), - (0xFA05, "M", "洞"), - (0xFA06, "M", "暴"), - (0xFA07, "M", "輻"), - (0xFA08, "M", "行"), - (0xFA09, "M", "降"), - (0xFA0A, "M", "見"), - (0xFA0B, "M", "廓"), - (0xFA0C, "M", "兀"), - (0xFA0D, "M", "嗀"), - (0xFA0E, "V"), - (0xFA10, "M", "塚"), - (0xFA11, "V"), - (0xFA12, "M", "晴"), - (0xFA13, "V"), - (0xFA15, "M", "凞"), - (0xFA16, "M", "猪"), - (0xFA17, "M", "益"), - (0xFA18, "M", "礼"), - (0xFA19, "M", "神"), - (0xFA1A, "M", "祥"), - (0xFA1B, "M", "福"), - (0xFA1C, "M", "靖"), - (0xFA1D, "M", "精"), - (0xFA1E, "M", "羽"), - (0xFA1F, "V"), - (0xFA20, "M", "蘒"), - (0xFA21, "V"), - (0xFA22, "M", "諸"), - (0xFA23, "V"), - (0xFA25, "M", "逸"), - (0xFA26, "M", "都"), - (0xFA27, "V"), - (0xFA2A, "M", "飯"), - (0xFA2B, "M", "飼"), - (0xFA2C, "M", "館"), - (0xFA2D, "M", "鶴"), - (0xFA2E, "M", "郞"), - (0xFA2F, "M", "隷"), - (0xFA30, "M", "侮"), - (0xFA31, "M", "僧"), - (0xFA32, "M", "免"), - (0xFA33, "M", "勉"), - (0xFA34, "M", "勤"), - (0xFA35, "M", "卑"), - (0xFA36, "M", "喝"), - (0xFA37, "M", "嘆"), - (0xFA38, "M", "器"), - (0xFA39, "M", "塀"), - (0xFA3A, "M", "墨"), - (0xFA3B, "M", "層"), - (0xFA3C, "M", "屮"), - (0xFA3D, "M", "悔"), - (0xFA3E, "M", "慨"), - (0xFA3F, "M", "憎"), - (0xFA40, "M", "懲"), - (0xFA41, "M", "敏"), - (0xFA42, "M", "既"), - (0xFA43, "M", "暑"), - (0xFA44, "M", "梅"), - (0xFA45, "M", "海"), - (0xFA46, "M", "渚"), - (0xFA47, "M", "漢"), - (0xFA48, "M", "煮"), - (0xFA49, "M", "爫"), - (0xFA4A, "M", "琢"), - (0xFA4B, "M", "碑"), - (0xFA4C, "M", "社"), - (0xFA4D, "M", "祉"), - (0xFA4E, "M", "祈"), - (0xFA4F, "M", "祐"), - (0xFA50, "M", "祖"), - (0xFA51, "M", "祝"), - (0xFA52, "M", "禍"), - (0xFA53, "M", "禎"), - (0xFA54, "M", "穀"), - (0xFA55, "M", "突"), - (0xFA56, "M", "節"), - (0xFA57, "M", "練"), - (0xFA58, "M", "縉"), - (0xFA59, "M", "繁"), - (0xFA5A, "M", "署"), - (0xFA5B, "M", "者"), - (0xFA5C, "M", "臭"), - (0xFA5D, "M", "艹"), - (0xFA5F, "M", "著"), - (0xFA60, "M", "褐"), - (0xFA61, "M", "視"), - (0xFA62, "M", "謁"), - (0xFA63, "M", "謹"), - (0xFA64, "M", "賓"), - (0xFA65, "M", "贈"), - (0xFA66, "M", "辶"), - (0xFA67, "M", "逸"), - (0xFA68, "M", "難"), - (0xFA69, "M", "響"), - (0xFA6A, "M", "頻"), - (0xFA6B, "M", "恵"), - (0xFA6C, "M", "𤋮"), - (0xFA6D, "M", "舘"), - (0xFA6E, "X"), - (0xFA70, "M", "並"), - (0xFA71, "M", "况"), - (0xFA72, "M", "全"), - (0xFA73, "M", "侀"), - (0xFA74, "M", "充"), - (0xFA75, "M", "冀"), - (0xFA76, "M", "勇"), - (0xFA77, "M", "勺"), - (0xFA78, "M", "喝"), - (0xFA79, "M", "啕"), - (0xFA7A, "M", "喙"), - (0xFA7B, "M", "嗢"), - (0xFA7C, "M", "塚"), - (0xFA7D, "M", "墳"), - (0xFA7E, "M", "奄"), - (0xFA7F, "M", "奔"), - (0xFA80, "M", "婢"), - (0xFA81, "M", "嬨"), - (0xFA82, "M", "廒"), - (0xFA83, "M", "廙"), - (0xFA84, "M", "彩"), - (0xFA85, "M", "徭"), - (0xFA86, "M", "惘"), - (0xFA87, "M", "慎"), - (0xFA88, "M", "愈"), - (0xFA89, "M", "憎"), - (0xFA8A, "M", "慠"), - (0xFA8B, "M", "懲"), - (0xFA8C, "M", "戴"), - (0xFA8D, "M", "揄"), - (0xFA8E, "M", "搜"), - (0xFA8F, "M", "摒"), - (0xFA90, "M", "敖"), - (0xFA91, "M", "晴"), - (0xFA92, "M", "朗"), - (0xFA93, "M", "望"), - (0xFA94, "M", "杖"), - (0xFA95, "M", "歹"), - (0xFA96, "M", "殺"), - (0xFA97, "M", "流"), - (0xFA98, "M", "滛"), - (0xFA99, "M", "滋"), - (0xFA9A, "M", "漢"), - (0xFA9B, "M", "瀞"), - (0xFA9C, "M", "煮"), - (0xFA9D, "M", "瞧"), - (0xFA9E, "M", "爵"), - (0xFA9F, "M", "犯"), - (0xFAA0, "M", "猪"), - (0xFAA1, "M", "瑱"), - (0xFAA2, "M", "甆"), - (0xFAA3, "M", "画"), - (0xFAA4, "M", "瘝"), - (0xFAA5, "M", "瘟"), - (0xFAA6, "M", "益"), - (0xFAA7, "M", "盛"), - (0xFAA8, "M", "直"), - (0xFAA9, "M", "睊"), - (0xFAAA, "M", "着"), - (0xFAAB, "M", "磌"), - (0xFAAC, "M", "窱"), - (0xFAAD, "M", "節"), - (0xFAAE, "M", "类"), - (0xFAAF, "M", "絛"), - (0xFAB0, "M", "練"), - (0xFAB1, "M", "缾"), - (0xFAB2, "M", "者"), - (0xFAB3, "M", "荒"), - (0xFAB4, "M", "華"), - (0xFAB5, "M", "蝹"), - (0xFAB6, "M", "襁"), - (0xFAB7, "M", "覆"), - (0xFAB8, "M", "視"), - (0xFAB9, "M", "調"), - (0xFABA, "M", "諸"), - (0xFABB, "M", "請"), - (0xFABC, "M", "謁"), - (0xFABD, "M", "諾"), - (0xFABE, "M", "諭"), - (0xFABF, "M", "謹"), - (0xFAC0, "M", "變"), - (0xFAC1, "M", "贈"), - (0xFAC2, "M", "輸"), - (0xFAC3, "M", "遲"), - (0xFAC4, "M", "醙"), - (0xFAC5, "M", "鉶"), - (0xFAC6, "M", "陼"), - (0xFAC7, "M", "難"), - (0xFAC8, "M", "靖"), - (0xFAC9, "M", "韛"), - (0xFACA, "M", "響"), - (0xFACB, "M", "頋"), - (0xFACC, "M", "頻"), - (0xFACD, "M", "鬒"), - (0xFACE, "M", "龜"), - (0xFACF, "M", "𢡊"), - (0xFAD0, "M", "𢡄"), - (0xFAD1, "M", "𣏕"), - (0xFAD2, "M", "㮝"), - (0xFAD3, "M", "䀘"), - (0xFAD4, "M", "䀹"), - (0xFAD5, "M", "𥉉"), - (0xFAD6, "M", "𥳐"), - (0xFAD7, "M", "𧻓"), - (0xFAD8, "M", "齃"), - (0xFAD9, "M", "龎"), - (0xFADA, "X"), - (0xFB00, "M", "ff"), - (0xFB01, "M", "fi"), - (0xFB02, "M", "fl"), - (0xFB03, "M", "ffi"), - (0xFB04, "M", "ffl"), - (0xFB05, "M", "st"), - (0xFB07, "X"), - (0xFB13, "M", "մն"), - (0xFB14, "M", "մե"), - (0xFB15, "M", "մի"), - (0xFB16, "M", "վն"), - (0xFB17, "M", "մխ"), - (0xFB18, "X"), - (0xFB1D, "M", "יִ"), - (0xFB1E, "V"), - (0xFB1F, "M", "ײַ"), - (0xFB20, "M", "ע"), - (0xFB21, "M", "א"), - (0xFB22, "M", "ד"), - (0xFB23, "M", "ה"), - (0xFB24, "M", "כ"), - (0xFB25, "M", "ל"), - (0xFB26, "M", "ם"), - (0xFB27, "M", "ר"), - (0xFB28, "M", "ת"), - (0xFB29, "M", "+"), - (0xFB2A, "M", "שׁ"), - (0xFB2B, "M", "שׂ"), - (0xFB2C, "M", "שּׁ"), - (0xFB2D, "M", "שּׂ"), - (0xFB2E, "M", "אַ"), - (0xFB2F, "M", "אָ"), - (0xFB30, "M", "אּ"), - (0xFB31, "M", "בּ"), - (0xFB32, "M", "גּ"), - (0xFB33, "M", "דּ"), - (0xFB34, "M", "הּ"), - (0xFB35, "M", "וּ"), - (0xFB36, "M", "זּ"), - (0xFB37, "X"), - (0xFB38, "M", "טּ"), - (0xFB39, "M", "יּ"), - (0xFB3A, "M", "ךּ"), - (0xFB3B, "M", "כּ"), - (0xFB3C, "M", "לּ"), - (0xFB3D, "X"), - (0xFB3E, "M", "מּ"), - (0xFB3F, "X"), - (0xFB40, "M", "נּ"), - (0xFB41, "M", "סּ"), - (0xFB42, "X"), - (0xFB43, "M", "ףּ"), - (0xFB44, "M", "פּ"), - (0xFB45, "X"), - (0xFB46, "M", "צּ"), - (0xFB47, "M", "קּ"), - (0xFB48, "M", "רּ"), - (0xFB49, "M", "שּ"), - (0xFB4A, "M", "תּ"), - (0xFB4B, "M", "וֹ"), - (0xFB4C, "M", "בֿ"), - (0xFB4D, "M", "כֿ"), - (0xFB4E, "M", "פֿ"), - (0xFB4F, "M", "אל"), - (0xFB50, "M", "ٱ"), - (0xFB52, "M", "ٻ"), - (0xFB56, "M", "پ"), - (0xFB5A, "M", "ڀ"), - (0xFB5E, "M", "ٺ"), - (0xFB62, "M", "ٿ"), - (0xFB66, "M", "ٹ"), - (0xFB6A, "M", "ڤ"), - (0xFB6E, "M", "ڦ"), - (0xFB72, "M", "ڄ"), - (0xFB76, "M", "ڃ"), - (0xFB7A, "M", "چ"), - (0xFB7E, "M", "ڇ"), - (0xFB82, "M", "ڍ"), - (0xFB84, "M", "ڌ"), - (0xFB86, "M", "ڎ"), - (0xFB88, "M", "ڈ"), - (0xFB8A, "M", "ژ"), - (0xFB8C, "M", "ڑ"), - (0xFB8E, "M", "ک"), - (0xFB92, "M", "گ"), - (0xFB96, "M", "ڳ"), - (0xFB9A, "M", "ڱ"), - (0xFB9E, "M", "ں"), - (0xFBA0, "M", "ڻ"), - (0xFBA4, "M", "ۀ"), - (0xFBA6, "M", "ہ"), - (0xFBAA, "M", "ھ"), - (0xFBAE, "M", "ے"), - (0xFBB0, "M", "ۓ"), - (0xFBB2, "V"), - (0xFBD3, "M", "ڭ"), - (0xFBD7, "M", "ۇ"), - (0xFBD9, "M", "ۆ"), - (0xFBDB, "M", "ۈ"), - (0xFBDD, "M", "ۇٴ"), - (0xFBDE, "M", "ۋ"), - (0xFBE0, "M", "ۅ"), - (0xFBE2, "M", "ۉ"), - (0xFBE4, "M", "ې"), - (0xFBE8, "M", "ى"), - (0xFBEA, "M", "ئا"), - (0xFBEC, "M", "ئە"), - (0xFBEE, "M", "ئو"), - (0xFBF0, "M", "ئۇ"), - (0xFBF2, "M", "ئۆ"), - (0xFBF4, "M", "ئۈ"), - (0xFBF6, "M", "ئې"), - (0xFBF9, "M", "ئى"), - (0xFBFC, "M", "ی"), - (0xFC00, "M", "ئج"), - (0xFC01, "M", "ئح"), - (0xFC02, "M", "ئم"), - (0xFC03, "M", "ئى"), - (0xFC04, "M", "ئي"), - (0xFC05, "M", "بج"), - (0xFC06, "M", "بح"), - (0xFC07, "M", "بخ"), - (0xFC08, "M", "بم"), - (0xFC09, "M", "بى"), - (0xFC0A, "M", "بي"), - (0xFC0B, "M", "تج"), - (0xFC0C, "M", "تح"), - (0xFC0D, "M", "تخ"), - (0xFC0E, "M", "تم"), - (0xFC0F, "M", "تى"), - (0xFC10, "M", "تي"), - (0xFC11, "M", "ثج"), - (0xFC12, "M", "ثم"), - (0xFC13, "M", "ثى"), - (0xFC14, "M", "ثي"), - (0xFC15, "M", "جح"), - (0xFC16, "M", "جم"), - (0xFC17, "M", "حج"), - (0xFC18, "M", "حم"), - (0xFC19, "M", "خج"), - (0xFC1A, "M", "خح"), - (0xFC1B, "M", "خم"), - (0xFC1C, "M", "سج"), - (0xFC1D, "M", "سح"), - (0xFC1E, "M", "سخ"), - (0xFC1F, "M", "سم"), - (0xFC20, "M", "صح"), - (0xFC21, "M", "صم"), - (0xFC22, "M", "ضج"), - (0xFC23, "M", "ضح"), - (0xFC24, "M", "ضخ"), - (0xFC25, "M", "ضم"), - (0xFC26, "M", "طح"), - (0xFC27, "M", "طم"), - (0xFC28, "M", "ظم"), - (0xFC29, "M", "عج"), - (0xFC2A, "M", "عم"), - (0xFC2B, "M", "غج"), - (0xFC2C, "M", "غم"), - (0xFC2D, "M", "فج"), - (0xFC2E, "M", "فح"), - (0xFC2F, "M", "فخ"), - (0xFC30, "M", "فم"), - (0xFC31, "M", "فى"), - (0xFC32, "M", "في"), - (0xFC33, "M", "قح"), - (0xFC34, "M", "قم"), - (0xFC35, "M", "قى"), - (0xFC36, "M", "قي"), - (0xFC37, "M", "كا"), - (0xFC38, "M", "كج"), - (0xFC39, "M", "كح"), - (0xFC3A, "M", "كخ"), - (0xFC3B, "M", "كل"), - (0xFC3C, "M", "كم"), - (0xFC3D, "M", "كى"), - (0xFC3E, "M", "كي"), - (0xFC3F, "M", "لج"), - (0xFC40, "M", "لح"), - (0xFC41, "M", "لخ"), - (0xFC42, "M", "لم"), - (0xFC43, "M", "لى"), - (0xFC44, "M", "لي"), - (0xFC45, "M", "مج"), - (0xFC46, "M", "مح"), - (0xFC47, "M", "مخ"), - (0xFC48, "M", "مم"), - (0xFC49, "M", "مى"), - (0xFC4A, "M", "مي"), - (0xFC4B, "M", "نج"), - (0xFC4C, "M", "نح"), - (0xFC4D, "M", "نخ"), - (0xFC4E, "M", "نم"), - (0xFC4F, "M", "نى"), - (0xFC50, "M", "ني"), - (0xFC51, "M", "هج"), - (0xFC52, "M", "هم"), - (0xFC53, "M", "هى"), - (0xFC54, "M", "هي"), - (0xFC55, "M", "يج"), - (0xFC56, "M", "يح"), - (0xFC57, "M", "يخ"), - (0xFC58, "M", "يم"), - (0xFC59, "M", "يى"), - (0xFC5A, "M", "يي"), - (0xFC5B, "M", "ذٰ"), - (0xFC5C, "M", "رٰ"), - (0xFC5D, "M", "ىٰ"), - (0xFC5E, "M", " ٌّ"), - (0xFC5F, "M", " ٍّ"), - (0xFC60, "M", " َّ"), - (0xFC61, "M", " ُّ"), - (0xFC62, "M", " ِّ"), - (0xFC63, "M", " ّٰ"), - (0xFC64, "M", "ئر"), - (0xFC65, "M", "ئز"), - (0xFC66, "M", "ئم"), - (0xFC67, "M", "ئن"), - (0xFC68, "M", "ئى"), - (0xFC69, "M", "ئي"), - (0xFC6A, "M", "بر"), - (0xFC6B, "M", "بز"), - (0xFC6C, "M", "بم"), - (0xFC6D, "M", "بن"), - (0xFC6E, "M", "بى"), - (0xFC6F, "M", "بي"), - (0xFC70, "M", "تر"), - (0xFC71, "M", "تز"), - (0xFC72, "M", "تم"), - (0xFC73, "M", "تن"), - (0xFC74, "M", "تى"), - (0xFC75, "M", "تي"), - (0xFC76, "M", "ثر"), - (0xFC77, "M", "ثز"), - (0xFC78, "M", "ثم"), - (0xFC79, "M", "ثن"), - (0xFC7A, "M", "ثى"), - (0xFC7B, "M", "ثي"), - (0xFC7C, "M", "فى"), - (0xFC7D, "M", "في"), - (0xFC7E, "M", "قى"), - (0xFC7F, "M", "قي"), - (0xFC80, "M", "كا"), - (0xFC81, "M", "كل"), - (0xFC82, "M", "كم"), - (0xFC83, "M", "كى"), - (0xFC84, "M", "كي"), - (0xFC85, "M", "لم"), - (0xFC86, "M", "لى"), - (0xFC87, "M", "لي"), - (0xFC88, "M", "ما"), - (0xFC89, "M", "مم"), - (0xFC8A, "M", "نر"), - (0xFC8B, "M", "نز"), - (0xFC8C, "M", "نم"), - (0xFC8D, "M", "نن"), - (0xFC8E, "M", "نى"), - (0xFC8F, "M", "ني"), - (0xFC90, "M", "ىٰ"), - (0xFC91, "M", "ير"), - (0xFC92, "M", "يز"), - (0xFC93, "M", "يم"), - (0xFC94, "M", "ين"), - (0xFC95, "M", "يى"), - (0xFC96, "M", "يي"), - (0xFC97, "M", "ئج"), - (0xFC98, "M", "ئح"), - (0xFC99, "M", "ئخ"), - (0xFC9A, "M", "ئم"), - (0xFC9B, "M", "ئه"), - (0xFC9C, "M", "بج"), - (0xFC9D, "M", "بح"), - (0xFC9E, "M", "بخ"), - (0xFC9F, "M", "بم"), - (0xFCA0, "M", "به"), - (0xFCA1, "M", "تج"), - (0xFCA2, "M", "تح"), - (0xFCA3, "M", "تخ"), - (0xFCA4, "M", "تم"), - (0xFCA5, "M", "ته"), - (0xFCA6, "M", "ثم"), - (0xFCA7, "M", "جح"), - (0xFCA8, "M", "جم"), - (0xFCA9, "M", "حج"), - (0xFCAA, "M", "حم"), - (0xFCAB, "M", "خج"), - (0xFCAC, "M", "خم"), - (0xFCAD, "M", "سج"), - (0xFCAE, "M", "سح"), - (0xFCAF, "M", "سخ"), - (0xFCB0, "M", "سم"), - (0xFCB1, "M", "صح"), - (0xFCB2, "M", "صخ"), - (0xFCB3, "M", "صم"), - (0xFCB4, "M", "ضج"), - (0xFCB5, "M", "ضح"), - (0xFCB6, "M", "ضخ"), - (0xFCB7, "M", "ضم"), - (0xFCB8, "M", "طح"), - (0xFCB9, "M", "ظم"), - (0xFCBA, "M", "عج"), - (0xFCBB, "M", "عم"), - (0xFCBC, "M", "غج"), - (0xFCBD, "M", "غم"), - (0xFCBE, "M", "فج"), - (0xFCBF, "M", "فح"), - (0xFCC0, "M", "فخ"), - (0xFCC1, "M", "فم"), - (0xFCC2, "M", "قح"), - (0xFCC3, "M", "قم"), - (0xFCC4, "M", "كج"), - (0xFCC5, "M", "كح"), - (0xFCC6, "M", "كخ"), - (0xFCC7, "M", "كل"), - (0xFCC8, "M", "كم"), - (0xFCC9, "M", "لج"), - (0xFCCA, "M", "لح"), - (0xFCCB, "M", "لخ"), - (0xFCCC, "M", "لم"), - (0xFCCD, "M", "له"), - (0xFCCE, "M", "مج"), - (0xFCCF, "M", "مح"), - (0xFCD0, "M", "مخ"), - (0xFCD1, "M", "مم"), - (0xFCD2, "M", "نج"), - (0xFCD3, "M", "نح"), - (0xFCD4, "M", "نخ"), - (0xFCD5, "M", "نم"), - (0xFCD6, "M", "نه"), - (0xFCD7, "M", "هج"), - (0xFCD8, "M", "هم"), - (0xFCD9, "M", "هٰ"), - (0xFCDA, "M", "يج"), - (0xFCDB, "M", "يح"), - (0xFCDC, "M", "يخ"), - (0xFCDD, "M", "يم"), - (0xFCDE, "M", "يه"), - (0xFCDF, "M", "ئم"), - (0xFCE0, "M", "ئه"), - (0xFCE1, "M", "بم"), - (0xFCE2, "M", "به"), - (0xFCE3, "M", "تم"), - (0xFCE4, "M", "ته"), - (0xFCE5, "M", "ثم"), - (0xFCE6, "M", "ثه"), - (0xFCE7, "M", "سم"), - (0xFCE8, "M", "سه"), - (0xFCE9, "M", "شم"), - (0xFCEA, "M", "شه"), - (0xFCEB, "M", "كل"), - (0xFCEC, "M", "كم"), - (0xFCED, "M", "لم"), - (0xFCEE, "M", "نم"), - (0xFCEF, "M", "نه"), - (0xFCF0, "M", "يم"), - (0xFCF1, "M", "يه"), - (0xFCF2, "M", "ـَّ"), - (0xFCF3, "M", "ـُّ"), - (0xFCF4, "M", "ـِّ"), - (0xFCF5, "M", "طى"), - (0xFCF6, "M", "طي"), - (0xFCF7, "M", "عى"), - (0xFCF8, "M", "عي"), - (0xFCF9, "M", "غى"), - (0xFCFA, "M", "غي"), - (0xFCFB, "M", "سى"), - (0xFCFC, "M", "سي"), - (0xFCFD, "M", "شى"), - (0xFCFE, "M", "شي"), - (0xFCFF, "M", "حى"), - (0xFD00, "M", "حي"), - (0xFD01, "M", "جى"), - (0xFD02, "M", "جي"), - (0xFD03, "M", "خى"), - (0xFD04, "M", "خي"), - (0xFD05, "M", "صى"), - (0xFD06, "M", "صي"), - (0xFD07, "M", "ضى"), - (0xFD08, "M", "ضي"), - (0xFD09, "M", "شج"), - (0xFD0A, "M", "شح"), - (0xFD0B, "M", "شخ"), - (0xFD0C, "M", "شم"), - (0xFD0D, "M", "شر"), - (0xFD0E, "M", "سر"), - (0xFD0F, "M", "صر"), - (0xFD10, "M", "ضر"), - (0xFD11, "M", "طى"), - (0xFD12, "M", "طي"), - (0xFD13, "M", "عى"), - (0xFD14, "M", "عي"), - (0xFD15, "M", "غى"), - (0xFD16, "M", "غي"), - (0xFD17, "M", "سى"), - (0xFD18, "M", "سي"), - (0xFD19, "M", "شى"), - (0xFD1A, "M", "شي"), - (0xFD1B, "M", "حى"), - (0xFD1C, "M", "حي"), - (0xFD1D, "M", "جى"), - (0xFD1E, "M", "جي"), - (0xFD1F, "M", "خى"), - (0xFD20, "M", "خي"), - (0xFD21, "M", "صى"), - (0xFD22, "M", "صي"), - (0xFD23, "M", "ضى"), - (0xFD24, "M", "ضي"), - (0xFD25, "M", "شج"), - (0xFD26, "M", "شح"), - (0xFD27, "M", "شخ"), - (0xFD28, "M", "شم"), - (0xFD29, "M", "شر"), - (0xFD2A, "M", "سر"), - (0xFD2B, "M", "صر"), - (0xFD2C, "M", "ضر"), - (0xFD2D, "M", "شج"), - (0xFD2E, "M", "شح"), - (0xFD2F, "M", "شخ"), - (0xFD30, "M", "شم"), - (0xFD31, "M", "سه"), - (0xFD32, "M", "شه"), - (0xFD33, "M", "طم"), - (0xFD34, "M", "سج"), - (0xFD35, "M", "سح"), - (0xFD36, "M", "سخ"), - (0xFD37, "M", "شج"), - (0xFD38, "M", "شح"), - (0xFD39, "M", "شخ"), - (0xFD3A, "M", "طم"), - (0xFD3B, "M", "ظم"), - (0xFD3C, "M", "اً"), - (0xFD3E, "V"), - (0xFD50, "M", "تجم"), - (0xFD51, "M", "تحج"), - (0xFD53, "M", "تحم"), - (0xFD54, "M", "تخم"), - (0xFD55, "M", "تمج"), - (0xFD56, "M", "تمح"), - (0xFD57, "M", "تمخ"), - (0xFD58, "M", "جمح"), - (0xFD5A, "M", "حمي"), - (0xFD5B, "M", "حمى"), - (0xFD5C, "M", "سحج"), - (0xFD5D, "M", "سجح"), - (0xFD5E, "M", "سجى"), - (0xFD5F, "M", "سمح"), - (0xFD61, "M", "سمج"), - (0xFD62, "M", "سمم"), - (0xFD64, "M", "صحح"), - (0xFD66, "M", "صمم"), - (0xFD67, "M", "شحم"), - (0xFD69, "M", "شجي"), - (0xFD6A, "M", "شمخ"), - (0xFD6C, "M", "شمم"), - (0xFD6E, "M", "ضحى"), - (0xFD6F, "M", "ضخم"), - (0xFD71, "M", "طمح"), - (0xFD73, "M", "طمم"), - (0xFD74, "M", "طمي"), - (0xFD75, "M", "عجم"), - (0xFD76, "M", "عمم"), - (0xFD78, "M", "عمى"), - (0xFD79, "M", "غمم"), - (0xFD7A, "M", "غمي"), - (0xFD7B, "M", "غمى"), - (0xFD7C, "M", "فخم"), - (0xFD7E, "M", "قمح"), - (0xFD7F, "M", "قمم"), - (0xFD80, "M", "لحم"), - (0xFD81, "M", "لحي"), - (0xFD82, "M", "لحى"), - (0xFD83, "M", "لجج"), - (0xFD85, "M", "لخم"), - (0xFD87, "M", "لمح"), - (0xFD89, "M", "محج"), - (0xFD8A, "M", "محم"), - (0xFD8B, "M", "محي"), - (0xFD8C, "M", "مجح"), - (0xFD8D, "M", "مجم"), - (0xFD8E, "M", "مخج"), - (0xFD8F, "M", "مخم"), - (0xFD90, "V"), - (0xFD92, "M", "مجخ"), - (0xFD93, "M", "همج"), - (0xFD94, "M", "همم"), - (0xFD95, "M", "نحم"), - (0xFD96, "M", "نحى"), - (0xFD97, "M", "نجم"), - (0xFD99, "M", "نجى"), - (0xFD9A, "M", "نمي"), - (0xFD9B, "M", "نمى"), - (0xFD9C, "M", "يمم"), - (0xFD9E, "M", "بخي"), - (0xFD9F, "M", "تجي"), - (0xFDA0, "M", "تجى"), - (0xFDA1, "M", "تخي"), - (0xFDA2, "M", "تخى"), - (0xFDA3, "M", "تمي"), - (0xFDA4, "M", "تمى"), - (0xFDA5, "M", "جمي"), - (0xFDA6, "M", "جحى"), - (0xFDA7, "M", "جمى"), - (0xFDA8, "M", "سخى"), - (0xFDA9, "M", "صحي"), - (0xFDAA, "M", "شحي"), - (0xFDAB, "M", "ضحي"), - (0xFDAC, "M", "لجي"), - (0xFDAD, "M", "لمي"), - (0xFDAE, "M", "يحي"), - (0xFDAF, "M", "يجي"), - (0xFDB0, "M", "يمي"), - (0xFDB1, "M", "ممي"), - (0xFDB2, "M", "قمي"), - (0xFDB3, "M", "نحي"), - (0xFDB4, "M", "قمح"), - (0xFDB5, "M", "لحم"), - (0xFDB6, "M", "عمي"), - (0xFDB7, "M", "كمي"), - (0xFDB8, "M", "نجح"), - (0xFDB9, "M", "مخي"), - (0xFDBA, "M", "لجم"), - (0xFDBB, "M", "كمم"), - (0xFDBC, "M", "لجم"), - (0xFDBD, "M", "نجح"), - (0xFDBE, "M", "جحي"), - (0xFDBF, "M", "حجي"), - (0xFDC0, "M", "مجي"), - (0xFDC1, "M", "فمي"), - (0xFDC2, "M", "بحي"), - (0xFDC3, "M", "كمم"), - (0xFDC4, "M", "عجم"), - (0xFDC5, "M", "صمم"), - (0xFDC6, "M", "سخي"), - (0xFDC7, "M", "نجي"), - (0xFDC8, "V"), - (0xFDD0, "X"), - (0xFDF0, "M", "صلے"), - (0xFDF1, "M", "قلے"), - (0xFDF2, "M", "الله"), - (0xFDF3, "M", "اكبر"), - (0xFDF4, "M", "محمد"), - (0xFDF5, "M", "صلعم"), - (0xFDF6, "M", "رسول"), - (0xFDF7, "M", "عليه"), - (0xFDF8, "M", "وسلم"), - (0xFDF9, "M", "صلى"), - (0xFDFA, "M", "صلى الله عليه وسلم"), - (0xFDFB, "M", "جل جلاله"), - (0xFDFC, "M", "ریال"), - (0xFDFD, "V"), - (0xFE00, "I"), - (0xFE10, "M", ","), - (0xFE11, "M", "、"), - (0xFE12, "X"), - (0xFE13, "M", ":"), - (0xFE14, "M", ";"), - (0xFE15, "M", "!"), - (0xFE16, "M", "?"), - (0xFE17, "M", "〖"), - (0xFE18, "M", "〗"), - (0xFE19, "X"), - (0xFE20, "V"), - (0xFE30, "X"), - (0xFE31, "M", "—"), - (0xFE32, "M", "–"), - (0xFE33, "M", "_"), - (0xFE35, "M", "("), - (0xFE36, "M", ")"), - (0xFE37, "M", "{"), - (0xFE38, "M", "}"), - (0xFE39, "M", "〔"), - (0xFE3A, "M", "〕"), - (0xFE3B, "M", "【"), - (0xFE3C, "M", "】"), - (0xFE3D, "M", "《"), - (0xFE3E, "M", "》"), - (0xFE3F, "M", "〈"), - (0xFE40, "M", "〉"), - (0xFE41, "M", "「"), - (0xFE42, "M", "」"), - (0xFE43, "M", "『"), - (0xFE44, "M", "』"), - (0xFE45, "V"), - (0xFE47, "M", "["), - (0xFE48, "M", "]"), - (0xFE49, "M", " ̅"), - (0xFE4D, "M", "_"), - (0xFE50, "M", ","), - (0xFE51, "M", "、"), - (0xFE52, "X"), - (0xFE54, "M", ";"), - (0xFE55, "M", ":"), - (0xFE56, "M", "?"), - (0xFE57, "M", "!"), - (0xFE58, "M", "—"), - (0xFE59, "M", "("), - (0xFE5A, "M", ")"), - (0xFE5B, "M", "{"), - (0xFE5C, "M", "}"), - (0xFE5D, "M", "〔"), - (0xFE5E, "M", "〕"), - (0xFE5F, "M", "#"), - (0xFE60, "M", "&"), - (0xFE61, "M", "*"), - (0xFE62, "M", "+"), - (0xFE63, "M", "-"), - (0xFE64, "M", "<"), - (0xFE65, "M", ">"), - (0xFE66, "M", "="), - (0xFE67, "X"), - (0xFE68, "M", "\\"), - (0xFE69, "M", "$"), - (0xFE6A, "M", "%"), - (0xFE6B, "M", "@"), - (0xFE6C, "X"), - (0xFE70, "M", " ً"), - (0xFE71, "M", "ـً"), - (0xFE72, "M", " ٌ"), - (0xFE73, "V"), - (0xFE74, "M", " ٍ"), - (0xFE75, "X"), - (0xFE76, "M", " َ"), - (0xFE77, "M", "ـَ"), - (0xFE78, "M", " ُ"), - (0xFE79, "M", "ـُ"), - (0xFE7A, "M", " ِ"), - (0xFE7B, "M", "ـِ"), - (0xFE7C, "M", " ّ"), - (0xFE7D, "M", "ـّ"), - (0xFE7E, "M", " ْ"), - (0xFE7F, "M", "ـْ"), - (0xFE80, "M", "ء"), - (0xFE81, "M", "آ"), - (0xFE83, "M", "أ"), - (0xFE85, "M", "ؤ"), - (0xFE87, "M", "إ"), - (0xFE89, "M", "ئ"), - (0xFE8D, "M", "ا"), - (0xFE8F, "M", "ب"), - (0xFE93, "M", "ة"), - (0xFE95, "M", "ت"), - (0xFE99, "M", "ث"), - (0xFE9D, "M", "ج"), - (0xFEA1, "M", "ح"), - (0xFEA5, "M", "خ"), - (0xFEA9, "M", "د"), - (0xFEAB, "M", "ذ"), - (0xFEAD, "M", "ر"), - (0xFEAF, "M", "ز"), - (0xFEB1, "M", "س"), - (0xFEB5, "M", "ش"), - (0xFEB9, "M", "ص"), - (0xFEBD, "M", "ض"), - (0xFEC1, "M", "ط"), - (0xFEC5, "M", "ظ"), - (0xFEC9, "M", "ع"), - (0xFECD, "M", "غ"), - (0xFED1, "M", "ف"), - (0xFED5, "M", "ق"), - (0xFED9, "M", "ك"), - (0xFEDD, "M", "ل"), - (0xFEE1, "M", "م"), - (0xFEE5, "M", "ن"), - (0xFEE9, "M", "ه"), - (0xFEED, "M", "و"), - (0xFEEF, "M", "ى"), - (0xFEF1, "M", "ي"), - (0xFEF5, "M", "لآ"), - (0xFEF7, "M", "لأ"), - (0xFEF9, "M", "لإ"), - (0xFEFB, "M", "لا"), - (0xFEFD, "X"), - (0xFEFF, "I"), - (0xFF00, "X"), - (0xFF01, "M", "!"), - (0xFF02, "M", '"'), - (0xFF03, "M", "#"), - (0xFF04, "M", "$"), - (0xFF05, "M", "%"), - (0xFF06, "M", "&"), - (0xFF07, "M", "'"), - (0xFF08, "M", "("), - (0xFF09, "M", ")"), - (0xFF0A, "M", "*"), - (0xFF0B, "M", "+"), - (0xFF0C, "M", ","), - (0xFF0D, "M", "-"), - (0xFF0E, "M", "."), - (0xFF0F, "M", "/"), - (0xFF10, "M", "0"), - (0xFF11, "M", "1"), - (0xFF12, "M", "2"), - (0xFF13, "M", "3"), - (0xFF14, "M", "4"), - (0xFF15, "M", "5"), - (0xFF16, "M", "6"), - (0xFF17, "M", "7"), - (0xFF18, "M", "8"), - (0xFF19, "M", "9"), - (0xFF1A, "M", ":"), - (0xFF1B, "M", ";"), - (0xFF1C, "M", "<"), - (0xFF1D, "M", "="), - (0xFF1E, "M", ">"), - (0xFF1F, "M", "?"), - (0xFF20, "M", "@"), - (0xFF21, "M", "a"), - (0xFF22, "M", "b"), - (0xFF23, "M", "c"), - (0xFF24, "M", "d"), - (0xFF25, "M", "e"), - (0xFF26, "M", "f"), - (0xFF27, "M", "g"), - (0xFF28, "M", "h"), - (0xFF29, "M", "i"), - (0xFF2A, "M", "j"), - (0xFF2B, "M", "k"), - (0xFF2C, "M", "l"), - (0xFF2D, "M", "m"), - (0xFF2E, "M", "n"), - (0xFF2F, "M", "o"), - (0xFF30, "M", "p"), - (0xFF31, "M", "q"), - (0xFF32, "M", "r"), - (0xFF33, "M", "s"), - (0xFF34, "M", "t"), - (0xFF35, "M", "u"), - (0xFF36, "M", "v"), - (0xFF37, "M", "w"), - (0xFF38, "M", "x"), - (0xFF39, "M", "y"), - (0xFF3A, "M", "z"), - (0xFF3B, "M", "["), - (0xFF3C, "M", "\\"), - (0xFF3D, "M", "]"), - (0xFF3E, "M", "^"), - (0xFF3F, "M", "_"), - (0xFF40, "M", "`"), - (0xFF41, "M", "a"), - (0xFF42, "M", "b"), - (0xFF43, "M", "c"), - (0xFF44, "M", "d"), - (0xFF45, "M", "e"), - (0xFF46, "M", "f"), - (0xFF47, "M", "g"), - (0xFF48, "M", "h"), - (0xFF49, "M", "i"), - (0xFF4A, "M", "j"), - (0xFF4B, "M", "k"), - (0xFF4C, "M", "l"), - (0xFF4D, "M", "m"), - (0xFF4E, "M", "n"), - (0xFF4F, "M", "o"), - (0xFF50, "M", "p"), - (0xFF51, "M", "q"), - (0xFF52, "M", "r"), - (0xFF53, "M", "s"), - (0xFF54, "M", "t"), - (0xFF55, "M", "u"), - (0xFF56, "M", "v"), - (0xFF57, "M", "w"), - (0xFF58, "M", "x"), - (0xFF59, "M", "y"), - (0xFF5A, "M", "z"), - (0xFF5B, "M", "{"), - (0xFF5C, "M", "|"), - (0xFF5D, "M", "}"), - (0xFF5E, "M", "~"), - (0xFF5F, "M", "⦅"), - (0xFF60, "M", "⦆"), - (0xFF61, "M", "."), - (0xFF62, "M", "「"), - (0xFF63, "M", "」"), - (0xFF64, "M", "、"), - (0xFF65, "M", "・"), - (0xFF66, "M", "ヲ"), - (0xFF67, "M", "ァ"), - (0xFF68, "M", "ィ"), - (0xFF69, "M", "ゥ"), - (0xFF6A, "M", "ェ"), - (0xFF6B, "M", "ォ"), - (0xFF6C, "M", "ャ"), - (0xFF6D, "M", "ュ"), - (0xFF6E, "M", "ョ"), - (0xFF6F, "M", "ッ"), - (0xFF70, "M", "ー"), - (0xFF71, "M", "ア"), - (0xFF72, "M", "イ"), - (0xFF73, "M", "ウ"), - (0xFF74, "M", "エ"), - (0xFF75, "M", "オ"), - (0xFF76, "M", "カ"), - (0xFF77, "M", "キ"), - (0xFF78, "M", "ク"), - (0xFF79, "M", "ケ"), - (0xFF7A, "M", "コ"), - (0xFF7B, "M", "サ"), - (0xFF7C, "M", "シ"), - (0xFF7D, "M", "ス"), - (0xFF7E, "M", "セ"), - (0xFF7F, "M", "ソ"), - (0xFF80, "M", "タ"), - (0xFF81, "M", "チ"), - (0xFF82, "M", "ツ"), - (0xFF83, "M", "テ"), - (0xFF84, "M", "ト"), - (0xFF85, "M", "ナ"), - (0xFF86, "M", "ニ"), - (0xFF87, "M", "ヌ"), - (0xFF88, "M", "ネ"), - (0xFF89, "M", "ノ"), - (0xFF8A, "M", "ハ"), - (0xFF8B, "M", "ヒ"), - (0xFF8C, "M", "フ"), - (0xFF8D, "M", "ヘ"), - (0xFF8E, "M", "ホ"), - (0xFF8F, "M", "マ"), - (0xFF90, "M", "ミ"), - (0xFF91, "M", "ム"), - (0xFF92, "M", "メ"), - (0xFF93, "M", "モ"), - (0xFF94, "M", "ヤ"), - (0xFF95, "M", "ユ"), - (0xFF96, "M", "ヨ"), - (0xFF97, "M", "ラ"), - (0xFF98, "M", "リ"), - (0xFF99, "M", "ル"), - (0xFF9A, "M", "レ"), - (0xFF9B, "M", "ロ"), - (0xFF9C, "M", "ワ"), - (0xFF9D, "M", "ン"), - (0xFF9E, "M", "゙"), - (0xFF9F, "M", "゚"), - (0xFFA0, "I"), - (0xFFA1, "M", "ᄀ"), - (0xFFA2, "M", "ᄁ"), - (0xFFA3, "M", "ᆪ"), - (0xFFA4, "M", "ᄂ"), - (0xFFA5, "M", "ᆬ"), - (0xFFA6, "M", "ᆭ"), - (0xFFA7, "M", "ᄃ"), - (0xFFA8, "M", "ᄄ"), - (0xFFA9, "M", "ᄅ"), - (0xFFAA, "M", "ᆰ"), - (0xFFAB, "M", "ᆱ"), - (0xFFAC, "M", "ᆲ"), - (0xFFAD, "M", "ᆳ"), - (0xFFAE, "M", "ᆴ"), - (0xFFAF, "M", "ᆵ"), - (0xFFB0, "M", "ᄚ"), - (0xFFB1, "M", "ᄆ"), - (0xFFB2, "M", "ᄇ"), - (0xFFB3, "M", "ᄈ"), - (0xFFB4, "M", "ᄡ"), - (0xFFB5, "M", "ᄉ"), - (0xFFB6, "M", "ᄊ"), - (0xFFB7, "M", "ᄋ"), - (0xFFB8, "M", "ᄌ"), - (0xFFB9, "M", "ᄍ"), - (0xFFBA, "M", "ᄎ"), - (0xFFBB, "M", "ᄏ"), - (0xFFBC, "M", "ᄐ"), - (0xFFBD, "M", "ᄑ"), - (0xFFBE, "M", "ᄒ"), - (0xFFBF, "X"), - (0xFFC2, "M", "ᅡ"), - (0xFFC3, "M", "ᅢ"), - (0xFFC4, "M", "ᅣ"), - (0xFFC5, "M", "ᅤ"), - (0xFFC6, "M", "ᅥ"), - (0xFFC7, "M", "ᅦ"), - (0xFFC8, "X"), - (0xFFCA, "M", "ᅧ"), - (0xFFCB, "M", "ᅨ"), - (0xFFCC, "M", "ᅩ"), - (0xFFCD, "M", "ᅪ"), - (0xFFCE, "M", "ᅫ"), - (0xFFCF, "M", "ᅬ"), - (0xFFD0, "X"), - (0xFFD2, "M", "ᅭ"), - (0xFFD3, "M", "ᅮ"), - (0xFFD4, "M", "ᅯ"), - (0xFFD5, "M", "ᅰ"), - (0xFFD6, "M", "ᅱ"), - (0xFFD7, "M", "ᅲ"), - (0xFFD8, "X"), - (0xFFDA, "M", "ᅳ"), - (0xFFDB, "M", "ᅴ"), - (0xFFDC, "M", "ᅵ"), - (0xFFDD, "X"), - (0xFFE0, "M", "¢"), - (0xFFE1, "M", "£"), - (0xFFE2, "M", "¬"), - (0xFFE3, "M", " ̄"), - (0xFFE4, "M", "¦"), - (0xFFE5, "M", "¥"), - (0xFFE6, "M", "₩"), - (0xFFE7, "X"), - (0xFFE8, "M", "│"), - (0xFFE9, "M", "←"), - (0xFFEA, "M", "↑"), - (0xFFEB, "M", "→"), - (0xFFEC, "M", "↓"), - (0xFFED, "M", "■"), - (0xFFEE, "M", "○"), - (0xFFEF, "X"), - (0x10000, "V"), - (0x1000C, "X"), - (0x1000D, "V"), - (0x10027, "X"), - (0x10028, "V"), - (0x1003B, "X"), - (0x1003C, "V"), - (0x1003E, "X"), - (0x1003F, "V"), - (0x1004E, "X"), - (0x10050, "V"), - (0x1005E, "X"), - (0x10080, "V"), - (0x100FB, "X"), - (0x10100, "V"), - (0x10103, "X"), - (0x10107, "V"), - (0x10134, "X"), - (0x10137, "V"), - (0x1018F, "X"), - (0x10190, "V"), - (0x1019D, "X"), - (0x101A0, "V"), - (0x101A1, "X"), - (0x101D0, "V"), - (0x101FE, "X"), - (0x10280, "V"), - (0x1029D, "X"), - (0x102A0, "V"), - (0x102D1, "X"), - (0x102E0, "V"), - (0x102FC, "X"), - (0x10300, "V"), - (0x10324, "X"), - (0x1032D, "V"), - (0x1034B, "X"), - (0x10350, "V"), - (0x1037B, "X"), - (0x10380, "V"), - (0x1039E, "X"), - (0x1039F, "V"), - (0x103C4, "X"), - (0x103C8, "V"), - (0x103D6, "X"), - (0x10400, "M", "𐐨"), - (0x10401, "M", "𐐩"), - (0x10402, "M", "𐐪"), - (0x10403, "M", "𐐫"), - (0x10404, "M", "𐐬"), - (0x10405, "M", "𐐭"), - (0x10406, "M", "𐐮"), - (0x10407, "M", "𐐯"), - (0x10408, "M", "𐐰"), - (0x10409, "M", "𐐱"), - (0x1040A, "M", "𐐲"), - (0x1040B, "M", "𐐳"), - (0x1040C, "M", "𐐴"), - (0x1040D, "M", "𐐵"), - (0x1040E, "M", "𐐶"), - (0x1040F, "M", "𐐷"), - (0x10410, "M", "𐐸"), - (0x10411, "M", "𐐹"), - (0x10412, "M", "𐐺"), - (0x10413, "M", "𐐻"), - (0x10414, "M", "𐐼"), - (0x10415, "M", "𐐽"), - (0x10416, "M", "𐐾"), - (0x10417, "M", "𐐿"), - (0x10418, "M", "𐑀"), - (0x10419, "M", "𐑁"), - (0x1041A, "M", "𐑂"), - (0x1041B, "M", "𐑃"), - (0x1041C, "M", "𐑄"), - (0x1041D, "M", "𐑅"), - (0x1041E, "M", "𐑆"), - (0x1041F, "M", "𐑇"), - (0x10420, "M", "𐑈"), - (0x10421, "M", "𐑉"), - (0x10422, "M", "𐑊"), - (0x10423, "M", "𐑋"), - (0x10424, "M", "𐑌"), - (0x10425, "M", "𐑍"), - (0x10426, "M", "𐑎"), - (0x10427, "M", "𐑏"), - (0x10428, "V"), - (0x1049E, "X"), - (0x104A0, "V"), - (0x104AA, "X"), - (0x104B0, "M", "𐓘"), - (0x104B1, "M", "𐓙"), - (0x104B2, "M", "𐓚"), - (0x104B3, "M", "𐓛"), - (0x104B4, "M", "𐓜"), - (0x104B5, "M", "𐓝"), - (0x104B6, "M", "𐓞"), - (0x104B7, "M", "𐓟"), - (0x104B8, "M", "𐓠"), - (0x104B9, "M", "𐓡"), - (0x104BA, "M", "𐓢"), - (0x104BB, "M", "𐓣"), - (0x104BC, "M", "𐓤"), - (0x104BD, "M", "𐓥"), - (0x104BE, "M", "𐓦"), - (0x104BF, "M", "𐓧"), - (0x104C0, "M", "𐓨"), - (0x104C1, "M", "𐓩"), - (0x104C2, "M", "𐓪"), - (0x104C3, "M", "𐓫"), - (0x104C4, "M", "𐓬"), - (0x104C5, "M", "𐓭"), - (0x104C6, "M", "𐓮"), - (0x104C7, "M", "𐓯"), - (0x104C8, "M", "𐓰"), - (0x104C9, "M", "𐓱"), - (0x104CA, "M", "𐓲"), - (0x104CB, "M", "𐓳"), - (0x104CC, "M", "𐓴"), - (0x104CD, "M", "𐓵"), - (0x104CE, "M", "𐓶"), - (0x104CF, "M", "𐓷"), - (0x104D0, "M", "𐓸"), - (0x104D1, "M", "𐓹"), - (0x104D2, "M", "𐓺"), - (0x104D3, "M", "𐓻"), - (0x104D4, "X"), - (0x104D8, "V"), - (0x104FC, "X"), - (0x10500, "V"), - (0x10528, "X"), - (0x10530, "V"), - (0x10564, "X"), - (0x1056F, "V"), - (0x10570, "M", "𐖗"), - (0x10571, "M", "𐖘"), - (0x10572, "M", "𐖙"), - (0x10573, "M", "𐖚"), - (0x10574, "M", "𐖛"), - (0x10575, "M", "𐖜"), - (0x10576, "M", "𐖝"), - (0x10577, "M", "𐖞"), - (0x10578, "M", "𐖟"), - (0x10579, "M", "𐖠"), - (0x1057A, "M", "𐖡"), - (0x1057B, "X"), - (0x1057C, "M", "𐖣"), - (0x1057D, "M", "𐖤"), - (0x1057E, "M", "𐖥"), - (0x1057F, "M", "𐖦"), - (0x10580, "M", "𐖧"), - (0x10581, "M", "𐖨"), - (0x10582, "M", "𐖩"), - (0x10583, "M", "𐖪"), - (0x10584, "M", "𐖫"), - (0x10585, "M", "𐖬"), - (0x10586, "M", "𐖭"), - (0x10587, "M", "𐖮"), - (0x10588, "M", "𐖯"), - (0x10589, "M", "𐖰"), - (0x1058A, "M", "𐖱"), - (0x1058B, "X"), - (0x1058C, "M", "𐖳"), - (0x1058D, "M", "𐖴"), - (0x1058E, "M", "𐖵"), - (0x1058F, "M", "𐖶"), - (0x10590, "M", "𐖷"), - (0x10591, "M", "𐖸"), - (0x10592, "M", "𐖹"), - (0x10593, "X"), - (0x10594, "M", "𐖻"), - (0x10595, "M", "𐖼"), - (0x10596, "X"), - (0x10597, "V"), - (0x105A2, "X"), - (0x105A3, "V"), - (0x105B2, "X"), - (0x105B3, "V"), - (0x105BA, "X"), - (0x105BB, "V"), - (0x105BD, "X"), - (0x105C0, "V"), - (0x105F4, "X"), - (0x10600, "V"), - (0x10737, "X"), - (0x10740, "V"), - (0x10756, "X"), - (0x10760, "V"), - (0x10768, "X"), - (0x10780, "V"), - (0x10781, "M", "ː"), - (0x10782, "M", "ˑ"), - (0x10783, "M", "æ"), - (0x10784, "M", "ʙ"), - (0x10785, "M", "ɓ"), - (0x10786, "X"), - (0x10787, "M", "ʣ"), - (0x10788, "M", "ꭦ"), - (0x10789, "M", "ʥ"), - (0x1078A, "M", "ʤ"), - (0x1078B, "M", "ɖ"), - (0x1078C, "M", "ɗ"), - (0x1078D, "M", "ᶑ"), - (0x1078E, "M", "ɘ"), - (0x1078F, "M", "ɞ"), - (0x10790, "M", "ʩ"), - (0x10791, "M", "ɤ"), - (0x10792, "M", "ɢ"), - (0x10793, "M", "ɠ"), - (0x10794, "M", "ʛ"), - (0x10795, "M", "ħ"), - (0x10796, "M", "ʜ"), - (0x10797, "M", "ɧ"), - (0x10798, "M", "ʄ"), - (0x10799, "M", "ʪ"), - (0x1079A, "M", "ʫ"), - (0x1079B, "M", "ɬ"), - (0x1079C, "M", "𝼄"), - (0x1079D, "M", "ꞎ"), - (0x1079E, "M", "ɮ"), - (0x1079F, "M", "𝼅"), - (0x107A0, "M", "ʎ"), - (0x107A1, "M", "𝼆"), - (0x107A2, "M", "ø"), - (0x107A3, "M", "ɶ"), - (0x107A4, "M", "ɷ"), - (0x107A5, "M", "q"), - (0x107A6, "M", "ɺ"), - (0x107A7, "M", "𝼈"), - (0x107A8, "M", "ɽ"), - (0x107A9, "M", "ɾ"), - (0x107AA, "M", "ʀ"), - (0x107AB, "M", "ʨ"), - (0x107AC, "M", "ʦ"), - (0x107AD, "M", "ꭧ"), - (0x107AE, "M", "ʧ"), - (0x107AF, "M", "ʈ"), - (0x107B0, "M", "ⱱ"), - (0x107B1, "X"), - (0x107B2, "M", "ʏ"), - (0x107B3, "M", "ʡ"), - (0x107B4, "M", "ʢ"), - (0x107B5, "M", "ʘ"), - (0x107B6, "M", "ǀ"), - (0x107B7, "M", "ǁ"), - (0x107B8, "M", "ǂ"), - (0x107B9, "M", "𝼊"), - (0x107BA, "M", "𝼞"), - (0x107BB, "X"), - (0x10800, "V"), - (0x10806, "X"), - (0x10808, "V"), - (0x10809, "X"), - (0x1080A, "V"), - (0x10836, "X"), - (0x10837, "V"), - (0x10839, "X"), - (0x1083C, "V"), - (0x1083D, "X"), - (0x1083F, "V"), - (0x10856, "X"), - (0x10857, "V"), - (0x1089F, "X"), - (0x108A7, "V"), - (0x108B0, "X"), - (0x108E0, "V"), - (0x108F3, "X"), - (0x108F4, "V"), - (0x108F6, "X"), - (0x108FB, "V"), - (0x1091C, "X"), - (0x1091F, "V"), - (0x1093A, "X"), - (0x1093F, "V"), - (0x1095A, "X"), - (0x10980, "V"), - (0x109B8, "X"), - (0x109BC, "V"), - (0x109D0, "X"), - (0x109D2, "V"), - (0x10A04, "X"), - (0x10A05, "V"), - (0x10A07, "X"), - (0x10A0C, "V"), - (0x10A14, "X"), - (0x10A15, "V"), - (0x10A18, "X"), - (0x10A19, "V"), - (0x10A36, "X"), - (0x10A38, "V"), - (0x10A3B, "X"), - (0x10A3F, "V"), - (0x10A49, "X"), - (0x10A50, "V"), - (0x10A59, "X"), - (0x10A60, "V"), - (0x10AA0, "X"), - (0x10AC0, "V"), - (0x10AE7, "X"), - (0x10AEB, "V"), - (0x10AF7, "X"), - (0x10B00, "V"), - (0x10B36, "X"), - (0x10B39, "V"), - (0x10B56, "X"), - (0x10B58, "V"), - (0x10B73, "X"), - (0x10B78, "V"), - (0x10B92, "X"), - (0x10B99, "V"), - (0x10B9D, "X"), - (0x10BA9, "V"), - (0x10BB0, "X"), - (0x10C00, "V"), - (0x10C49, "X"), - (0x10C80, "M", "𐳀"), - (0x10C81, "M", "𐳁"), - (0x10C82, "M", "𐳂"), - (0x10C83, "M", "𐳃"), - (0x10C84, "M", "𐳄"), - (0x10C85, "M", "𐳅"), - (0x10C86, "M", "𐳆"), - (0x10C87, "M", "𐳇"), - (0x10C88, "M", "𐳈"), - (0x10C89, "M", "𐳉"), - (0x10C8A, "M", "𐳊"), - (0x10C8B, "M", "𐳋"), - (0x10C8C, "M", "𐳌"), - (0x10C8D, "M", "𐳍"), - (0x10C8E, "M", "𐳎"), - (0x10C8F, "M", "𐳏"), - (0x10C90, "M", "𐳐"), - (0x10C91, "M", "𐳑"), - (0x10C92, "M", "𐳒"), - (0x10C93, "M", "𐳓"), - (0x10C94, "M", "𐳔"), - (0x10C95, "M", "𐳕"), - (0x10C96, "M", "𐳖"), - (0x10C97, "M", "𐳗"), - (0x10C98, "M", "𐳘"), - (0x10C99, "M", "𐳙"), - (0x10C9A, "M", "𐳚"), - (0x10C9B, "M", "𐳛"), - (0x10C9C, "M", "𐳜"), - (0x10C9D, "M", "𐳝"), - (0x10C9E, "M", "𐳞"), - (0x10C9F, "M", "𐳟"), - (0x10CA0, "M", "𐳠"), - (0x10CA1, "M", "𐳡"), - (0x10CA2, "M", "𐳢"), - (0x10CA3, "M", "𐳣"), - (0x10CA4, "M", "𐳤"), - (0x10CA5, "M", "𐳥"), - (0x10CA6, "M", "𐳦"), - (0x10CA7, "M", "𐳧"), - (0x10CA8, "M", "𐳨"), - (0x10CA9, "M", "𐳩"), - (0x10CAA, "M", "𐳪"), - (0x10CAB, "M", "𐳫"), - (0x10CAC, "M", "𐳬"), - (0x10CAD, "M", "𐳭"), - (0x10CAE, "M", "𐳮"), - (0x10CAF, "M", "𐳯"), - (0x10CB0, "M", "𐳰"), - (0x10CB1, "M", "𐳱"), - (0x10CB2, "M", "𐳲"), - (0x10CB3, "X"), - (0x10CC0, "V"), - (0x10CF3, "X"), - (0x10CFA, "V"), - (0x10D28, "X"), - (0x10D30, "V"), - (0x10D3A, "X"), - (0x10D40, "V"), - (0x10D50, "M", ""), - (0x10D51, "M", ""), - (0x10D52, "M", ""), - (0x10D53, "M", ""), - (0x10D54, "M", ""), - (0x10D55, "M", ""), - (0x10D56, "M", ""), - (0x10D57, "M", ""), - (0x10D58, "M", ""), - (0x10D59, "M", ""), - (0x10D5A, "M", ""), - (0x10D5B, "M", ""), - (0x10D5C, "M", ""), - (0x10D5D, "M", ""), - (0x10D5E, "M", ""), - (0x10D5F, "M", ""), - (0x10D60, "M", ""), - (0x10D61, "M", ""), - (0x10D62, "M", ""), - (0x10D63, "M", ""), - (0x10D64, "M", ""), - (0x10D65, "M", ""), - (0x10D66, "X"), - (0x10D69, "V"), - (0x10D86, "X"), - (0x10D8E, "V"), - (0x10D90, "X"), - (0x10E60, "V"), - (0x10E7F, "X"), - (0x10E80, "V"), - (0x10EAA, "X"), - (0x10EAB, "V"), - (0x10EAE, "X"), - (0x10EB0, "V"), - (0x10EB2, "X"), - (0x10EC2, "V"), - (0x10EC8, "X"), - (0x10ED0, "V"), - (0x10ED9, "X"), - (0x10EFA, "V"), - (0x10F28, "X"), - (0x10F30, "V"), - (0x10F5A, "X"), - (0x10F70, "V"), - (0x10F8A, "X"), - (0x10FB0, "V"), - (0x10FCC, "X"), - (0x10FE0, "V"), - (0x10FF7, "X"), - (0x11000, "V"), - (0x1104E, "X"), - (0x11052, "V"), - (0x11076, "X"), - (0x1107F, "V"), - (0x110BD, "X"), - (0x110BE, "V"), - (0x110C3, "X"), - (0x110D0, "V"), - (0x110E9, "X"), - (0x110F0, "V"), - (0x110FA, "X"), - (0x11100, "V"), - (0x11135, "X"), - (0x11136, "V"), - (0x11148, "X"), - (0x11150, "V"), - (0x11177, "X"), - (0x11180, "V"), - (0x111E0, "X"), - (0x111E1, "V"), - (0x111F5, "X"), - (0x11200, "V"), - (0x11212, "X"), - (0x11213, "V"), - (0x11242, "X"), - (0x11280, "V"), - (0x11287, "X"), - (0x11288, "V"), - (0x11289, "X"), - (0x1128A, "V"), - (0x1128E, "X"), - (0x1128F, "V"), - (0x1129E, "X"), - (0x1129F, "V"), - (0x112AA, "X"), - (0x112B0, "V"), - (0x112EB, "X"), - (0x112F0, "V"), - (0x112FA, "X"), - (0x11300, "V"), - (0x11304, "X"), - (0x11305, "V"), - (0x1130D, "X"), - (0x1130F, "V"), - (0x11311, "X"), - (0x11313, "V"), - (0x11329, "X"), - (0x1132A, "V"), - (0x11331, "X"), - (0x11332, "V"), - (0x11334, "X"), - (0x11335, "V"), - (0x1133A, "X"), - (0x1133B, "V"), - (0x11345, "X"), - (0x11347, "V"), - (0x11349, "X"), - (0x1134B, "V"), - (0x1134E, "X"), - (0x11350, "V"), - (0x11351, "X"), - (0x11357, "V"), - (0x11358, "X"), - (0x1135D, "V"), - (0x11364, "X"), - (0x11366, "V"), - (0x1136D, "X"), - (0x11370, "V"), - (0x11375, "X"), - (0x11380, "V"), - (0x1138A, "X"), - (0x1138B, "V"), - (0x1138C, "X"), - (0x1138E, "V"), - (0x1138F, "X"), - (0x11390, "V"), - (0x113B6, "X"), - (0x113B7, "V"), - (0x113C1, "X"), - (0x113C2, "V"), - (0x113C3, "X"), - (0x113C5, "V"), - (0x113C6, "X"), - (0x113C7, "V"), - (0x113CB, "X"), - (0x113CC, "V"), - (0x113D6, "X"), - (0x113D7, "V"), - (0x113D9, "X"), - (0x113E1, "V"), - (0x113E3, "X"), - (0x11400, "V"), - (0x1145C, "X"), - (0x1145D, "V"), - (0x11462, "X"), - (0x11480, "V"), - (0x114C8, "X"), - (0x114D0, "V"), - (0x114DA, "X"), - (0x11580, "V"), - (0x115B6, "X"), - (0x115B8, "V"), - (0x115DE, "X"), - (0x11600, "V"), - (0x11645, "X"), - (0x11650, "V"), - (0x1165A, "X"), - (0x11660, "V"), - (0x1166D, "X"), - (0x11680, "V"), - (0x116BA, "X"), - (0x116C0, "V"), - (0x116CA, "X"), - (0x116D0, "V"), - (0x116E4, "X"), - (0x11700, "V"), - (0x1171B, "X"), - (0x1171D, "V"), - (0x1172C, "X"), - (0x11730, "V"), - (0x11747, "X"), - (0x11800, "V"), - (0x1183C, "X"), - (0x118A0, "M", "𑣀"), - (0x118A1, "M", "𑣁"), - (0x118A2, "M", "𑣂"), - (0x118A3, "M", "𑣃"), - (0x118A4, "M", "𑣄"), - (0x118A5, "M", "𑣅"), - (0x118A6, "M", "𑣆"), - (0x118A7, "M", "𑣇"), - (0x118A8, "M", "𑣈"), - (0x118A9, "M", "𑣉"), - (0x118AA, "M", "𑣊"), - (0x118AB, "M", "𑣋"), - (0x118AC, "M", "𑣌"), - (0x118AD, "M", "𑣍"), - (0x118AE, "M", "𑣎"), - (0x118AF, "M", "𑣏"), - (0x118B0, "M", "𑣐"), - (0x118B1, "M", "𑣑"), - (0x118B2, "M", "𑣒"), - (0x118B3, "M", "𑣓"), - (0x118B4, "M", "𑣔"), - (0x118B5, "M", "𑣕"), - (0x118B6, "M", "𑣖"), - (0x118B7, "M", "𑣗"), - (0x118B8, "M", "𑣘"), - (0x118B9, "M", "𑣙"), - (0x118BA, "M", "𑣚"), - (0x118BB, "M", "𑣛"), - (0x118BC, "M", "𑣜"), - (0x118BD, "M", "𑣝"), - (0x118BE, "M", "𑣞"), - (0x118BF, "M", "𑣟"), - (0x118C0, "V"), - (0x118F3, "X"), - (0x118FF, "V"), - (0x11907, "X"), - (0x11909, "V"), - (0x1190A, "X"), - (0x1190C, "V"), - (0x11914, "X"), - (0x11915, "V"), - (0x11917, "X"), - (0x11918, "V"), - (0x11936, "X"), - (0x11937, "V"), - (0x11939, "X"), - (0x1193B, "V"), - (0x11947, "X"), - (0x11950, "V"), - (0x1195A, "X"), - (0x119A0, "V"), - (0x119A8, "X"), - (0x119AA, "V"), - (0x119D8, "X"), - (0x119DA, "V"), - (0x119E5, "X"), - (0x11A00, "V"), - (0x11A48, "X"), - (0x11A50, "V"), - (0x11AA3, "X"), - (0x11AB0, "V"), - (0x11AF9, "X"), - (0x11B00, "V"), - (0x11B0A, "X"), - (0x11B60, "V"), - (0x11B68, "X"), - (0x11BC0, "V"), - (0x11BE2, "X"), - (0x11BF0, "V"), - (0x11BFA, "X"), - (0x11C00, "V"), - (0x11C09, "X"), - (0x11C0A, "V"), - (0x11C37, "X"), - (0x11C38, "V"), - (0x11C46, "X"), - (0x11C50, "V"), - (0x11C6D, "X"), - (0x11C70, "V"), - (0x11C90, "X"), - (0x11C92, "V"), - (0x11CA8, "X"), - (0x11CA9, "V"), - (0x11CB7, "X"), - (0x11D00, "V"), - (0x11D07, "X"), - (0x11D08, "V"), - (0x11D0A, "X"), - (0x11D0B, "V"), - (0x11D37, "X"), - (0x11D3A, "V"), - (0x11D3B, "X"), - (0x11D3C, "V"), - (0x11D3E, "X"), - (0x11D3F, "V"), - (0x11D48, "X"), - (0x11D50, "V"), - (0x11D5A, "X"), - (0x11D60, "V"), - (0x11D66, "X"), - (0x11D67, "V"), - (0x11D69, "X"), - (0x11D6A, "V"), - (0x11D8F, "X"), - (0x11D90, "V"), - (0x11D92, "X"), - (0x11D93, "V"), - (0x11D99, "X"), - (0x11DA0, "V"), - (0x11DAA, "X"), - (0x11DB0, "V"), - (0x11DDC, "X"), - (0x11DE0, "V"), - (0x11DEA, "X"), - (0x11EE0, "V"), - (0x11EF9, "X"), - (0x11F00, "V"), - (0x11F11, "X"), - (0x11F12, "V"), - (0x11F3B, "X"), - (0x11F3E, "V"), - (0x11F5B, "X"), - (0x11FB0, "V"), - (0x11FB1, "X"), - (0x11FC0, "V"), - (0x11FF2, "X"), - (0x11FFF, "V"), - (0x1239A, "X"), - (0x12400, "V"), - (0x1246F, "X"), - (0x12470, "V"), - (0x12475, "X"), - (0x12480, "V"), - (0x12544, "X"), - (0x12F90, "V"), - (0x12FF3, "X"), - (0x13000, "V"), - (0x13430, "X"), - (0x13440, "V"), - (0x13456, "X"), - (0x13460, "V"), - (0x143FB, "X"), - (0x14400, "V"), - (0x14647, "X"), - (0x16100, "V"), - (0x1613A, "X"), - (0x16800, "V"), - (0x16A39, "X"), - (0x16A40, "V"), - (0x16A5F, "X"), - (0x16A60, "V"), - (0x16A6A, "X"), - (0x16A6E, "V"), - (0x16ABF, "X"), - (0x16AC0, "V"), - (0x16ACA, "X"), - (0x16AD0, "V"), - (0x16AEE, "X"), - (0x16AF0, "V"), - (0x16AF6, "X"), - (0x16B00, "V"), - (0x16B46, "X"), - (0x16B50, "V"), - (0x16B5A, "X"), - (0x16B5B, "V"), - (0x16B62, "X"), - (0x16B63, "V"), - (0x16B78, "X"), - (0x16B7D, "V"), - (0x16B90, "X"), - (0x16D40, "V"), - (0x16D7A, "X"), - (0x16E40, "M", "𖹠"), - (0x16E41, "M", "𖹡"), - (0x16E42, "M", "𖹢"), - (0x16E43, "M", "𖹣"), - (0x16E44, "M", "𖹤"), - (0x16E45, "M", "𖹥"), - (0x16E46, "M", "𖹦"), - (0x16E47, "M", "𖹧"), - (0x16E48, "M", "𖹨"), - (0x16E49, "M", "𖹩"), - (0x16E4A, "M", "𖹪"), - (0x16E4B, "M", "𖹫"), - (0x16E4C, "M", "𖹬"), - (0x16E4D, "M", "𖹭"), - (0x16E4E, "M", "𖹮"), - (0x16E4F, "M", "𖹯"), - (0x16E50, "M", "𖹰"), - (0x16E51, "M", "𖹱"), - (0x16E52, "M", "𖹲"), - (0x16E53, "M", "𖹳"), - (0x16E54, "M", "𖹴"), - (0x16E55, "M", "𖹵"), - (0x16E56, "M", "𖹶"), - (0x16E57, "M", "𖹷"), - (0x16E58, "M", "𖹸"), - (0x16E59, "M", "𖹹"), - (0x16E5A, "M", "𖹺"), - (0x16E5B, "M", "𖹻"), - (0x16E5C, "M", "𖹼"), - (0x16E5D, "M", "𖹽"), - (0x16E5E, "M", "𖹾"), - (0x16E5F, "M", "𖹿"), - (0x16E60, "V"), - (0x16E9B, "X"), - (0x16EA0, "M", ""), - (0x16EA1, "M", ""), - (0x16EA2, "M", ""), - (0x16EA3, "M", ""), - (0x16EA4, "M", ""), - (0x16EA5, "M", ""), - (0x16EA6, "M", ""), - (0x16EA7, "M", ""), - (0x16EA8, "M", ""), - (0x16EA9, "M", ""), - (0x16EAA, "M", ""), - (0x16EAB, "M", ""), - (0x16EAC, "M", ""), - (0x16EAD, "M", ""), - (0x16EAE, "M", ""), - (0x16EAF, "M", ""), - (0x16EB0, "M", ""), - (0x16EB1, "M", ""), - (0x16EB2, "M", ""), - (0x16EB3, "M", ""), - (0x16EB4, "M", ""), - (0x16EB5, "M", ""), - (0x16EB6, "M", ""), - (0x16EB7, "M", ""), - (0x16EB8, "M", ""), - (0x16EB9, "X"), - (0x16EBB, "V"), - (0x16ED4, "X"), - (0x16F00, "V"), - (0x16F4B, "X"), - (0x16F4F, "V"), - (0x16F88, "X"), - (0x16F8F, "V"), - (0x16FA0, "X"), - (0x16FE0, "V"), - (0x16FE5, "X"), - (0x16FF0, "V"), - (0x16FF7, "X"), - (0x17000, "V"), - (0x18CD6, "X"), - (0x18CFF, "V"), - (0x18D1F, "X"), - (0x18D80, "V"), - (0x18DF3, "X"), - (0x1AFF0, "V"), - (0x1AFF4, "X"), - (0x1AFF5, "V"), - (0x1AFFC, "X"), - (0x1AFFD, "V"), - (0x1AFFF, "X"), - (0x1B000, "V"), - (0x1B123, "X"), - (0x1B132, "V"), - (0x1B133, "X"), - (0x1B150, "V"), - (0x1B153, "X"), - (0x1B155, "V"), - (0x1B156, "X"), - (0x1B164, "V"), - (0x1B168, "X"), - (0x1B170, "V"), - (0x1B2FC, "X"), - (0x1BC00, "V"), - (0x1BC6B, "X"), - (0x1BC70, "V"), - (0x1BC7D, "X"), - (0x1BC80, "V"), - (0x1BC89, "X"), - (0x1BC90, "V"), - (0x1BC9A, "X"), - (0x1BC9C, "V"), - (0x1BCA0, "I"), - (0x1BCA4, "X"), - (0x1CC00, "V"), - (0x1CCD6, "M", "a"), - (0x1CCD7, "M", "b"), - (0x1CCD8, "M", "c"), - (0x1CCD9, "M", "d"), - (0x1CCDA, "M", "e"), - (0x1CCDB, "M", "f"), - (0x1CCDC, "M", "g"), - (0x1CCDD, "M", "h"), - (0x1CCDE, "M", "i"), - (0x1CCDF, "M", "j"), - (0x1CCE0, "M", "k"), - (0x1CCE1, "M", "l"), - (0x1CCE2, "M", "m"), - (0x1CCE3, "M", "n"), - (0x1CCE4, "M", "o"), - (0x1CCE5, "M", "p"), - (0x1CCE6, "M", "q"), - (0x1CCE7, "M", "r"), - (0x1CCE8, "M", "s"), - (0x1CCE9, "M", "t"), - (0x1CCEA, "M", "u"), - (0x1CCEB, "M", "v"), - (0x1CCEC, "M", "w"), - (0x1CCED, "M", "x"), - (0x1CCEE, "M", "y"), - (0x1CCEF, "M", "z"), - (0x1CCF0, "M", "0"), - (0x1CCF1, "M", "1"), - (0x1CCF2, "M", "2"), - (0x1CCF3, "M", "3"), - (0x1CCF4, "M", "4"), - (0x1CCF5, "M", "5"), - (0x1CCF6, "M", "6"), - (0x1CCF7, "M", "7"), - (0x1CCF8, "M", "8"), - (0x1CCF9, "M", "9"), - (0x1CCFA, "V"), - (0x1CCFD, "X"), - (0x1CD00, "V"), - (0x1CEB4, "X"), - (0x1CEBA, "V"), - (0x1CED1, "X"), - (0x1CEE0, "V"), - (0x1CEF1, "X"), - (0x1CF00, "V"), - (0x1CF2E, "X"), - (0x1CF30, "V"), - (0x1CF47, "X"), - (0x1CF50, "V"), - (0x1CFC4, "X"), - (0x1D000, "V"), - (0x1D0F6, "X"), - (0x1D100, "V"), - (0x1D127, "X"), - (0x1D129, "V"), - (0x1D15E, "M", "𝅗𝅥"), - (0x1D15F, "M", "𝅘𝅥"), - (0x1D160, "M", "𝅘𝅥𝅮"), - (0x1D161, "M", "𝅘𝅥𝅯"), - (0x1D162, "M", "𝅘𝅥𝅰"), - (0x1D163, "M", "𝅘𝅥𝅱"), - (0x1D164, "M", "𝅘𝅥𝅲"), - (0x1D165, "V"), - (0x1D173, "I"), - (0x1D17B, "V"), - (0x1D1BB, "M", "𝆹𝅥"), - (0x1D1BC, "M", "𝆺𝅥"), - (0x1D1BD, "M", "𝆹𝅥𝅮"), - (0x1D1BE, "M", "𝆺𝅥𝅮"), - (0x1D1BF, "M", "𝆹𝅥𝅯"), - (0x1D1C0, "M", "𝆺𝅥𝅯"), - (0x1D1C1, "V"), - (0x1D1EB, "X"), - (0x1D200, "V"), - (0x1D246, "X"), - (0x1D2C0, "V"), - (0x1D2D4, "X"), - (0x1D2E0, "V"), - (0x1D2F4, "X"), - (0x1D300, "V"), - (0x1D357, "X"), - (0x1D360, "V"), - (0x1D379, "X"), - (0x1D400, "M", "a"), - (0x1D401, "M", "b"), - (0x1D402, "M", "c"), - (0x1D403, "M", "d"), - (0x1D404, "M", "e"), - (0x1D405, "M", "f"), - (0x1D406, "M", "g"), - (0x1D407, "M", "h"), - (0x1D408, "M", "i"), - (0x1D409, "M", "j"), - (0x1D40A, "M", "k"), - (0x1D40B, "M", "l"), - (0x1D40C, "M", "m"), - (0x1D40D, "M", "n"), - (0x1D40E, "M", "o"), - (0x1D40F, "M", "p"), - (0x1D410, "M", "q"), - (0x1D411, "M", "r"), - (0x1D412, "M", "s"), - (0x1D413, "M", "t"), - (0x1D414, "M", "u"), - (0x1D415, "M", "v"), - (0x1D416, "M", "w"), - (0x1D417, "M", "x"), - (0x1D418, "M", "y"), - (0x1D419, "M", "z"), - (0x1D41A, "M", "a"), - (0x1D41B, "M", "b"), - (0x1D41C, "M", "c"), - (0x1D41D, "M", "d"), - (0x1D41E, "M", "e"), - (0x1D41F, "M", "f"), - (0x1D420, "M", "g"), - (0x1D421, "M", "h"), - (0x1D422, "M", "i"), - (0x1D423, "M", "j"), - (0x1D424, "M", "k"), - (0x1D425, "M", "l"), - (0x1D426, "M", "m"), - (0x1D427, "M", "n"), - (0x1D428, "M", "o"), - (0x1D429, "M", "p"), - (0x1D42A, "M", "q"), - (0x1D42B, "M", "r"), - (0x1D42C, "M", "s"), - (0x1D42D, "M", "t"), - (0x1D42E, "M", "u"), - (0x1D42F, "M", "v"), - (0x1D430, "M", "w"), - (0x1D431, "M", "x"), - (0x1D432, "M", "y"), - (0x1D433, "M", "z"), - (0x1D434, "M", "a"), - (0x1D435, "M", "b"), - (0x1D436, "M", "c"), - (0x1D437, "M", "d"), - (0x1D438, "M", "e"), - (0x1D439, "M", "f"), - (0x1D43A, "M", "g"), - (0x1D43B, "M", "h"), - (0x1D43C, "M", "i"), - (0x1D43D, "M", "j"), - (0x1D43E, "M", "k"), - (0x1D43F, "M", "l"), - (0x1D440, "M", "m"), - (0x1D441, "M", "n"), - (0x1D442, "M", "o"), - (0x1D443, "M", "p"), - (0x1D444, "M", "q"), - (0x1D445, "M", "r"), - (0x1D446, "M", "s"), - (0x1D447, "M", "t"), - (0x1D448, "M", "u"), - (0x1D449, "M", "v"), - (0x1D44A, "M", "w"), - (0x1D44B, "M", "x"), - (0x1D44C, "M", "y"), - (0x1D44D, "M", "z"), - (0x1D44E, "M", "a"), - (0x1D44F, "M", "b"), - (0x1D450, "M", "c"), - (0x1D451, "M", "d"), - (0x1D452, "M", "e"), - (0x1D453, "M", "f"), - (0x1D454, "M", "g"), - (0x1D455, "X"), - (0x1D456, "M", "i"), - (0x1D457, "M", "j"), - (0x1D458, "M", "k"), - (0x1D459, "M", "l"), - (0x1D45A, "M", "m"), - (0x1D45B, "M", "n"), - (0x1D45C, "M", "o"), - (0x1D45D, "M", "p"), - (0x1D45E, "M", "q"), - (0x1D45F, "M", "r"), - (0x1D460, "M", "s"), - (0x1D461, "M", "t"), - (0x1D462, "M", "u"), - (0x1D463, "M", "v"), - (0x1D464, "M", "w"), - (0x1D465, "M", "x"), - (0x1D466, "M", "y"), - (0x1D467, "M", "z"), - (0x1D468, "M", "a"), - (0x1D469, "M", "b"), - (0x1D46A, "M", "c"), - (0x1D46B, "M", "d"), - (0x1D46C, "M", "e"), - (0x1D46D, "M", "f"), - (0x1D46E, "M", "g"), - (0x1D46F, "M", "h"), - (0x1D470, "M", "i"), - (0x1D471, "M", "j"), - (0x1D472, "M", "k"), - (0x1D473, "M", "l"), - (0x1D474, "M", "m"), - (0x1D475, "M", "n"), - (0x1D476, "M", "o"), - (0x1D477, "M", "p"), - (0x1D478, "M", "q"), - (0x1D479, "M", "r"), - (0x1D47A, "M", "s"), - (0x1D47B, "M", "t"), - (0x1D47C, "M", "u"), - (0x1D47D, "M", "v"), - (0x1D47E, "M", "w"), - (0x1D47F, "M", "x"), - (0x1D480, "M", "y"), - (0x1D481, "M", "z"), - (0x1D482, "M", "a"), - (0x1D483, "M", "b"), - (0x1D484, "M", "c"), - (0x1D485, "M", "d"), - (0x1D486, "M", "e"), - (0x1D487, "M", "f"), - (0x1D488, "M", "g"), - (0x1D489, "M", "h"), - (0x1D48A, "M", "i"), - (0x1D48B, "M", "j"), - (0x1D48C, "M", "k"), - (0x1D48D, "M", "l"), - (0x1D48E, "M", "m"), - (0x1D48F, "M", "n"), - (0x1D490, "M", "o"), - (0x1D491, "M", "p"), - (0x1D492, "M", "q"), - (0x1D493, "M", "r"), - (0x1D494, "M", "s"), - (0x1D495, "M", "t"), - (0x1D496, "M", "u"), - (0x1D497, "M", "v"), - (0x1D498, "M", "w"), - (0x1D499, "M", "x"), - (0x1D49A, "M", "y"), - (0x1D49B, "M", "z"), - (0x1D49C, "M", "a"), - (0x1D49D, "X"), - (0x1D49E, "M", "c"), - (0x1D49F, "M", "d"), - (0x1D4A0, "X"), - (0x1D4A2, "M", "g"), - (0x1D4A3, "X"), - (0x1D4A5, "M", "j"), - (0x1D4A6, "M", "k"), - (0x1D4A7, "X"), - (0x1D4A9, "M", "n"), - (0x1D4AA, "M", "o"), - (0x1D4AB, "M", "p"), - (0x1D4AC, "M", "q"), - (0x1D4AD, "X"), - (0x1D4AE, "M", "s"), - (0x1D4AF, "M", "t"), - (0x1D4B0, "M", "u"), - (0x1D4B1, "M", "v"), - (0x1D4B2, "M", "w"), - (0x1D4B3, "M", "x"), - (0x1D4B4, "M", "y"), - (0x1D4B5, "M", "z"), - (0x1D4B6, "M", "a"), - (0x1D4B7, "M", "b"), - (0x1D4B8, "M", "c"), - (0x1D4B9, "M", "d"), - (0x1D4BA, "X"), - (0x1D4BB, "M", "f"), - (0x1D4BC, "X"), - (0x1D4BD, "M", "h"), - (0x1D4BE, "M", "i"), - (0x1D4BF, "M", "j"), - (0x1D4C0, "M", "k"), - (0x1D4C1, "M", "l"), - (0x1D4C2, "M", "m"), - (0x1D4C3, "M", "n"), - (0x1D4C4, "X"), - (0x1D4C5, "M", "p"), - (0x1D4C6, "M", "q"), - (0x1D4C7, "M", "r"), - (0x1D4C8, "M", "s"), - (0x1D4C9, "M", "t"), - (0x1D4CA, "M", "u"), - (0x1D4CB, "M", "v"), - (0x1D4CC, "M", "w"), - (0x1D4CD, "M", "x"), - (0x1D4CE, "M", "y"), - (0x1D4CF, "M", "z"), - (0x1D4D0, "M", "a"), - (0x1D4D1, "M", "b"), - (0x1D4D2, "M", "c"), - (0x1D4D3, "M", "d"), - (0x1D4D4, "M", "e"), - (0x1D4D5, "M", "f"), - (0x1D4D6, "M", "g"), - (0x1D4D7, "M", "h"), - (0x1D4D8, "M", "i"), - (0x1D4D9, "M", "j"), - (0x1D4DA, "M", "k"), - (0x1D4DB, "M", "l"), - (0x1D4DC, "M", "m"), - (0x1D4DD, "M", "n"), - (0x1D4DE, "M", "o"), - (0x1D4DF, "M", "p"), - (0x1D4E0, "M", "q"), - (0x1D4E1, "M", "r"), - (0x1D4E2, "M", "s"), - (0x1D4E3, "M", "t"), - (0x1D4E4, "M", "u"), - (0x1D4E5, "M", "v"), - (0x1D4E6, "M", "w"), - (0x1D4E7, "M", "x"), - (0x1D4E8, "M", "y"), - (0x1D4E9, "M", "z"), - (0x1D4EA, "M", "a"), - (0x1D4EB, "M", "b"), - (0x1D4EC, "M", "c"), - (0x1D4ED, "M", "d"), - (0x1D4EE, "M", "e"), - (0x1D4EF, "M", "f"), - (0x1D4F0, "M", "g"), - (0x1D4F1, "M", "h"), - (0x1D4F2, "M", "i"), - (0x1D4F3, "M", "j"), - (0x1D4F4, "M", "k"), - (0x1D4F5, "M", "l"), - (0x1D4F6, "M", "m"), - (0x1D4F7, "M", "n"), - (0x1D4F8, "M", "o"), - (0x1D4F9, "M", "p"), - (0x1D4FA, "M", "q"), - (0x1D4FB, "M", "r"), - (0x1D4FC, "M", "s"), - (0x1D4FD, "M", "t"), - (0x1D4FE, "M", "u"), - (0x1D4FF, "M", "v"), - (0x1D500, "M", "w"), - (0x1D501, "M", "x"), - (0x1D502, "M", "y"), - (0x1D503, "M", "z"), - (0x1D504, "M", "a"), - (0x1D505, "M", "b"), - (0x1D506, "X"), - (0x1D507, "M", "d"), - (0x1D508, "M", "e"), - (0x1D509, "M", "f"), - (0x1D50A, "M", "g"), - (0x1D50B, "X"), - (0x1D50D, "M", "j"), - (0x1D50E, "M", "k"), - (0x1D50F, "M", "l"), - (0x1D510, "M", "m"), - (0x1D511, "M", "n"), - (0x1D512, "M", "o"), - (0x1D513, "M", "p"), - (0x1D514, "M", "q"), - (0x1D515, "X"), - (0x1D516, "M", "s"), - (0x1D517, "M", "t"), - (0x1D518, "M", "u"), - (0x1D519, "M", "v"), - (0x1D51A, "M", "w"), - (0x1D51B, "M", "x"), - (0x1D51C, "M", "y"), - (0x1D51D, "X"), - (0x1D51E, "M", "a"), - (0x1D51F, "M", "b"), - (0x1D520, "M", "c"), - (0x1D521, "M", "d"), - (0x1D522, "M", "e"), - (0x1D523, "M", "f"), - (0x1D524, "M", "g"), - (0x1D525, "M", "h"), - (0x1D526, "M", "i"), - (0x1D527, "M", "j"), - (0x1D528, "M", "k"), - (0x1D529, "M", "l"), - (0x1D52A, "M", "m"), - (0x1D52B, "M", "n"), - (0x1D52C, "M", "o"), - (0x1D52D, "M", "p"), - (0x1D52E, "M", "q"), - (0x1D52F, "M", "r"), - (0x1D530, "M", "s"), - (0x1D531, "M", "t"), - (0x1D532, "M", "u"), - (0x1D533, "M", "v"), - (0x1D534, "M", "w"), - (0x1D535, "M", "x"), - (0x1D536, "M", "y"), - (0x1D537, "M", "z"), - (0x1D538, "M", "a"), - (0x1D539, "M", "b"), - (0x1D53A, "X"), - (0x1D53B, "M", "d"), - (0x1D53C, "M", "e"), - (0x1D53D, "M", "f"), - (0x1D53E, "M", "g"), - (0x1D53F, "X"), - (0x1D540, "M", "i"), - (0x1D541, "M", "j"), - (0x1D542, "M", "k"), - (0x1D543, "M", "l"), - (0x1D544, "M", "m"), - (0x1D545, "X"), - (0x1D546, "M", "o"), - (0x1D547, "X"), - (0x1D54A, "M", "s"), - (0x1D54B, "M", "t"), - (0x1D54C, "M", "u"), - (0x1D54D, "M", "v"), - (0x1D54E, "M", "w"), - (0x1D54F, "M", "x"), - (0x1D550, "M", "y"), - (0x1D551, "X"), - (0x1D552, "M", "a"), - (0x1D553, "M", "b"), - (0x1D554, "M", "c"), - (0x1D555, "M", "d"), - (0x1D556, "M", "e"), - (0x1D557, "M", "f"), - (0x1D558, "M", "g"), - (0x1D559, "M", "h"), - (0x1D55A, "M", "i"), - (0x1D55B, "M", "j"), - (0x1D55C, "M", "k"), - (0x1D55D, "M", "l"), - (0x1D55E, "M", "m"), - (0x1D55F, "M", "n"), - (0x1D560, "M", "o"), - (0x1D561, "M", "p"), - (0x1D562, "M", "q"), - (0x1D563, "M", "r"), - (0x1D564, "M", "s"), - (0x1D565, "M", "t"), - (0x1D566, "M", "u"), - (0x1D567, "M", "v"), - (0x1D568, "M", "w"), - (0x1D569, "M", "x"), - (0x1D56A, "M", "y"), - (0x1D56B, "M", "z"), - (0x1D56C, "M", "a"), - (0x1D56D, "M", "b"), - (0x1D56E, "M", "c"), - (0x1D56F, "M", "d"), - (0x1D570, "M", "e"), - (0x1D571, "M", "f"), - (0x1D572, "M", "g"), - (0x1D573, "M", "h"), - (0x1D574, "M", "i"), - (0x1D575, "M", "j"), - (0x1D576, "M", "k"), - (0x1D577, "M", "l"), - (0x1D578, "M", "m"), - (0x1D579, "M", "n"), - (0x1D57A, "M", "o"), - (0x1D57B, "M", "p"), - (0x1D57C, "M", "q"), - (0x1D57D, "M", "r"), - (0x1D57E, "M", "s"), - (0x1D57F, "M", "t"), - (0x1D580, "M", "u"), - (0x1D581, "M", "v"), - (0x1D582, "M", "w"), - (0x1D583, "M", "x"), - (0x1D584, "M", "y"), - (0x1D585, "M", "z"), - (0x1D586, "M", "a"), - (0x1D587, "M", "b"), - (0x1D588, "M", "c"), - (0x1D589, "M", "d"), - (0x1D58A, "M", "e"), - (0x1D58B, "M", "f"), - (0x1D58C, "M", "g"), - (0x1D58D, "M", "h"), - (0x1D58E, "M", "i"), - (0x1D58F, "M", "j"), - (0x1D590, "M", "k"), - (0x1D591, "M", "l"), - (0x1D592, "M", "m"), - (0x1D593, "M", "n"), - (0x1D594, "M", "o"), - (0x1D595, "M", "p"), - (0x1D596, "M", "q"), - (0x1D597, "M", "r"), - (0x1D598, "M", "s"), - (0x1D599, "M", "t"), - (0x1D59A, "M", "u"), - (0x1D59B, "M", "v"), - (0x1D59C, "M", "w"), - (0x1D59D, "M", "x"), - (0x1D59E, "M", "y"), - (0x1D59F, "M", "z"), - (0x1D5A0, "M", "a"), - (0x1D5A1, "M", "b"), - (0x1D5A2, "M", "c"), - (0x1D5A3, "M", "d"), - (0x1D5A4, "M", "e"), - (0x1D5A5, "M", "f"), - (0x1D5A6, "M", "g"), - (0x1D5A7, "M", "h"), - (0x1D5A8, "M", "i"), - (0x1D5A9, "M", "j"), - (0x1D5AA, "M", "k"), - (0x1D5AB, "M", "l"), - (0x1D5AC, "M", "m"), - (0x1D5AD, "M", "n"), - (0x1D5AE, "M", "o"), - (0x1D5AF, "M", "p"), - (0x1D5B0, "M", "q"), - (0x1D5B1, "M", "r"), - (0x1D5B2, "M", "s"), - (0x1D5B3, "M", "t"), - (0x1D5B4, "M", "u"), - (0x1D5B5, "M", "v"), - (0x1D5B6, "M", "w"), - (0x1D5B7, "M", "x"), - (0x1D5B8, "M", "y"), - (0x1D5B9, "M", "z"), - (0x1D5BA, "M", "a"), - (0x1D5BB, "M", "b"), - (0x1D5BC, "M", "c"), - (0x1D5BD, "M", "d"), - (0x1D5BE, "M", "e"), - (0x1D5BF, "M", "f"), - (0x1D5C0, "M", "g"), - (0x1D5C1, "M", "h"), - (0x1D5C2, "M", "i"), - (0x1D5C3, "M", "j"), - (0x1D5C4, "M", "k"), - (0x1D5C5, "M", "l"), - (0x1D5C6, "M", "m"), - (0x1D5C7, "M", "n"), - (0x1D5C8, "M", "o"), - (0x1D5C9, "M", "p"), - (0x1D5CA, "M", "q"), - (0x1D5CB, "M", "r"), - (0x1D5CC, "M", "s"), - (0x1D5CD, "M", "t"), - (0x1D5CE, "M", "u"), - (0x1D5CF, "M", "v"), - (0x1D5D0, "M", "w"), - (0x1D5D1, "M", "x"), - (0x1D5D2, "M", "y"), - (0x1D5D3, "M", "z"), - (0x1D5D4, "M", "a"), - (0x1D5D5, "M", "b"), - (0x1D5D6, "M", "c"), - (0x1D5D7, "M", "d"), - (0x1D5D8, "M", "e"), - (0x1D5D9, "M", "f"), - (0x1D5DA, "M", "g"), - (0x1D5DB, "M", "h"), - (0x1D5DC, "M", "i"), - (0x1D5DD, "M", "j"), - (0x1D5DE, "M", "k"), - (0x1D5DF, "M", "l"), - (0x1D5E0, "M", "m"), - (0x1D5E1, "M", "n"), - (0x1D5E2, "M", "o"), - (0x1D5E3, "M", "p"), - (0x1D5E4, "M", "q"), - (0x1D5E5, "M", "r"), - (0x1D5E6, "M", "s"), - (0x1D5E7, "M", "t"), - (0x1D5E8, "M", "u"), - (0x1D5E9, "M", "v"), - (0x1D5EA, "M", "w"), - (0x1D5EB, "M", "x"), - (0x1D5EC, "M", "y"), - (0x1D5ED, "M", "z"), - (0x1D5EE, "M", "a"), - (0x1D5EF, "M", "b"), - (0x1D5F0, "M", "c"), - (0x1D5F1, "M", "d"), - (0x1D5F2, "M", "e"), - (0x1D5F3, "M", "f"), - (0x1D5F4, "M", "g"), - (0x1D5F5, "M", "h"), - (0x1D5F6, "M", "i"), - (0x1D5F7, "M", "j"), - (0x1D5F8, "M", "k"), - (0x1D5F9, "M", "l"), - (0x1D5FA, "M", "m"), - (0x1D5FB, "M", "n"), - (0x1D5FC, "M", "o"), - (0x1D5FD, "M", "p"), - (0x1D5FE, "M", "q"), - (0x1D5FF, "M", "r"), - (0x1D600, "M", "s"), - (0x1D601, "M", "t"), - (0x1D602, "M", "u"), - (0x1D603, "M", "v"), - (0x1D604, "M", "w"), - (0x1D605, "M", "x"), - (0x1D606, "M", "y"), - (0x1D607, "M", "z"), - (0x1D608, "M", "a"), - (0x1D609, "M", "b"), - (0x1D60A, "M", "c"), - (0x1D60B, "M", "d"), - (0x1D60C, "M", "e"), - (0x1D60D, "M", "f"), - (0x1D60E, "M", "g"), - (0x1D60F, "M", "h"), - (0x1D610, "M", "i"), - (0x1D611, "M", "j"), - (0x1D612, "M", "k"), - (0x1D613, "M", "l"), - (0x1D614, "M", "m"), - (0x1D615, "M", "n"), - (0x1D616, "M", "o"), - (0x1D617, "M", "p"), - (0x1D618, "M", "q"), - (0x1D619, "M", "r"), - (0x1D61A, "M", "s"), - (0x1D61B, "M", "t"), - (0x1D61C, "M", "u"), - (0x1D61D, "M", "v"), - (0x1D61E, "M", "w"), - (0x1D61F, "M", "x"), - (0x1D620, "M", "y"), - (0x1D621, "M", "z"), - (0x1D622, "M", "a"), - (0x1D623, "M", "b"), - (0x1D624, "M", "c"), - (0x1D625, "M", "d"), - (0x1D626, "M", "e"), - (0x1D627, "M", "f"), - (0x1D628, "M", "g"), - (0x1D629, "M", "h"), - (0x1D62A, "M", "i"), - (0x1D62B, "M", "j"), - (0x1D62C, "M", "k"), - (0x1D62D, "M", "l"), - (0x1D62E, "M", "m"), - (0x1D62F, "M", "n"), - (0x1D630, "M", "o"), - (0x1D631, "M", "p"), - (0x1D632, "M", "q"), - (0x1D633, "M", "r"), - (0x1D634, "M", "s"), - (0x1D635, "M", "t"), - (0x1D636, "M", "u"), - (0x1D637, "M", "v"), - (0x1D638, "M", "w"), - (0x1D639, "M", "x"), - (0x1D63A, "M", "y"), - (0x1D63B, "M", "z"), - (0x1D63C, "M", "a"), - (0x1D63D, "M", "b"), - (0x1D63E, "M", "c"), - (0x1D63F, "M", "d"), - (0x1D640, "M", "e"), - (0x1D641, "M", "f"), - (0x1D642, "M", "g"), - (0x1D643, "M", "h"), - (0x1D644, "M", "i"), - (0x1D645, "M", "j"), - (0x1D646, "M", "k"), - (0x1D647, "M", "l"), - (0x1D648, "M", "m"), - (0x1D649, "M", "n"), - (0x1D64A, "M", "o"), - (0x1D64B, "M", "p"), - (0x1D64C, "M", "q"), - (0x1D64D, "M", "r"), - (0x1D64E, "M", "s"), - (0x1D64F, "M", "t"), - (0x1D650, "M", "u"), - (0x1D651, "M", "v"), - (0x1D652, "M", "w"), - (0x1D653, "M", "x"), - (0x1D654, "M", "y"), - (0x1D655, "M", "z"), - (0x1D656, "M", "a"), - (0x1D657, "M", "b"), - (0x1D658, "M", "c"), - (0x1D659, "M", "d"), - (0x1D65A, "M", "e"), - (0x1D65B, "M", "f"), - (0x1D65C, "M", "g"), - (0x1D65D, "M", "h"), - (0x1D65E, "M", "i"), - (0x1D65F, "M", "j"), - (0x1D660, "M", "k"), - (0x1D661, "M", "l"), - (0x1D662, "M", "m"), - (0x1D663, "M", "n"), - (0x1D664, "M", "o"), - (0x1D665, "M", "p"), - (0x1D666, "M", "q"), - (0x1D667, "M", "r"), - (0x1D668, "M", "s"), - (0x1D669, "M", "t"), - (0x1D66A, "M", "u"), - (0x1D66B, "M", "v"), - (0x1D66C, "M", "w"), - (0x1D66D, "M", "x"), - (0x1D66E, "M", "y"), - (0x1D66F, "M", "z"), - (0x1D670, "M", "a"), - (0x1D671, "M", "b"), - (0x1D672, "M", "c"), - (0x1D673, "M", "d"), - (0x1D674, "M", "e"), - (0x1D675, "M", "f"), - (0x1D676, "M", "g"), - (0x1D677, "M", "h"), - (0x1D678, "M", "i"), - (0x1D679, "M", "j"), - (0x1D67A, "M", "k"), - (0x1D67B, "M", "l"), - (0x1D67C, "M", "m"), - (0x1D67D, "M", "n"), - (0x1D67E, "M", "o"), - (0x1D67F, "M", "p"), - (0x1D680, "M", "q"), - (0x1D681, "M", "r"), - (0x1D682, "M", "s"), - (0x1D683, "M", "t"), - (0x1D684, "M", "u"), - (0x1D685, "M", "v"), - (0x1D686, "M", "w"), - (0x1D687, "M", "x"), - (0x1D688, "M", "y"), - (0x1D689, "M", "z"), - (0x1D68A, "M", "a"), - (0x1D68B, "M", "b"), - (0x1D68C, "M", "c"), - (0x1D68D, "M", "d"), - (0x1D68E, "M", "e"), - (0x1D68F, "M", "f"), - (0x1D690, "M", "g"), - (0x1D691, "M", "h"), - (0x1D692, "M", "i"), - (0x1D693, "M", "j"), - (0x1D694, "M", "k"), - (0x1D695, "M", "l"), - (0x1D696, "M", "m"), - (0x1D697, "M", "n"), - (0x1D698, "M", "o"), - (0x1D699, "M", "p"), - (0x1D69A, "M", "q"), - (0x1D69B, "M", "r"), - (0x1D69C, "M", "s"), - (0x1D69D, "M", "t"), - (0x1D69E, "M", "u"), - (0x1D69F, "M", "v"), - (0x1D6A0, "M", "w"), - (0x1D6A1, "M", "x"), - (0x1D6A2, "M", "y"), - (0x1D6A3, "M", "z"), - (0x1D6A4, "M", "ı"), - (0x1D6A5, "M", "ȷ"), - (0x1D6A6, "X"), - (0x1D6A8, "M", "α"), - (0x1D6A9, "M", "β"), - (0x1D6AA, "M", "γ"), - (0x1D6AB, "M", "δ"), - (0x1D6AC, "M", "ε"), - (0x1D6AD, "M", "ζ"), - (0x1D6AE, "M", "η"), - (0x1D6AF, "M", "θ"), - (0x1D6B0, "M", "ι"), - (0x1D6B1, "M", "κ"), - (0x1D6B2, "M", "λ"), - (0x1D6B3, "M", "μ"), - (0x1D6B4, "M", "ν"), - (0x1D6B5, "M", "ξ"), - (0x1D6B6, "M", "ο"), - (0x1D6B7, "M", "π"), - (0x1D6B8, "M", "ρ"), - (0x1D6B9, "M", "θ"), - (0x1D6BA, "M", "σ"), - (0x1D6BB, "M", "τ"), - (0x1D6BC, "M", "υ"), - (0x1D6BD, "M", "φ"), - (0x1D6BE, "M", "χ"), - (0x1D6BF, "M", "ψ"), - (0x1D6C0, "M", "ω"), - (0x1D6C1, "M", "∇"), - (0x1D6C2, "M", "α"), - (0x1D6C3, "M", "β"), - (0x1D6C4, "M", "γ"), - (0x1D6C5, "M", "δ"), - (0x1D6C6, "M", "ε"), - (0x1D6C7, "M", "ζ"), - (0x1D6C8, "M", "η"), - (0x1D6C9, "M", "θ"), - (0x1D6CA, "M", "ι"), - (0x1D6CB, "M", "κ"), - (0x1D6CC, "M", "λ"), - (0x1D6CD, "M", "μ"), - (0x1D6CE, "M", "ν"), - (0x1D6CF, "M", "ξ"), - (0x1D6D0, "M", "ο"), - (0x1D6D1, "M", "π"), - (0x1D6D2, "M", "ρ"), - (0x1D6D3, "M", "σ"), - (0x1D6D5, "M", "τ"), - (0x1D6D6, "M", "υ"), - (0x1D6D7, "M", "φ"), - (0x1D6D8, "M", "χ"), - (0x1D6D9, "M", "ψ"), - (0x1D6DA, "M", "ω"), - (0x1D6DB, "M", "∂"), - (0x1D6DC, "M", "ε"), - (0x1D6DD, "M", "θ"), - (0x1D6DE, "M", "κ"), - (0x1D6DF, "M", "φ"), - (0x1D6E0, "M", "ρ"), - (0x1D6E1, "M", "π"), - (0x1D6E2, "M", "α"), - (0x1D6E3, "M", "β"), - (0x1D6E4, "M", "γ"), - (0x1D6E5, "M", "δ"), - (0x1D6E6, "M", "ε"), - (0x1D6E7, "M", "ζ"), - (0x1D6E8, "M", "η"), - (0x1D6E9, "M", "θ"), - (0x1D6EA, "M", "ι"), - (0x1D6EB, "M", "κ"), - (0x1D6EC, "M", "λ"), - (0x1D6ED, "M", "μ"), - (0x1D6EE, "M", "ν"), - (0x1D6EF, "M", "ξ"), - (0x1D6F0, "M", "ο"), - (0x1D6F1, "M", "π"), - (0x1D6F2, "M", "ρ"), - (0x1D6F3, "M", "θ"), - (0x1D6F4, "M", "σ"), - (0x1D6F5, "M", "τ"), - (0x1D6F6, "M", "υ"), - (0x1D6F7, "M", "φ"), - (0x1D6F8, "M", "χ"), - (0x1D6F9, "M", "ψ"), - (0x1D6FA, "M", "ω"), - (0x1D6FB, "M", "∇"), - (0x1D6FC, "M", "α"), - (0x1D6FD, "M", "β"), - (0x1D6FE, "M", "γ"), - (0x1D6FF, "M", "δ"), - (0x1D700, "M", "ε"), - (0x1D701, "M", "ζ"), - (0x1D702, "M", "η"), - (0x1D703, "M", "θ"), - (0x1D704, "M", "ι"), - (0x1D705, "M", "κ"), - (0x1D706, "M", "λ"), - (0x1D707, "M", "μ"), - (0x1D708, "M", "ν"), - (0x1D709, "M", "ξ"), - (0x1D70A, "M", "ο"), - (0x1D70B, "M", "π"), - (0x1D70C, "M", "ρ"), - (0x1D70D, "M", "σ"), - (0x1D70F, "M", "τ"), - (0x1D710, "M", "υ"), - (0x1D711, "M", "φ"), - (0x1D712, "M", "χ"), - (0x1D713, "M", "ψ"), - (0x1D714, "M", "ω"), - (0x1D715, "M", "∂"), - (0x1D716, "M", "ε"), - (0x1D717, "M", "θ"), - (0x1D718, "M", "κ"), - (0x1D719, "M", "φ"), - (0x1D71A, "M", "ρ"), - (0x1D71B, "M", "π"), - (0x1D71C, "M", "α"), - (0x1D71D, "M", "β"), - (0x1D71E, "M", "γ"), - (0x1D71F, "M", "δ"), - (0x1D720, "M", "ε"), - (0x1D721, "M", "ζ"), - (0x1D722, "M", "η"), - (0x1D723, "M", "θ"), - (0x1D724, "M", "ι"), - (0x1D725, "M", "κ"), - (0x1D726, "M", "λ"), - (0x1D727, "M", "μ"), - (0x1D728, "M", "ν"), - (0x1D729, "M", "ξ"), - (0x1D72A, "M", "ο"), - (0x1D72B, "M", "π"), - (0x1D72C, "M", "ρ"), - (0x1D72D, "M", "θ"), - (0x1D72E, "M", "σ"), - (0x1D72F, "M", "τ"), - (0x1D730, "M", "υ"), - (0x1D731, "M", "φ"), - (0x1D732, "M", "χ"), - (0x1D733, "M", "ψ"), - (0x1D734, "M", "ω"), - (0x1D735, "M", "∇"), - (0x1D736, "M", "α"), - (0x1D737, "M", "β"), - (0x1D738, "M", "γ"), - (0x1D739, "M", "δ"), - (0x1D73A, "M", "ε"), - (0x1D73B, "M", "ζ"), - (0x1D73C, "M", "η"), - (0x1D73D, "M", "θ"), - (0x1D73E, "M", "ι"), - (0x1D73F, "M", "κ"), - (0x1D740, "M", "λ"), - (0x1D741, "M", "μ"), - (0x1D742, "M", "ν"), - (0x1D743, "M", "ξ"), - (0x1D744, "M", "ο"), - (0x1D745, "M", "π"), - (0x1D746, "M", "ρ"), - (0x1D747, "M", "σ"), - (0x1D749, "M", "τ"), - (0x1D74A, "M", "υ"), - (0x1D74B, "M", "φ"), - (0x1D74C, "M", "χ"), - (0x1D74D, "M", "ψ"), - (0x1D74E, "M", "ω"), - (0x1D74F, "M", "∂"), - (0x1D750, "M", "ε"), - (0x1D751, "M", "θ"), - (0x1D752, "M", "κ"), - (0x1D753, "M", "φ"), - (0x1D754, "M", "ρ"), - (0x1D755, "M", "π"), - (0x1D756, "M", "α"), - (0x1D757, "M", "β"), - (0x1D758, "M", "γ"), - (0x1D759, "M", "δ"), - (0x1D75A, "M", "ε"), - (0x1D75B, "M", "ζ"), - (0x1D75C, "M", "η"), - (0x1D75D, "M", "θ"), - (0x1D75E, "M", "ι"), - (0x1D75F, "M", "κ"), - (0x1D760, "M", "λ"), - (0x1D761, "M", "μ"), - (0x1D762, "M", "ν"), - (0x1D763, "M", "ξ"), - (0x1D764, "M", "ο"), - (0x1D765, "M", "π"), - (0x1D766, "M", "ρ"), - (0x1D767, "M", "θ"), - (0x1D768, "M", "σ"), - (0x1D769, "M", "τ"), - (0x1D76A, "M", "υ"), - (0x1D76B, "M", "φ"), - (0x1D76C, "M", "χ"), - (0x1D76D, "M", "ψ"), - (0x1D76E, "M", "ω"), - (0x1D76F, "M", "∇"), - (0x1D770, "M", "α"), - (0x1D771, "M", "β"), - (0x1D772, "M", "γ"), - (0x1D773, "M", "δ"), - (0x1D774, "M", "ε"), - (0x1D775, "M", "ζ"), - (0x1D776, "M", "η"), - (0x1D777, "M", "θ"), - (0x1D778, "M", "ι"), - (0x1D779, "M", "κ"), - (0x1D77A, "M", "λ"), - (0x1D77B, "M", "μ"), - (0x1D77C, "M", "ν"), - (0x1D77D, "M", "ξ"), - (0x1D77E, "M", "ο"), - (0x1D77F, "M", "π"), - (0x1D780, "M", "ρ"), - (0x1D781, "M", "σ"), - (0x1D783, "M", "τ"), - (0x1D784, "M", "υ"), - (0x1D785, "M", "φ"), - (0x1D786, "M", "χ"), - (0x1D787, "M", "ψ"), - (0x1D788, "M", "ω"), - (0x1D789, "M", "∂"), - (0x1D78A, "M", "ε"), - (0x1D78B, "M", "θ"), - (0x1D78C, "M", "κ"), - (0x1D78D, "M", "φ"), - (0x1D78E, "M", "ρ"), - (0x1D78F, "M", "π"), - (0x1D790, "M", "α"), - (0x1D791, "M", "β"), - (0x1D792, "M", "γ"), - (0x1D793, "M", "δ"), - (0x1D794, "M", "ε"), - (0x1D795, "M", "ζ"), - (0x1D796, "M", "η"), - (0x1D797, "M", "θ"), - (0x1D798, "M", "ι"), - (0x1D799, "M", "κ"), - (0x1D79A, "M", "λ"), - (0x1D79B, "M", "μ"), - (0x1D79C, "M", "ν"), - (0x1D79D, "M", "ξ"), - (0x1D79E, "M", "ο"), - (0x1D79F, "M", "π"), - (0x1D7A0, "M", "ρ"), - (0x1D7A1, "M", "θ"), - (0x1D7A2, "M", "σ"), - (0x1D7A3, "M", "τ"), - (0x1D7A4, "M", "υ"), - (0x1D7A5, "M", "φ"), - (0x1D7A6, "M", "χ"), - (0x1D7A7, "M", "ψ"), - (0x1D7A8, "M", "ω"), - (0x1D7A9, "M", "∇"), - (0x1D7AA, "M", "α"), - (0x1D7AB, "M", "β"), - (0x1D7AC, "M", "γ"), - (0x1D7AD, "M", "δ"), - (0x1D7AE, "M", "ε"), - (0x1D7AF, "M", "ζ"), - (0x1D7B0, "M", "η"), - (0x1D7B1, "M", "θ"), - (0x1D7B2, "M", "ι"), - (0x1D7B3, "M", "κ"), - (0x1D7B4, "M", "λ"), - (0x1D7B5, "M", "μ"), - (0x1D7B6, "M", "ν"), - (0x1D7B7, "M", "ξ"), - (0x1D7B8, "M", "ο"), - (0x1D7B9, "M", "π"), - (0x1D7BA, "M", "ρ"), - (0x1D7BB, "M", "σ"), - (0x1D7BD, "M", "τ"), - (0x1D7BE, "M", "υ"), - (0x1D7BF, "M", "φ"), - (0x1D7C0, "M", "χ"), - (0x1D7C1, "M", "ψ"), - (0x1D7C2, "M", "ω"), - (0x1D7C3, "M", "∂"), - (0x1D7C4, "M", "ε"), - (0x1D7C5, "M", "θ"), - (0x1D7C6, "M", "κ"), - (0x1D7C7, "M", "φ"), - (0x1D7C8, "M", "ρ"), - (0x1D7C9, "M", "π"), - (0x1D7CA, "M", "ϝ"), - (0x1D7CC, "X"), - (0x1D7CE, "M", "0"), - (0x1D7CF, "M", "1"), - (0x1D7D0, "M", "2"), - (0x1D7D1, "M", "3"), - (0x1D7D2, "M", "4"), - (0x1D7D3, "M", "5"), - (0x1D7D4, "M", "6"), - (0x1D7D5, "M", "7"), - (0x1D7D6, "M", "8"), - (0x1D7D7, "M", "9"), - (0x1D7D8, "M", "0"), - (0x1D7D9, "M", "1"), - (0x1D7DA, "M", "2"), - (0x1D7DB, "M", "3"), - (0x1D7DC, "M", "4"), - (0x1D7DD, "M", "5"), - (0x1D7DE, "M", "6"), - (0x1D7DF, "M", "7"), - (0x1D7E0, "M", "8"), - (0x1D7E1, "M", "9"), - (0x1D7E2, "M", "0"), - (0x1D7E3, "M", "1"), - (0x1D7E4, "M", "2"), - (0x1D7E5, "M", "3"), - (0x1D7E6, "M", "4"), - (0x1D7E7, "M", "5"), - (0x1D7E8, "M", "6"), - (0x1D7E9, "M", "7"), - (0x1D7EA, "M", "8"), - (0x1D7EB, "M", "9"), - (0x1D7EC, "M", "0"), - (0x1D7ED, "M", "1"), - (0x1D7EE, "M", "2"), - (0x1D7EF, "M", "3"), - (0x1D7F0, "M", "4"), - (0x1D7F1, "M", "5"), - (0x1D7F2, "M", "6"), - (0x1D7F3, "M", "7"), - (0x1D7F4, "M", "8"), - (0x1D7F5, "M", "9"), - (0x1D7F6, "M", "0"), - (0x1D7F7, "M", "1"), - (0x1D7F8, "M", "2"), - (0x1D7F9, "M", "3"), - (0x1D7FA, "M", "4"), - (0x1D7FB, "M", "5"), - (0x1D7FC, "M", "6"), - (0x1D7FD, "M", "7"), - (0x1D7FE, "M", "8"), - (0x1D7FF, "M", "9"), - (0x1D800, "V"), - (0x1DA8C, "X"), - (0x1DA9B, "V"), - (0x1DAA0, "X"), - (0x1DAA1, "V"), - (0x1DAB0, "X"), - (0x1DF00, "V"), - (0x1DF1F, "X"), - (0x1DF25, "V"), - (0x1DF2B, "X"), - (0x1E000, "V"), - (0x1E007, "X"), - (0x1E008, "V"), - (0x1E019, "X"), - (0x1E01B, "V"), - (0x1E022, "X"), - (0x1E023, "V"), - (0x1E025, "X"), - (0x1E026, "V"), - (0x1E02B, "X"), - (0x1E030, "M", "а"), - (0x1E031, "M", "б"), - (0x1E032, "M", "в"), - (0x1E033, "M", "г"), - (0x1E034, "M", "д"), - (0x1E035, "M", "е"), - (0x1E036, "M", "ж"), - (0x1E037, "M", "з"), - (0x1E038, "M", "и"), - (0x1E039, "M", "к"), - (0x1E03A, "M", "л"), - (0x1E03B, "M", "м"), - (0x1E03C, "M", "о"), - (0x1E03D, "M", "п"), - (0x1E03E, "M", "р"), - (0x1E03F, "M", "с"), - (0x1E040, "M", "т"), - (0x1E041, "M", "у"), - (0x1E042, "M", "ф"), - (0x1E043, "M", "х"), - (0x1E044, "M", "ц"), - (0x1E045, "M", "ч"), - (0x1E046, "M", "ш"), - (0x1E047, "M", "ы"), - (0x1E048, "M", "э"), - (0x1E049, "M", "ю"), - (0x1E04A, "M", "ꚉ"), - (0x1E04B, "M", "ә"), - (0x1E04C, "M", "і"), - (0x1E04D, "M", "ј"), - (0x1E04E, "M", "ө"), - (0x1E04F, "M", "ү"), - (0x1E050, "M", "ӏ"), - (0x1E051, "M", "а"), - (0x1E052, "M", "б"), - (0x1E053, "M", "в"), - (0x1E054, "M", "г"), - (0x1E055, "M", "д"), - (0x1E056, "M", "е"), - (0x1E057, "M", "ж"), - (0x1E058, "M", "з"), - (0x1E059, "M", "и"), - (0x1E05A, "M", "к"), - (0x1E05B, "M", "л"), - (0x1E05C, "M", "о"), - (0x1E05D, "M", "п"), - (0x1E05E, "M", "с"), - (0x1E05F, "M", "у"), - (0x1E060, "M", "ф"), - (0x1E061, "M", "х"), - (0x1E062, "M", "ц"), - (0x1E063, "M", "ч"), - (0x1E064, "M", "ш"), - (0x1E065, "M", "ъ"), - (0x1E066, "M", "ы"), - (0x1E067, "M", "ґ"), - (0x1E068, "M", "і"), - (0x1E069, "M", "ѕ"), - (0x1E06A, "M", "џ"), - (0x1E06B, "M", "ҫ"), - (0x1E06C, "M", "ꙑ"), - (0x1E06D, "M", "ұ"), - (0x1E06E, "X"), - (0x1E08F, "V"), - (0x1E090, "X"), - (0x1E100, "V"), - (0x1E12D, "X"), - (0x1E130, "V"), - (0x1E13E, "X"), - (0x1E140, "V"), - (0x1E14A, "X"), - (0x1E14E, "V"), - (0x1E150, "X"), - (0x1E290, "V"), - (0x1E2AF, "X"), - (0x1E2C0, "V"), - (0x1E2FA, "X"), - (0x1E2FF, "V"), - (0x1E300, "X"), - (0x1E4D0, "V"), - (0x1E4FA, "X"), - (0x1E5D0, "V"), - (0x1E5FB, "X"), - (0x1E5FF, "V"), - (0x1E600, "X"), - (0x1E6C0, "V"), - (0x1E6DF, "X"), - (0x1E6E0, "V"), - (0x1E6F6, "X"), - (0x1E6FE, "V"), - (0x1E700, "X"), - (0x1E7E0, "V"), - (0x1E7E7, "X"), - (0x1E7E8, "V"), - (0x1E7EC, "X"), - (0x1E7ED, "V"), - (0x1E7EF, "X"), - (0x1E7F0, "V"), - (0x1E7FF, "X"), - (0x1E800, "V"), - (0x1E8C5, "X"), - (0x1E8C7, "V"), - (0x1E8D7, "X"), - (0x1E900, "M", "𞤢"), - (0x1E901, "M", "𞤣"), - (0x1E902, "M", "𞤤"), - (0x1E903, "M", "𞤥"), - (0x1E904, "M", "𞤦"), - (0x1E905, "M", "𞤧"), - (0x1E906, "M", "𞤨"), - (0x1E907, "M", "𞤩"), - (0x1E908, "M", "𞤪"), - (0x1E909, "M", "𞤫"), - (0x1E90A, "M", "𞤬"), - (0x1E90B, "M", "𞤭"), - (0x1E90C, "M", "𞤮"), - (0x1E90D, "M", "𞤯"), - (0x1E90E, "M", "𞤰"), - (0x1E90F, "M", "𞤱"), - (0x1E910, "M", "𞤲"), - (0x1E911, "M", "𞤳"), - (0x1E912, "M", "𞤴"), - (0x1E913, "M", "𞤵"), - (0x1E914, "M", "𞤶"), - (0x1E915, "M", "𞤷"), - (0x1E916, "M", "𞤸"), - (0x1E917, "M", "𞤹"), - (0x1E918, "M", "𞤺"), - (0x1E919, "M", "𞤻"), - (0x1E91A, "M", "𞤼"), - (0x1E91B, "M", "𞤽"), - (0x1E91C, "M", "𞤾"), - (0x1E91D, "M", "𞤿"), - (0x1E91E, "M", "𞥀"), - (0x1E91F, "M", "𞥁"), - (0x1E920, "M", "𞥂"), - (0x1E921, "M", "𞥃"), - (0x1E922, "V"), - (0x1E94C, "X"), - (0x1E950, "V"), - (0x1E95A, "X"), - (0x1E95E, "V"), - (0x1E960, "X"), - (0x1EC71, "V"), - (0x1ECB5, "X"), - (0x1ED01, "V"), - (0x1ED3E, "X"), - (0x1EE00, "M", "ا"), - (0x1EE01, "M", "ب"), - (0x1EE02, "M", "ج"), - (0x1EE03, "M", "د"), - (0x1EE04, "X"), - (0x1EE05, "M", "و"), - (0x1EE06, "M", "ز"), - (0x1EE07, "M", "ح"), - (0x1EE08, "M", "ط"), - (0x1EE09, "M", "ي"), - (0x1EE0A, "M", "ك"), - (0x1EE0B, "M", "ل"), - (0x1EE0C, "M", "م"), - (0x1EE0D, "M", "ن"), - (0x1EE0E, "M", "س"), - (0x1EE0F, "M", "ع"), - (0x1EE10, "M", "ف"), - (0x1EE11, "M", "ص"), - (0x1EE12, "M", "ق"), - (0x1EE13, "M", "ر"), - (0x1EE14, "M", "ش"), - (0x1EE15, "M", "ت"), - (0x1EE16, "M", "ث"), - (0x1EE17, "M", "خ"), - (0x1EE18, "M", "ذ"), - (0x1EE19, "M", "ض"), - (0x1EE1A, "M", "ظ"), - (0x1EE1B, "M", "غ"), - (0x1EE1C, "M", "ٮ"), - (0x1EE1D, "M", "ں"), - (0x1EE1E, "M", "ڡ"), - (0x1EE1F, "M", "ٯ"), - (0x1EE20, "X"), - (0x1EE21, "M", "ب"), - (0x1EE22, "M", "ج"), - (0x1EE23, "X"), - (0x1EE24, "M", "ه"), - (0x1EE25, "X"), - (0x1EE27, "M", "ح"), - (0x1EE28, "X"), - (0x1EE29, "M", "ي"), - (0x1EE2A, "M", "ك"), - (0x1EE2B, "M", "ل"), - (0x1EE2C, "M", "م"), - (0x1EE2D, "M", "ن"), - (0x1EE2E, "M", "س"), - (0x1EE2F, "M", "ع"), - (0x1EE30, "M", "ف"), - (0x1EE31, "M", "ص"), - (0x1EE32, "M", "ق"), - (0x1EE33, "X"), - (0x1EE34, "M", "ش"), - (0x1EE35, "M", "ت"), - (0x1EE36, "M", "ث"), - (0x1EE37, "M", "خ"), - (0x1EE38, "X"), - (0x1EE39, "M", "ض"), - (0x1EE3A, "X"), - (0x1EE3B, "M", "غ"), - (0x1EE3C, "X"), - (0x1EE42, "M", "ج"), - (0x1EE43, "X"), - (0x1EE47, "M", "ح"), - (0x1EE48, "X"), - (0x1EE49, "M", "ي"), - (0x1EE4A, "X"), - (0x1EE4B, "M", "ل"), - (0x1EE4C, "X"), - (0x1EE4D, "M", "ن"), - (0x1EE4E, "M", "س"), - (0x1EE4F, "M", "ع"), - (0x1EE50, "X"), - (0x1EE51, "M", "ص"), - (0x1EE52, "M", "ق"), - (0x1EE53, "X"), - (0x1EE54, "M", "ش"), - (0x1EE55, "X"), - (0x1EE57, "M", "خ"), - (0x1EE58, "X"), - (0x1EE59, "M", "ض"), - (0x1EE5A, "X"), - (0x1EE5B, "M", "غ"), - (0x1EE5C, "X"), - (0x1EE5D, "M", "ں"), - (0x1EE5E, "X"), - (0x1EE5F, "M", "ٯ"), - (0x1EE60, "X"), - (0x1EE61, "M", "ب"), - (0x1EE62, "M", "ج"), - (0x1EE63, "X"), - (0x1EE64, "M", "ه"), - (0x1EE65, "X"), - (0x1EE67, "M", "ح"), - (0x1EE68, "M", "ط"), - (0x1EE69, "M", "ي"), - (0x1EE6A, "M", "ك"), - (0x1EE6B, "X"), - (0x1EE6C, "M", "م"), - (0x1EE6D, "M", "ن"), - (0x1EE6E, "M", "س"), - (0x1EE6F, "M", "ع"), - (0x1EE70, "M", "ف"), - (0x1EE71, "M", "ص"), - (0x1EE72, "M", "ق"), - (0x1EE73, "X"), - (0x1EE74, "M", "ش"), - (0x1EE75, "M", "ت"), - (0x1EE76, "M", "ث"), - (0x1EE77, "M", "خ"), - (0x1EE78, "X"), - (0x1EE79, "M", "ض"), - (0x1EE7A, "M", "ظ"), - (0x1EE7B, "M", "غ"), - (0x1EE7C, "M", "ٮ"), - (0x1EE7D, "X"), - (0x1EE7E, "M", "ڡ"), - (0x1EE7F, "X"), - (0x1EE80, "M", "ا"), - (0x1EE81, "M", "ب"), - (0x1EE82, "M", "ج"), - (0x1EE83, "M", "د"), - (0x1EE84, "M", "ه"), - (0x1EE85, "M", "و"), - (0x1EE86, "M", "ز"), - (0x1EE87, "M", "ح"), - (0x1EE88, "M", "ط"), - (0x1EE89, "M", "ي"), - (0x1EE8A, "X"), - (0x1EE8B, "M", "ل"), - (0x1EE8C, "M", "م"), - (0x1EE8D, "M", "ن"), - (0x1EE8E, "M", "س"), - (0x1EE8F, "M", "ع"), - (0x1EE90, "M", "ف"), - (0x1EE91, "M", "ص"), - (0x1EE92, "M", "ق"), - (0x1EE93, "M", "ر"), - (0x1EE94, "M", "ش"), - (0x1EE95, "M", "ت"), - (0x1EE96, "M", "ث"), - (0x1EE97, "M", "خ"), - (0x1EE98, "M", "ذ"), - (0x1EE99, "M", "ض"), - (0x1EE9A, "M", "ظ"), - (0x1EE9B, "M", "غ"), - (0x1EE9C, "X"), - (0x1EEA1, "M", "ب"), - (0x1EEA2, "M", "ج"), - (0x1EEA3, "M", "د"), - (0x1EEA4, "X"), - (0x1EEA5, "M", "و"), - (0x1EEA6, "M", "ز"), - (0x1EEA7, "M", "ح"), - (0x1EEA8, "M", "ط"), - (0x1EEA9, "M", "ي"), - (0x1EEAA, "X"), - (0x1EEAB, "M", "ل"), - (0x1EEAC, "M", "م"), - (0x1EEAD, "M", "ن"), - (0x1EEAE, "M", "س"), - (0x1EEAF, "M", "ع"), - (0x1EEB0, "M", "ف"), - (0x1EEB1, "M", "ص"), - (0x1EEB2, "M", "ق"), - (0x1EEB3, "M", "ر"), - (0x1EEB4, "M", "ش"), - (0x1EEB5, "M", "ت"), - (0x1EEB6, "M", "ث"), - (0x1EEB7, "M", "خ"), - (0x1EEB8, "M", "ذ"), - (0x1EEB9, "M", "ض"), - (0x1EEBA, "M", "ظ"), - (0x1EEBB, "M", "غ"), - (0x1EEBC, "X"), - (0x1EEF0, "V"), - (0x1EEF2, "X"), - (0x1F000, "V"), - (0x1F02C, "X"), - (0x1F030, "V"), - (0x1F094, "X"), - (0x1F0A0, "V"), - (0x1F0AF, "X"), - (0x1F0B1, "V"), - (0x1F0C0, "X"), - (0x1F0C1, "V"), - (0x1F0D0, "X"), - (0x1F0D1, "V"), - (0x1F0F6, "X"), - (0x1F101, "M", "0,"), - (0x1F102, "M", "1,"), - (0x1F103, "M", "2,"), - (0x1F104, "M", "3,"), - (0x1F105, "M", "4,"), - (0x1F106, "M", "5,"), - (0x1F107, "M", "6,"), - (0x1F108, "M", "7,"), - (0x1F109, "M", "8,"), - (0x1F10A, "M", "9,"), - (0x1F10B, "V"), - (0x1F110, "M", "(a)"), - (0x1F111, "M", "(b)"), - (0x1F112, "M", "(c)"), - (0x1F113, "M", "(d)"), - (0x1F114, "M", "(e)"), - (0x1F115, "M", "(f)"), - (0x1F116, "M", "(g)"), - (0x1F117, "M", "(h)"), - (0x1F118, "M", "(i)"), - (0x1F119, "M", "(j)"), - (0x1F11A, "M", "(k)"), - (0x1F11B, "M", "(l)"), - (0x1F11C, "M", "(m)"), - (0x1F11D, "M", "(n)"), - (0x1F11E, "M", "(o)"), - (0x1F11F, "M", "(p)"), - (0x1F120, "M", "(q)"), - (0x1F121, "M", "(r)"), - (0x1F122, "M", "(s)"), - (0x1F123, "M", "(t)"), - (0x1F124, "M", "(u)"), - (0x1F125, "M", "(v)"), - (0x1F126, "M", "(w)"), - (0x1F127, "M", "(x)"), - (0x1F128, "M", "(y)"), - (0x1F129, "M", "(z)"), - (0x1F12A, "M", "〔s〕"), - (0x1F12B, "M", "c"), - (0x1F12C, "M", "r"), - (0x1F12D, "M", "cd"), - (0x1F12E, "M", "wz"), - (0x1F12F, "V"), - (0x1F130, "M", "a"), - (0x1F131, "M", "b"), - (0x1F132, "M", "c"), - (0x1F133, "M", "d"), - (0x1F134, "M", "e"), - (0x1F135, "M", "f"), - (0x1F136, "M", "g"), - (0x1F137, "M", "h"), - (0x1F138, "M", "i"), - (0x1F139, "M", "j"), - (0x1F13A, "M", "k"), - (0x1F13B, "M", "l"), - (0x1F13C, "M", "m"), - (0x1F13D, "M", "n"), - (0x1F13E, "M", "o"), - (0x1F13F, "M", "p"), - (0x1F140, "M", "q"), - (0x1F141, "M", "r"), - (0x1F142, "M", "s"), - (0x1F143, "M", "t"), - (0x1F144, "M", "u"), - (0x1F145, "M", "v"), - (0x1F146, "M", "w"), - (0x1F147, "M", "x"), - (0x1F148, "M", "y"), - (0x1F149, "M", "z"), - (0x1F14A, "M", "hv"), - (0x1F14B, "M", "mv"), - (0x1F14C, "M", "sd"), - (0x1F14D, "M", "ss"), - (0x1F14E, "M", "ppv"), - (0x1F14F, "M", "wc"), - (0x1F150, "V"), - (0x1F16A, "M", "mc"), - (0x1F16B, "M", "md"), - (0x1F16C, "M", "mr"), - (0x1F16D, "V"), - (0x1F190, "M", "dj"), - (0x1F191, "V"), - (0x1F1AE, "X"), - (0x1F1E6, "V"), - (0x1F200, "M", "ほか"), - (0x1F201, "M", "ココ"), - (0x1F202, "M", "サ"), - (0x1F203, "X"), - (0x1F210, "M", "手"), - (0x1F211, "M", "字"), - (0x1F212, "M", "双"), - (0x1F213, "M", "デ"), - (0x1F214, "M", "二"), - (0x1F215, "M", "多"), - (0x1F216, "M", "解"), - (0x1F217, "M", "天"), - (0x1F218, "M", "交"), - (0x1F219, "M", "映"), - (0x1F21A, "M", "無"), - (0x1F21B, "M", "料"), - (0x1F21C, "M", "前"), - (0x1F21D, "M", "後"), - (0x1F21E, "M", "再"), - (0x1F21F, "M", "新"), - (0x1F220, "M", "初"), - (0x1F221, "M", "終"), - (0x1F222, "M", "生"), - (0x1F223, "M", "販"), - (0x1F224, "M", "声"), - (0x1F225, "M", "吹"), - (0x1F226, "M", "演"), - (0x1F227, "M", "投"), - (0x1F228, "M", "捕"), - (0x1F229, "M", "一"), - (0x1F22A, "M", "三"), - (0x1F22B, "M", "遊"), - (0x1F22C, "M", "左"), - (0x1F22D, "M", "中"), - (0x1F22E, "M", "右"), - (0x1F22F, "M", "指"), - (0x1F230, "M", "走"), - (0x1F231, "M", "打"), - (0x1F232, "M", "禁"), - (0x1F233, "M", "空"), - (0x1F234, "M", "合"), - (0x1F235, "M", "満"), - (0x1F236, "M", "有"), - (0x1F237, "M", "月"), - (0x1F238, "M", "申"), - (0x1F239, "M", "割"), - (0x1F23A, "M", "営"), - (0x1F23B, "M", "配"), - (0x1F23C, "X"), - (0x1F240, "M", "〔本〕"), - (0x1F241, "M", "〔三〕"), - (0x1F242, "M", "〔二〕"), - (0x1F243, "M", "〔安〕"), - (0x1F244, "M", "〔点〕"), - (0x1F245, "M", "〔打〕"), - (0x1F246, "M", "〔盗〕"), - (0x1F247, "M", "〔勝〕"), - (0x1F248, "M", "〔敗〕"), - (0x1F249, "X"), - (0x1F250, "M", "得"), - (0x1F251, "M", "可"), - (0x1F252, "X"), - (0x1F260, "V"), - (0x1F266, "X"), - (0x1F300, "V"), - (0x1F6D9, "X"), - (0x1F6DC, "V"), - (0x1F6ED, "X"), - (0x1F6F0, "V"), - (0x1F6FD, "X"), - (0x1F700, "V"), - (0x1F7DA, "X"), - (0x1F7E0, "V"), - (0x1F7EC, "X"), - (0x1F7F0, "V"), - (0x1F7F1, "X"), - (0x1F800, "V"), - (0x1F80C, "X"), - (0x1F810, "V"), - (0x1F848, "X"), - (0x1F850, "V"), - (0x1F85A, "X"), - (0x1F860, "V"), - (0x1F888, "X"), - (0x1F890, "V"), - (0x1F8AE, "X"), - (0x1F8B0, "V"), - (0x1F8BC, "X"), - (0x1F8C0, "V"), - (0x1F8C2, "X"), - (0x1F8D0, "V"), - (0x1F8D9, "X"), - (0x1F900, "V"), - (0x1FA58, "X"), - (0x1FA60, "V"), - (0x1FA6E, "X"), - (0x1FA70, "V"), - (0x1FA7D, "X"), - (0x1FA80, "V"), - (0x1FA8B, "X"), - (0x1FA8E, "V"), - (0x1FAC7, "X"), - (0x1FAC8, "V"), - (0x1FAC9, "X"), - (0x1FACD, "V"), - (0x1FADD, "X"), - (0x1FADF, "V"), - (0x1FAEB, "X"), - (0x1FAEF, "V"), - (0x1FAF9, "X"), - (0x1FB00, "V"), - (0x1FB93, "X"), - (0x1FB94, "V"), - (0x1FBF0, "M", "0"), - (0x1FBF1, "M", "1"), - (0x1FBF2, "M", "2"), - (0x1FBF3, "M", "3"), - (0x1FBF4, "M", "4"), - (0x1FBF5, "M", "5"), - (0x1FBF6, "M", "6"), - (0x1FBF7, "M", "7"), - (0x1FBF8, "M", "8"), - (0x1FBF9, "M", "9"), - (0x1FBFA, "V"), - (0x1FBFB, "X"), - (0x20000, "V"), - (0x2A6E0, "X"), - (0x2A700, "V"), - (0x2B81E, "X"), - (0x2B820, "V"), - (0x2CEAE, "X"), - (0x2CEB0, "V"), - (0x2EBE1, "X"), - (0x2EBF0, "V"), - (0x2EE5E, "X"), - (0x2F800, "M", "丽"), - (0x2F801, "M", "丸"), - (0x2F802, "M", "乁"), - (0x2F803, "M", "𠄢"), - (0x2F804, "M", "你"), - (0x2F805, "M", "侮"), - (0x2F806, "M", "侻"), - (0x2F807, "M", "倂"), - (0x2F808, "M", "偺"), - (0x2F809, "M", "備"), - (0x2F80A, "M", "僧"), - (0x2F80B, "M", "像"), - (0x2F80C, "M", "㒞"), - (0x2F80D, "M", "𠘺"), - (0x2F80E, "M", "免"), - (0x2F80F, "M", "兔"), - (0x2F810, "M", "兤"), - (0x2F811, "M", "具"), - (0x2F812, "M", "𠔜"), - (0x2F813, "M", "㒹"), - (0x2F814, "M", "內"), - (0x2F815, "M", "再"), - (0x2F816, "M", "𠕋"), - (0x2F817, "M", "冗"), - (0x2F818, "M", "冤"), - (0x2F819, "M", "仌"), - (0x2F81A, "M", "冬"), - (0x2F81B, "M", "况"), - (0x2F81C, "M", "𩇟"), - (0x2F81D, "M", "凵"), - (0x2F81E, "M", "刃"), - (0x2F81F, "M", "㓟"), - (0x2F820, "M", "刻"), - (0x2F821, "M", "剆"), - (0x2F822, "M", "割"), - (0x2F823, "M", "剷"), - (0x2F824, "M", "㔕"), - (0x2F825, "M", "勇"), - (0x2F826, "M", "勉"), - (0x2F827, "M", "勤"), - (0x2F828, "M", "勺"), - (0x2F829, "M", "包"), - (0x2F82A, "M", "匆"), - (0x2F82B, "M", "北"), - (0x2F82C, "M", "卉"), - (0x2F82D, "M", "卑"), - (0x2F82E, "M", "博"), - (0x2F82F, "M", "即"), - (0x2F830, "M", "卽"), - (0x2F831, "M", "卿"), - (0x2F834, "M", "𠨬"), - (0x2F835, "M", "灰"), - (0x2F836, "M", "及"), - (0x2F837, "M", "叟"), - (0x2F838, "M", "𠭣"), - (0x2F839, "M", "叫"), - (0x2F83A, "M", "叱"), - (0x2F83B, "M", "吆"), - (0x2F83C, "M", "咞"), - (0x2F83D, "M", "吸"), - (0x2F83E, "M", "呈"), - (0x2F83F, "M", "周"), - (0x2F840, "M", "咢"), - (0x2F841, "M", "哶"), - (0x2F842, "M", "唐"), - (0x2F843, "M", "啓"), - (0x2F844, "M", "啣"), - (0x2F845, "M", "善"), - (0x2F847, "M", "喙"), - (0x2F848, "M", "喫"), - (0x2F849, "M", "喳"), - (0x2F84A, "M", "嗂"), - (0x2F84B, "M", "圖"), - (0x2F84C, "M", "嘆"), - (0x2F84D, "M", "圗"), - (0x2F84E, "M", "噑"), - (0x2F84F, "M", "噴"), - (0x2F850, "M", "切"), - (0x2F851, "M", "壮"), - (0x2F852, "M", "城"), - (0x2F853, "M", "埴"), - (0x2F854, "M", "堍"), - (0x2F855, "M", "型"), - (0x2F856, "M", "堲"), - (0x2F857, "M", "報"), - (0x2F858, "M", "墬"), - (0x2F859, "M", "𡓤"), - (0x2F85A, "M", "売"), - (0x2F85B, "M", "壷"), - (0x2F85C, "M", "夆"), - (0x2F85D, "M", "多"), - (0x2F85E, "M", "夢"), - (0x2F85F, "M", "奢"), - (0x2F860, "M", "𡚨"), - (0x2F861, "M", "𡛪"), - (0x2F862, "M", "姬"), - (0x2F863, "M", "娛"), - (0x2F864, "M", "娧"), - (0x2F865, "M", "姘"), - (0x2F866, "M", "婦"), - (0x2F867, "M", "㛮"), - (0x2F868, "M", "㛼"), - (0x2F869, "M", "嬈"), - (0x2F86A, "M", "嬾"), - (0x2F86C, "M", "𡧈"), - (0x2F86D, "M", "寃"), - (0x2F86E, "M", "寘"), - (0x2F86F, "M", "寧"), - (0x2F870, "M", "寳"), - (0x2F871, "M", "𡬘"), - (0x2F872, "M", "寿"), - (0x2F873, "M", "将"), - (0x2F874, "M", "当"), - (0x2F875, "M", "尢"), - (0x2F876, "M", "㞁"), - (0x2F877, "M", "屠"), - (0x2F878, "M", "屮"), - (0x2F879, "M", "峀"), - (0x2F87A, "M", "岍"), - (0x2F87B, "M", "𡷤"), - (0x2F87C, "M", "嵃"), - (0x2F87D, "M", "𡷦"), - (0x2F87E, "M", "嵮"), - (0x2F87F, "M", "嵫"), - (0x2F880, "M", "嵼"), - (0x2F881, "M", "巡"), - (0x2F882, "M", "巢"), - (0x2F883, "M", "㠯"), - (0x2F884, "M", "巽"), - (0x2F885, "M", "帨"), - (0x2F886, "M", "帽"), - (0x2F887, "M", "幩"), - (0x2F888, "M", "㡢"), - (0x2F889, "M", "𢆃"), - (0x2F88A, "M", "㡼"), - (0x2F88B, "M", "庰"), - (0x2F88C, "M", "庳"), - (0x2F88D, "M", "庶"), - (0x2F88E, "M", "廊"), - (0x2F88F, "M", "𪎒"), - (0x2F890, "M", "廾"), - (0x2F891, "M", "𢌱"), - (0x2F893, "M", "舁"), - (0x2F894, "M", "弢"), - (0x2F896, "M", "㣇"), - (0x2F897, "M", "𣊸"), - (0x2F898, "M", "𦇚"), - (0x2F899, "M", "形"), - (0x2F89A, "M", "彫"), - (0x2F89B, "M", "㣣"), - (0x2F89C, "M", "徚"), - (0x2F89D, "M", "忍"), - (0x2F89E, "M", "志"), - (0x2F89F, "M", "忹"), - (0x2F8A0, "M", "悁"), - (0x2F8A1, "M", "㤺"), - (0x2F8A2, "M", "㤜"), - (0x2F8A3, "M", "悔"), - (0x2F8A4, "M", "𢛔"), - (0x2F8A5, "M", "惇"), - (0x2F8A6, "M", "慈"), - (0x2F8A7, "M", "慌"), - (0x2F8A8, "M", "慎"), - (0x2F8A9, "M", "慌"), - (0x2F8AA, "M", "慺"), - (0x2F8AB, "M", "憎"), - (0x2F8AC, "M", "憲"), - (0x2F8AD, "M", "憤"), - (0x2F8AE, "M", "憯"), - (0x2F8AF, "M", "懞"), - (0x2F8B0, "M", "懲"), - (0x2F8B1, "M", "懶"), - (0x2F8B2, "M", "成"), - (0x2F8B3, "M", "戛"), - (0x2F8B4, "M", "扝"), - (0x2F8B5, "M", "抱"), - (0x2F8B6, "M", "拔"), - (0x2F8B7, "M", "捐"), - (0x2F8B8, "M", "𢬌"), - (0x2F8B9, "M", "挽"), - (0x2F8BA, "M", "拼"), - (0x2F8BB, "M", "捨"), - (0x2F8BC, "M", "掃"), - (0x2F8BD, "M", "揤"), - (0x2F8BE, "M", "𢯱"), - (0x2F8BF, "M", "搢"), - (0x2F8C0, "M", "揅"), - (0x2F8C1, "M", "掩"), - (0x2F8C2, "M", "㨮"), - (0x2F8C3, "M", "摩"), - (0x2F8C4, "M", "摾"), - (0x2F8C5, "M", "撝"), - (0x2F8C6, "M", "摷"), - (0x2F8C7, "M", "㩬"), - (0x2F8C8, "M", "敏"), - (0x2F8C9, "M", "敬"), - (0x2F8CA, "M", "𣀊"), - (0x2F8CB, "M", "旣"), - (0x2F8CC, "M", "書"), - (0x2F8CD, "M", "晉"), - (0x2F8CE, "M", "㬙"), - (0x2F8CF, "M", "暑"), - (0x2F8D0, "M", "㬈"), - (0x2F8D1, "M", "㫤"), - (0x2F8D2, "M", "冒"), - (0x2F8D3, "M", "冕"), - (0x2F8D4, "M", "最"), - (0x2F8D5, "M", "暜"), - (0x2F8D6, "M", "肭"), - (0x2F8D7, "M", "䏙"), - (0x2F8D8, "M", "朗"), - (0x2F8D9, "M", "望"), - (0x2F8DA, "M", "朡"), - (0x2F8DB, "M", "杞"), - (0x2F8DC, "M", "杓"), - (0x2F8DD, "M", "𣏃"), - (0x2F8DE, "M", "㭉"), - (0x2F8DF, "M", "柺"), - (0x2F8E0, "M", "枅"), - (0x2F8E1, "M", "桒"), - (0x2F8E2, "M", "梅"), - (0x2F8E3, "M", "𣑭"), - (0x2F8E4, "M", "梎"), - (0x2F8E5, "M", "栟"), - (0x2F8E6, "M", "椔"), - (0x2F8E7, "M", "㮝"), - (0x2F8E8, "M", "楂"), - (0x2F8E9, "M", "榣"), - (0x2F8EA, "M", "槪"), - (0x2F8EB, "M", "檨"), - (0x2F8EC, "M", "𣚣"), - (0x2F8ED, "M", "櫛"), - (0x2F8EE, "M", "㰘"), - (0x2F8EF, "M", "次"), - (0x2F8F0, "M", "𣢧"), - (0x2F8F1, "M", "歔"), - (0x2F8F2, "M", "㱎"), - (0x2F8F3, "M", "歲"), - (0x2F8F4, "M", "殟"), - (0x2F8F5, "M", "殺"), - (0x2F8F6, "M", "殻"), - (0x2F8F7, "M", "𣪍"), - (0x2F8F8, "M", "𡴋"), - (0x2F8F9, "M", "𣫺"), - (0x2F8FA, "M", "汎"), - (0x2F8FB, "M", "𣲼"), - (0x2F8FC, "M", "沿"), - (0x2F8FD, "M", "泍"), - (0x2F8FE, "M", "汧"), - (0x2F8FF, "M", "洖"), - (0x2F900, "M", "派"), - (0x2F901, "M", "海"), - (0x2F902, "M", "流"), - (0x2F903, "M", "浩"), - (0x2F904, "M", "浸"), - (0x2F905, "M", "涅"), - (0x2F906, "M", "𣴞"), - (0x2F907, "M", "洴"), - (0x2F908, "M", "港"), - (0x2F909, "M", "湮"), - (0x2F90A, "M", "㴳"), - (0x2F90B, "M", "滋"), - (0x2F90C, "M", "滇"), - (0x2F90D, "M", "𣻑"), - (0x2F90E, "M", "淹"), - (0x2F90F, "M", "潮"), - (0x2F910, "M", "𣽞"), - (0x2F911, "M", "𣾎"), - (0x2F912, "M", "濆"), - (0x2F913, "M", "瀹"), - (0x2F914, "M", "瀞"), - (0x2F915, "M", "瀛"), - (0x2F916, "M", "㶖"), - (0x2F917, "M", "灊"), - (0x2F918, "M", "災"), - (0x2F919, "M", "灷"), - (0x2F91A, "M", "炭"), - (0x2F91B, "M", "𠔥"), - (0x2F91C, "M", "煅"), - (0x2F91D, "M", "𤉣"), - (0x2F91E, "M", "熜"), - (0x2F91F, "M", "𤎫"), - (0x2F920, "M", "爨"), - (0x2F921, "M", "爵"), - (0x2F922, "M", "牐"), - (0x2F923, "M", "𤘈"), - (0x2F924, "M", "犀"), - (0x2F925, "M", "犕"), - (0x2F926, "M", "𤜵"), - (0x2F927, "M", "𤠔"), - (0x2F928, "M", "獺"), - (0x2F929, "M", "王"), - (0x2F92A, "M", "㺬"), - (0x2F92B, "M", "玥"), - (0x2F92C, "M", "㺸"), - (0x2F92E, "M", "瑇"), - (0x2F92F, "M", "瑜"), - (0x2F930, "M", "瑱"), - (0x2F931, "M", "璅"), - (0x2F932, "M", "瓊"), - (0x2F933, "M", "㼛"), - (0x2F934, "M", "甤"), - (0x2F935, "M", "𤰶"), - (0x2F936, "M", "甾"), - (0x2F937, "M", "𤲒"), - (0x2F938, "M", "異"), - (0x2F939, "M", "𢆟"), - (0x2F93A, "M", "瘐"), - (0x2F93B, "M", "𤾡"), - (0x2F93C, "M", "𤾸"), - (0x2F93D, "M", "𥁄"), - (0x2F93E, "M", "㿼"), - (0x2F93F, "M", "䀈"), - (0x2F940, "M", "直"), - (0x2F941, "M", "𥃳"), - (0x2F942, "M", "𥃲"), - (0x2F943, "M", "𥄙"), - (0x2F944, "M", "𥄳"), - (0x2F945, "M", "眞"), - (0x2F946, "M", "真"), - (0x2F948, "M", "睊"), - (0x2F949, "M", "䀹"), - (0x2F94A, "M", "瞋"), - (0x2F94B, "M", "䁆"), - (0x2F94C, "M", "䂖"), - (0x2F94D, "M", "𥐝"), - (0x2F94E, "M", "硎"), - (0x2F94F, "M", "碌"), - (0x2F950, "M", "磌"), - (0x2F951, "M", "䃣"), - (0x2F952, "M", "𥘦"), - (0x2F953, "M", "祖"), - (0x2F954, "M", "𥚚"), - (0x2F955, "M", "𥛅"), - (0x2F956, "M", "福"), - (0x2F957, "M", "秫"), - (0x2F958, "M", "䄯"), - (0x2F959, "M", "穀"), - (0x2F95A, "M", "穊"), - (0x2F95B, "M", "穏"), - (0x2F95C, "M", "𥥼"), - (0x2F95D, "M", "𥪧"), - (0x2F95F, "M", "竮"), - (0x2F960, "M", "䈂"), - (0x2F961, "M", "𥮫"), - (0x2F962, "M", "篆"), - (0x2F963, "M", "築"), - (0x2F964, "M", "䈧"), - (0x2F965, "M", "𥲀"), - (0x2F966, "M", "糒"), - (0x2F967, "M", "䊠"), - (0x2F968, "M", "糨"), - (0x2F969, "M", "糣"), - (0x2F96A, "M", "紀"), - (0x2F96B, "M", "𥾆"), - (0x2F96C, "M", "絣"), - (0x2F96D, "M", "䌁"), - (0x2F96E, "M", "緇"), - (0x2F96F, "M", "縂"), - (0x2F970, "M", "繅"), - (0x2F971, "M", "䌴"), - (0x2F972, "M", "𦈨"), - (0x2F973, "M", "𦉇"), - (0x2F974, "M", "䍙"), - (0x2F975, "M", "𦋙"), - (0x2F976, "M", "罺"), - (0x2F977, "M", "𦌾"), - (0x2F978, "M", "羕"), - (0x2F979, "M", "翺"), - (0x2F97A, "M", "者"), - (0x2F97B, "M", "𦓚"), - (0x2F97C, "M", "𦔣"), - (0x2F97D, "M", "聠"), - (0x2F97E, "M", "𦖨"), - (0x2F97F, "M", "聰"), - (0x2F980, "M", "𣍟"), - (0x2F981, "M", "䏕"), - (0x2F982, "M", "育"), - (0x2F983, "M", "脃"), - (0x2F984, "M", "䐋"), - (0x2F985, "M", "脾"), - (0x2F986, "M", "媵"), - (0x2F987, "M", "𦞧"), - (0x2F988, "M", "𦞵"), - (0x2F989, "M", "𣎓"), - (0x2F98A, "M", "𣎜"), - (0x2F98B, "M", "舁"), - (0x2F98C, "M", "舄"), - (0x2F98D, "M", "辞"), - (0x2F98E, "M", "䑫"), - (0x2F98F, "M", "芑"), - (0x2F990, "M", "芋"), - (0x2F991, "M", "芝"), - (0x2F992, "M", "劳"), - (0x2F993, "M", "花"), - (0x2F994, "M", "芳"), - (0x2F995, "M", "芽"), - (0x2F996, "M", "苦"), - (0x2F997, "M", "𦬼"), - (0x2F998, "M", "若"), - (0x2F999, "M", "茝"), - (0x2F99A, "M", "荣"), - (0x2F99B, "M", "莭"), - (0x2F99C, "M", "茣"), - (0x2F99D, "M", "莽"), - (0x2F99E, "M", "菧"), - (0x2F99F, "M", "著"), - (0x2F9A0, "M", "荓"), - (0x2F9A1, "M", "菊"), - (0x2F9A2, "M", "菌"), - (0x2F9A3, "M", "菜"), - (0x2F9A4, "M", "𦰶"), - (0x2F9A5, "M", "𦵫"), - (0x2F9A6, "M", "𦳕"), - (0x2F9A7, "M", "䔫"), - (0x2F9A8, "M", "蓱"), - (0x2F9A9, "M", "蓳"), - (0x2F9AA, "M", "蔖"), - (0x2F9AB, "M", "𧏊"), - (0x2F9AC, "M", "蕤"), - (0x2F9AD, "M", "𦼬"), - (0x2F9AE, "M", "䕝"), - (0x2F9AF, "M", "䕡"), - (0x2F9B0, "M", "𦾱"), - (0x2F9B1, "M", "𧃒"), - (0x2F9B2, "M", "䕫"), - (0x2F9B3, "M", "虐"), - (0x2F9B4, "M", "虜"), - (0x2F9B5, "M", "虧"), - (0x2F9B6, "M", "虩"), - (0x2F9B7, "M", "蚩"), - (0x2F9B8, "M", "蚈"), - (0x2F9B9, "M", "蜎"), - (0x2F9BA, "M", "蛢"), - (0x2F9BB, "M", "蝹"), - (0x2F9BC, "M", "蜨"), - (0x2F9BD, "M", "蝫"), - (0x2F9BE, "M", "螆"), - (0x2F9BF, "M", "䗗"), - (0x2F9C0, "M", "蟡"), - (0x2F9C1, "M", "蠁"), - (0x2F9C2, "M", "䗹"), - (0x2F9C3, "M", "衠"), - (0x2F9C4, "M", "衣"), - (0x2F9C5, "M", "𧙧"), - (0x2F9C6, "M", "裗"), - (0x2F9C7, "M", "裞"), - (0x2F9C8, "M", "䘵"), - (0x2F9C9, "M", "裺"), - (0x2F9CA, "M", "㒻"), - (0x2F9CB, "M", "𧢮"), - (0x2F9CC, "M", "𧥦"), - (0x2F9CD, "M", "䚾"), - (0x2F9CE, "M", "䛇"), - (0x2F9CF, "M", "誠"), - (0x2F9D0, "M", "諭"), - (0x2F9D1, "M", "變"), - (0x2F9D2, "M", "豕"), - (0x2F9D3, "M", "𧲨"), - (0x2F9D4, "M", "貫"), - (0x2F9D5, "M", "賁"), - (0x2F9D6, "M", "贛"), - (0x2F9D7, "M", "起"), - (0x2F9D8, "M", "𧼯"), - (0x2F9D9, "M", "𠠄"), - (0x2F9DA, "M", "跋"), - (0x2F9DB, "M", "趼"), - (0x2F9DC, "M", "跰"), - (0x2F9DD, "M", "𠣞"), - (0x2F9DE, "M", "軔"), - (0x2F9DF, "M", "輸"), - (0x2F9E0, "M", "𨗒"), - (0x2F9E1, "M", "𨗭"), - (0x2F9E2, "M", "邔"), - (0x2F9E3, "M", "郱"), - (0x2F9E4, "M", "鄑"), - (0x2F9E5, "M", "𨜮"), - (0x2F9E6, "M", "鄛"), - (0x2F9E7, "M", "鈸"), - (0x2F9E8, "M", "鋗"), - (0x2F9E9, "M", "鋘"), - (0x2F9EA, "M", "鉼"), - (0x2F9EB, "M", "鏹"), - (0x2F9EC, "M", "鐕"), - (0x2F9ED, "M", "𨯺"), - (0x2F9EE, "M", "開"), - (0x2F9EF, "M", "䦕"), - (0x2F9F0, "M", "閷"), - (0x2F9F1, "M", "𨵷"), - (0x2F9F2, "M", "䧦"), - (0x2F9F3, "M", "雃"), - (0x2F9F4, "M", "嶲"), - (0x2F9F5, "M", "霣"), - (0x2F9F6, "M", "𩅅"), - (0x2F9F7, "M", "𩈚"), - (0x2F9F8, "M", "䩮"), - (0x2F9F9, "M", "䩶"), - (0x2F9FA, "M", "韠"), - (0x2F9FB, "M", "𩐊"), - (0x2F9FC, "M", "䪲"), - (0x2F9FD, "M", "𩒖"), - (0x2F9FE, "M", "頋"), - (0x2FA00, "M", "頩"), - (0x2FA01, "M", "𩖶"), - (0x2FA02, "M", "飢"), - (0x2FA03, "M", "䬳"), - (0x2FA04, "M", "餩"), - (0x2FA05, "M", "馧"), - (0x2FA06, "M", "駂"), - (0x2FA07, "M", "駾"), - (0x2FA08, "M", "䯎"), - (0x2FA09, "M", "𩬰"), - (0x2FA0A, "M", "鬒"), - (0x2FA0B, "M", "鱀"), - (0x2FA0C, "M", "鳽"), - (0x2FA0D, "M", "䳎"), - (0x2FA0E, "M", "䳭"), - (0x2FA0F, "M", "鵧"), - (0x2FA10, "M", "𪃎"), - (0x2FA11, "M", "䳸"), - (0x2FA12, "M", "𪄅"), - (0x2FA13, "M", "𪈎"), - (0x2FA14, "M", "𪊑"), - (0x2FA15, "M", "麻"), - (0x2FA16, "M", "䵖"), - (0x2FA17, "M", "黹"), - (0x2FA18, "M", "黾"), - (0x2FA19, "M", "鼅"), - (0x2FA1A, "M", "鼏"), - (0x2FA1B, "M", "鼖"), - (0x2FA1C, "M", "鼻"), - (0x2FA1D, "M", "𪘀"), - (0x2FA1E, "X"), - (0x30000, "V"), - (0x3134B, "X"), - (0x31350, "V"), - (0x3347A, "X"), - (0xE0100, "I"), - (0xE01F0, "X"), +uts46_starts: "array[int]" = array( + "I", + ( + 0x0, + 0x1, + 0x2, + 0x3, + 0x4, + 0x5, + 0x6, + 0x7, + 0x8, + 0x9, + 0xA, + 0xB, + 0xC, + 0xD, + 0xE, + 0xF, + 0x10, + 0x11, + 0x12, + 0x13, + 0x14, + 0x15, + 0x16, + 0x17, + 0x18, + 0x19, + 0x1A, + 0x1B, + 0x1C, + 0x1D, + 0x1E, + 0x1F, + 0x20, + 0x21, + 0x22, + 0x23, + 0x24, + 0x25, + 0x26, + 0x27, + 0x28, + 0x29, + 0x2A, + 0x2B, + 0x2C, + 0x2D, + 0x2E, + 0x2F, + 0x30, + 0x31, + 0x32, + 0x33, + 0x34, + 0x35, + 0x36, + 0x37, + 0x38, + 0x39, + 0x3A, + 0x3B, + 0x3C, + 0x3D, + 0x3E, + 0x3F, + 0x40, + 0x41, + 0x42, + 0x43, + 0x44, + 0x45, + 0x46, + 0x47, + 0x48, + 0x49, + 0x4A, + 0x4B, + 0x4C, + 0x4D, + 0x4E, + 0x4F, + 0x50, + 0x51, + 0x52, + 0x53, + 0x54, + 0x55, + 0x56, + 0x57, + 0x58, + 0x59, + 0x5A, + 0x5B, + 0x5C, + 0x5D, + 0x5E, + 0x5F, + 0x60, + 0x61, + 0x62, + 0x63, + 0x64, + 0x65, + 0x66, + 0x67, + 0x68, + 0x69, + 0x6A, + 0x6B, + 0x6C, + 0x6D, + 0x6E, + 0x6F, + 0x70, + 0x71, + 0x72, + 0x73, + 0x74, + 0x75, + 0x76, + 0x77, + 0x78, + 0x79, + 0x7A, + 0x7B, + 0x7C, + 0x7D, + 0x7E, + 0x7F, + 0x80, + 0x81, + 0x82, + 0x83, + 0x84, + 0x85, + 0x86, + 0x87, + 0x88, + 0x89, + 0x8A, + 0x8B, + 0x8C, + 0x8D, + 0x8E, + 0x8F, + 0x90, + 0x91, + 0x92, + 0x93, + 0x94, + 0x95, + 0x96, + 0x97, + 0x98, + 0x99, + 0x9A, + 0x9B, + 0x9C, + 0x9D, + 0x9E, + 0x9F, + 0xA0, + 0xA1, + 0xA2, + 0xA3, + 0xA4, + 0xA5, + 0xA6, + 0xA7, + 0xA8, + 0xA9, + 0xAA, + 0xAB, + 0xAC, + 0xAD, + 0xAE, + 0xAF, + 0xB0, + 0xB1, + 0xB2, + 0xB3, + 0xB4, + 0xB5, + 0xB6, + 0xB7, + 0xB8, + 0xB9, + 0xBA, + 0xBB, + 0xBC, + 0xBD, + 0xBE, + 0xBF, + 0xC0, + 0xC1, + 0xC2, + 0xC3, + 0xC4, + 0xC5, + 0xC6, + 0xC7, + 0xC8, + 0xC9, + 0xCA, + 0xCB, + 0xCC, + 0xCD, + 0xCE, + 0xCF, + 0xD0, + 0xD1, + 0xD2, + 0xD3, + 0xD4, + 0xD5, + 0xD6, + 0xD7, + 0xD8, + 0xD9, + 0xDA, + 0xDB, + 0xDC, + 0xDD, + 0xDE, + 0xDF, + 0xE0, + 0xE1, + 0xE2, + 0xE3, + 0xE4, + 0xE5, + 0xE6, + 0xE7, + 0xE8, + 0xE9, + 0xEA, + 0xEB, + 0xEC, + 0xED, + 0xEE, + 0xEF, + 0xF0, + 0xF1, + 0xF2, + 0xF3, + 0xF4, + 0xF5, + 0xF6, + 0xF7, + 0xF8, + 0xF9, + 0xFA, + 0xFB, + 0xFC, + 0xFD, + 0xFE, + 0xFF, + 0x100, + 0x101, + 0x102, + 0x103, + 0x104, + 0x105, + 0x106, + 0x107, + 0x108, + 0x109, + 0x10A, + 0x10B, + 0x10C, + 0x10D, + 0x10E, + 0x10F, + 0x110, + 0x111, + 0x112, + 0x113, + 0x114, + 0x115, + 0x116, + 0x117, + 0x118, + 0x119, + 0x11A, + 0x11B, + 0x11C, + 0x11D, + 0x11E, + 0x11F, + 0x120, + 0x121, + 0x122, + 0x123, + 0x124, + 0x125, + 0x126, + 0x127, + 0x128, + 0x129, + 0x12A, + 0x12B, + 0x12C, + 0x12D, + 0x12E, + 0x12F, + 0x130, + 0x131, + 0x132, + 0x134, + 0x135, + 0x136, + 0x137, + 0x139, + 0x13A, + 0x13B, + 0x13C, + 0x13D, + 0x13E, + 0x13F, + 0x141, + 0x142, + 0x143, + 0x144, + 0x145, + 0x146, + 0x147, + 0x148, + 0x149, + 0x14A, + 0x14B, + 0x14C, + 0x14D, + 0x14E, + 0x14F, + 0x150, + 0x151, + 0x152, + 0x153, + 0x154, + 0x155, + 0x156, + 0x157, + 0x158, + 0x159, + 0x15A, + 0x15B, + 0x15C, + 0x15D, + 0x15E, + 0x15F, + 0x160, + 0x161, + 0x162, + 0x163, + 0x164, + 0x165, + 0x166, + 0x167, + 0x168, + 0x169, + 0x16A, + 0x16B, + 0x16C, + 0x16D, + 0x16E, + 0x16F, + 0x170, + 0x171, + 0x172, + 0x173, + 0x174, + 0x175, + 0x176, + 0x177, + 0x178, + 0x179, + 0x17A, + 0x17B, + 0x17C, + 0x17D, + 0x17E, + 0x17F, + 0x180, + 0x181, + 0x182, + 0x183, + 0x184, + 0x185, + 0x186, + 0x187, + 0x188, + 0x189, + 0x18A, + 0x18B, + 0x18C, + 0x18E, + 0x18F, + 0x190, + 0x191, + 0x192, + 0x193, + 0x194, + 0x195, + 0x196, + 0x197, + 0x198, + 0x199, + 0x19C, + 0x19D, + 0x19E, + 0x19F, + 0x1A0, + 0x1A1, + 0x1A2, + 0x1A3, + 0x1A4, + 0x1A5, + 0x1A6, + 0x1A7, + 0x1A8, + 0x1A9, + 0x1AA, + 0x1AC, + 0x1AD, + 0x1AE, + 0x1AF, + 0x1B0, + 0x1B1, + 0x1B2, + 0x1B3, + 0x1B4, + 0x1B5, + 0x1B6, + 0x1B7, + 0x1B8, + 0x1B9, + 0x1BC, + 0x1BD, + 0x1C4, + 0x1C7, + 0x1CA, + 0x1CD, + 0x1CE, + 0x1CF, + 0x1D0, + 0x1D1, + 0x1D2, + 0x1D3, + 0x1D4, + 0x1D5, + 0x1D6, + 0x1D7, + 0x1D8, + 0x1D9, + 0x1DA, + 0x1DB, + 0x1DC, + 0x1DE, + 0x1DF, + 0x1E0, + 0x1E1, + 0x1E2, + 0x1E3, + 0x1E4, + 0x1E5, + 0x1E6, + 0x1E7, + 0x1E8, + 0x1E9, + 0x1EA, + 0x1EB, + 0x1EC, + 0x1ED, + 0x1EE, + 0x1EF, + 0x1F1, + 0x1F4, + 0x1F5, + 0x1F6, + 0x1F7, + 0x1F8, + 0x1F9, + 0x1FA, + 0x1FB, + 0x1FC, + 0x1FD, + 0x1FE, + 0x1FF, + 0x200, + 0x201, + 0x202, + 0x203, + 0x204, + 0x205, + 0x206, + 0x207, + 0x208, + 0x209, + 0x20A, + 0x20B, + 0x20C, + 0x20D, + 0x20E, + 0x20F, + 0x210, + 0x211, + 0x212, + 0x213, + 0x214, + 0x215, + 0x216, + 0x217, + 0x218, + 0x219, + 0x21A, + 0x21B, + 0x21C, + 0x21D, + 0x21E, + 0x21F, + 0x220, + 0x221, + 0x222, + 0x223, + 0x224, + 0x225, + 0x226, + 0x227, + 0x228, + 0x229, + 0x22A, + 0x22B, + 0x22C, + 0x22D, + 0x22E, + 0x22F, + 0x230, + 0x231, + 0x232, + 0x233, + 0x23A, + 0x23B, + 0x23C, + 0x23D, + 0x23E, + 0x23F, + 0x241, + 0x242, + 0x243, + 0x244, + 0x245, + 0x246, + 0x247, + 0x248, + 0x249, + 0x24A, + 0x24B, + 0x24C, + 0x24D, + 0x24E, + 0x24F, + 0x2B0, + 0x2B1, + 0x2B2, + 0x2B3, + 0x2B4, + 0x2B5, + 0x2B6, + 0x2B7, + 0x2B8, + 0x2B9, + 0x2D8, + 0x2D9, + 0x2DA, + 0x2DB, + 0x2DC, + 0x2DD, + 0x2DE, + 0x2E0, + 0x2E1, + 0x2E2, + 0x2E3, + 0x2E4, + 0x2E5, + 0x340, + 0x341, + 0x342, + 0x343, + 0x344, + 0x345, + 0x346, + 0x34F, + 0x350, + 0x370, + 0x371, + 0x372, + 0x373, + 0x374, + 0x375, + 0x376, + 0x377, + 0x378, + 0x37A, + 0x37B, + 0x37E, + 0x37F, + 0x380, + 0x384, + 0x385, + 0x386, + 0x387, + 0x388, + 0x389, + 0x38A, + 0x38B, + 0x38C, + 0x38D, + 0x38E, + 0x38F, + 0x390, + 0x391, + 0x392, + 0x393, + 0x394, + 0x395, + 0x396, + 0x397, + 0x398, + 0x399, + 0x39A, + 0x39B, + 0x39C, + 0x39D, + 0x39E, + 0x39F, + 0x3A0, + 0x3A1, + 0x3A2, + 0x3A3, + 0x3A4, + 0x3A5, + 0x3A6, + 0x3A7, + 0x3A8, + 0x3A9, + 0x3AA, + 0x3AB, + 0x3AC, + 0x3C2, + 0x3C3, + 0x3CF, + 0x3D0, + 0x3D1, + 0x3D2, + 0x3D3, + 0x3D4, + 0x3D5, + 0x3D6, + 0x3D7, + 0x3D8, + 0x3D9, + 0x3DA, + 0x3DB, + 0x3DC, + 0x3DD, + 0x3DE, + 0x3DF, + 0x3E0, + 0x3E1, + 0x3E2, + 0x3E3, + 0x3E4, + 0x3E5, + 0x3E6, + 0x3E7, + 0x3E8, + 0x3E9, + 0x3EA, + 0x3EB, + 0x3EC, + 0x3ED, + 0x3EE, + 0x3EF, + 0x3F0, + 0x3F1, + 0x3F2, + 0x3F3, + 0x3F4, + 0x3F5, + 0x3F6, + 0x3F7, + 0x3F8, + 0x3F9, + 0x3FA, + 0x3FB, + 0x3FD, + 0x3FE, + 0x3FF, + 0x400, + 0x401, + 0x402, + 0x403, + 0x404, + 0x405, + 0x406, + 0x407, + 0x408, + 0x409, + 0x40A, + 0x40B, + 0x40C, + 0x40D, + 0x40E, + 0x40F, + 0x410, + 0x411, + 0x412, + 0x413, + 0x414, + 0x415, + 0x416, + 0x417, + 0x418, + 0x419, + 0x41A, + 0x41B, + 0x41C, + 0x41D, + 0x41E, + 0x41F, + 0x420, + 0x421, + 0x422, + 0x423, + 0x424, + 0x425, + 0x426, + 0x427, + 0x428, + 0x429, + 0x42A, + 0x42B, + 0x42C, + 0x42D, + 0x42E, + 0x42F, + 0x430, + 0x460, + 0x461, + 0x462, + 0x463, + 0x464, + 0x465, + 0x466, + 0x467, + 0x468, + 0x469, + 0x46A, + 0x46B, + 0x46C, + 0x46D, + 0x46E, + 0x46F, + 0x470, + 0x471, + 0x472, + 0x473, + 0x474, + 0x475, + 0x476, + 0x477, + 0x478, + 0x479, + 0x47A, + 0x47B, + 0x47C, + 0x47D, + 0x47E, + 0x47F, + 0x480, + 0x481, + 0x48A, + 0x48B, + 0x48C, + 0x48D, + 0x48E, + 0x48F, + 0x490, + 0x491, + 0x492, + 0x493, + 0x494, + 0x495, + 0x496, + 0x497, + 0x498, + 0x499, + 0x49A, + 0x49B, + 0x49C, + 0x49D, + 0x49E, + 0x49F, + 0x4A0, + 0x4A1, + 0x4A2, + 0x4A3, + 0x4A4, + 0x4A5, + 0x4A6, + 0x4A7, + 0x4A8, + 0x4A9, + 0x4AA, + 0x4AB, + 0x4AC, + 0x4AD, + 0x4AE, + 0x4AF, + 0x4B0, + 0x4B1, + 0x4B2, + 0x4B3, + 0x4B4, + 0x4B5, + 0x4B6, + 0x4B7, + 0x4B8, + 0x4B9, + 0x4BA, + 0x4BB, + 0x4BC, + 0x4BD, + 0x4BE, + 0x4BF, + 0x4C0, + 0x4C1, + 0x4C2, + 0x4C3, + 0x4C4, + 0x4C5, + 0x4C6, + 0x4C7, + 0x4C8, + 0x4C9, + 0x4CA, + 0x4CB, + 0x4CC, + 0x4CD, + 0x4CE, + 0x4D0, + 0x4D1, + 0x4D2, + 0x4D3, + 0x4D4, + 0x4D5, + 0x4D6, + 0x4D7, + 0x4D8, + 0x4D9, + 0x4DA, + 0x4DB, + 0x4DC, + 0x4DD, + 0x4DE, + 0x4DF, + 0x4E0, + 0x4E1, + 0x4E2, + 0x4E3, + 0x4E4, + 0x4E5, + 0x4E6, + 0x4E7, + 0x4E8, + 0x4E9, + 0x4EA, + 0x4EB, + 0x4EC, + 0x4ED, + 0x4EE, + 0x4EF, + 0x4F0, + 0x4F1, + 0x4F2, + 0x4F3, + 0x4F4, + 0x4F5, + 0x4F6, + 0x4F7, + 0x4F8, + 0x4F9, + 0x4FA, + 0x4FB, + 0x4FC, + 0x4FD, + 0x4FE, + 0x4FF, + 0x500, + 0x501, + 0x502, + 0x503, + 0x504, + 0x505, + 0x506, + 0x507, + 0x508, + 0x509, + 0x50A, + 0x50B, + 0x50C, + 0x50D, + 0x50E, + 0x50F, + 0x510, + 0x511, + 0x512, + 0x513, + 0x514, + 0x515, + 0x516, + 0x517, + 0x518, + 0x519, + 0x51A, + 0x51B, + 0x51C, + 0x51D, + 0x51E, + 0x51F, + 0x520, + 0x521, + 0x522, + 0x523, + 0x524, + 0x525, + 0x526, + 0x527, + 0x528, + 0x529, + 0x52A, + 0x52B, + 0x52C, + 0x52D, + 0x52E, + 0x52F, + 0x530, + 0x531, + 0x532, + 0x533, + 0x534, + 0x535, + 0x536, + 0x537, + 0x538, + 0x539, + 0x53A, + 0x53B, + 0x53C, + 0x53D, + 0x53E, + 0x53F, + 0x540, + 0x541, + 0x542, + 0x543, + 0x544, + 0x545, + 0x546, + 0x547, + 0x548, + 0x549, + 0x54A, + 0x54B, + 0x54C, + 0x54D, + 0x54E, + 0x54F, + 0x550, + 0x551, + 0x552, + 0x553, + 0x554, + 0x555, + 0x556, + 0x557, + 0x559, + 0x587, + 0x588, + 0x58B, + 0x58D, + 0x590, + 0x591, + 0x5C8, + 0x5D0, + 0x5EB, + 0x5EF, + 0x5F5, + 0x606, + 0x61C, + 0x61D, + 0x675, + 0x676, + 0x677, + 0x678, + 0x679, + 0x6DD, + 0x6DE, + 0x70E, + 0x710, + 0x74B, + 0x74D, + 0x7B2, + 0x7C0, + 0x7FB, + 0x7FD, + 0x82E, + 0x830, + 0x83F, + 0x840, + 0x85C, + 0x85E, + 0x85F, + 0x860, + 0x86B, + 0x870, + 0x890, + 0x897, + 0x8E2, + 0x8E3, + 0x958, + 0x959, + 0x95A, + 0x95B, + 0x95C, + 0x95D, + 0x95E, + 0x95F, + 0x960, + 0x984, + 0x985, + 0x98D, + 0x98F, + 0x991, + 0x993, + 0x9A9, + 0x9AA, + 0x9B1, + 0x9B2, + 0x9B3, + 0x9B6, + 0x9BA, + 0x9BC, + 0x9C5, + 0x9C7, + 0x9C9, + 0x9CB, + 0x9CF, + 0x9D7, + 0x9D8, + 0x9DC, + 0x9DD, + 0x9DE, + 0x9DF, + 0x9E0, + 0x9E4, + 0x9E6, + 0x9FF, + 0xA01, + 0xA04, + 0xA05, + 0xA0B, + 0xA0F, + 0xA11, + 0xA13, + 0xA29, + 0xA2A, + 0xA31, + 0xA32, + 0xA33, + 0xA34, + 0xA35, + 0xA36, + 0xA37, + 0xA38, + 0xA3A, + 0xA3C, + 0xA3D, + 0xA3E, + 0xA43, + 0xA47, + 0xA49, + 0xA4B, + 0xA4E, + 0xA51, + 0xA52, + 0xA59, + 0xA5A, + 0xA5B, + 0xA5C, + 0xA5D, + 0xA5E, + 0xA5F, + 0xA66, + 0xA77, + 0xA81, + 0xA84, + 0xA85, + 0xA8E, + 0xA8F, + 0xA92, + 0xA93, + 0xAA9, + 0xAAA, + 0xAB1, + 0xAB2, + 0xAB4, + 0xAB5, + 0xABA, + 0xABC, + 0xAC6, + 0xAC7, + 0xACA, + 0xACB, + 0xACE, + 0xAD0, + 0xAD1, + 0xAE0, + 0xAE4, + 0xAE6, + 0xAF2, + 0xAF9, + 0xB00, + 0xB01, + 0xB04, + 0xB05, + 0xB0D, + 0xB0F, + 0xB11, + 0xB13, + 0xB29, + 0xB2A, + 0xB31, + 0xB32, + 0xB34, + 0xB35, + 0xB3A, + 0xB3C, + 0xB45, + 0xB47, + 0xB49, + 0xB4B, + 0xB4E, + 0xB55, + 0xB58, + 0xB5C, + 0xB5D, + 0xB5E, + 0xB5F, + 0xB64, + 0xB66, + 0xB78, + 0xB82, + 0xB84, + 0xB85, + 0xB8B, + 0xB8E, + 0xB91, + 0xB92, + 0xB96, + 0xB99, + 0xB9B, + 0xB9C, + 0xB9D, + 0xB9E, + 0xBA0, + 0xBA3, + 0xBA5, + 0xBA8, + 0xBAB, + 0xBAE, + 0xBBA, + 0xBBE, + 0xBC3, + 0xBC6, + 0xBC9, + 0xBCA, + 0xBCE, + 0xBD0, + 0xBD1, + 0xBD7, + 0xBD8, + 0xBE6, + 0xBFB, + 0xC00, + 0xC0D, + 0xC0E, + 0xC11, + 0xC12, + 0xC29, + 0xC2A, + 0xC3A, + 0xC3C, + 0xC45, + 0xC46, + 0xC49, + 0xC4A, + 0xC4E, + 0xC55, + 0xC57, + 0xC58, + 0xC5B, + 0xC5C, + 0xC5E, + 0xC60, + 0xC64, + 0xC66, + 0xC70, + 0xC77, + 0xC8D, + 0xC8E, + 0xC91, + 0xC92, + 0xCA9, + 0xCAA, + 0xCB4, + 0xCB5, + 0xCBA, + 0xCBC, + 0xCC5, + 0xCC6, + 0xCC9, + 0xCCA, + 0xCCE, + 0xCD5, + 0xCD7, + 0xCDC, + 0xCDF, + 0xCE0, + 0xCE4, + 0xCE6, + 0xCF0, + 0xCF1, + 0xCF4, + 0xD00, + 0xD0D, + 0xD0E, + 0xD11, + 0xD12, + 0xD45, + 0xD46, + 0xD49, + 0xD4A, + 0xD50, + 0xD54, + 0xD64, + 0xD66, + 0xD80, + 0xD81, + 0xD84, + 0xD85, + 0xD97, + 0xD9A, + 0xDB2, + 0xDB3, + 0xDBC, + 0xDBD, + 0xDBE, + 0xDC0, + 0xDC7, + 0xDCA, + 0xDCB, + 0xDCF, + 0xDD5, + 0xDD6, + 0xDD7, + 0xDD8, + 0xDE0, + 0xDE6, + 0xDF0, + 0xDF2, + 0xDF5, + 0xE01, + 0xE33, + 0xE34, + 0xE3B, + 0xE3F, + 0xE5C, + 0xE81, + 0xE83, + 0xE84, + 0xE85, + 0xE86, + 0xE8B, + 0xE8C, + 0xEA4, + 0xEA5, + 0xEA6, + 0xEA7, + 0xEB3, + 0xEB4, + 0xEBE, + 0xEC0, + 0xEC5, + 0xEC6, + 0xEC7, + 0xEC8, + 0xECF, + 0xED0, + 0xEDA, + 0xEDC, + 0xEDD, + 0xEDE, + 0xEE0, + 0xF00, + 0xF0C, + 0xF0D, + 0xF43, + 0xF44, + 0xF48, + 0xF49, + 0xF4D, + 0xF4E, + 0xF52, + 0xF53, + 0xF57, + 0xF58, + 0xF5C, + 0xF5D, + 0xF69, + 0xF6A, + 0xF6D, + 0xF71, + 0xF73, + 0xF74, + 0xF75, + 0xF76, + 0xF77, + 0xF78, + 0xF79, + 0xF7A, + 0xF81, + 0xF82, + 0xF93, + 0xF94, + 0xF98, + 0xF99, + 0xF9D, + 0xF9E, + 0xFA2, + 0xFA3, + 0xFA7, + 0xFA8, + 0xFAC, + 0xFAD, + 0xFB9, + 0xFBA, + 0xFBD, + 0xFBE, + 0xFCD, + 0xFCE, + 0xFDB, + 0x1000, + 0x10A0, + 0x10A1, + 0x10A2, + 0x10A3, + 0x10A4, + 0x10A5, + 0x10A6, + 0x10A7, + 0x10A8, + 0x10A9, + 0x10AA, + 0x10AB, + 0x10AC, + 0x10AD, + 0x10AE, + 0x10AF, + 0x10B0, + 0x10B1, + 0x10B2, + 0x10B3, + 0x10B4, + 0x10B5, + 0x10B6, + 0x10B7, + 0x10B8, + 0x10B9, + 0x10BA, + 0x10BB, + 0x10BC, + 0x10BD, + 0x10BE, + 0x10BF, + 0x10C0, + 0x10C1, + 0x10C2, + 0x10C3, + 0x10C4, + 0x10C5, + 0x10C6, + 0x10C7, + 0x10C8, + 0x10CD, + 0x10CE, + 0x10D0, + 0x10FC, + 0x10FD, + 0x115F, + 0x1161, + 0x1249, + 0x124A, + 0x124E, + 0x1250, + 0x1257, + 0x1258, + 0x1259, + 0x125A, + 0x125E, + 0x1260, + 0x1289, + 0x128A, + 0x128E, + 0x1290, + 0x12B1, + 0x12B2, + 0x12B6, + 0x12B8, + 0x12BF, + 0x12C0, + 0x12C1, + 0x12C2, + 0x12C6, + 0x12C8, + 0x12D7, + 0x12D8, + 0x1311, + 0x1312, + 0x1316, + 0x1318, + 0x135B, + 0x135D, + 0x137D, + 0x1380, + 0x139A, + 0x13A0, + 0x13F6, + 0x13F8, + 0x13F9, + 0x13FA, + 0x13FB, + 0x13FC, + 0x13FD, + 0x13FE, + 0x1400, + 0x1680, + 0x1681, + 0x169D, + 0x16A0, + 0x16F9, + 0x1700, + 0x1716, + 0x171F, + 0x1737, + 0x1740, + 0x1754, + 0x1760, + 0x176D, + 0x176E, + 0x1771, + 0x1772, + 0x1774, + 0x1780, + 0x17B4, + 0x17B6, + 0x17DE, + 0x17E0, + 0x17EA, + 0x17F0, + 0x17FA, + 0x1800, + 0x180B, + 0x1810, + 0x181A, + 0x1820, + 0x1879, + 0x1880, + 0x18AB, + 0x18B0, + 0x18F6, + 0x1900, + 0x191F, + 0x1920, + 0x192C, + 0x1930, + 0x193C, + 0x1940, + 0x1941, + 0x1944, + 0x196E, + 0x1970, + 0x1975, + 0x1980, + 0x19AC, + 0x19B0, + 0x19CA, + 0x19D0, + 0x19DB, + 0x19DE, + 0x1A1C, + 0x1A1E, + 0x1A5F, + 0x1A60, + 0x1A7D, + 0x1A7F, + 0x1A8A, + 0x1A90, + 0x1A9A, + 0x1AA0, + 0x1AAE, + 0x1AB0, + 0x1ADE, + 0x1AE0, + 0x1AEC, + 0x1B00, + 0x1B4D, + 0x1B4E, + 0x1BF4, + 0x1BFC, + 0x1C38, + 0x1C3B, + 0x1C4A, + 0x1C4D, + 0x1C80, + 0x1C81, + 0x1C82, + 0x1C83, + 0x1C84, + 0x1C86, + 0x1C87, + 0x1C88, + 0x1C89, + 0x1C8A, + 0x1C8B, + 0x1C90, + 0x1C91, + 0x1C92, + 0x1C93, + 0x1C94, + 0x1C95, + 0x1C96, + 0x1C97, + 0x1C98, + 0x1C99, + 0x1C9A, + 0x1C9B, + 0x1C9C, + 0x1C9D, + 0x1C9E, + 0x1C9F, + 0x1CA0, + 0x1CA1, + 0x1CA2, + 0x1CA3, + 0x1CA4, + 0x1CA5, + 0x1CA6, + 0x1CA7, + 0x1CA8, + 0x1CA9, + 0x1CAA, + 0x1CAB, + 0x1CAC, + 0x1CAD, + 0x1CAE, + 0x1CAF, + 0x1CB0, + 0x1CB1, + 0x1CB2, + 0x1CB3, + 0x1CB4, + 0x1CB5, + 0x1CB6, + 0x1CB7, + 0x1CB8, + 0x1CB9, + 0x1CBA, + 0x1CBB, + 0x1CBD, + 0x1CBE, + 0x1CBF, + 0x1CC0, + 0x1CC8, + 0x1CD0, + 0x1CFB, + 0x1D00, + 0x1D2C, + 0x1D2D, + 0x1D2E, + 0x1D2F, + 0x1D30, + 0x1D31, + 0x1D32, + 0x1D33, + 0x1D34, + 0x1D35, + 0x1D36, + 0x1D37, + 0x1D38, + 0x1D39, + 0x1D3A, + 0x1D3B, + 0x1D3C, + 0x1D3D, + 0x1D3E, + 0x1D3F, + 0x1D40, + 0x1D41, + 0x1D42, + 0x1D43, + 0x1D44, + 0x1D45, + 0x1D46, + 0x1D47, + 0x1D48, + 0x1D49, + 0x1D4A, + 0x1D4B, + 0x1D4C, + 0x1D4D, + 0x1D4E, + 0x1D4F, + 0x1D50, + 0x1D51, + 0x1D52, + 0x1D53, + 0x1D54, + 0x1D55, + 0x1D56, + 0x1D57, + 0x1D58, + 0x1D59, + 0x1D5A, + 0x1D5B, + 0x1D5C, + 0x1D5D, + 0x1D5E, + 0x1D5F, + 0x1D60, + 0x1D61, + 0x1D62, + 0x1D63, + 0x1D64, + 0x1D65, + 0x1D66, + 0x1D67, + 0x1D68, + 0x1D69, + 0x1D6A, + 0x1D6B, + 0x1D78, + 0x1D79, + 0x1D9B, + 0x1D9C, + 0x1D9D, + 0x1D9E, + 0x1D9F, + 0x1DA0, + 0x1DA1, + 0x1DA2, + 0x1DA3, + 0x1DA4, + 0x1DA5, + 0x1DA6, + 0x1DA7, + 0x1DA8, + 0x1DA9, + 0x1DAA, + 0x1DAB, + 0x1DAC, + 0x1DAD, + 0x1DAE, + 0x1DAF, + 0x1DB0, + 0x1DB1, + 0x1DB2, + 0x1DB3, + 0x1DB4, + 0x1DB5, + 0x1DB6, + 0x1DB7, + 0x1DB8, + 0x1DB9, + 0x1DBA, + 0x1DBB, + 0x1DBC, + 0x1DBD, + 0x1DBE, + 0x1DBF, + 0x1DC0, + 0x1E00, + 0x1E01, + 0x1E02, + 0x1E03, + 0x1E04, + 0x1E05, + 0x1E06, + 0x1E07, + 0x1E08, + 0x1E09, + 0x1E0A, + 0x1E0B, + 0x1E0C, + 0x1E0D, + 0x1E0E, + 0x1E0F, + 0x1E10, + 0x1E11, + 0x1E12, + 0x1E13, + 0x1E14, + 0x1E15, + 0x1E16, + 0x1E17, + 0x1E18, + 0x1E19, + 0x1E1A, + 0x1E1B, + 0x1E1C, + 0x1E1D, + 0x1E1E, + 0x1E1F, + 0x1E20, + 0x1E21, + 0x1E22, + 0x1E23, + 0x1E24, + 0x1E25, + 0x1E26, + 0x1E27, + 0x1E28, + 0x1E29, + 0x1E2A, + 0x1E2B, + 0x1E2C, + 0x1E2D, + 0x1E2E, + 0x1E2F, + 0x1E30, + 0x1E31, + 0x1E32, + 0x1E33, + 0x1E34, + 0x1E35, + 0x1E36, + 0x1E37, + 0x1E38, + 0x1E39, + 0x1E3A, + 0x1E3B, + 0x1E3C, + 0x1E3D, + 0x1E3E, + 0x1E3F, + 0x1E40, + 0x1E41, + 0x1E42, + 0x1E43, + 0x1E44, + 0x1E45, + 0x1E46, + 0x1E47, + 0x1E48, + 0x1E49, + 0x1E4A, + 0x1E4B, + 0x1E4C, + 0x1E4D, + 0x1E4E, + 0x1E4F, + 0x1E50, + 0x1E51, + 0x1E52, + 0x1E53, + 0x1E54, + 0x1E55, + 0x1E56, + 0x1E57, + 0x1E58, + 0x1E59, + 0x1E5A, + 0x1E5B, + 0x1E5C, + 0x1E5D, + 0x1E5E, + 0x1E5F, + 0x1E60, + 0x1E61, + 0x1E62, + 0x1E63, + 0x1E64, + 0x1E65, + 0x1E66, + 0x1E67, + 0x1E68, + 0x1E69, + 0x1E6A, + 0x1E6B, + 0x1E6C, + 0x1E6D, + 0x1E6E, + 0x1E6F, + 0x1E70, + 0x1E71, + 0x1E72, + 0x1E73, + 0x1E74, + 0x1E75, + 0x1E76, + 0x1E77, + 0x1E78, + 0x1E79, + 0x1E7A, + 0x1E7B, + 0x1E7C, + 0x1E7D, + 0x1E7E, + 0x1E7F, + 0x1E80, + 0x1E81, + 0x1E82, + 0x1E83, + 0x1E84, + 0x1E85, + 0x1E86, + 0x1E87, + 0x1E88, + 0x1E89, + 0x1E8A, + 0x1E8B, + 0x1E8C, + 0x1E8D, + 0x1E8E, + 0x1E8F, + 0x1E90, + 0x1E91, + 0x1E92, + 0x1E93, + 0x1E94, + 0x1E95, + 0x1E9A, + 0x1E9B, + 0x1E9C, + 0x1E9E, + 0x1E9F, + 0x1EA0, + 0x1EA1, + 0x1EA2, + 0x1EA3, + 0x1EA4, + 0x1EA5, + 0x1EA6, + 0x1EA7, + 0x1EA8, + 0x1EA9, + 0x1EAA, + 0x1EAB, + 0x1EAC, + 0x1EAD, + 0x1EAE, + 0x1EAF, + 0x1EB0, + 0x1EB1, + 0x1EB2, + 0x1EB3, + 0x1EB4, + 0x1EB5, + 0x1EB6, + 0x1EB7, + 0x1EB8, + 0x1EB9, + 0x1EBA, + 0x1EBB, + 0x1EBC, + 0x1EBD, + 0x1EBE, + 0x1EBF, + 0x1EC0, + 0x1EC1, + 0x1EC2, + 0x1EC3, + 0x1EC4, + 0x1EC5, + 0x1EC6, + 0x1EC7, + 0x1EC8, + 0x1EC9, + 0x1ECA, + 0x1ECB, + 0x1ECC, + 0x1ECD, + 0x1ECE, + 0x1ECF, + 0x1ED0, + 0x1ED1, + 0x1ED2, + 0x1ED3, + 0x1ED4, + 0x1ED5, + 0x1ED6, + 0x1ED7, + 0x1ED8, + 0x1ED9, + 0x1EDA, + 0x1EDB, + 0x1EDC, + 0x1EDD, + 0x1EDE, + 0x1EDF, + 0x1EE0, + 0x1EE1, + 0x1EE2, + 0x1EE3, + 0x1EE4, + 0x1EE5, + 0x1EE6, + 0x1EE7, + 0x1EE8, + 0x1EE9, + 0x1EEA, + 0x1EEB, + 0x1EEC, + 0x1EED, + 0x1EEE, + 0x1EEF, + 0x1EF0, + 0x1EF1, + 0x1EF2, + 0x1EF3, + 0x1EF4, + 0x1EF5, + 0x1EF6, + 0x1EF7, + 0x1EF8, + 0x1EF9, + 0x1EFA, + 0x1EFB, + 0x1EFC, + 0x1EFD, + 0x1EFE, + 0x1EFF, + 0x1F08, + 0x1F09, + 0x1F0A, + 0x1F0B, + 0x1F0C, + 0x1F0D, + 0x1F0E, + 0x1F0F, + 0x1F10, + 0x1F16, + 0x1F18, + 0x1F19, + 0x1F1A, + 0x1F1B, + 0x1F1C, + 0x1F1D, + 0x1F1E, + 0x1F20, + 0x1F28, + 0x1F29, + 0x1F2A, + 0x1F2B, + 0x1F2C, + 0x1F2D, + 0x1F2E, + 0x1F2F, + 0x1F30, + 0x1F38, + 0x1F39, + 0x1F3A, + 0x1F3B, + 0x1F3C, + 0x1F3D, + 0x1F3E, + 0x1F3F, + 0x1F40, + 0x1F46, + 0x1F48, + 0x1F49, + 0x1F4A, + 0x1F4B, + 0x1F4C, + 0x1F4D, + 0x1F4E, + 0x1F50, + 0x1F58, + 0x1F59, + 0x1F5A, + 0x1F5B, + 0x1F5C, + 0x1F5D, + 0x1F5E, + 0x1F5F, + 0x1F60, + 0x1F68, + 0x1F69, + 0x1F6A, + 0x1F6B, + 0x1F6C, + 0x1F6D, + 0x1F6E, + 0x1F6F, + 0x1F70, + 0x1F71, + 0x1F72, + 0x1F73, + 0x1F74, + 0x1F75, + 0x1F76, + 0x1F77, + 0x1F78, + 0x1F79, + 0x1F7A, + 0x1F7B, + 0x1F7C, + 0x1F7D, + 0x1F7E, + 0x1F80, + 0x1F81, + 0x1F82, + 0x1F83, + 0x1F84, + 0x1F85, + 0x1F86, + 0x1F87, + 0x1F88, + 0x1F89, + 0x1F8A, + 0x1F8B, + 0x1F8C, + 0x1F8D, + 0x1F8E, + 0x1F8F, + 0x1F90, + 0x1F91, + 0x1F92, + 0x1F93, + 0x1F94, + 0x1F95, + 0x1F96, + 0x1F97, + 0x1F98, + 0x1F99, + 0x1F9A, + 0x1F9B, + 0x1F9C, + 0x1F9D, + 0x1F9E, + 0x1F9F, + 0x1FA0, + 0x1FA1, + 0x1FA2, + 0x1FA3, + 0x1FA4, + 0x1FA5, + 0x1FA6, + 0x1FA7, + 0x1FA8, + 0x1FA9, + 0x1FAA, + 0x1FAB, + 0x1FAC, + 0x1FAD, + 0x1FAE, + 0x1FAF, + 0x1FB0, + 0x1FB2, + 0x1FB3, + 0x1FB4, + 0x1FB5, + 0x1FB6, + 0x1FB7, + 0x1FB8, + 0x1FB9, + 0x1FBA, + 0x1FBB, + 0x1FBC, + 0x1FBD, + 0x1FBE, + 0x1FBF, + 0x1FC0, + 0x1FC1, + 0x1FC2, + 0x1FC3, + 0x1FC4, + 0x1FC5, + 0x1FC6, + 0x1FC7, + 0x1FC8, + 0x1FC9, + 0x1FCA, + 0x1FCB, + 0x1FCC, + 0x1FCD, + 0x1FCE, + 0x1FCF, + 0x1FD0, + 0x1FD3, + 0x1FD4, + 0x1FD6, + 0x1FD8, + 0x1FD9, + 0x1FDA, + 0x1FDB, + 0x1FDC, + 0x1FDD, + 0x1FDE, + 0x1FDF, + 0x1FE0, + 0x1FE3, + 0x1FE4, + 0x1FE8, + 0x1FE9, + 0x1FEA, + 0x1FEB, + 0x1FEC, + 0x1FED, + 0x1FEE, + 0x1FEF, + 0x1FF0, + 0x1FF2, + 0x1FF3, + 0x1FF4, + 0x1FF5, + 0x1FF6, + 0x1FF7, + 0x1FF8, + 0x1FF9, + 0x1FFA, + 0x1FFB, + 0x1FFC, + 0x1FFD, + 0x1FFE, + 0x1FFF, + 0x2000, + 0x200B, + 0x200C, + 0x200E, + 0x2010, + 0x2011, + 0x2012, + 0x2017, + 0x2018, + 0x2024, + 0x2027, + 0x2028, + 0x202F, + 0x2030, + 0x2033, + 0x2034, + 0x2035, + 0x2036, + 0x2037, + 0x2038, + 0x203C, + 0x203D, + 0x203E, + 0x203F, + 0x2047, + 0x2048, + 0x2049, + 0x204A, + 0x2057, + 0x2058, + 0x205F, + 0x2060, + 0x2065, + 0x206A, + 0x2070, + 0x2071, + 0x2072, + 0x2074, + 0x2075, + 0x2076, + 0x2077, + 0x2078, + 0x2079, + 0x207A, + 0x207B, + 0x207C, + 0x207D, + 0x207E, + 0x207F, + 0x2080, + 0x2081, + 0x2082, + 0x2083, + 0x2084, + 0x2085, + 0x2086, + 0x2087, + 0x2088, + 0x2089, + 0x208A, + 0x208B, + 0x208C, + 0x208D, + 0x208E, + 0x208F, + 0x2090, + 0x2091, + 0x2092, + 0x2093, + 0x2094, + 0x2095, + 0x2096, + 0x2097, + 0x2098, + 0x2099, + 0x209A, + 0x209B, + 0x209C, + 0x209D, + 0x20A0, + 0x20A8, + 0x20A9, + 0x20C2, + 0x20D0, + 0x20F1, + 0x2100, + 0x2101, + 0x2102, + 0x2103, + 0x2104, + 0x2105, + 0x2106, + 0x2107, + 0x2108, + 0x2109, + 0x210A, + 0x210B, + 0x210F, + 0x2110, + 0x2112, + 0x2114, + 0x2115, + 0x2116, + 0x2117, + 0x2119, + 0x211A, + 0x211B, + 0x211E, + 0x2120, + 0x2121, + 0x2122, + 0x2123, + 0x2124, + 0x2125, + 0x2126, + 0x2127, + 0x2128, + 0x2129, + 0x212A, + 0x212B, + 0x212C, + 0x212D, + 0x212E, + 0x212F, + 0x2131, + 0x2132, + 0x2133, + 0x2134, + 0x2135, + 0x2136, + 0x2137, + 0x2138, + 0x2139, + 0x213A, + 0x213B, + 0x213C, + 0x213D, + 0x213F, + 0x2140, + 0x2141, + 0x2145, + 0x2147, + 0x2148, + 0x2149, + 0x214A, + 0x2150, + 0x2151, + 0x2152, + 0x2153, + 0x2154, + 0x2155, + 0x2156, + 0x2157, + 0x2158, + 0x2159, + 0x215A, + 0x215B, + 0x215C, + 0x215D, + 0x215E, + 0x215F, + 0x2160, + 0x2161, + 0x2162, + 0x2163, + 0x2164, + 0x2165, + 0x2166, + 0x2167, + 0x2168, + 0x2169, + 0x216A, + 0x216B, + 0x216C, + 0x216D, + 0x216E, + 0x216F, + 0x2170, + 0x2171, + 0x2172, + 0x2173, + 0x2174, + 0x2175, + 0x2176, + 0x2177, + 0x2178, + 0x2179, + 0x217A, + 0x217B, + 0x217C, + 0x217D, + 0x217E, + 0x217F, + 0x2180, + 0x2183, + 0x2184, + 0x2189, + 0x218A, + 0x218C, + 0x2190, + 0x222C, + 0x222D, + 0x222E, + 0x222F, + 0x2230, + 0x2231, + 0x2329, + 0x232A, + 0x232B, + 0x242A, + 0x2440, + 0x244B, + 0x2460, + 0x2461, + 0x2462, + 0x2463, + 0x2464, + 0x2465, + 0x2466, + 0x2467, + 0x2468, + 0x2469, + 0x246A, + 0x246B, + 0x246C, + 0x246D, + 0x246E, + 0x246F, + 0x2470, + 0x2471, + 0x2472, + 0x2473, + 0x2474, + 0x2475, + 0x2476, + 0x2477, + 0x2478, + 0x2479, + 0x247A, + 0x247B, + 0x247C, + 0x247D, + 0x247E, + 0x247F, + 0x2480, + 0x2481, + 0x2482, + 0x2483, + 0x2484, + 0x2485, + 0x2486, + 0x2487, + 0x2488, + 0x249C, + 0x249D, + 0x249E, + 0x249F, + 0x24A0, + 0x24A1, + 0x24A2, + 0x24A3, + 0x24A4, + 0x24A5, + 0x24A6, + 0x24A7, + 0x24A8, + 0x24A9, + 0x24AA, + 0x24AB, + 0x24AC, + 0x24AD, + 0x24AE, + 0x24AF, + 0x24B0, + 0x24B1, + 0x24B2, + 0x24B3, + 0x24B4, + 0x24B5, + 0x24B6, + 0x24B7, + 0x24B8, + 0x24B9, + 0x24BA, + 0x24BB, + 0x24BC, + 0x24BD, + 0x24BE, + 0x24BF, + 0x24C0, + 0x24C1, + 0x24C2, + 0x24C3, + 0x24C4, + 0x24C5, + 0x24C6, + 0x24C7, + 0x24C8, + 0x24C9, + 0x24CA, + 0x24CB, + 0x24CC, + 0x24CD, + 0x24CE, + 0x24CF, + 0x24D0, + 0x24D1, + 0x24D2, + 0x24D3, + 0x24D4, + 0x24D5, + 0x24D6, + 0x24D7, + 0x24D8, + 0x24D9, + 0x24DA, + 0x24DB, + 0x24DC, + 0x24DD, + 0x24DE, + 0x24DF, + 0x24E0, + 0x24E1, + 0x24E2, + 0x24E3, + 0x24E4, + 0x24E5, + 0x24E6, + 0x24E7, + 0x24E8, + 0x24E9, + 0x24EA, + 0x24EB, + 0x2A0C, + 0x2A0D, + 0x2A74, + 0x2A75, + 0x2A76, + 0x2A77, + 0x2ADC, + 0x2ADD, + 0x2B74, + 0x2B76, + 0x2C00, + 0x2C01, + 0x2C02, + 0x2C03, + 0x2C04, + 0x2C05, + 0x2C06, + 0x2C07, + 0x2C08, + 0x2C09, + 0x2C0A, + 0x2C0B, + 0x2C0C, + 0x2C0D, + 0x2C0E, + 0x2C0F, + 0x2C10, + 0x2C11, + 0x2C12, + 0x2C13, + 0x2C14, + 0x2C15, + 0x2C16, + 0x2C17, + 0x2C18, + 0x2C19, + 0x2C1A, + 0x2C1B, + 0x2C1C, + 0x2C1D, + 0x2C1E, + 0x2C1F, + 0x2C20, + 0x2C21, + 0x2C22, + 0x2C23, + 0x2C24, + 0x2C25, + 0x2C26, + 0x2C27, + 0x2C28, + 0x2C29, + 0x2C2A, + 0x2C2B, + 0x2C2C, + 0x2C2D, + 0x2C2E, + 0x2C2F, + 0x2C30, + 0x2C60, + 0x2C61, + 0x2C62, + 0x2C63, + 0x2C64, + 0x2C65, + 0x2C67, + 0x2C68, + 0x2C69, + 0x2C6A, + 0x2C6B, + 0x2C6C, + 0x2C6D, + 0x2C6E, + 0x2C6F, + 0x2C70, + 0x2C71, + 0x2C72, + 0x2C73, + 0x2C75, + 0x2C76, + 0x2C7C, + 0x2C7D, + 0x2C7E, + 0x2C7F, + 0x2C80, + 0x2C81, + 0x2C82, + 0x2C83, + 0x2C84, + 0x2C85, + 0x2C86, + 0x2C87, + 0x2C88, + 0x2C89, + 0x2C8A, + 0x2C8B, + 0x2C8C, + 0x2C8D, + 0x2C8E, + 0x2C8F, + 0x2C90, + 0x2C91, + 0x2C92, + 0x2C93, + 0x2C94, + 0x2C95, + 0x2C96, + 0x2C97, + 0x2C98, + 0x2C99, + 0x2C9A, + 0x2C9B, + 0x2C9C, + 0x2C9D, + 0x2C9E, + 0x2C9F, + 0x2CA0, + 0x2CA1, + 0x2CA2, + 0x2CA3, + 0x2CA4, + 0x2CA5, + 0x2CA6, + 0x2CA7, + 0x2CA8, + 0x2CA9, + 0x2CAA, + 0x2CAB, + 0x2CAC, + 0x2CAD, + 0x2CAE, + 0x2CAF, + 0x2CB0, + 0x2CB1, + 0x2CB2, + 0x2CB3, + 0x2CB4, + 0x2CB5, + 0x2CB6, + 0x2CB7, + 0x2CB8, + 0x2CB9, + 0x2CBA, + 0x2CBB, + 0x2CBC, + 0x2CBD, + 0x2CBE, + 0x2CBF, + 0x2CC0, + 0x2CC1, + 0x2CC2, + 0x2CC3, + 0x2CC4, + 0x2CC5, + 0x2CC6, + 0x2CC7, + 0x2CC8, + 0x2CC9, + 0x2CCA, + 0x2CCB, + 0x2CCC, + 0x2CCD, + 0x2CCE, + 0x2CCF, + 0x2CD0, + 0x2CD1, + 0x2CD2, + 0x2CD3, + 0x2CD4, + 0x2CD5, + 0x2CD6, + 0x2CD7, + 0x2CD8, + 0x2CD9, + 0x2CDA, + 0x2CDB, + 0x2CDC, + 0x2CDD, + 0x2CDE, + 0x2CDF, + 0x2CE0, + 0x2CE1, + 0x2CE2, + 0x2CE3, + 0x2CEB, + 0x2CEC, + 0x2CED, + 0x2CEE, + 0x2CF2, + 0x2CF3, + 0x2CF4, + 0x2CF9, + 0x2D26, + 0x2D27, + 0x2D28, + 0x2D2D, + 0x2D2E, + 0x2D30, + 0x2D68, + 0x2D6F, + 0x2D70, + 0x2D71, + 0x2D7F, + 0x2D97, + 0x2DA0, + 0x2DA7, + 0x2DA8, + 0x2DAF, + 0x2DB0, + 0x2DB7, + 0x2DB8, + 0x2DBF, + 0x2DC0, + 0x2DC7, + 0x2DC8, + 0x2DCF, + 0x2DD0, + 0x2DD7, + 0x2DD8, + 0x2DDF, + 0x2DE0, + 0x2E5E, + 0x2E80, + 0x2E9A, + 0x2E9B, + 0x2E9F, + 0x2EA0, + 0x2EF3, + 0x2EF4, + 0x2F00, + 0x2F01, + 0x2F02, + 0x2F03, + 0x2F04, + 0x2F05, + 0x2F06, + 0x2F07, + 0x2F08, + 0x2F09, + 0x2F0A, + 0x2F0B, + 0x2F0C, + 0x2F0D, + 0x2F0E, + 0x2F0F, + 0x2F10, + 0x2F11, + 0x2F12, + 0x2F13, + 0x2F14, + 0x2F15, + 0x2F16, + 0x2F17, + 0x2F18, + 0x2F19, + 0x2F1A, + 0x2F1B, + 0x2F1C, + 0x2F1D, + 0x2F1E, + 0x2F1F, + 0x2F20, + 0x2F21, + 0x2F22, + 0x2F23, + 0x2F24, + 0x2F25, + 0x2F26, + 0x2F27, + 0x2F28, + 0x2F29, + 0x2F2A, + 0x2F2B, + 0x2F2C, + 0x2F2D, + 0x2F2E, + 0x2F2F, + 0x2F30, + 0x2F31, + 0x2F32, + 0x2F33, + 0x2F34, + 0x2F35, + 0x2F36, + 0x2F37, + 0x2F38, + 0x2F39, + 0x2F3A, + 0x2F3B, + 0x2F3C, + 0x2F3D, + 0x2F3E, + 0x2F3F, + 0x2F40, + 0x2F41, + 0x2F42, + 0x2F43, + 0x2F44, + 0x2F45, + 0x2F46, + 0x2F47, + 0x2F48, + 0x2F49, + 0x2F4A, + 0x2F4B, + 0x2F4C, + 0x2F4D, + 0x2F4E, + 0x2F4F, + 0x2F50, + 0x2F51, + 0x2F52, + 0x2F53, + 0x2F54, + 0x2F55, + 0x2F56, + 0x2F57, + 0x2F58, + 0x2F59, + 0x2F5A, + 0x2F5B, + 0x2F5C, + 0x2F5D, + 0x2F5E, + 0x2F5F, + 0x2F60, + 0x2F61, + 0x2F62, + 0x2F63, + 0x2F64, + 0x2F65, + 0x2F66, + 0x2F67, + 0x2F68, + 0x2F69, + 0x2F6A, + 0x2F6B, + 0x2F6C, + 0x2F6D, + 0x2F6E, + 0x2F6F, + 0x2F70, + 0x2F71, + 0x2F72, + 0x2F73, + 0x2F74, + 0x2F75, + 0x2F76, + 0x2F77, + 0x2F78, + 0x2F79, + 0x2F7A, + 0x2F7B, + 0x2F7C, + 0x2F7D, + 0x2F7E, + 0x2F7F, + 0x2F80, + 0x2F81, + 0x2F82, + 0x2F83, + 0x2F84, + 0x2F85, + 0x2F86, + 0x2F87, + 0x2F88, + 0x2F89, + 0x2F8A, + 0x2F8B, + 0x2F8C, + 0x2F8D, + 0x2F8E, + 0x2F8F, + 0x2F90, + 0x2F91, + 0x2F92, + 0x2F93, + 0x2F94, + 0x2F95, + 0x2F96, + 0x2F97, + 0x2F98, + 0x2F99, + 0x2F9A, + 0x2F9B, + 0x2F9C, + 0x2F9D, + 0x2F9E, + 0x2F9F, + 0x2FA0, + 0x2FA1, + 0x2FA2, + 0x2FA3, + 0x2FA4, + 0x2FA5, + 0x2FA6, + 0x2FA7, + 0x2FA8, + 0x2FA9, + 0x2FAA, + 0x2FAB, + 0x2FAC, + 0x2FAD, + 0x2FAE, + 0x2FAF, + 0x2FB0, + 0x2FB1, + 0x2FB2, + 0x2FB3, + 0x2FB4, + 0x2FB5, + 0x2FB6, + 0x2FB7, + 0x2FB8, + 0x2FB9, + 0x2FBA, + 0x2FBB, + 0x2FBC, + 0x2FBD, + 0x2FBE, + 0x2FBF, + 0x2FC0, + 0x2FC1, + 0x2FC2, + 0x2FC3, + 0x2FC4, + 0x2FC5, + 0x2FC6, + 0x2FC7, + 0x2FC8, + 0x2FC9, + 0x2FCA, + 0x2FCB, + 0x2FCC, + 0x2FCD, + 0x2FCE, + 0x2FCF, + 0x2FD0, + 0x2FD1, + 0x2FD2, + 0x2FD3, + 0x2FD4, + 0x2FD5, + 0x2FD6, + 0x3000, + 0x3001, + 0x3002, + 0x3003, + 0x3036, + 0x3037, + 0x3038, + 0x3039, + 0x303A, + 0x303B, + 0x3040, + 0x3041, + 0x3097, + 0x3099, + 0x309B, + 0x309C, + 0x309D, + 0x309F, + 0x30A0, + 0x30FF, + 0x3100, + 0x3105, + 0x3130, + 0x3131, + 0x3132, + 0x3133, + 0x3134, + 0x3135, + 0x3136, + 0x3137, + 0x3138, + 0x3139, + 0x313A, + 0x313B, + 0x313C, + 0x313D, + 0x313E, + 0x313F, + 0x3140, + 0x3141, + 0x3142, + 0x3143, + 0x3144, + 0x3145, + 0x3146, + 0x3147, + 0x3148, + 0x3149, + 0x314A, + 0x314B, + 0x314C, + 0x314D, + 0x314E, + 0x314F, + 0x3150, + 0x3151, + 0x3152, + 0x3153, + 0x3154, + 0x3155, + 0x3156, + 0x3157, + 0x3158, + 0x3159, + 0x315A, + 0x315B, + 0x315C, + 0x315D, + 0x315E, + 0x315F, + 0x3160, + 0x3161, + 0x3162, + 0x3163, + 0x3164, + 0x3165, + 0x3166, + 0x3167, + 0x3168, + 0x3169, + 0x316A, + 0x316B, + 0x316C, + 0x316D, + 0x316E, + 0x316F, + 0x3170, + 0x3171, + 0x3172, + 0x3173, + 0x3174, + 0x3175, + 0x3176, + 0x3177, + 0x3178, + 0x3179, + 0x317A, + 0x317B, + 0x317C, + 0x317D, + 0x317E, + 0x317F, + 0x3180, + 0x3181, + 0x3182, + 0x3183, + 0x3184, + 0x3185, + 0x3186, + 0x3187, + 0x3188, + 0x3189, + 0x318A, + 0x318B, + 0x318C, + 0x318D, + 0x318E, + 0x318F, + 0x3190, + 0x3192, + 0x3193, + 0x3194, + 0x3195, + 0x3196, + 0x3197, + 0x3198, + 0x3199, + 0x319A, + 0x319B, + 0x319C, + 0x319D, + 0x319E, + 0x319F, + 0x31A0, + 0x31E6, + 0x31F0, + 0x3200, + 0x3201, + 0x3202, + 0x3203, + 0x3204, + 0x3205, + 0x3206, + 0x3207, + 0x3208, + 0x3209, + 0x320A, + 0x320B, + 0x320C, + 0x320D, + 0x320E, + 0x320F, + 0x3210, + 0x3211, + 0x3212, + 0x3213, + 0x3214, + 0x3215, + 0x3216, + 0x3217, + 0x3218, + 0x3219, + 0x321A, + 0x321B, + 0x321C, + 0x321D, + 0x321E, + 0x321F, + 0x3220, + 0x3221, + 0x3222, + 0x3223, + 0x3224, + 0x3225, + 0x3226, + 0x3227, + 0x3228, + 0x3229, + 0x322A, + 0x322B, + 0x322C, + 0x322D, + 0x322E, + 0x322F, + 0x3230, + 0x3231, + 0x3232, + 0x3233, + 0x3234, + 0x3235, + 0x3236, + 0x3237, + 0x3238, + 0x3239, + 0x323A, + 0x323B, + 0x323C, + 0x323D, + 0x323E, + 0x323F, + 0x3240, + 0x3241, + 0x3242, + 0x3243, + 0x3244, + 0x3245, + 0x3246, + 0x3247, + 0x3248, + 0x3250, + 0x3251, + 0x3252, + 0x3253, + 0x3254, + 0x3255, + 0x3256, + 0x3257, + 0x3258, + 0x3259, + 0x325A, + 0x325B, + 0x325C, + 0x325D, + 0x325E, + 0x325F, + 0x3260, + 0x3261, + 0x3262, + 0x3263, + 0x3264, + 0x3265, + 0x3266, + 0x3267, + 0x3268, + 0x3269, + 0x326A, + 0x326B, + 0x326C, + 0x326D, + 0x326E, + 0x326F, + 0x3270, + 0x3271, + 0x3272, + 0x3273, + 0x3274, + 0x3275, + 0x3276, + 0x3277, + 0x3278, + 0x3279, + 0x327A, + 0x327B, + 0x327C, + 0x327D, + 0x327E, + 0x327F, + 0x3280, + 0x3281, + 0x3282, + 0x3283, + 0x3284, + 0x3285, + 0x3286, + 0x3287, + 0x3288, + 0x3289, + 0x328A, + 0x328B, + 0x328C, + 0x328D, + 0x328E, + 0x328F, + 0x3290, + 0x3291, + 0x3292, + 0x3293, + 0x3294, + 0x3295, + 0x3296, + 0x3297, + 0x3298, + 0x3299, + 0x329A, + 0x329B, + 0x329C, + 0x329D, + 0x329E, + 0x329F, + 0x32A0, + 0x32A1, + 0x32A2, + 0x32A3, + 0x32A4, + 0x32A5, + 0x32A6, + 0x32A7, + 0x32A8, + 0x32A9, + 0x32AA, + 0x32AB, + 0x32AC, + 0x32AD, + 0x32AE, + 0x32AF, + 0x32B0, + 0x32B1, + 0x32B2, + 0x32B3, + 0x32B4, + 0x32B5, + 0x32B6, + 0x32B7, + 0x32B8, + 0x32B9, + 0x32BA, + 0x32BB, + 0x32BC, + 0x32BD, + 0x32BE, + 0x32BF, + 0x32C0, + 0x32C1, + 0x32C2, + 0x32C3, + 0x32C4, + 0x32C5, + 0x32C6, + 0x32C7, + 0x32C8, + 0x32C9, + 0x32CA, + 0x32CB, + 0x32CC, + 0x32CD, + 0x32CE, + 0x32CF, + 0x32D0, + 0x32D1, + 0x32D2, + 0x32D3, + 0x32D4, + 0x32D5, + 0x32D6, + 0x32D7, + 0x32D8, + 0x32D9, + 0x32DA, + 0x32DB, + 0x32DC, + 0x32DD, + 0x32DE, + 0x32DF, + 0x32E0, + 0x32E1, + 0x32E2, + 0x32E3, + 0x32E4, + 0x32E5, + 0x32E6, + 0x32E7, + 0x32E8, + 0x32E9, + 0x32EA, + 0x32EB, + 0x32EC, + 0x32ED, + 0x32EE, + 0x32EF, + 0x32F0, + 0x32F1, + 0x32F2, + 0x32F3, + 0x32F4, + 0x32F5, + 0x32F6, + 0x32F7, + 0x32F8, + 0x32F9, + 0x32FA, + 0x32FB, + 0x32FC, + 0x32FD, + 0x32FE, + 0x32FF, + 0x3300, + 0x3301, + 0x3302, + 0x3303, + 0x3304, + 0x3305, + 0x3306, + 0x3307, + 0x3308, + 0x3309, + 0x330A, + 0x330B, + 0x330C, + 0x330D, + 0x330E, + 0x330F, + 0x3310, + 0x3311, + 0x3312, + 0x3313, + 0x3314, + 0x3315, + 0x3316, + 0x3317, + 0x3318, + 0x3319, + 0x331A, + 0x331B, + 0x331C, + 0x331D, + 0x331E, + 0x331F, + 0x3320, + 0x3321, + 0x3322, + 0x3323, + 0x3324, + 0x3325, + 0x3326, + 0x3327, + 0x3328, + 0x3329, + 0x332A, + 0x332B, + 0x332C, + 0x332D, + 0x332E, + 0x332F, + 0x3330, + 0x3331, + 0x3332, + 0x3333, + 0x3334, + 0x3335, + 0x3336, + 0x3337, + 0x3338, + 0x3339, + 0x333A, + 0x333B, + 0x333C, + 0x333D, + 0x333E, + 0x333F, + 0x3340, + 0x3341, + 0x3342, + 0x3343, + 0x3344, + 0x3345, + 0x3346, + 0x3347, + 0x3348, + 0x3349, + 0x334A, + 0x334B, + 0x334C, + 0x334D, + 0x334E, + 0x334F, + 0x3350, + 0x3351, + 0x3352, + 0x3353, + 0x3354, + 0x3355, + 0x3356, + 0x3357, + 0x3358, + 0x3359, + 0x335A, + 0x335B, + 0x335C, + 0x335D, + 0x335E, + 0x335F, + 0x3360, + 0x3361, + 0x3362, + 0x3363, + 0x3364, + 0x3365, + 0x3366, + 0x3367, + 0x3368, + 0x3369, + 0x336A, + 0x336B, + 0x336C, + 0x336D, + 0x336E, + 0x336F, + 0x3370, + 0x3371, + 0x3372, + 0x3373, + 0x3374, + 0x3375, + 0x3376, + 0x3377, + 0x3378, + 0x3379, + 0x337A, + 0x337B, + 0x337C, + 0x337D, + 0x337E, + 0x337F, + 0x3380, + 0x3381, + 0x3382, + 0x3383, + 0x3384, + 0x3385, + 0x3386, + 0x3387, + 0x3388, + 0x3389, + 0x338A, + 0x338B, + 0x338C, + 0x338D, + 0x338E, + 0x338F, + 0x3390, + 0x3391, + 0x3392, + 0x3393, + 0x3394, + 0x3395, + 0x3396, + 0x3397, + 0x3398, + 0x3399, + 0x339A, + 0x339B, + 0x339C, + 0x339D, + 0x339E, + 0x339F, + 0x33A0, + 0x33A1, + 0x33A2, + 0x33A3, + 0x33A4, + 0x33A5, + 0x33A6, + 0x33A7, + 0x33A8, + 0x33A9, + 0x33AA, + 0x33AB, + 0x33AC, + 0x33AD, + 0x33AE, + 0x33AF, + 0x33B0, + 0x33B1, + 0x33B2, + 0x33B3, + 0x33B4, + 0x33B5, + 0x33B6, + 0x33B7, + 0x33B8, + 0x33B9, + 0x33BA, + 0x33BB, + 0x33BC, + 0x33BD, + 0x33BE, + 0x33BF, + 0x33C0, + 0x33C1, + 0x33C2, + 0x33C3, + 0x33C4, + 0x33C5, + 0x33C6, + 0x33C7, + 0x33C8, + 0x33C9, + 0x33CA, + 0x33CB, + 0x33CC, + 0x33CD, + 0x33CE, + 0x33CF, + 0x33D0, + 0x33D1, + 0x33D2, + 0x33D3, + 0x33D4, + 0x33D5, + 0x33D6, + 0x33D7, + 0x33D8, + 0x33D9, + 0x33DA, + 0x33DB, + 0x33DC, + 0x33DD, + 0x33DE, + 0x33DF, + 0x33E0, + 0x33E1, + 0x33E2, + 0x33E3, + 0x33E4, + 0x33E5, + 0x33E6, + 0x33E7, + 0x33E8, + 0x33E9, + 0x33EA, + 0x33EB, + 0x33EC, + 0x33ED, + 0x33EE, + 0x33EF, + 0x33F0, + 0x33F1, + 0x33F2, + 0x33F3, + 0x33F4, + 0x33F5, + 0x33F6, + 0x33F7, + 0x33F8, + 0x33F9, + 0x33FA, + 0x33FB, + 0x33FC, + 0x33FD, + 0x33FE, + 0x33FF, + 0x3400, + 0xA48D, + 0xA490, + 0xA4C7, + 0xA4D0, + 0xA62C, + 0xA640, + 0xA641, + 0xA642, + 0xA643, + 0xA644, + 0xA645, + 0xA646, + 0xA647, + 0xA648, + 0xA649, + 0xA64A, + 0xA64B, + 0xA64C, + 0xA64D, + 0xA64E, + 0xA64F, + 0xA650, + 0xA651, + 0xA652, + 0xA653, + 0xA654, + 0xA655, + 0xA656, + 0xA657, + 0xA658, + 0xA659, + 0xA65A, + 0xA65B, + 0xA65C, + 0xA65D, + 0xA65E, + 0xA65F, + 0xA660, + 0xA661, + 0xA662, + 0xA663, + 0xA664, + 0xA665, + 0xA666, + 0xA667, + 0xA668, + 0xA669, + 0xA66A, + 0xA66B, + 0xA66C, + 0xA66D, + 0xA680, + 0xA681, + 0xA682, + 0xA683, + 0xA684, + 0xA685, + 0xA686, + 0xA687, + 0xA688, + 0xA689, + 0xA68A, + 0xA68B, + 0xA68C, + 0xA68D, + 0xA68E, + 0xA68F, + 0xA690, + 0xA691, + 0xA692, + 0xA693, + 0xA694, + 0xA695, + 0xA696, + 0xA697, + 0xA698, + 0xA699, + 0xA69A, + 0xA69B, + 0xA69C, + 0xA69D, + 0xA69E, + 0xA6F8, + 0xA700, + 0xA722, + 0xA723, + 0xA724, + 0xA725, + 0xA726, + 0xA727, + 0xA728, + 0xA729, + 0xA72A, + 0xA72B, + 0xA72C, + 0xA72D, + 0xA72E, + 0xA72F, + 0xA732, + 0xA733, + 0xA734, + 0xA735, + 0xA736, + 0xA737, + 0xA738, + 0xA739, + 0xA73A, + 0xA73B, + 0xA73C, + 0xA73D, + 0xA73E, + 0xA73F, + 0xA740, + 0xA741, + 0xA742, + 0xA743, + 0xA744, + 0xA745, + 0xA746, + 0xA747, + 0xA748, + 0xA749, + 0xA74A, + 0xA74B, + 0xA74C, + 0xA74D, + 0xA74E, + 0xA74F, + 0xA750, + 0xA751, + 0xA752, + 0xA753, + 0xA754, + 0xA755, + 0xA756, + 0xA757, + 0xA758, + 0xA759, + 0xA75A, + 0xA75B, + 0xA75C, + 0xA75D, + 0xA75E, + 0xA75F, + 0xA760, + 0xA761, + 0xA762, + 0xA763, + 0xA764, + 0xA765, + 0xA766, + 0xA767, + 0xA768, + 0xA769, + 0xA76A, + 0xA76B, + 0xA76C, + 0xA76D, + 0xA76E, + 0xA76F, + 0xA770, + 0xA771, + 0xA779, + 0xA77A, + 0xA77B, + 0xA77C, + 0xA77D, + 0xA77E, + 0xA77F, + 0xA780, + 0xA781, + 0xA782, + 0xA783, + 0xA784, + 0xA785, + 0xA786, + 0xA787, + 0xA78B, + 0xA78C, + 0xA78D, + 0xA78E, + 0xA790, + 0xA791, + 0xA792, + 0xA793, + 0xA796, + 0xA797, + 0xA798, + 0xA799, + 0xA79A, + 0xA79B, + 0xA79C, + 0xA79D, + 0xA79E, + 0xA79F, + 0xA7A0, + 0xA7A1, + 0xA7A2, + 0xA7A3, + 0xA7A4, + 0xA7A5, + 0xA7A6, + 0xA7A7, + 0xA7A8, + 0xA7A9, + 0xA7AA, + 0xA7AB, + 0xA7AC, + 0xA7AD, + 0xA7AE, + 0xA7AF, + 0xA7B0, + 0xA7B1, + 0xA7B2, + 0xA7B3, + 0xA7B4, + 0xA7B5, + 0xA7B6, + 0xA7B7, + 0xA7B8, + 0xA7B9, + 0xA7BA, + 0xA7BB, + 0xA7BC, + 0xA7BD, + 0xA7BE, + 0xA7BF, + 0xA7C0, + 0xA7C1, + 0xA7C2, + 0xA7C3, + 0xA7C4, + 0xA7C5, + 0xA7C6, + 0xA7C7, + 0xA7C8, + 0xA7C9, + 0xA7CA, + 0xA7CB, + 0xA7CC, + 0xA7CD, + 0xA7CE, + 0xA7CF, + 0xA7D0, + 0xA7D1, + 0xA7D2, + 0xA7D3, + 0xA7D4, + 0xA7D5, + 0xA7D6, + 0xA7D7, + 0xA7D8, + 0xA7D9, + 0xA7DA, + 0xA7DB, + 0xA7DC, + 0xA7DD, + 0xA7F1, + 0xA7F2, + 0xA7F3, + 0xA7F4, + 0xA7F5, + 0xA7F6, + 0xA7F8, + 0xA7F9, + 0xA7FA, + 0xA82D, + 0xA830, + 0xA83A, + 0xA840, + 0xA878, + 0xA880, + 0xA8C6, + 0xA8CE, + 0xA8DA, + 0xA8E0, + 0xA954, + 0xA95F, + 0xA97D, + 0xA980, + 0xA9CE, + 0xA9CF, + 0xA9DA, + 0xA9DE, + 0xA9FF, + 0xAA00, + 0xAA37, + 0xAA40, + 0xAA4E, + 0xAA50, + 0xAA5A, + 0xAA5C, + 0xAAC3, + 0xAADB, + 0xAAF7, + 0xAB01, + 0xAB07, + 0xAB09, + 0xAB0F, + 0xAB11, + 0xAB17, + 0xAB20, + 0xAB27, + 0xAB28, + 0xAB2F, + 0xAB30, + 0xAB5C, + 0xAB5D, + 0xAB5E, + 0xAB5F, + 0xAB60, + 0xAB69, + 0xAB6A, + 0xAB6C, + 0xAB70, + 0xAB71, + 0xAB72, + 0xAB73, + 0xAB74, + 0xAB75, + 0xAB76, + 0xAB77, + 0xAB78, + 0xAB79, + 0xAB7A, + 0xAB7B, + 0xAB7C, + 0xAB7D, + 0xAB7E, + 0xAB7F, + 0xAB80, + 0xAB81, + 0xAB82, + 0xAB83, + 0xAB84, + 0xAB85, + 0xAB86, + 0xAB87, + 0xAB88, + 0xAB89, + 0xAB8A, + 0xAB8B, + 0xAB8C, + 0xAB8D, + 0xAB8E, + 0xAB8F, + 0xAB90, + 0xAB91, + 0xAB92, + 0xAB93, + 0xAB94, + 0xAB95, + 0xAB96, + 0xAB97, + 0xAB98, + 0xAB99, + 0xAB9A, + 0xAB9B, + 0xAB9C, + 0xAB9D, + 0xAB9E, + 0xAB9F, + 0xABA0, + 0xABA1, + 0xABA2, + 0xABA3, + 0xABA4, + 0xABA5, + 0xABA6, + 0xABA7, + 0xABA8, + 0xABA9, + 0xABAA, + 0xABAB, + 0xABAC, + 0xABAD, + 0xABAE, + 0xABAF, + 0xABB0, + 0xABB1, + 0xABB2, + 0xABB3, + 0xABB4, + 0xABB5, + 0xABB6, + 0xABB7, + 0xABB8, + 0xABB9, + 0xABBA, + 0xABBB, + 0xABBC, + 0xABBD, + 0xABBE, + 0xABBF, + 0xABC0, + 0xABEE, + 0xABF0, + 0xABFA, + 0xAC00, + 0xD7A4, + 0xD7B0, + 0xD7C7, + 0xD7CB, + 0xD7FC, + 0xF900, + 0xF901, + 0xF902, + 0xF903, + 0xF904, + 0xF905, + 0xF906, + 0xF907, + 0xF909, + 0xF90A, + 0xF90B, + 0xF90C, + 0xF90D, + 0xF90E, + 0xF90F, + 0xF910, + 0xF911, + 0xF912, + 0xF913, + 0xF914, + 0xF915, + 0xF916, + 0xF917, + 0xF918, + 0xF919, + 0xF91A, + 0xF91B, + 0xF91C, + 0xF91D, + 0xF91E, + 0xF91F, + 0xF920, + 0xF921, + 0xF922, + 0xF923, + 0xF924, + 0xF925, + 0xF926, + 0xF927, + 0xF928, + 0xF929, + 0xF92A, + 0xF92B, + 0xF92C, + 0xF92D, + 0xF92E, + 0xF92F, + 0xF930, + 0xF931, + 0xF932, + 0xF933, + 0xF934, + 0xF935, + 0xF936, + 0xF937, + 0xF938, + 0xF939, + 0xF93A, + 0xF93B, + 0xF93C, + 0xF93D, + 0xF93E, + 0xF93F, + 0xF940, + 0xF941, + 0xF942, + 0xF943, + 0xF944, + 0xF945, + 0xF946, + 0xF947, + 0xF948, + 0xF949, + 0xF94A, + 0xF94B, + 0xF94C, + 0xF94D, + 0xF94E, + 0xF94F, + 0xF950, + 0xF951, + 0xF952, + 0xF953, + 0xF954, + 0xF955, + 0xF956, + 0xF957, + 0xF958, + 0xF959, + 0xF95A, + 0xF95B, + 0xF95C, + 0xF95D, + 0xF95E, + 0xF95F, + 0xF960, + 0xF961, + 0xF962, + 0xF963, + 0xF964, + 0xF965, + 0xF966, + 0xF967, + 0xF968, + 0xF969, + 0xF96A, + 0xF96B, + 0xF96C, + 0xF96D, + 0xF96E, + 0xF96F, + 0xF970, + 0xF971, + 0xF972, + 0xF973, + 0xF974, + 0xF975, + 0xF976, + 0xF977, + 0xF978, + 0xF979, + 0xF97A, + 0xF97B, + 0xF97C, + 0xF97D, + 0xF97E, + 0xF97F, + 0xF980, + 0xF981, + 0xF982, + 0xF983, + 0xF984, + 0xF985, + 0xF986, + 0xF987, + 0xF988, + 0xF989, + 0xF98A, + 0xF98B, + 0xF98C, + 0xF98D, + 0xF98E, + 0xF98F, + 0xF990, + 0xF991, + 0xF992, + 0xF993, + 0xF994, + 0xF995, + 0xF996, + 0xF997, + 0xF998, + 0xF999, + 0xF99A, + 0xF99B, + 0xF99C, + 0xF99D, + 0xF99E, + 0xF99F, + 0xF9A0, + 0xF9A1, + 0xF9A2, + 0xF9A3, + 0xF9A4, + 0xF9A5, + 0xF9A6, + 0xF9A7, + 0xF9A8, + 0xF9A9, + 0xF9AA, + 0xF9AB, + 0xF9AC, + 0xF9AD, + 0xF9AE, + 0xF9AF, + 0xF9B0, + 0xF9B1, + 0xF9B2, + 0xF9B3, + 0xF9B4, + 0xF9B5, + 0xF9B6, + 0xF9B7, + 0xF9B8, + 0xF9B9, + 0xF9BA, + 0xF9BB, + 0xF9BC, + 0xF9BD, + 0xF9BE, + 0xF9BF, + 0xF9C0, + 0xF9C1, + 0xF9C2, + 0xF9C3, + 0xF9C4, + 0xF9C5, + 0xF9C6, + 0xF9C7, + 0xF9C8, + 0xF9C9, + 0xF9CA, + 0xF9CB, + 0xF9CC, + 0xF9CD, + 0xF9CE, + 0xF9CF, + 0xF9D0, + 0xF9D1, + 0xF9D2, + 0xF9D3, + 0xF9D4, + 0xF9D5, + 0xF9D6, + 0xF9D7, + 0xF9D8, + 0xF9D9, + 0xF9DA, + 0xF9DB, + 0xF9DC, + 0xF9DD, + 0xF9DE, + 0xF9DF, + 0xF9E0, + 0xF9E1, + 0xF9E2, + 0xF9E3, + 0xF9E4, + 0xF9E5, + 0xF9E6, + 0xF9E7, + 0xF9E8, + 0xF9E9, + 0xF9EA, + 0xF9EB, + 0xF9EC, + 0xF9ED, + 0xF9EE, + 0xF9EF, + 0xF9F0, + 0xF9F1, + 0xF9F2, + 0xF9F3, + 0xF9F4, + 0xF9F5, + 0xF9F6, + 0xF9F7, + 0xF9F8, + 0xF9F9, + 0xF9FA, + 0xF9FB, + 0xF9FC, + 0xF9FD, + 0xF9FE, + 0xF9FF, + 0xFA00, + 0xFA01, + 0xFA02, + 0xFA03, + 0xFA04, + 0xFA05, + 0xFA06, + 0xFA07, + 0xFA08, + 0xFA09, + 0xFA0A, + 0xFA0B, + 0xFA0C, + 0xFA0D, + 0xFA0E, + 0xFA10, + 0xFA11, + 0xFA12, + 0xFA13, + 0xFA15, + 0xFA16, + 0xFA17, + 0xFA18, + 0xFA19, + 0xFA1A, + 0xFA1B, + 0xFA1C, + 0xFA1D, + 0xFA1E, + 0xFA1F, + 0xFA20, + 0xFA21, + 0xFA22, + 0xFA23, + 0xFA25, + 0xFA26, + 0xFA27, + 0xFA2A, + 0xFA2B, + 0xFA2C, + 0xFA2D, + 0xFA2E, + 0xFA2F, + 0xFA30, + 0xFA31, + 0xFA32, + 0xFA33, + 0xFA34, + 0xFA35, + 0xFA36, + 0xFA37, + 0xFA38, + 0xFA39, + 0xFA3A, + 0xFA3B, + 0xFA3C, + 0xFA3D, + 0xFA3E, + 0xFA3F, + 0xFA40, + 0xFA41, + 0xFA42, + 0xFA43, + 0xFA44, + 0xFA45, + 0xFA46, + 0xFA47, + 0xFA48, + 0xFA49, + 0xFA4A, + 0xFA4B, + 0xFA4C, + 0xFA4D, + 0xFA4E, + 0xFA4F, + 0xFA50, + 0xFA51, + 0xFA52, + 0xFA53, + 0xFA54, + 0xFA55, + 0xFA56, + 0xFA57, + 0xFA58, + 0xFA59, + 0xFA5A, + 0xFA5B, + 0xFA5C, + 0xFA5D, + 0xFA5F, + 0xFA60, + 0xFA61, + 0xFA62, + 0xFA63, + 0xFA64, + 0xFA65, + 0xFA66, + 0xFA67, + 0xFA68, + 0xFA69, + 0xFA6A, + 0xFA6B, + 0xFA6C, + 0xFA6D, + 0xFA6E, + 0xFA70, + 0xFA71, + 0xFA72, + 0xFA73, + 0xFA74, + 0xFA75, + 0xFA76, + 0xFA77, + 0xFA78, + 0xFA79, + 0xFA7A, + 0xFA7B, + 0xFA7C, + 0xFA7D, + 0xFA7E, + 0xFA7F, + 0xFA80, + 0xFA81, + 0xFA82, + 0xFA83, + 0xFA84, + 0xFA85, + 0xFA86, + 0xFA87, + 0xFA88, + 0xFA89, + 0xFA8A, + 0xFA8B, + 0xFA8C, + 0xFA8D, + 0xFA8E, + 0xFA8F, + 0xFA90, + 0xFA91, + 0xFA92, + 0xFA93, + 0xFA94, + 0xFA95, + 0xFA96, + 0xFA97, + 0xFA98, + 0xFA99, + 0xFA9A, + 0xFA9B, + 0xFA9C, + 0xFA9D, + 0xFA9E, + 0xFA9F, + 0xFAA0, + 0xFAA1, + 0xFAA2, + 0xFAA3, + 0xFAA4, + 0xFAA5, + 0xFAA6, + 0xFAA7, + 0xFAA8, + 0xFAA9, + 0xFAAA, + 0xFAAB, + 0xFAAC, + 0xFAAD, + 0xFAAE, + 0xFAAF, + 0xFAB0, + 0xFAB1, + 0xFAB2, + 0xFAB3, + 0xFAB4, + 0xFAB5, + 0xFAB6, + 0xFAB7, + 0xFAB8, + 0xFAB9, + 0xFABA, + 0xFABB, + 0xFABC, + 0xFABD, + 0xFABE, + 0xFABF, + 0xFAC0, + 0xFAC1, + 0xFAC2, + 0xFAC3, + 0xFAC4, + 0xFAC5, + 0xFAC6, + 0xFAC7, + 0xFAC8, + 0xFAC9, + 0xFACA, + 0xFACB, + 0xFACC, + 0xFACD, + 0xFACE, + 0xFACF, + 0xFAD0, + 0xFAD1, + 0xFAD2, + 0xFAD3, + 0xFAD4, + 0xFAD5, + 0xFAD6, + 0xFAD7, + 0xFAD8, + 0xFAD9, + 0xFADA, + 0xFB00, + 0xFB01, + 0xFB02, + 0xFB03, + 0xFB04, + 0xFB05, + 0xFB07, + 0xFB13, + 0xFB14, + 0xFB15, + 0xFB16, + 0xFB17, + 0xFB18, + 0xFB1D, + 0xFB1E, + 0xFB1F, + 0xFB20, + 0xFB21, + 0xFB22, + 0xFB23, + 0xFB24, + 0xFB25, + 0xFB26, + 0xFB27, + 0xFB28, + 0xFB29, + 0xFB2A, + 0xFB2B, + 0xFB2C, + 0xFB2D, + 0xFB2E, + 0xFB2F, + 0xFB30, + 0xFB31, + 0xFB32, + 0xFB33, + 0xFB34, + 0xFB35, + 0xFB36, + 0xFB37, + 0xFB38, + 0xFB39, + 0xFB3A, + 0xFB3B, + 0xFB3C, + 0xFB3D, + 0xFB3E, + 0xFB3F, + 0xFB40, + 0xFB41, + 0xFB42, + 0xFB43, + 0xFB44, + 0xFB45, + 0xFB46, + 0xFB47, + 0xFB48, + 0xFB49, + 0xFB4A, + 0xFB4B, + 0xFB4C, + 0xFB4D, + 0xFB4E, + 0xFB4F, + 0xFB50, + 0xFB52, + 0xFB56, + 0xFB5A, + 0xFB5E, + 0xFB62, + 0xFB66, + 0xFB6A, + 0xFB6E, + 0xFB72, + 0xFB76, + 0xFB7A, + 0xFB7E, + 0xFB82, + 0xFB84, + 0xFB86, + 0xFB88, + 0xFB8A, + 0xFB8C, + 0xFB8E, + 0xFB92, + 0xFB96, + 0xFB9A, + 0xFB9E, + 0xFBA0, + 0xFBA4, + 0xFBA6, + 0xFBAA, + 0xFBAE, + 0xFBB0, + 0xFBB2, + 0xFBD3, + 0xFBD7, + 0xFBD9, + 0xFBDB, + 0xFBDD, + 0xFBDE, + 0xFBE0, + 0xFBE2, + 0xFBE4, + 0xFBE8, + 0xFBEA, + 0xFBEC, + 0xFBEE, + 0xFBF0, + 0xFBF2, + 0xFBF4, + 0xFBF6, + 0xFBF9, + 0xFBFC, + 0xFC00, + 0xFC01, + 0xFC02, + 0xFC03, + 0xFC04, + 0xFC05, + 0xFC06, + 0xFC07, + 0xFC08, + 0xFC09, + 0xFC0A, + 0xFC0B, + 0xFC0C, + 0xFC0D, + 0xFC0E, + 0xFC0F, + 0xFC10, + 0xFC11, + 0xFC12, + 0xFC13, + 0xFC14, + 0xFC15, + 0xFC16, + 0xFC17, + 0xFC18, + 0xFC19, + 0xFC1A, + 0xFC1B, + 0xFC1C, + 0xFC1D, + 0xFC1E, + 0xFC1F, + 0xFC20, + 0xFC21, + 0xFC22, + 0xFC23, + 0xFC24, + 0xFC25, + 0xFC26, + 0xFC27, + 0xFC28, + 0xFC29, + 0xFC2A, + 0xFC2B, + 0xFC2C, + 0xFC2D, + 0xFC2E, + 0xFC2F, + 0xFC30, + 0xFC31, + 0xFC32, + 0xFC33, + 0xFC34, + 0xFC35, + 0xFC36, + 0xFC37, + 0xFC38, + 0xFC39, + 0xFC3A, + 0xFC3B, + 0xFC3C, + 0xFC3D, + 0xFC3E, + 0xFC3F, + 0xFC40, + 0xFC41, + 0xFC42, + 0xFC43, + 0xFC44, + 0xFC45, + 0xFC46, + 0xFC47, + 0xFC48, + 0xFC49, + 0xFC4A, + 0xFC4B, + 0xFC4C, + 0xFC4D, + 0xFC4E, + 0xFC4F, + 0xFC50, + 0xFC51, + 0xFC52, + 0xFC53, + 0xFC54, + 0xFC55, + 0xFC56, + 0xFC57, + 0xFC58, + 0xFC59, + 0xFC5A, + 0xFC5B, + 0xFC5C, + 0xFC5D, + 0xFC5E, + 0xFC5F, + 0xFC60, + 0xFC61, + 0xFC62, + 0xFC63, + 0xFC64, + 0xFC65, + 0xFC66, + 0xFC67, + 0xFC68, + 0xFC69, + 0xFC6A, + 0xFC6B, + 0xFC6C, + 0xFC6D, + 0xFC6E, + 0xFC6F, + 0xFC70, + 0xFC71, + 0xFC72, + 0xFC73, + 0xFC74, + 0xFC75, + 0xFC76, + 0xFC77, + 0xFC78, + 0xFC79, + 0xFC7A, + 0xFC7B, + 0xFC7C, + 0xFC7D, + 0xFC7E, + 0xFC7F, + 0xFC80, + 0xFC81, + 0xFC82, + 0xFC83, + 0xFC84, + 0xFC85, + 0xFC86, + 0xFC87, + 0xFC88, + 0xFC89, + 0xFC8A, + 0xFC8B, + 0xFC8C, + 0xFC8D, + 0xFC8E, + 0xFC8F, + 0xFC90, + 0xFC91, + 0xFC92, + 0xFC93, + 0xFC94, + 0xFC95, + 0xFC96, + 0xFC97, + 0xFC98, + 0xFC99, + 0xFC9A, + 0xFC9B, + 0xFC9C, + 0xFC9D, + 0xFC9E, + 0xFC9F, + 0xFCA0, + 0xFCA1, + 0xFCA2, + 0xFCA3, + 0xFCA4, + 0xFCA5, + 0xFCA6, + 0xFCA7, + 0xFCA8, + 0xFCA9, + 0xFCAA, + 0xFCAB, + 0xFCAC, + 0xFCAD, + 0xFCAE, + 0xFCAF, + 0xFCB0, + 0xFCB1, + 0xFCB2, + 0xFCB3, + 0xFCB4, + 0xFCB5, + 0xFCB6, + 0xFCB7, + 0xFCB8, + 0xFCB9, + 0xFCBA, + 0xFCBB, + 0xFCBC, + 0xFCBD, + 0xFCBE, + 0xFCBF, + 0xFCC0, + 0xFCC1, + 0xFCC2, + 0xFCC3, + 0xFCC4, + 0xFCC5, + 0xFCC6, + 0xFCC7, + 0xFCC8, + 0xFCC9, + 0xFCCA, + 0xFCCB, + 0xFCCC, + 0xFCCD, + 0xFCCE, + 0xFCCF, + 0xFCD0, + 0xFCD1, + 0xFCD2, + 0xFCD3, + 0xFCD4, + 0xFCD5, + 0xFCD6, + 0xFCD7, + 0xFCD8, + 0xFCD9, + 0xFCDA, + 0xFCDB, + 0xFCDC, + 0xFCDD, + 0xFCDE, + 0xFCDF, + 0xFCE0, + 0xFCE1, + 0xFCE2, + 0xFCE3, + 0xFCE4, + 0xFCE5, + 0xFCE6, + 0xFCE7, + 0xFCE8, + 0xFCE9, + 0xFCEA, + 0xFCEB, + 0xFCEC, + 0xFCED, + 0xFCEE, + 0xFCEF, + 0xFCF0, + 0xFCF1, + 0xFCF2, + 0xFCF3, + 0xFCF4, + 0xFCF5, + 0xFCF6, + 0xFCF7, + 0xFCF8, + 0xFCF9, + 0xFCFA, + 0xFCFB, + 0xFCFC, + 0xFCFD, + 0xFCFE, + 0xFCFF, + 0xFD00, + 0xFD01, + 0xFD02, + 0xFD03, + 0xFD04, + 0xFD05, + 0xFD06, + 0xFD07, + 0xFD08, + 0xFD09, + 0xFD0A, + 0xFD0B, + 0xFD0C, + 0xFD0D, + 0xFD0E, + 0xFD0F, + 0xFD10, + 0xFD11, + 0xFD12, + 0xFD13, + 0xFD14, + 0xFD15, + 0xFD16, + 0xFD17, + 0xFD18, + 0xFD19, + 0xFD1A, + 0xFD1B, + 0xFD1C, + 0xFD1D, + 0xFD1E, + 0xFD1F, + 0xFD20, + 0xFD21, + 0xFD22, + 0xFD23, + 0xFD24, + 0xFD25, + 0xFD26, + 0xFD27, + 0xFD28, + 0xFD29, + 0xFD2A, + 0xFD2B, + 0xFD2C, + 0xFD2D, + 0xFD2E, + 0xFD2F, + 0xFD30, + 0xFD31, + 0xFD32, + 0xFD33, + 0xFD34, + 0xFD35, + 0xFD36, + 0xFD37, + 0xFD38, + 0xFD39, + 0xFD3A, + 0xFD3B, + 0xFD3C, + 0xFD3E, + 0xFD50, + 0xFD51, + 0xFD53, + 0xFD54, + 0xFD55, + 0xFD56, + 0xFD57, + 0xFD58, + 0xFD5A, + 0xFD5B, + 0xFD5C, + 0xFD5D, + 0xFD5E, + 0xFD5F, + 0xFD61, + 0xFD62, + 0xFD64, + 0xFD66, + 0xFD67, + 0xFD69, + 0xFD6A, + 0xFD6C, + 0xFD6E, + 0xFD6F, + 0xFD71, + 0xFD73, + 0xFD74, + 0xFD75, + 0xFD76, + 0xFD78, + 0xFD79, + 0xFD7A, + 0xFD7B, + 0xFD7C, + 0xFD7E, + 0xFD7F, + 0xFD80, + 0xFD81, + 0xFD82, + 0xFD83, + 0xFD85, + 0xFD87, + 0xFD89, + 0xFD8A, + 0xFD8B, + 0xFD8C, + 0xFD8D, + 0xFD8E, + 0xFD8F, + 0xFD90, + 0xFD92, + 0xFD93, + 0xFD94, + 0xFD95, + 0xFD96, + 0xFD97, + 0xFD99, + 0xFD9A, + 0xFD9B, + 0xFD9C, + 0xFD9E, + 0xFD9F, + 0xFDA0, + 0xFDA1, + 0xFDA2, + 0xFDA3, + 0xFDA4, + 0xFDA5, + 0xFDA6, + 0xFDA7, + 0xFDA8, + 0xFDA9, + 0xFDAA, + 0xFDAB, + 0xFDAC, + 0xFDAD, + 0xFDAE, + 0xFDAF, + 0xFDB0, + 0xFDB1, + 0xFDB2, + 0xFDB3, + 0xFDB4, + 0xFDB5, + 0xFDB6, + 0xFDB7, + 0xFDB8, + 0xFDB9, + 0xFDBA, + 0xFDBB, + 0xFDBC, + 0xFDBD, + 0xFDBE, + 0xFDBF, + 0xFDC0, + 0xFDC1, + 0xFDC2, + 0xFDC3, + 0xFDC4, + 0xFDC5, + 0xFDC6, + 0xFDC7, + 0xFDC8, + 0xFDD0, + 0xFDF0, + 0xFDF1, + 0xFDF2, + 0xFDF3, + 0xFDF4, + 0xFDF5, + 0xFDF6, + 0xFDF7, + 0xFDF8, + 0xFDF9, + 0xFDFA, + 0xFDFB, + 0xFDFC, + 0xFDFD, + 0xFE00, + 0xFE10, + 0xFE11, + 0xFE12, + 0xFE13, + 0xFE14, + 0xFE15, + 0xFE16, + 0xFE17, + 0xFE18, + 0xFE19, + 0xFE20, + 0xFE30, + 0xFE31, + 0xFE32, + 0xFE33, + 0xFE35, + 0xFE36, + 0xFE37, + 0xFE38, + 0xFE39, + 0xFE3A, + 0xFE3B, + 0xFE3C, + 0xFE3D, + 0xFE3E, + 0xFE3F, + 0xFE40, + 0xFE41, + 0xFE42, + 0xFE43, + 0xFE44, + 0xFE45, + 0xFE47, + 0xFE48, + 0xFE49, + 0xFE4D, + 0xFE50, + 0xFE51, + 0xFE52, + 0xFE54, + 0xFE55, + 0xFE56, + 0xFE57, + 0xFE58, + 0xFE59, + 0xFE5A, + 0xFE5B, + 0xFE5C, + 0xFE5D, + 0xFE5E, + 0xFE5F, + 0xFE60, + 0xFE61, + 0xFE62, + 0xFE63, + 0xFE64, + 0xFE65, + 0xFE66, + 0xFE67, + 0xFE68, + 0xFE69, + 0xFE6A, + 0xFE6B, + 0xFE6C, + 0xFE70, + 0xFE71, + 0xFE72, + 0xFE73, + 0xFE74, + 0xFE75, + 0xFE76, + 0xFE77, + 0xFE78, + 0xFE79, + 0xFE7A, + 0xFE7B, + 0xFE7C, + 0xFE7D, + 0xFE7E, + 0xFE7F, + 0xFE80, + 0xFE81, + 0xFE83, + 0xFE85, + 0xFE87, + 0xFE89, + 0xFE8D, + 0xFE8F, + 0xFE93, + 0xFE95, + 0xFE99, + 0xFE9D, + 0xFEA1, + 0xFEA5, + 0xFEA9, + 0xFEAB, + 0xFEAD, + 0xFEAF, + 0xFEB1, + 0xFEB5, + 0xFEB9, + 0xFEBD, + 0xFEC1, + 0xFEC5, + 0xFEC9, + 0xFECD, + 0xFED1, + 0xFED5, + 0xFED9, + 0xFEDD, + 0xFEE1, + 0xFEE5, + 0xFEE9, + 0xFEED, + 0xFEEF, + 0xFEF1, + 0xFEF5, + 0xFEF7, + 0xFEF9, + 0xFEFB, + 0xFEFD, + 0xFEFF, + 0xFF00, + 0xFF01, + 0xFF02, + 0xFF03, + 0xFF04, + 0xFF05, + 0xFF06, + 0xFF07, + 0xFF08, + 0xFF09, + 0xFF0A, + 0xFF0B, + 0xFF0C, + 0xFF0D, + 0xFF0E, + 0xFF0F, + 0xFF10, + 0xFF11, + 0xFF12, + 0xFF13, + 0xFF14, + 0xFF15, + 0xFF16, + 0xFF17, + 0xFF18, + 0xFF19, + 0xFF1A, + 0xFF1B, + 0xFF1C, + 0xFF1D, + 0xFF1E, + 0xFF1F, + 0xFF20, + 0xFF21, + 0xFF22, + 0xFF23, + 0xFF24, + 0xFF25, + 0xFF26, + 0xFF27, + 0xFF28, + 0xFF29, + 0xFF2A, + 0xFF2B, + 0xFF2C, + 0xFF2D, + 0xFF2E, + 0xFF2F, + 0xFF30, + 0xFF31, + 0xFF32, + 0xFF33, + 0xFF34, + 0xFF35, + 0xFF36, + 0xFF37, + 0xFF38, + 0xFF39, + 0xFF3A, + 0xFF3B, + 0xFF3C, + 0xFF3D, + 0xFF3E, + 0xFF3F, + 0xFF40, + 0xFF41, + 0xFF42, + 0xFF43, + 0xFF44, + 0xFF45, + 0xFF46, + 0xFF47, + 0xFF48, + 0xFF49, + 0xFF4A, + 0xFF4B, + 0xFF4C, + 0xFF4D, + 0xFF4E, + 0xFF4F, + 0xFF50, + 0xFF51, + 0xFF52, + 0xFF53, + 0xFF54, + 0xFF55, + 0xFF56, + 0xFF57, + 0xFF58, + 0xFF59, + 0xFF5A, + 0xFF5B, + 0xFF5C, + 0xFF5D, + 0xFF5E, + 0xFF5F, + 0xFF60, + 0xFF61, + 0xFF62, + 0xFF63, + 0xFF64, + 0xFF65, + 0xFF66, + 0xFF67, + 0xFF68, + 0xFF69, + 0xFF6A, + 0xFF6B, + 0xFF6C, + 0xFF6D, + 0xFF6E, + 0xFF6F, + 0xFF70, + 0xFF71, + 0xFF72, + 0xFF73, + 0xFF74, + 0xFF75, + 0xFF76, + 0xFF77, + 0xFF78, + 0xFF79, + 0xFF7A, + 0xFF7B, + 0xFF7C, + 0xFF7D, + 0xFF7E, + 0xFF7F, + 0xFF80, + 0xFF81, + 0xFF82, + 0xFF83, + 0xFF84, + 0xFF85, + 0xFF86, + 0xFF87, + 0xFF88, + 0xFF89, + 0xFF8A, + 0xFF8B, + 0xFF8C, + 0xFF8D, + 0xFF8E, + 0xFF8F, + 0xFF90, + 0xFF91, + 0xFF92, + 0xFF93, + 0xFF94, + 0xFF95, + 0xFF96, + 0xFF97, + 0xFF98, + 0xFF99, + 0xFF9A, + 0xFF9B, + 0xFF9C, + 0xFF9D, + 0xFF9E, + 0xFF9F, + 0xFFA0, + 0xFFA1, + 0xFFA2, + 0xFFA3, + 0xFFA4, + 0xFFA5, + 0xFFA6, + 0xFFA7, + 0xFFA8, + 0xFFA9, + 0xFFAA, + 0xFFAB, + 0xFFAC, + 0xFFAD, + 0xFFAE, + 0xFFAF, + 0xFFB0, + 0xFFB1, + 0xFFB2, + 0xFFB3, + 0xFFB4, + 0xFFB5, + 0xFFB6, + 0xFFB7, + 0xFFB8, + 0xFFB9, + 0xFFBA, + 0xFFBB, + 0xFFBC, + 0xFFBD, + 0xFFBE, + 0xFFBF, + 0xFFC2, + 0xFFC3, + 0xFFC4, + 0xFFC5, + 0xFFC6, + 0xFFC7, + 0xFFC8, + 0xFFCA, + 0xFFCB, + 0xFFCC, + 0xFFCD, + 0xFFCE, + 0xFFCF, + 0xFFD0, + 0xFFD2, + 0xFFD3, + 0xFFD4, + 0xFFD5, + 0xFFD6, + 0xFFD7, + 0xFFD8, + 0xFFDA, + 0xFFDB, + 0xFFDC, + 0xFFDD, + 0xFFE0, + 0xFFE1, + 0xFFE2, + 0xFFE3, + 0xFFE4, + 0xFFE5, + 0xFFE6, + 0xFFE7, + 0xFFE8, + 0xFFE9, + 0xFFEA, + 0xFFEB, + 0xFFEC, + 0xFFED, + 0xFFEE, + 0xFFEF, + 0x10000, + 0x1000C, + 0x1000D, + 0x10027, + 0x10028, + 0x1003B, + 0x1003C, + 0x1003E, + 0x1003F, + 0x1004E, + 0x10050, + 0x1005E, + 0x10080, + 0x100FB, + 0x10100, + 0x10103, + 0x10107, + 0x10134, + 0x10137, + 0x1018F, + 0x10190, + 0x1019D, + 0x101A0, + 0x101A1, + 0x101D0, + 0x101FE, + 0x10280, + 0x1029D, + 0x102A0, + 0x102D1, + 0x102E0, + 0x102FC, + 0x10300, + 0x10324, + 0x1032D, + 0x1034B, + 0x10350, + 0x1037B, + 0x10380, + 0x1039E, + 0x1039F, + 0x103C4, + 0x103C8, + 0x103D6, + 0x10400, + 0x10401, + 0x10402, + 0x10403, + 0x10404, + 0x10405, + 0x10406, + 0x10407, + 0x10408, + 0x10409, + 0x1040A, + 0x1040B, + 0x1040C, + 0x1040D, + 0x1040E, + 0x1040F, + 0x10410, + 0x10411, + 0x10412, + 0x10413, + 0x10414, + 0x10415, + 0x10416, + 0x10417, + 0x10418, + 0x10419, + 0x1041A, + 0x1041B, + 0x1041C, + 0x1041D, + 0x1041E, + 0x1041F, + 0x10420, + 0x10421, + 0x10422, + 0x10423, + 0x10424, + 0x10425, + 0x10426, + 0x10427, + 0x10428, + 0x1049E, + 0x104A0, + 0x104AA, + 0x104B0, + 0x104B1, + 0x104B2, + 0x104B3, + 0x104B4, + 0x104B5, + 0x104B6, + 0x104B7, + 0x104B8, + 0x104B9, + 0x104BA, + 0x104BB, + 0x104BC, + 0x104BD, + 0x104BE, + 0x104BF, + 0x104C0, + 0x104C1, + 0x104C2, + 0x104C3, + 0x104C4, + 0x104C5, + 0x104C6, + 0x104C7, + 0x104C8, + 0x104C9, + 0x104CA, + 0x104CB, + 0x104CC, + 0x104CD, + 0x104CE, + 0x104CF, + 0x104D0, + 0x104D1, + 0x104D2, + 0x104D3, + 0x104D4, + 0x104D8, + 0x104FC, + 0x10500, + 0x10528, + 0x10530, + 0x10564, + 0x1056F, + 0x10570, + 0x10571, + 0x10572, + 0x10573, + 0x10574, + 0x10575, + 0x10576, + 0x10577, + 0x10578, + 0x10579, + 0x1057A, + 0x1057B, + 0x1057C, + 0x1057D, + 0x1057E, + 0x1057F, + 0x10580, + 0x10581, + 0x10582, + 0x10583, + 0x10584, + 0x10585, + 0x10586, + 0x10587, + 0x10588, + 0x10589, + 0x1058A, + 0x1058B, + 0x1058C, + 0x1058D, + 0x1058E, + 0x1058F, + 0x10590, + 0x10591, + 0x10592, + 0x10593, + 0x10594, + 0x10595, + 0x10596, + 0x10597, + 0x105A2, + 0x105A3, + 0x105B2, + 0x105B3, + 0x105BA, + 0x105BB, + 0x105BD, + 0x105C0, + 0x105F4, + 0x10600, + 0x10737, + 0x10740, + 0x10756, + 0x10760, + 0x10768, + 0x10780, + 0x10781, + 0x10782, + 0x10783, + 0x10784, + 0x10785, + 0x10786, + 0x10787, + 0x10788, + 0x10789, + 0x1078A, + 0x1078B, + 0x1078C, + 0x1078D, + 0x1078E, + 0x1078F, + 0x10790, + 0x10791, + 0x10792, + 0x10793, + 0x10794, + 0x10795, + 0x10796, + 0x10797, + 0x10798, + 0x10799, + 0x1079A, + 0x1079B, + 0x1079C, + 0x1079D, + 0x1079E, + 0x1079F, + 0x107A0, + 0x107A1, + 0x107A2, + 0x107A3, + 0x107A4, + 0x107A5, + 0x107A6, + 0x107A7, + 0x107A8, + 0x107A9, + 0x107AA, + 0x107AB, + 0x107AC, + 0x107AD, + 0x107AE, + 0x107AF, + 0x107B0, + 0x107B1, + 0x107B2, + 0x107B3, + 0x107B4, + 0x107B5, + 0x107B6, + 0x107B7, + 0x107B8, + 0x107B9, + 0x107BA, + 0x107BB, + 0x10800, + 0x10806, + 0x10808, + 0x10809, + 0x1080A, + 0x10836, + 0x10837, + 0x10839, + 0x1083C, + 0x1083D, + 0x1083F, + 0x10856, + 0x10857, + 0x1089F, + 0x108A7, + 0x108B0, + 0x108E0, + 0x108F3, + 0x108F4, + 0x108F6, + 0x108FB, + 0x1091C, + 0x1091F, + 0x1093A, + 0x1093F, + 0x1095A, + 0x10980, + 0x109B8, + 0x109BC, + 0x109D0, + 0x109D2, + 0x10A04, + 0x10A05, + 0x10A07, + 0x10A0C, + 0x10A14, + 0x10A15, + 0x10A18, + 0x10A19, + 0x10A36, + 0x10A38, + 0x10A3B, + 0x10A3F, + 0x10A49, + 0x10A50, + 0x10A59, + 0x10A60, + 0x10AA0, + 0x10AC0, + 0x10AE7, + 0x10AEB, + 0x10AF7, + 0x10B00, + 0x10B36, + 0x10B39, + 0x10B56, + 0x10B58, + 0x10B73, + 0x10B78, + 0x10B92, + 0x10B99, + 0x10B9D, + 0x10BA9, + 0x10BB0, + 0x10C00, + 0x10C49, + 0x10C80, + 0x10C81, + 0x10C82, + 0x10C83, + 0x10C84, + 0x10C85, + 0x10C86, + 0x10C87, + 0x10C88, + 0x10C89, + 0x10C8A, + 0x10C8B, + 0x10C8C, + 0x10C8D, + 0x10C8E, + 0x10C8F, + 0x10C90, + 0x10C91, + 0x10C92, + 0x10C93, + 0x10C94, + 0x10C95, + 0x10C96, + 0x10C97, + 0x10C98, + 0x10C99, + 0x10C9A, + 0x10C9B, + 0x10C9C, + 0x10C9D, + 0x10C9E, + 0x10C9F, + 0x10CA0, + 0x10CA1, + 0x10CA2, + 0x10CA3, + 0x10CA4, + 0x10CA5, + 0x10CA6, + 0x10CA7, + 0x10CA8, + 0x10CA9, + 0x10CAA, + 0x10CAB, + 0x10CAC, + 0x10CAD, + 0x10CAE, + 0x10CAF, + 0x10CB0, + 0x10CB1, + 0x10CB2, + 0x10CB3, + 0x10CC0, + 0x10CF3, + 0x10CFA, + 0x10D28, + 0x10D30, + 0x10D3A, + 0x10D40, + 0x10D50, + 0x10D51, + 0x10D52, + 0x10D53, + 0x10D54, + 0x10D55, + 0x10D56, + 0x10D57, + 0x10D58, + 0x10D59, + 0x10D5A, + 0x10D5B, + 0x10D5C, + 0x10D5D, + 0x10D5E, + 0x10D5F, + 0x10D60, + 0x10D61, + 0x10D62, + 0x10D63, + 0x10D64, + 0x10D65, + 0x10D66, + 0x10D69, + 0x10D86, + 0x10D8E, + 0x10D90, + 0x10E60, + 0x10E7F, + 0x10E80, + 0x10EAA, + 0x10EAB, + 0x10EAE, + 0x10EB0, + 0x10EB2, + 0x10EC2, + 0x10EC8, + 0x10ED0, + 0x10ED9, + 0x10EFA, + 0x10F28, + 0x10F30, + 0x10F5A, + 0x10F70, + 0x10F8A, + 0x10FB0, + 0x10FCC, + 0x10FE0, + 0x10FF7, + 0x11000, + 0x1104E, + 0x11052, + 0x11076, + 0x1107F, + 0x110BD, + 0x110BE, + 0x110C3, + 0x110D0, + 0x110E9, + 0x110F0, + 0x110FA, + 0x11100, + 0x11135, + 0x11136, + 0x11148, + 0x11150, + 0x11177, + 0x11180, + 0x111E0, + 0x111E1, + 0x111F5, + 0x11200, + 0x11212, + 0x11213, + 0x11242, + 0x11280, + 0x11287, + 0x11288, + 0x11289, + 0x1128A, + 0x1128E, + 0x1128F, + 0x1129E, + 0x1129F, + 0x112AA, + 0x112B0, + 0x112EB, + 0x112F0, + 0x112FA, + 0x11300, + 0x11304, + 0x11305, + 0x1130D, + 0x1130F, + 0x11311, + 0x11313, + 0x11329, + 0x1132A, + 0x11331, + 0x11332, + 0x11334, + 0x11335, + 0x1133A, + 0x1133B, + 0x11345, + 0x11347, + 0x11349, + 0x1134B, + 0x1134E, + 0x11350, + 0x11351, + 0x11357, + 0x11358, + 0x1135D, + 0x11364, + 0x11366, + 0x1136D, + 0x11370, + 0x11375, + 0x11380, + 0x1138A, + 0x1138B, + 0x1138C, + 0x1138E, + 0x1138F, + 0x11390, + 0x113B6, + 0x113B7, + 0x113C1, + 0x113C2, + 0x113C3, + 0x113C5, + 0x113C6, + 0x113C7, + 0x113CB, + 0x113CC, + 0x113D6, + 0x113D7, + 0x113D9, + 0x113E1, + 0x113E3, + 0x11400, + 0x1145C, + 0x1145D, + 0x11462, + 0x11480, + 0x114C8, + 0x114D0, + 0x114DA, + 0x11580, + 0x115B6, + 0x115B8, + 0x115DE, + 0x11600, + 0x11645, + 0x11650, + 0x1165A, + 0x11660, + 0x1166D, + 0x11680, + 0x116BA, + 0x116C0, + 0x116CA, + 0x116D0, + 0x116E4, + 0x11700, + 0x1171B, + 0x1171D, + 0x1172C, + 0x11730, + 0x11747, + 0x11800, + 0x1183C, + 0x118A0, + 0x118A1, + 0x118A2, + 0x118A3, + 0x118A4, + 0x118A5, + 0x118A6, + 0x118A7, + 0x118A8, + 0x118A9, + 0x118AA, + 0x118AB, + 0x118AC, + 0x118AD, + 0x118AE, + 0x118AF, + 0x118B0, + 0x118B1, + 0x118B2, + 0x118B3, + 0x118B4, + 0x118B5, + 0x118B6, + 0x118B7, + 0x118B8, + 0x118B9, + 0x118BA, + 0x118BB, + 0x118BC, + 0x118BD, + 0x118BE, + 0x118BF, + 0x118C0, + 0x118F3, + 0x118FF, + 0x11907, + 0x11909, + 0x1190A, + 0x1190C, + 0x11914, + 0x11915, + 0x11917, + 0x11918, + 0x11936, + 0x11937, + 0x11939, + 0x1193B, + 0x11947, + 0x11950, + 0x1195A, + 0x119A0, + 0x119A8, + 0x119AA, + 0x119D8, + 0x119DA, + 0x119E5, + 0x11A00, + 0x11A48, + 0x11A50, + 0x11AA3, + 0x11AB0, + 0x11AF9, + 0x11B00, + 0x11B0A, + 0x11B60, + 0x11B68, + 0x11BC0, + 0x11BE2, + 0x11BF0, + 0x11BFA, + 0x11C00, + 0x11C09, + 0x11C0A, + 0x11C37, + 0x11C38, + 0x11C46, + 0x11C50, + 0x11C6D, + 0x11C70, + 0x11C90, + 0x11C92, + 0x11CA8, + 0x11CA9, + 0x11CB7, + 0x11D00, + 0x11D07, + 0x11D08, + 0x11D0A, + 0x11D0B, + 0x11D37, + 0x11D3A, + 0x11D3B, + 0x11D3C, + 0x11D3E, + 0x11D3F, + 0x11D48, + 0x11D50, + 0x11D5A, + 0x11D60, + 0x11D66, + 0x11D67, + 0x11D69, + 0x11D6A, + 0x11D8F, + 0x11D90, + 0x11D92, + 0x11D93, + 0x11D99, + 0x11DA0, + 0x11DAA, + 0x11DB0, + 0x11DDC, + 0x11DE0, + 0x11DEA, + 0x11EE0, + 0x11EF9, + 0x11F00, + 0x11F11, + 0x11F12, + 0x11F3B, + 0x11F3E, + 0x11F5B, + 0x11FB0, + 0x11FB1, + 0x11FC0, + 0x11FF2, + 0x11FFF, + 0x1239A, + 0x12400, + 0x1246F, + 0x12470, + 0x12475, + 0x12480, + 0x12544, + 0x12F90, + 0x12FF3, + 0x13000, + 0x13430, + 0x13440, + 0x13456, + 0x13460, + 0x143FB, + 0x14400, + 0x14647, + 0x16100, + 0x1613A, + 0x16800, + 0x16A39, + 0x16A40, + 0x16A5F, + 0x16A60, + 0x16A6A, + 0x16A6E, + 0x16ABF, + 0x16AC0, + 0x16ACA, + 0x16AD0, + 0x16AEE, + 0x16AF0, + 0x16AF6, + 0x16B00, + 0x16B46, + 0x16B50, + 0x16B5A, + 0x16B5B, + 0x16B62, + 0x16B63, + 0x16B78, + 0x16B7D, + 0x16B90, + 0x16D40, + 0x16D7A, + 0x16E40, + 0x16E41, + 0x16E42, + 0x16E43, + 0x16E44, + 0x16E45, + 0x16E46, + 0x16E47, + 0x16E48, + 0x16E49, + 0x16E4A, + 0x16E4B, + 0x16E4C, + 0x16E4D, + 0x16E4E, + 0x16E4F, + 0x16E50, + 0x16E51, + 0x16E52, + 0x16E53, + 0x16E54, + 0x16E55, + 0x16E56, + 0x16E57, + 0x16E58, + 0x16E59, + 0x16E5A, + 0x16E5B, + 0x16E5C, + 0x16E5D, + 0x16E5E, + 0x16E5F, + 0x16E60, + 0x16E9B, + 0x16EA0, + 0x16EA1, + 0x16EA2, + 0x16EA3, + 0x16EA4, + 0x16EA5, + 0x16EA6, + 0x16EA7, + 0x16EA8, + 0x16EA9, + 0x16EAA, + 0x16EAB, + 0x16EAC, + 0x16EAD, + 0x16EAE, + 0x16EAF, + 0x16EB0, + 0x16EB1, + 0x16EB2, + 0x16EB3, + 0x16EB4, + 0x16EB5, + 0x16EB6, + 0x16EB7, + 0x16EB8, + 0x16EB9, + 0x16EBB, + 0x16ED4, + 0x16F00, + 0x16F4B, + 0x16F4F, + 0x16F88, + 0x16F8F, + 0x16FA0, + 0x16FE0, + 0x16FE5, + 0x16FF0, + 0x16FF7, + 0x17000, + 0x18CD6, + 0x18CFF, + 0x18D1F, + 0x18D80, + 0x18DF3, + 0x1AFF0, + 0x1AFF4, + 0x1AFF5, + 0x1AFFC, + 0x1AFFD, + 0x1AFFF, + 0x1B000, + 0x1B123, + 0x1B132, + 0x1B133, + 0x1B150, + 0x1B153, + 0x1B155, + 0x1B156, + 0x1B164, + 0x1B168, + 0x1B170, + 0x1B2FC, + 0x1BC00, + 0x1BC6B, + 0x1BC70, + 0x1BC7D, + 0x1BC80, + 0x1BC89, + 0x1BC90, + 0x1BC9A, + 0x1BC9C, + 0x1BCA0, + 0x1BCA4, + 0x1CC00, + 0x1CCD6, + 0x1CCD7, + 0x1CCD8, + 0x1CCD9, + 0x1CCDA, + 0x1CCDB, + 0x1CCDC, + 0x1CCDD, + 0x1CCDE, + 0x1CCDF, + 0x1CCE0, + 0x1CCE1, + 0x1CCE2, + 0x1CCE3, + 0x1CCE4, + 0x1CCE5, + 0x1CCE6, + 0x1CCE7, + 0x1CCE8, + 0x1CCE9, + 0x1CCEA, + 0x1CCEB, + 0x1CCEC, + 0x1CCED, + 0x1CCEE, + 0x1CCEF, + 0x1CCF0, + 0x1CCF1, + 0x1CCF2, + 0x1CCF3, + 0x1CCF4, + 0x1CCF5, + 0x1CCF6, + 0x1CCF7, + 0x1CCF8, + 0x1CCF9, + 0x1CCFA, + 0x1CCFD, + 0x1CD00, + 0x1CEB4, + 0x1CEBA, + 0x1CED1, + 0x1CEE0, + 0x1CEF1, + 0x1CF00, + 0x1CF2E, + 0x1CF30, + 0x1CF47, + 0x1CF50, + 0x1CFC4, + 0x1D000, + 0x1D0F6, + 0x1D100, + 0x1D127, + 0x1D129, + 0x1D15E, + 0x1D15F, + 0x1D160, + 0x1D161, + 0x1D162, + 0x1D163, + 0x1D164, + 0x1D165, + 0x1D173, + 0x1D17B, + 0x1D1BB, + 0x1D1BC, + 0x1D1BD, + 0x1D1BE, + 0x1D1BF, + 0x1D1C0, + 0x1D1C1, + 0x1D1EB, + 0x1D200, + 0x1D246, + 0x1D2C0, + 0x1D2D4, + 0x1D2E0, + 0x1D2F4, + 0x1D300, + 0x1D357, + 0x1D360, + 0x1D379, + 0x1D400, + 0x1D401, + 0x1D402, + 0x1D403, + 0x1D404, + 0x1D405, + 0x1D406, + 0x1D407, + 0x1D408, + 0x1D409, + 0x1D40A, + 0x1D40B, + 0x1D40C, + 0x1D40D, + 0x1D40E, + 0x1D40F, + 0x1D410, + 0x1D411, + 0x1D412, + 0x1D413, + 0x1D414, + 0x1D415, + 0x1D416, + 0x1D417, + 0x1D418, + 0x1D419, + 0x1D41A, + 0x1D41B, + 0x1D41C, + 0x1D41D, + 0x1D41E, + 0x1D41F, + 0x1D420, + 0x1D421, + 0x1D422, + 0x1D423, + 0x1D424, + 0x1D425, + 0x1D426, + 0x1D427, + 0x1D428, + 0x1D429, + 0x1D42A, + 0x1D42B, + 0x1D42C, + 0x1D42D, + 0x1D42E, + 0x1D42F, + 0x1D430, + 0x1D431, + 0x1D432, + 0x1D433, + 0x1D434, + 0x1D435, + 0x1D436, + 0x1D437, + 0x1D438, + 0x1D439, + 0x1D43A, + 0x1D43B, + 0x1D43C, + 0x1D43D, + 0x1D43E, + 0x1D43F, + 0x1D440, + 0x1D441, + 0x1D442, + 0x1D443, + 0x1D444, + 0x1D445, + 0x1D446, + 0x1D447, + 0x1D448, + 0x1D449, + 0x1D44A, + 0x1D44B, + 0x1D44C, + 0x1D44D, + 0x1D44E, + 0x1D44F, + 0x1D450, + 0x1D451, + 0x1D452, + 0x1D453, + 0x1D454, + 0x1D455, + 0x1D456, + 0x1D457, + 0x1D458, + 0x1D459, + 0x1D45A, + 0x1D45B, + 0x1D45C, + 0x1D45D, + 0x1D45E, + 0x1D45F, + 0x1D460, + 0x1D461, + 0x1D462, + 0x1D463, + 0x1D464, + 0x1D465, + 0x1D466, + 0x1D467, + 0x1D468, + 0x1D469, + 0x1D46A, + 0x1D46B, + 0x1D46C, + 0x1D46D, + 0x1D46E, + 0x1D46F, + 0x1D470, + 0x1D471, + 0x1D472, + 0x1D473, + 0x1D474, + 0x1D475, + 0x1D476, + 0x1D477, + 0x1D478, + 0x1D479, + 0x1D47A, + 0x1D47B, + 0x1D47C, + 0x1D47D, + 0x1D47E, + 0x1D47F, + 0x1D480, + 0x1D481, + 0x1D482, + 0x1D483, + 0x1D484, + 0x1D485, + 0x1D486, + 0x1D487, + 0x1D488, + 0x1D489, + 0x1D48A, + 0x1D48B, + 0x1D48C, + 0x1D48D, + 0x1D48E, + 0x1D48F, + 0x1D490, + 0x1D491, + 0x1D492, + 0x1D493, + 0x1D494, + 0x1D495, + 0x1D496, + 0x1D497, + 0x1D498, + 0x1D499, + 0x1D49A, + 0x1D49B, + 0x1D49C, + 0x1D49D, + 0x1D49E, + 0x1D49F, + 0x1D4A0, + 0x1D4A2, + 0x1D4A3, + 0x1D4A5, + 0x1D4A6, + 0x1D4A7, + 0x1D4A9, + 0x1D4AA, + 0x1D4AB, + 0x1D4AC, + 0x1D4AD, + 0x1D4AE, + 0x1D4AF, + 0x1D4B0, + 0x1D4B1, + 0x1D4B2, + 0x1D4B3, + 0x1D4B4, + 0x1D4B5, + 0x1D4B6, + 0x1D4B7, + 0x1D4B8, + 0x1D4B9, + 0x1D4BA, + 0x1D4BB, + 0x1D4BC, + 0x1D4BD, + 0x1D4BE, + 0x1D4BF, + 0x1D4C0, + 0x1D4C1, + 0x1D4C2, + 0x1D4C3, + 0x1D4C4, + 0x1D4C5, + 0x1D4C6, + 0x1D4C7, + 0x1D4C8, + 0x1D4C9, + 0x1D4CA, + 0x1D4CB, + 0x1D4CC, + 0x1D4CD, + 0x1D4CE, + 0x1D4CF, + 0x1D4D0, + 0x1D4D1, + 0x1D4D2, + 0x1D4D3, + 0x1D4D4, + 0x1D4D5, + 0x1D4D6, + 0x1D4D7, + 0x1D4D8, + 0x1D4D9, + 0x1D4DA, + 0x1D4DB, + 0x1D4DC, + 0x1D4DD, + 0x1D4DE, + 0x1D4DF, + 0x1D4E0, + 0x1D4E1, + 0x1D4E2, + 0x1D4E3, + 0x1D4E4, + 0x1D4E5, + 0x1D4E6, + 0x1D4E7, + 0x1D4E8, + 0x1D4E9, + 0x1D4EA, + 0x1D4EB, + 0x1D4EC, + 0x1D4ED, + 0x1D4EE, + 0x1D4EF, + 0x1D4F0, + 0x1D4F1, + 0x1D4F2, + 0x1D4F3, + 0x1D4F4, + 0x1D4F5, + 0x1D4F6, + 0x1D4F7, + 0x1D4F8, + 0x1D4F9, + 0x1D4FA, + 0x1D4FB, + 0x1D4FC, + 0x1D4FD, + 0x1D4FE, + 0x1D4FF, + 0x1D500, + 0x1D501, + 0x1D502, + 0x1D503, + 0x1D504, + 0x1D505, + 0x1D506, + 0x1D507, + 0x1D508, + 0x1D509, + 0x1D50A, + 0x1D50B, + 0x1D50D, + 0x1D50E, + 0x1D50F, + 0x1D510, + 0x1D511, + 0x1D512, + 0x1D513, + 0x1D514, + 0x1D515, + 0x1D516, + 0x1D517, + 0x1D518, + 0x1D519, + 0x1D51A, + 0x1D51B, + 0x1D51C, + 0x1D51D, + 0x1D51E, + 0x1D51F, + 0x1D520, + 0x1D521, + 0x1D522, + 0x1D523, + 0x1D524, + 0x1D525, + 0x1D526, + 0x1D527, + 0x1D528, + 0x1D529, + 0x1D52A, + 0x1D52B, + 0x1D52C, + 0x1D52D, + 0x1D52E, + 0x1D52F, + 0x1D530, + 0x1D531, + 0x1D532, + 0x1D533, + 0x1D534, + 0x1D535, + 0x1D536, + 0x1D537, + 0x1D538, + 0x1D539, + 0x1D53A, + 0x1D53B, + 0x1D53C, + 0x1D53D, + 0x1D53E, + 0x1D53F, + 0x1D540, + 0x1D541, + 0x1D542, + 0x1D543, + 0x1D544, + 0x1D545, + 0x1D546, + 0x1D547, + 0x1D54A, + 0x1D54B, + 0x1D54C, + 0x1D54D, + 0x1D54E, + 0x1D54F, + 0x1D550, + 0x1D551, + 0x1D552, + 0x1D553, + 0x1D554, + 0x1D555, + 0x1D556, + 0x1D557, + 0x1D558, + 0x1D559, + 0x1D55A, + 0x1D55B, + 0x1D55C, + 0x1D55D, + 0x1D55E, + 0x1D55F, + 0x1D560, + 0x1D561, + 0x1D562, + 0x1D563, + 0x1D564, + 0x1D565, + 0x1D566, + 0x1D567, + 0x1D568, + 0x1D569, + 0x1D56A, + 0x1D56B, + 0x1D56C, + 0x1D56D, + 0x1D56E, + 0x1D56F, + 0x1D570, + 0x1D571, + 0x1D572, + 0x1D573, + 0x1D574, + 0x1D575, + 0x1D576, + 0x1D577, + 0x1D578, + 0x1D579, + 0x1D57A, + 0x1D57B, + 0x1D57C, + 0x1D57D, + 0x1D57E, + 0x1D57F, + 0x1D580, + 0x1D581, + 0x1D582, + 0x1D583, + 0x1D584, + 0x1D585, + 0x1D586, + 0x1D587, + 0x1D588, + 0x1D589, + 0x1D58A, + 0x1D58B, + 0x1D58C, + 0x1D58D, + 0x1D58E, + 0x1D58F, + 0x1D590, + 0x1D591, + 0x1D592, + 0x1D593, + 0x1D594, + 0x1D595, + 0x1D596, + 0x1D597, + 0x1D598, + 0x1D599, + 0x1D59A, + 0x1D59B, + 0x1D59C, + 0x1D59D, + 0x1D59E, + 0x1D59F, + 0x1D5A0, + 0x1D5A1, + 0x1D5A2, + 0x1D5A3, + 0x1D5A4, + 0x1D5A5, + 0x1D5A6, + 0x1D5A7, + 0x1D5A8, + 0x1D5A9, + 0x1D5AA, + 0x1D5AB, + 0x1D5AC, + 0x1D5AD, + 0x1D5AE, + 0x1D5AF, + 0x1D5B0, + 0x1D5B1, + 0x1D5B2, + 0x1D5B3, + 0x1D5B4, + 0x1D5B5, + 0x1D5B6, + 0x1D5B7, + 0x1D5B8, + 0x1D5B9, + 0x1D5BA, + 0x1D5BB, + 0x1D5BC, + 0x1D5BD, + 0x1D5BE, + 0x1D5BF, + 0x1D5C0, + 0x1D5C1, + 0x1D5C2, + 0x1D5C3, + 0x1D5C4, + 0x1D5C5, + 0x1D5C6, + 0x1D5C7, + 0x1D5C8, + 0x1D5C9, + 0x1D5CA, + 0x1D5CB, + 0x1D5CC, + 0x1D5CD, + 0x1D5CE, + 0x1D5CF, + 0x1D5D0, + 0x1D5D1, + 0x1D5D2, + 0x1D5D3, + 0x1D5D4, + 0x1D5D5, + 0x1D5D6, + 0x1D5D7, + 0x1D5D8, + 0x1D5D9, + 0x1D5DA, + 0x1D5DB, + 0x1D5DC, + 0x1D5DD, + 0x1D5DE, + 0x1D5DF, + 0x1D5E0, + 0x1D5E1, + 0x1D5E2, + 0x1D5E3, + 0x1D5E4, + 0x1D5E5, + 0x1D5E6, + 0x1D5E7, + 0x1D5E8, + 0x1D5E9, + 0x1D5EA, + 0x1D5EB, + 0x1D5EC, + 0x1D5ED, + 0x1D5EE, + 0x1D5EF, + 0x1D5F0, + 0x1D5F1, + 0x1D5F2, + 0x1D5F3, + 0x1D5F4, + 0x1D5F5, + 0x1D5F6, + 0x1D5F7, + 0x1D5F8, + 0x1D5F9, + 0x1D5FA, + 0x1D5FB, + 0x1D5FC, + 0x1D5FD, + 0x1D5FE, + 0x1D5FF, + 0x1D600, + 0x1D601, + 0x1D602, + 0x1D603, + 0x1D604, + 0x1D605, + 0x1D606, + 0x1D607, + 0x1D608, + 0x1D609, + 0x1D60A, + 0x1D60B, + 0x1D60C, + 0x1D60D, + 0x1D60E, + 0x1D60F, + 0x1D610, + 0x1D611, + 0x1D612, + 0x1D613, + 0x1D614, + 0x1D615, + 0x1D616, + 0x1D617, + 0x1D618, + 0x1D619, + 0x1D61A, + 0x1D61B, + 0x1D61C, + 0x1D61D, + 0x1D61E, + 0x1D61F, + 0x1D620, + 0x1D621, + 0x1D622, + 0x1D623, + 0x1D624, + 0x1D625, + 0x1D626, + 0x1D627, + 0x1D628, + 0x1D629, + 0x1D62A, + 0x1D62B, + 0x1D62C, + 0x1D62D, + 0x1D62E, + 0x1D62F, + 0x1D630, + 0x1D631, + 0x1D632, + 0x1D633, + 0x1D634, + 0x1D635, + 0x1D636, + 0x1D637, + 0x1D638, + 0x1D639, + 0x1D63A, + 0x1D63B, + 0x1D63C, + 0x1D63D, + 0x1D63E, + 0x1D63F, + 0x1D640, + 0x1D641, + 0x1D642, + 0x1D643, + 0x1D644, + 0x1D645, + 0x1D646, + 0x1D647, + 0x1D648, + 0x1D649, + 0x1D64A, + 0x1D64B, + 0x1D64C, + 0x1D64D, + 0x1D64E, + 0x1D64F, + 0x1D650, + 0x1D651, + 0x1D652, + 0x1D653, + 0x1D654, + 0x1D655, + 0x1D656, + 0x1D657, + 0x1D658, + 0x1D659, + 0x1D65A, + 0x1D65B, + 0x1D65C, + 0x1D65D, + 0x1D65E, + 0x1D65F, + 0x1D660, + 0x1D661, + 0x1D662, + 0x1D663, + 0x1D664, + 0x1D665, + 0x1D666, + 0x1D667, + 0x1D668, + 0x1D669, + 0x1D66A, + 0x1D66B, + 0x1D66C, + 0x1D66D, + 0x1D66E, + 0x1D66F, + 0x1D670, + 0x1D671, + 0x1D672, + 0x1D673, + 0x1D674, + 0x1D675, + 0x1D676, + 0x1D677, + 0x1D678, + 0x1D679, + 0x1D67A, + 0x1D67B, + 0x1D67C, + 0x1D67D, + 0x1D67E, + 0x1D67F, + 0x1D680, + 0x1D681, + 0x1D682, + 0x1D683, + 0x1D684, + 0x1D685, + 0x1D686, + 0x1D687, + 0x1D688, + 0x1D689, + 0x1D68A, + 0x1D68B, + 0x1D68C, + 0x1D68D, + 0x1D68E, + 0x1D68F, + 0x1D690, + 0x1D691, + 0x1D692, + 0x1D693, + 0x1D694, + 0x1D695, + 0x1D696, + 0x1D697, + 0x1D698, + 0x1D699, + 0x1D69A, + 0x1D69B, + 0x1D69C, + 0x1D69D, + 0x1D69E, + 0x1D69F, + 0x1D6A0, + 0x1D6A1, + 0x1D6A2, + 0x1D6A3, + 0x1D6A4, + 0x1D6A5, + 0x1D6A6, + 0x1D6A8, + 0x1D6A9, + 0x1D6AA, + 0x1D6AB, + 0x1D6AC, + 0x1D6AD, + 0x1D6AE, + 0x1D6AF, + 0x1D6B0, + 0x1D6B1, + 0x1D6B2, + 0x1D6B3, + 0x1D6B4, + 0x1D6B5, + 0x1D6B6, + 0x1D6B7, + 0x1D6B8, + 0x1D6B9, + 0x1D6BA, + 0x1D6BB, + 0x1D6BC, + 0x1D6BD, + 0x1D6BE, + 0x1D6BF, + 0x1D6C0, + 0x1D6C1, + 0x1D6C2, + 0x1D6C3, + 0x1D6C4, + 0x1D6C5, + 0x1D6C6, + 0x1D6C7, + 0x1D6C8, + 0x1D6C9, + 0x1D6CA, + 0x1D6CB, + 0x1D6CC, + 0x1D6CD, + 0x1D6CE, + 0x1D6CF, + 0x1D6D0, + 0x1D6D1, + 0x1D6D2, + 0x1D6D3, + 0x1D6D5, + 0x1D6D6, + 0x1D6D7, + 0x1D6D8, + 0x1D6D9, + 0x1D6DA, + 0x1D6DB, + 0x1D6DC, + 0x1D6DD, + 0x1D6DE, + 0x1D6DF, + 0x1D6E0, + 0x1D6E1, + 0x1D6E2, + 0x1D6E3, + 0x1D6E4, + 0x1D6E5, + 0x1D6E6, + 0x1D6E7, + 0x1D6E8, + 0x1D6E9, + 0x1D6EA, + 0x1D6EB, + 0x1D6EC, + 0x1D6ED, + 0x1D6EE, + 0x1D6EF, + 0x1D6F0, + 0x1D6F1, + 0x1D6F2, + 0x1D6F3, + 0x1D6F4, + 0x1D6F5, + 0x1D6F6, + 0x1D6F7, + 0x1D6F8, + 0x1D6F9, + 0x1D6FA, + 0x1D6FB, + 0x1D6FC, + 0x1D6FD, + 0x1D6FE, + 0x1D6FF, + 0x1D700, + 0x1D701, + 0x1D702, + 0x1D703, + 0x1D704, + 0x1D705, + 0x1D706, + 0x1D707, + 0x1D708, + 0x1D709, + 0x1D70A, + 0x1D70B, + 0x1D70C, + 0x1D70D, + 0x1D70F, + 0x1D710, + 0x1D711, + 0x1D712, + 0x1D713, + 0x1D714, + 0x1D715, + 0x1D716, + 0x1D717, + 0x1D718, + 0x1D719, + 0x1D71A, + 0x1D71B, + 0x1D71C, + 0x1D71D, + 0x1D71E, + 0x1D71F, + 0x1D720, + 0x1D721, + 0x1D722, + 0x1D723, + 0x1D724, + 0x1D725, + 0x1D726, + 0x1D727, + 0x1D728, + 0x1D729, + 0x1D72A, + 0x1D72B, + 0x1D72C, + 0x1D72D, + 0x1D72E, + 0x1D72F, + 0x1D730, + 0x1D731, + 0x1D732, + 0x1D733, + 0x1D734, + 0x1D735, + 0x1D736, + 0x1D737, + 0x1D738, + 0x1D739, + 0x1D73A, + 0x1D73B, + 0x1D73C, + 0x1D73D, + 0x1D73E, + 0x1D73F, + 0x1D740, + 0x1D741, + 0x1D742, + 0x1D743, + 0x1D744, + 0x1D745, + 0x1D746, + 0x1D747, + 0x1D749, + 0x1D74A, + 0x1D74B, + 0x1D74C, + 0x1D74D, + 0x1D74E, + 0x1D74F, + 0x1D750, + 0x1D751, + 0x1D752, + 0x1D753, + 0x1D754, + 0x1D755, + 0x1D756, + 0x1D757, + 0x1D758, + 0x1D759, + 0x1D75A, + 0x1D75B, + 0x1D75C, + 0x1D75D, + 0x1D75E, + 0x1D75F, + 0x1D760, + 0x1D761, + 0x1D762, + 0x1D763, + 0x1D764, + 0x1D765, + 0x1D766, + 0x1D767, + 0x1D768, + 0x1D769, + 0x1D76A, + 0x1D76B, + 0x1D76C, + 0x1D76D, + 0x1D76E, + 0x1D76F, + 0x1D770, + 0x1D771, + 0x1D772, + 0x1D773, + 0x1D774, + 0x1D775, + 0x1D776, + 0x1D777, + 0x1D778, + 0x1D779, + 0x1D77A, + 0x1D77B, + 0x1D77C, + 0x1D77D, + 0x1D77E, + 0x1D77F, + 0x1D780, + 0x1D781, + 0x1D783, + 0x1D784, + 0x1D785, + 0x1D786, + 0x1D787, + 0x1D788, + 0x1D789, + 0x1D78A, + 0x1D78B, + 0x1D78C, + 0x1D78D, + 0x1D78E, + 0x1D78F, + 0x1D790, + 0x1D791, + 0x1D792, + 0x1D793, + 0x1D794, + 0x1D795, + 0x1D796, + 0x1D797, + 0x1D798, + 0x1D799, + 0x1D79A, + 0x1D79B, + 0x1D79C, + 0x1D79D, + 0x1D79E, + 0x1D79F, + 0x1D7A0, + 0x1D7A1, + 0x1D7A2, + 0x1D7A3, + 0x1D7A4, + 0x1D7A5, + 0x1D7A6, + 0x1D7A7, + 0x1D7A8, + 0x1D7A9, + 0x1D7AA, + 0x1D7AB, + 0x1D7AC, + 0x1D7AD, + 0x1D7AE, + 0x1D7AF, + 0x1D7B0, + 0x1D7B1, + 0x1D7B2, + 0x1D7B3, + 0x1D7B4, + 0x1D7B5, + 0x1D7B6, + 0x1D7B7, + 0x1D7B8, + 0x1D7B9, + 0x1D7BA, + 0x1D7BB, + 0x1D7BD, + 0x1D7BE, + 0x1D7BF, + 0x1D7C0, + 0x1D7C1, + 0x1D7C2, + 0x1D7C3, + 0x1D7C4, + 0x1D7C5, + 0x1D7C6, + 0x1D7C7, + 0x1D7C8, + 0x1D7C9, + 0x1D7CA, + 0x1D7CC, + 0x1D7CE, + 0x1D7CF, + 0x1D7D0, + 0x1D7D1, + 0x1D7D2, + 0x1D7D3, + 0x1D7D4, + 0x1D7D5, + 0x1D7D6, + 0x1D7D7, + 0x1D7D8, + 0x1D7D9, + 0x1D7DA, + 0x1D7DB, + 0x1D7DC, + 0x1D7DD, + 0x1D7DE, + 0x1D7DF, + 0x1D7E0, + 0x1D7E1, + 0x1D7E2, + 0x1D7E3, + 0x1D7E4, + 0x1D7E5, + 0x1D7E6, + 0x1D7E7, + 0x1D7E8, + 0x1D7E9, + 0x1D7EA, + 0x1D7EB, + 0x1D7EC, + 0x1D7ED, + 0x1D7EE, + 0x1D7EF, + 0x1D7F0, + 0x1D7F1, + 0x1D7F2, + 0x1D7F3, + 0x1D7F4, + 0x1D7F5, + 0x1D7F6, + 0x1D7F7, + 0x1D7F8, + 0x1D7F9, + 0x1D7FA, + 0x1D7FB, + 0x1D7FC, + 0x1D7FD, + 0x1D7FE, + 0x1D7FF, + 0x1D800, + 0x1DA8C, + 0x1DA9B, + 0x1DAA0, + 0x1DAA1, + 0x1DAB0, + 0x1DF00, + 0x1DF1F, + 0x1DF25, + 0x1DF2B, + 0x1E000, + 0x1E007, + 0x1E008, + 0x1E019, + 0x1E01B, + 0x1E022, + 0x1E023, + 0x1E025, + 0x1E026, + 0x1E02B, + 0x1E030, + 0x1E031, + 0x1E032, + 0x1E033, + 0x1E034, + 0x1E035, + 0x1E036, + 0x1E037, + 0x1E038, + 0x1E039, + 0x1E03A, + 0x1E03B, + 0x1E03C, + 0x1E03D, + 0x1E03E, + 0x1E03F, + 0x1E040, + 0x1E041, + 0x1E042, + 0x1E043, + 0x1E044, + 0x1E045, + 0x1E046, + 0x1E047, + 0x1E048, + 0x1E049, + 0x1E04A, + 0x1E04B, + 0x1E04C, + 0x1E04D, + 0x1E04E, + 0x1E04F, + 0x1E050, + 0x1E051, + 0x1E052, + 0x1E053, + 0x1E054, + 0x1E055, + 0x1E056, + 0x1E057, + 0x1E058, + 0x1E059, + 0x1E05A, + 0x1E05B, + 0x1E05C, + 0x1E05D, + 0x1E05E, + 0x1E05F, + 0x1E060, + 0x1E061, + 0x1E062, + 0x1E063, + 0x1E064, + 0x1E065, + 0x1E066, + 0x1E067, + 0x1E068, + 0x1E069, + 0x1E06A, + 0x1E06B, + 0x1E06C, + 0x1E06D, + 0x1E06E, + 0x1E08F, + 0x1E090, + 0x1E100, + 0x1E12D, + 0x1E130, + 0x1E13E, + 0x1E140, + 0x1E14A, + 0x1E14E, + 0x1E150, + 0x1E290, + 0x1E2AF, + 0x1E2C0, + 0x1E2FA, + 0x1E2FF, + 0x1E300, + 0x1E4D0, + 0x1E4FA, + 0x1E5D0, + 0x1E5FB, + 0x1E5FF, + 0x1E600, + 0x1E6C0, + 0x1E6DF, + 0x1E6E0, + 0x1E6F6, + 0x1E6FE, + 0x1E700, + 0x1E7E0, + 0x1E7E7, + 0x1E7E8, + 0x1E7EC, + 0x1E7ED, + 0x1E7EF, + 0x1E7F0, + 0x1E7FF, + 0x1E800, + 0x1E8C5, + 0x1E8C7, + 0x1E8D7, + 0x1E900, + 0x1E901, + 0x1E902, + 0x1E903, + 0x1E904, + 0x1E905, + 0x1E906, + 0x1E907, + 0x1E908, + 0x1E909, + 0x1E90A, + 0x1E90B, + 0x1E90C, + 0x1E90D, + 0x1E90E, + 0x1E90F, + 0x1E910, + 0x1E911, + 0x1E912, + 0x1E913, + 0x1E914, + 0x1E915, + 0x1E916, + 0x1E917, + 0x1E918, + 0x1E919, + 0x1E91A, + 0x1E91B, + 0x1E91C, + 0x1E91D, + 0x1E91E, + 0x1E91F, + 0x1E920, + 0x1E921, + 0x1E922, + 0x1E94C, + 0x1E950, + 0x1E95A, + 0x1E95E, + 0x1E960, + 0x1EC71, + 0x1ECB5, + 0x1ED01, + 0x1ED3E, + 0x1EE00, + 0x1EE01, + 0x1EE02, + 0x1EE03, + 0x1EE04, + 0x1EE05, + 0x1EE06, + 0x1EE07, + 0x1EE08, + 0x1EE09, + 0x1EE0A, + 0x1EE0B, + 0x1EE0C, + 0x1EE0D, + 0x1EE0E, + 0x1EE0F, + 0x1EE10, + 0x1EE11, + 0x1EE12, + 0x1EE13, + 0x1EE14, + 0x1EE15, + 0x1EE16, + 0x1EE17, + 0x1EE18, + 0x1EE19, + 0x1EE1A, + 0x1EE1B, + 0x1EE1C, + 0x1EE1D, + 0x1EE1E, + 0x1EE1F, + 0x1EE20, + 0x1EE21, + 0x1EE22, + 0x1EE23, + 0x1EE24, + 0x1EE25, + 0x1EE27, + 0x1EE28, + 0x1EE29, + 0x1EE2A, + 0x1EE2B, + 0x1EE2C, + 0x1EE2D, + 0x1EE2E, + 0x1EE2F, + 0x1EE30, + 0x1EE31, + 0x1EE32, + 0x1EE33, + 0x1EE34, + 0x1EE35, + 0x1EE36, + 0x1EE37, + 0x1EE38, + 0x1EE39, + 0x1EE3A, + 0x1EE3B, + 0x1EE3C, + 0x1EE42, + 0x1EE43, + 0x1EE47, + 0x1EE48, + 0x1EE49, + 0x1EE4A, + 0x1EE4B, + 0x1EE4C, + 0x1EE4D, + 0x1EE4E, + 0x1EE4F, + 0x1EE50, + 0x1EE51, + 0x1EE52, + 0x1EE53, + 0x1EE54, + 0x1EE55, + 0x1EE57, + 0x1EE58, + 0x1EE59, + 0x1EE5A, + 0x1EE5B, + 0x1EE5C, + 0x1EE5D, + 0x1EE5E, + 0x1EE5F, + 0x1EE60, + 0x1EE61, + 0x1EE62, + 0x1EE63, + 0x1EE64, + 0x1EE65, + 0x1EE67, + 0x1EE68, + 0x1EE69, + 0x1EE6A, + 0x1EE6B, + 0x1EE6C, + 0x1EE6D, + 0x1EE6E, + 0x1EE6F, + 0x1EE70, + 0x1EE71, + 0x1EE72, + 0x1EE73, + 0x1EE74, + 0x1EE75, + 0x1EE76, + 0x1EE77, + 0x1EE78, + 0x1EE79, + 0x1EE7A, + 0x1EE7B, + 0x1EE7C, + 0x1EE7D, + 0x1EE7E, + 0x1EE7F, + 0x1EE80, + 0x1EE81, + 0x1EE82, + 0x1EE83, + 0x1EE84, + 0x1EE85, + 0x1EE86, + 0x1EE87, + 0x1EE88, + 0x1EE89, + 0x1EE8A, + 0x1EE8B, + 0x1EE8C, + 0x1EE8D, + 0x1EE8E, + 0x1EE8F, + 0x1EE90, + 0x1EE91, + 0x1EE92, + 0x1EE93, + 0x1EE94, + 0x1EE95, + 0x1EE96, + 0x1EE97, + 0x1EE98, + 0x1EE99, + 0x1EE9A, + 0x1EE9B, + 0x1EE9C, + 0x1EEA1, + 0x1EEA2, + 0x1EEA3, + 0x1EEA4, + 0x1EEA5, + 0x1EEA6, + 0x1EEA7, + 0x1EEA8, + 0x1EEA9, + 0x1EEAA, + 0x1EEAB, + 0x1EEAC, + 0x1EEAD, + 0x1EEAE, + 0x1EEAF, + 0x1EEB0, + 0x1EEB1, + 0x1EEB2, + 0x1EEB3, + 0x1EEB4, + 0x1EEB5, + 0x1EEB6, + 0x1EEB7, + 0x1EEB8, + 0x1EEB9, + 0x1EEBA, + 0x1EEBB, + 0x1EEBC, + 0x1EEF0, + 0x1EEF2, + 0x1F000, + 0x1F02C, + 0x1F030, + 0x1F094, + 0x1F0A0, + 0x1F0AF, + 0x1F0B1, + 0x1F0C0, + 0x1F0C1, + 0x1F0D0, + 0x1F0D1, + 0x1F0F6, + 0x1F101, + 0x1F102, + 0x1F103, + 0x1F104, + 0x1F105, + 0x1F106, + 0x1F107, + 0x1F108, + 0x1F109, + 0x1F10A, + 0x1F10B, + 0x1F110, + 0x1F111, + 0x1F112, + 0x1F113, + 0x1F114, + 0x1F115, + 0x1F116, + 0x1F117, + 0x1F118, + 0x1F119, + 0x1F11A, + 0x1F11B, + 0x1F11C, + 0x1F11D, + 0x1F11E, + 0x1F11F, + 0x1F120, + 0x1F121, + 0x1F122, + 0x1F123, + 0x1F124, + 0x1F125, + 0x1F126, + 0x1F127, + 0x1F128, + 0x1F129, + 0x1F12A, + 0x1F12B, + 0x1F12C, + 0x1F12D, + 0x1F12E, + 0x1F12F, + 0x1F130, + 0x1F131, + 0x1F132, + 0x1F133, + 0x1F134, + 0x1F135, + 0x1F136, + 0x1F137, + 0x1F138, + 0x1F139, + 0x1F13A, + 0x1F13B, + 0x1F13C, + 0x1F13D, + 0x1F13E, + 0x1F13F, + 0x1F140, + 0x1F141, + 0x1F142, + 0x1F143, + 0x1F144, + 0x1F145, + 0x1F146, + 0x1F147, + 0x1F148, + 0x1F149, + 0x1F14A, + 0x1F14B, + 0x1F14C, + 0x1F14D, + 0x1F14E, + 0x1F14F, + 0x1F150, + 0x1F16A, + 0x1F16B, + 0x1F16C, + 0x1F16D, + 0x1F190, + 0x1F191, + 0x1F1AE, + 0x1F1E6, + 0x1F200, + 0x1F201, + 0x1F202, + 0x1F203, + 0x1F210, + 0x1F211, + 0x1F212, + 0x1F213, + 0x1F214, + 0x1F215, + 0x1F216, + 0x1F217, + 0x1F218, + 0x1F219, + 0x1F21A, + 0x1F21B, + 0x1F21C, + 0x1F21D, + 0x1F21E, + 0x1F21F, + 0x1F220, + 0x1F221, + 0x1F222, + 0x1F223, + 0x1F224, + 0x1F225, + 0x1F226, + 0x1F227, + 0x1F228, + 0x1F229, + 0x1F22A, + 0x1F22B, + 0x1F22C, + 0x1F22D, + 0x1F22E, + 0x1F22F, + 0x1F230, + 0x1F231, + 0x1F232, + 0x1F233, + 0x1F234, + 0x1F235, + 0x1F236, + 0x1F237, + 0x1F238, + 0x1F239, + 0x1F23A, + 0x1F23B, + 0x1F23C, + 0x1F240, + 0x1F241, + 0x1F242, + 0x1F243, + 0x1F244, + 0x1F245, + 0x1F246, + 0x1F247, + 0x1F248, + 0x1F249, + 0x1F250, + 0x1F251, + 0x1F252, + 0x1F260, + 0x1F266, + 0x1F300, + 0x1F6D9, + 0x1F6DC, + 0x1F6ED, + 0x1F6F0, + 0x1F6FD, + 0x1F700, + 0x1F7DA, + 0x1F7E0, + 0x1F7EC, + 0x1F7F0, + 0x1F7F1, + 0x1F800, + 0x1F80C, + 0x1F810, + 0x1F848, + 0x1F850, + 0x1F85A, + 0x1F860, + 0x1F888, + 0x1F890, + 0x1F8AE, + 0x1F8B0, + 0x1F8BC, + 0x1F8C0, + 0x1F8C2, + 0x1F8D0, + 0x1F8D9, + 0x1F900, + 0x1FA58, + 0x1FA60, + 0x1FA6E, + 0x1FA70, + 0x1FA7D, + 0x1FA80, + 0x1FA8B, + 0x1FA8E, + 0x1FAC7, + 0x1FAC8, + 0x1FAC9, + 0x1FACD, + 0x1FADD, + 0x1FADF, + 0x1FAEB, + 0x1FAEF, + 0x1FAF9, + 0x1FB00, + 0x1FB93, + 0x1FB94, + 0x1FBF0, + 0x1FBF1, + 0x1FBF2, + 0x1FBF3, + 0x1FBF4, + 0x1FBF5, + 0x1FBF6, + 0x1FBF7, + 0x1FBF8, + 0x1FBF9, + 0x1FBFA, + 0x1FBFB, + 0x20000, + 0x2A6E0, + 0x2A700, + 0x2B81E, + 0x2B820, + 0x2CEAE, + 0x2CEB0, + 0x2EBE1, + 0x2EBF0, + 0x2EE5E, + 0x2F800, + 0x2F801, + 0x2F802, + 0x2F803, + 0x2F804, + 0x2F805, + 0x2F806, + 0x2F807, + 0x2F808, + 0x2F809, + 0x2F80A, + 0x2F80B, + 0x2F80C, + 0x2F80D, + 0x2F80E, + 0x2F80F, + 0x2F810, + 0x2F811, + 0x2F812, + 0x2F813, + 0x2F814, + 0x2F815, + 0x2F816, + 0x2F817, + 0x2F818, + 0x2F819, + 0x2F81A, + 0x2F81B, + 0x2F81C, + 0x2F81D, + 0x2F81E, + 0x2F81F, + 0x2F820, + 0x2F821, + 0x2F822, + 0x2F823, + 0x2F824, + 0x2F825, + 0x2F826, + 0x2F827, + 0x2F828, + 0x2F829, + 0x2F82A, + 0x2F82B, + 0x2F82C, + 0x2F82D, + 0x2F82E, + 0x2F82F, + 0x2F830, + 0x2F831, + 0x2F834, + 0x2F835, + 0x2F836, + 0x2F837, + 0x2F838, + 0x2F839, + 0x2F83A, + 0x2F83B, + 0x2F83C, + 0x2F83D, + 0x2F83E, + 0x2F83F, + 0x2F840, + 0x2F841, + 0x2F842, + 0x2F843, + 0x2F844, + 0x2F845, + 0x2F847, + 0x2F848, + 0x2F849, + 0x2F84A, + 0x2F84B, + 0x2F84C, + 0x2F84D, + 0x2F84E, + 0x2F84F, + 0x2F850, + 0x2F851, + 0x2F852, + 0x2F853, + 0x2F854, + 0x2F855, + 0x2F856, + 0x2F857, + 0x2F858, + 0x2F859, + 0x2F85A, + 0x2F85B, + 0x2F85C, + 0x2F85D, + 0x2F85E, + 0x2F85F, + 0x2F860, + 0x2F861, + 0x2F862, + 0x2F863, + 0x2F864, + 0x2F865, + 0x2F866, + 0x2F867, + 0x2F868, + 0x2F869, + 0x2F86A, + 0x2F86C, + 0x2F86D, + 0x2F86E, + 0x2F86F, + 0x2F870, + 0x2F871, + 0x2F872, + 0x2F873, + 0x2F874, + 0x2F875, + 0x2F876, + 0x2F877, + 0x2F878, + 0x2F879, + 0x2F87A, + 0x2F87B, + 0x2F87C, + 0x2F87D, + 0x2F87E, + 0x2F87F, + 0x2F880, + 0x2F881, + 0x2F882, + 0x2F883, + 0x2F884, + 0x2F885, + 0x2F886, + 0x2F887, + 0x2F888, + 0x2F889, + 0x2F88A, + 0x2F88B, + 0x2F88C, + 0x2F88D, + 0x2F88E, + 0x2F88F, + 0x2F890, + 0x2F891, + 0x2F893, + 0x2F894, + 0x2F896, + 0x2F897, + 0x2F898, + 0x2F899, + 0x2F89A, + 0x2F89B, + 0x2F89C, + 0x2F89D, + 0x2F89E, + 0x2F89F, + 0x2F8A0, + 0x2F8A1, + 0x2F8A2, + 0x2F8A3, + 0x2F8A4, + 0x2F8A5, + 0x2F8A6, + 0x2F8A7, + 0x2F8A8, + 0x2F8A9, + 0x2F8AA, + 0x2F8AB, + 0x2F8AC, + 0x2F8AD, + 0x2F8AE, + 0x2F8AF, + 0x2F8B0, + 0x2F8B1, + 0x2F8B2, + 0x2F8B3, + 0x2F8B4, + 0x2F8B5, + 0x2F8B6, + 0x2F8B7, + 0x2F8B8, + 0x2F8B9, + 0x2F8BA, + 0x2F8BB, + 0x2F8BC, + 0x2F8BD, + 0x2F8BE, + 0x2F8BF, + 0x2F8C0, + 0x2F8C1, + 0x2F8C2, + 0x2F8C3, + 0x2F8C4, + 0x2F8C5, + 0x2F8C6, + 0x2F8C7, + 0x2F8C8, + 0x2F8C9, + 0x2F8CA, + 0x2F8CB, + 0x2F8CC, + 0x2F8CD, + 0x2F8CE, + 0x2F8CF, + 0x2F8D0, + 0x2F8D1, + 0x2F8D2, + 0x2F8D3, + 0x2F8D4, + 0x2F8D5, + 0x2F8D6, + 0x2F8D7, + 0x2F8D8, + 0x2F8D9, + 0x2F8DA, + 0x2F8DB, + 0x2F8DC, + 0x2F8DD, + 0x2F8DE, + 0x2F8DF, + 0x2F8E0, + 0x2F8E1, + 0x2F8E2, + 0x2F8E3, + 0x2F8E4, + 0x2F8E5, + 0x2F8E6, + 0x2F8E7, + 0x2F8E8, + 0x2F8E9, + 0x2F8EA, + 0x2F8EB, + 0x2F8EC, + 0x2F8ED, + 0x2F8EE, + 0x2F8EF, + 0x2F8F0, + 0x2F8F1, + 0x2F8F2, + 0x2F8F3, + 0x2F8F4, + 0x2F8F5, + 0x2F8F6, + 0x2F8F7, + 0x2F8F8, + 0x2F8F9, + 0x2F8FA, + 0x2F8FB, + 0x2F8FC, + 0x2F8FD, + 0x2F8FE, + 0x2F8FF, + 0x2F900, + 0x2F901, + 0x2F902, + 0x2F903, + 0x2F904, + 0x2F905, + 0x2F906, + 0x2F907, + 0x2F908, + 0x2F909, + 0x2F90A, + 0x2F90B, + 0x2F90C, + 0x2F90D, + 0x2F90E, + 0x2F90F, + 0x2F910, + 0x2F911, + 0x2F912, + 0x2F913, + 0x2F914, + 0x2F915, + 0x2F916, + 0x2F917, + 0x2F918, + 0x2F919, + 0x2F91A, + 0x2F91B, + 0x2F91C, + 0x2F91D, + 0x2F91E, + 0x2F91F, + 0x2F920, + 0x2F921, + 0x2F922, + 0x2F923, + 0x2F924, + 0x2F925, + 0x2F926, + 0x2F927, + 0x2F928, + 0x2F929, + 0x2F92A, + 0x2F92B, + 0x2F92C, + 0x2F92E, + 0x2F92F, + 0x2F930, + 0x2F931, + 0x2F932, + 0x2F933, + 0x2F934, + 0x2F935, + 0x2F936, + 0x2F937, + 0x2F938, + 0x2F939, + 0x2F93A, + 0x2F93B, + 0x2F93C, + 0x2F93D, + 0x2F93E, + 0x2F93F, + 0x2F940, + 0x2F941, + 0x2F942, + 0x2F943, + 0x2F944, + 0x2F945, + 0x2F946, + 0x2F948, + 0x2F949, + 0x2F94A, + 0x2F94B, + 0x2F94C, + 0x2F94D, + 0x2F94E, + 0x2F94F, + 0x2F950, + 0x2F951, + 0x2F952, + 0x2F953, + 0x2F954, + 0x2F955, + 0x2F956, + 0x2F957, + 0x2F958, + 0x2F959, + 0x2F95A, + 0x2F95B, + 0x2F95C, + 0x2F95D, + 0x2F95F, + 0x2F960, + 0x2F961, + 0x2F962, + 0x2F963, + 0x2F964, + 0x2F965, + 0x2F966, + 0x2F967, + 0x2F968, + 0x2F969, + 0x2F96A, + 0x2F96B, + 0x2F96C, + 0x2F96D, + 0x2F96E, + 0x2F96F, + 0x2F970, + 0x2F971, + 0x2F972, + 0x2F973, + 0x2F974, + 0x2F975, + 0x2F976, + 0x2F977, + 0x2F978, + 0x2F979, + 0x2F97A, + 0x2F97B, + 0x2F97C, + 0x2F97D, + 0x2F97E, + 0x2F97F, + 0x2F980, + 0x2F981, + 0x2F982, + 0x2F983, + 0x2F984, + 0x2F985, + 0x2F986, + 0x2F987, + 0x2F988, + 0x2F989, + 0x2F98A, + 0x2F98B, + 0x2F98C, + 0x2F98D, + 0x2F98E, + 0x2F98F, + 0x2F990, + 0x2F991, + 0x2F992, + 0x2F993, + 0x2F994, + 0x2F995, + 0x2F996, + 0x2F997, + 0x2F998, + 0x2F999, + 0x2F99A, + 0x2F99B, + 0x2F99C, + 0x2F99D, + 0x2F99E, + 0x2F99F, + 0x2F9A0, + 0x2F9A1, + 0x2F9A2, + 0x2F9A3, + 0x2F9A4, + 0x2F9A5, + 0x2F9A6, + 0x2F9A7, + 0x2F9A8, + 0x2F9A9, + 0x2F9AA, + 0x2F9AB, + 0x2F9AC, + 0x2F9AD, + 0x2F9AE, + 0x2F9AF, + 0x2F9B0, + 0x2F9B1, + 0x2F9B2, + 0x2F9B3, + 0x2F9B4, + 0x2F9B5, + 0x2F9B6, + 0x2F9B7, + 0x2F9B8, + 0x2F9B9, + 0x2F9BA, + 0x2F9BB, + 0x2F9BC, + 0x2F9BD, + 0x2F9BE, + 0x2F9BF, + 0x2F9C0, + 0x2F9C1, + 0x2F9C2, + 0x2F9C3, + 0x2F9C4, + 0x2F9C5, + 0x2F9C6, + 0x2F9C7, + 0x2F9C8, + 0x2F9C9, + 0x2F9CA, + 0x2F9CB, + 0x2F9CC, + 0x2F9CD, + 0x2F9CE, + 0x2F9CF, + 0x2F9D0, + 0x2F9D1, + 0x2F9D2, + 0x2F9D3, + 0x2F9D4, + 0x2F9D5, + 0x2F9D6, + 0x2F9D7, + 0x2F9D8, + 0x2F9D9, + 0x2F9DA, + 0x2F9DB, + 0x2F9DC, + 0x2F9DD, + 0x2F9DE, + 0x2F9DF, + 0x2F9E0, + 0x2F9E1, + 0x2F9E2, + 0x2F9E3, + 0x2F9E4, + 0x2F9E5, + 0x2F9E6, + 0x2F9E7, + 0x2F9E8, + 0x2F9E9, + 0x2F9EA, + 0x2F9EB, + 0x2F9EC, + 0x2F9ED, + 0x2F9EE, + 0x2F9EF, + 0x2F9F0, + 0x2F9F1, + 0x2F9F2, + 0x2F9F3, + 0x2F9F4, + 0x2F9F5, + 0x2F9F6, + 0x2F9F7, + 0x2F9F8, + 0x2F9F9, + 0x2F9FA, + 0x2F9FB, + 0x2F9FC, + 0x2F9FD, + 0x2F9FE, + 0x2FA00, + 0x2FA01, + 0x2FA02, + 0x2FA03, + 0x2FA04, + 0x2FA05, + 0x2FA06, + 0x2FA07, + 0x2FA08, + 0x2FA09, + 0x2FA0A, + 0x2FA0B, + 0x2FA0C, + 0x2FA0D, + 0x2FA0E, + 0x2FA0F, + 0x2FA10, + 0x2FA11, + 0x2FA12, + 0x2FA13, + 0x2FA14, + 0x2FA15, + 0x2FA16, + 0x2FA17, + 0x2FA18, + 0x2FA19, + 0x2FA1A, + 0x2FA1B, + 0x2FA1C, + 0x2FA1D, + 0x2FA1E, + 0x30000, + 0x3134B, + 0x31350, + 0x3347A, + 0xE0100, + 0xE01F0, + ), +) + +uts46_statuses: bytes = ( + b"VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" + b"VMMMMMMMMMMMMMMMMMMMMMMMMMMVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" + b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXMVVVVVVVMVMVVIVMVVMMMMVVMMMVMMMV" + b"MMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMDVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVMVMVMVMMV" + b"MVMVMVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVMVMVMM" + b"VMVMMVMMMVMMMMVMMVMMMVMMVMMVMVMVMMVMVMVMMVMMMVMVMMVMVMMMMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMMVMMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMMVMVMMMMVMVMVMVMVMMMMMMMMMVMMMMMM" + b"VMMMMMVMMVMMMVIVMVMVMVMVXMVMMXMMMMMMMXMXMMVMMMMMMMMMMMMMMMMMXMMM" + b"MMMMMMVDVMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMVMVMMMVMMVMVMMVMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVXMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMXVMVXVXVXVXVXVXVMMMMVXVXVXVXVXVXVXVXVXVX" + b"VXVXVMMMMMMMMVXVXVXVXVXVXVXVXVXVXVXMMXMVXVXVXVXVXVXVXVMXVMXVXVXV" + b"XVXVXVXMMMVXMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" + b"XVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" + b"XVMVXVXVXVXVXVXVXVMVXVXVXVXVXMMVXVMVMVXVMVMVMVMVMVXVMVMMMMMVMVMV" + b"XVMVMVMVMVMVXVXVXVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMXMXVMV" + b"IVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMMMMMMXVXVXVXVXVXVXVXVXVX" + b"VIVXVXVXVIVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVMMM" + b"MMMMMMVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMVXVXVMMMV" + b"MMMMMMMMMMMVMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMMMMMMMVXMMMMMMXVMMMMMMMMVMMMMMM" + b"MMVXMMMMMMXVXMXMXMXMVMMMMMMMMVMVMVMVMVMVMVMXMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMXVMMMMMMMMMMMMMMXVMMMMMMMMMVMXVM" + b"MMMXMMMVMVMMMMMMMMXMMMXVMMMMMMMMXMIDXVMVMVXVXMVMMVMMVMVMVMMMVMVM" + b"IXIMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMXVMVXVXMMMMVMMMVM" + b"MMMMMVMMVMMMVMMMVMVMVMVMMMMVMMMMMMMMMMVMMMMMVMMMMVMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVXVMMVMMVMMVXVXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMMMVMVXVMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMMMVMVMVMVMMMMVMVMVMM" + b"MMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVXVXVXVXVXMVXVXVXVXVX" + b"VXVXVXVXVXVXVXVMVMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMVMVMVMMMVXVXVMMVMVMXV" + b"XMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMIMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVMMMMMMMMMMMMMMVXVMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMMMMMMMMM" + b"MMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXMVMVMVMVMVMVMVM" + b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVXV" + b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" + b"MVMVMVMVMVMVMVMVMVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMMMMVM" + b"MMMMVMVMVMVMVMVMVMVMMMMVMVMMVMVMVMVMVMVMVMVMXMMMMMVMMVXVXVXVXVXV" + b"XVXVXVXVXVXVXVXVXVXVXVXVXVXVXVMMMMVMVXMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVX" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMVMVMVMMMMMMMMMMVMVMVMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMXMMMMMMXMMMMMXMVMMMMMMMMMMMMMMMMMMMMMMMMXMMMMM" + b"XMXMMXMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXMMMMMMMMMMMMMVIMMXMMM" + b"MMMXVXMMMMMMMMMMMMMMMMMMMVMMMMMMXMMMMMMMMMMMMMMMMMMMXMMMMXMMMVMX" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXIXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMIMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMXMMMMM" + b"MXMMMMMMXMMMXMMMMMMMXMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" + b"XVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVMMMMMMMMMMMXMMMMMMMMMMMMMMMXMMM" + b"MMMMXMMXVXVXVXVXVXVXVXVXVMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMXMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMXVXVXVXVMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMVXMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVIXVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVX" + b"VXVXVXVXVXVXVMMMMMMMVIVMMMMMMVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMXMMXMXMMXMMMMXMMMMMMMMMMMMXMXMMMMMMMXMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMMMMXMMMMM" + b"MMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMXMXMMMMMMMXMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVX" + b"VXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVX" + b"VXMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMXMXMXMMMMMMMMMMXMMMMXMXMXMX" + b"MXMXMXMMMXMMXMXMXMXMXMXMXMMXMXMMMMXMMMMMMMXMMMMXMMMMXMXMMMMMMMMM" + b"MXMMMMMMMMMMMMMMMMMXMMMXMMMMMXMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXMM" + b"MMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMVMMMVMVXVMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMXMMMMMMMMMXMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" + b"VXVMMMMMMMMMMVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" + b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXIX" +) + +uts46_replacements: tuple[Optional[str], ...] = ( + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + " ", + None, + None, + None, + None, + None, + None, + None, + " ̈", + None, + "a", + None, + None, + None, + None, + " ̄", + None, + None, + "2", + "3", + " ́", + "μ", + None, + None, + " ̧", + "1", + "o", + None, + "1⁄4", + "1⁄2", + "3⁄4", + None, + "à", + "á", + "â", + "ã", + "ä", + "å", + "æ", + "ç", + "è", + "é", + "ê", + "ë", + "ì", + "í", + "î", + "ï", + "ð", + "ñ", + "ò", + "ó", + "ô", + "õ", + "ö", + None, + "ø", + "ù", + "ú", + "û", + "ü", + "ý", + "þ", + "ss", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ā", + None, + "ă", + None, + "ą", + None, + "ć", + None, + "ĉ", + None, + "ċ", + None, + "č", + None, + "ď", + None, + "đ", + None, + "ē", + None, + "ĕ", + None, + "ė", + None, + "ę", + None, + "ě", + None, + "ĝ", + None, + "ğ", + None, + "ġ", + None, + "ģ", + None, + "ĥ", + None, + "ħ", + None, + "ĩ", + None, + "ī", + None, + "ĭ", + None, + "į", + None, + "i̇", + None, + "ij", + "ĵ", + None, + "ķ", + None, + "ĺ", + None, + "ļ", + None, + "ľ", + None, + "l·", + "ł", + None, + "ń", + None, + "ņ", + None, + "ň", + None, + "ʼn", + "ŋ", + None, + "ō", + None, + "ŏ", + None, + "ő", + None, + "œ", + None, + "ŕ", + None, + "ŗ", + None, + "ř", + None, + "ś", + None, + "ŝ", + None, + "ş", + None, + "š", + None, + "ţ", + None, + "ť", + None, + "ŧ", + None, + "ũ", + None, + "ū", + None, + "ŭ", + None, + "ů", + None, + "ű", + None, + "ų", + None, + "ŵ", + None, + "ŷ", + None, + "ÿ", + "ź", + None, + "ż", + None, + "ž", + None, + "s", + None, + "ɓ", + "ƃ", + None, + "ƅ", + None, + "ɔ", + "ƈ", + None, + "ɖ", + "ɗ", + "ƌ", + None, + "ǝ", + "ə", + "ɛ", + "ƒ", + None, + "ɠ", + "ɣ", + None, + "ɩ", + "ɨ", + "ƙ", + None, + "ɯ", + "ɲ", + None, + "ɵ", + "ơ", + None, + "ƣ", + None, + "ƥ", + None, + "ʀ", + "ƨ", + None, + "ʃ", + None, + "ƭ", + None, + "ʈ", + "ư", + None, + "ʊ", + "ʋ", + "ƴ", + None, + "ƶ", + None, + "ʒ", + "ƹ", + None, + "ƽ", + None, + "dž", + "lj", + "nj", + "ǎ", + None, + "ǐ", + None, + "ǒ", + None, + "ǔ", + None, + "ǖ", + None, + "ǘ", + None, + "ǚ", + None, + "ǜ", + None, + "ǟ", + None, + "ǡ", + None, + "ǣ", + None, + "ǥ", + None, + "ǧ", + None, + "ǩ", + None, + "ǫ", + None, + "ǭ", + None, + "ǯ", + None, + "dz", + "ǵ", + None, + "ƕ", + "ƿ", + "ǹ", + None, + "ǻ", + None, + "ǽ", + None, + "ǿ", + None, + "ȁ", + None, + "ȃ", + None, + "ȅ", + None, + "ȇ", + None, + "ȉ", + None, + "ȋ", + None, + "ȍ", + None, + "ȏ", + None, + "ȑ", + None, + "ȓ", + None, + "ȕ", + None, + "ȗ", + None, + "ș", + None, + "ț", + None, + "ȝ", + None, + "ȟ", + None, + "ƞ", + None, + "ȣ", + None, + "ȥ", + None, + "ȧ", + None, + "ȩ", + None, + "ȫ", + None, + "ȭ", + None, + "ȯ", + None, + "ȱ", + None, + "ȳ", + None, + "ⱥ", + "ȼ", + None, + "ƚ", + "ⱦ", + None, + "ɂ", + None, + "ƀ", + "ʉ", + "ʌ", + "ɇ", + None, + "ɉ", + None, + "ɋ", + None, + "ɍ", + None, + "ɏ", + None, + "h", + "ɦ", + "j", + "r", + "ɹ", + "ɻ", + "ʁ", + "w", + "y", + None, + " ̆", + " ̇", + " ̊", + " ̨", + " ̃", + " ̋", + None, + "ɣ", + "l", + "s", + "x", + "ʕ", + None, + "̀", + "́", + None, + "̓", + "̈́", + "ι", + None, + None, + None, + "ͱ", + None, + "ͳ", + None, + "ʹ", + None, + "ͷ", + None, + None, + " ι", + None, + ";", + "ϳ", + None, + " ́", + " ̈́", + "ά", + "·", + "έ", + "ή", + "ί", + None, + "ό", + None, + "ύ", + "ώ", + None, + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + None, + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "ϊ", + "ϋ", + None, + "σ", + None, + "ϗ", + "β", + "θ", + "υ", + "ύ", + "ϋ", + "φ", + "π", + None, + "ϙ", + None, + "ϛ", + None, + "ϝ", + None, + "ϟ", + None, + "ϡ", + None, + "ϣ", + None, + "ϥ", + None, + "ϧ", + None, + "ϩ", + None, + "ϫ", + None, + "ϭ", + None, + "ϯ", + None, + "κ", + "ρ", + "σ", + None, + "θ", + "ε", + None, + "ϸ", + None, + "σ", + "ϻ", + None, + "ͻ", + "ͼ", + "ͽ", + "ѐ", + "ё", + "ђ", + "ѓ", + "є", + "ѕ", + "і", + "ї", + "ј", + "љ", + "њ", + "ћ", + "ќ", + "ѝ", + "ў", + "џ", + "а", + "б", + "в", + "г", + "д", + "е", + "ж", + "з", + "и", + "й", + "к", + "л", + "м", + "н", + "о", + "п", + "р", + "с", + "т", + "у", + "ф", + "х", + "ц", + "ч", + "ш", + "щ", + "ъ", + "ы", + "ь", + "э", + "ю", + "я", + None, + "ѡ", + None, + "ѣ", + None, + "ѥ", + None, + "ѧ", + None, + "ѩ", + None, + "ѫ", + None, + "ѭ", + None, + "ѯ", + None, + "ѱ", + None, + "ѳ", + None, + "ѵ", + None, + "ѷ", + None, + "ѹ", + None, + "ѻ", + None, + "ѽ", + None, + "ѿ", + None, + "ҁ", + None, + "ҋ", + None, + "ҍ", + None, + "ҏ", + None, + "ґ", + None, + "ғ", + None, + "ҕ", + None, + "җ", + None, + "ҙ", + None, + "қ", + None, + "ҝ", + None, + "ҟ", + None, + "ҡ", + None, + "ң", + None, + "ҥ", + None, + "ҧ", + None, + "ҩ", + None, + "ҫ", + None, + "ҭ", + None, + "ү", + None, + "ұ", + None, + "ҳ", + None, + "ҵ", + None, + "ҷ", + None, + "ҹ", + None, + "һ", + None, + "ҽ", + None, + "ҿ", + None, + "ӏ", + "ӂ", + None, + "ӄ", + None, + "ӆ", + None, + "ӈ", + None, + "ӊ", + None, + "ӌ", + None, + "ӎ", + None, + "ӑ", + None, + "ӓ", + None, + "ӕ", + None, + "ӗ", + None, + "ә", + None, + "ӛ", + None, + "ӝ", + None, + "ӟ", + None, + "ӡ", + None, + "ӣ", + None, + "ӥ", + None, + "ӧ", + None, + "ө", + None, + "ӫ", + None, + "ӭ", + None, + "ӯ", + None, + "ӱ", + None, + "ӳ", + None, + "ӵ", + None, + "ӷ", + None, + "ӹ", + None, + "ӻ", + None, + "ӽ", + None, + "ӿ", + None, + "ԁ", + None, + "ԃ", + None, + "ԅ", + None, + "ԇ", + None, + "ԉ", + None, + "ԋ", + None, + "ԍ", + None, + "ԏ", + None, + "ԑ", + None, + "ԓ", + None, + "ԕ", + None, + "ԗ", + None, + "ԙ", + None, + "ԛ", + None, + "ԝ", + None, + "ԟ", + None, + "ԡ", + None, + "ԣ", + None, + "ԥ", + None, + "ԧ", + None, + "ԩ", + None, + "ԫ", + None, + "ԭ", + None, + "ԯ", + None, + None, + "ա", + "բ", + "գ", + "դ", + "ե", + "զ", + "է", + "ը", + "թ", + "ժ", + "ի", + "լ", + "խ", + "ծ", + "կ", + "հ", + "ձ", + "ղ", + "ճ", + "մ", + "յ", + "ն", + "շ", + "ո", + "չ", + "պ", + "ջ", + "ռ", + "ս", + "վ", + "տ", + "ր", + "ց", + "ւ", + "փ", + "ք", + "օ", + "ֆ", + None, + None, + "եւ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "اٴ", + "وٴ", + "ۇٴ", + "يٴ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "क़", + "ख़", + "ग़", + "ज़", + "ड़", + "ढ़", + "फ़", + "य़", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ড়", + "ঢ়", + None, + "য়", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ਲ਼", + None, + None, + "ਸ਼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ਖ਼", + "ਗ਼", + "ਜ਼", + None, + None, + "ਫ਼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ଡ଼", + "ଢ଼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ํา", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ໍາ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ຫນ", + "ຫມ", + None, + None, + None, + "་", + None, + "གྷ", + None, + None, + None, + "ཌྷ", + None, + "དྷ", + None, + "བྷ", + None, + "ཛྷ", + None, + "ཀྵ", + None, + None, + None, + "ཱི", + None, + "ཱུ", + "ྲྀ", + "ྲཱྀ", + "ླྀ", + "ླཱྀ", + None, + "ཱྀ", + None, + "ྒྷ", + None, + None, + None, + "ྜྷ", + None, + "ྡྷ", + None, + "ྦྷ", + None, + "ྫྷ", + None, + "ྐྵ", + None, + None, + None, + None, + None, + None, + None, + "ⴀ", + "ⴁ", + "ⴂ", + "ⴃ", + "ⴄ", + "ⴅ", + "ⴆ", + "ⴇ", + "ⴈ", + "ⴉ", + "ⴊ", + "ⴋ", + "ⴌ", + "ⴍ", + "ⴎ", + "ⴏ", + "ⴐ", + "ⴑ", + "ⴒ", + "ⴓ", + "ⴔ", + "ⴕ", + "ⴖ", + "ⴗ", + "ⴘ", + "ⴙ", + "ⴚ", + "ⴛ", + "ⴜ", + "ⴝ", + "ⴞ", + "ⴟ", + "ⴠ", + "ⴡ", + "ⴢ", + "ⴣ", + "ⴤ", + "ⴥ", + None, + "ⴧ", + None, + "ⴭ", + None, + None, + "ნ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "Ᏸ", + "Ᏹ", + "Ᏺ", + "Ᏻ", + "Ᏼ", + "Ᏽ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "в", + "д", + "о", + "с", + "т", + "ъ", + "ѣ", + "ꙋ", + "\u1c8a", + None, + None, + "ა", + "ბ", + "გ", + "დ", + "ე", + "ვ", + "ზ", + "თ", + "ი", + "კ", + "ლ", + "მ", + "ნ", + "ო", + "პ", + "ჟ", + "რ", + "ს", + "ტ", + "უ", + "ფ", + "ქ", + "ღ", + "ყ", + "შ", + "ჩ", + "ც", + "ძ", + "წ", + "ჭ", + "ხ", + "ჯ", + "ჰ", + "ჱ", + "ჲ", + "ჳ", + "ჴ", + "ჵ", + "ჶ", + "ჷ", + "ჸ", + "ჹ", + "ჺ", + None, + "ჽ", + "ჾ", + "ჿ", + None, + None, + None, + None, + None, + "a", + "æ", + "b", + None, + "d", + "e", + "ǝ", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + None, + "o", + "ȣ", + "p", + "r", + "t", + "u", + "w", + "a", + "ɐ", + "ɑ", + "ᴂ", + "b", + "d", + "e", + "ə", + "ɛ", + "ɜ", + "g", + None, + "k", + "m", + "ŋ", + "o", + "ɔ", + "ᴖ", + "ᴗ", + "p", + "t", + "u", + "ᴝ", + "ɯ", + "v", + "ᴥ", + "β", + "γ", + "δ", + "φ", + "χ", + "i", + "r", + "u", + "v", + "β", + "γ", + "ρ", + "φ", + "χ", + None, + "н", + None, + "ɒ", + "c", + "ɕ", + "ð", + "ɜ", + "f", + "ɟ", + "ɡ", + "ɥ", + "ɨ", + "ɩ", + "ɪ", + "ᵻ", + "ʝ", + "ɭ", + "ᶅ", + "ʟ", + "ɱ", + "ɰ", + "ɲ", + "ɳ", + "ɴ", + "ɵ", + "ɸ", + "ʂ", + "ʃ", + "ƫ", + "ʉ", + "ʊ", + "ᴜ", + "ʋ", + "ʌ", + "z", + "ʐ", + "ʑ", + "ʒ", + "θ", + None, + "ḁ", + None, + "ḃ", + None, + "ḅ", + None, + "ḇ", + None, + "ḉ", + None, + "ḋ", + None, + "ḍ", + None, + "ḏ", + None, + "ḑ", + None, + "ḓ", + None, + "ḕ", + None, + "ḗ", + None, + "ḙ", + None, + "ḛ", + None, + "ḝ", + None, + "ḟ", + None, + "ḡ", + None, + "ḣ", + None, + "ḥ", + None, + "ḧ", + None, + "ḩ", + None, + "ḫ", + None, + "ḭ", + None, + "ḯ", + None, + "ḱ", + None, + "ḳ", + None, + "ḵ", + None, + "ḷ", + None, + "ḹ", + None, + "ḻ", + None, + "ḽ", + None, + "ḿ", + None, + "ṁ", + None, + "ṃ", + None, + "ṅ", + None, + "ṇ", + None, + "ṉ", + None, + "ṋ", + None, + "ṍ", + None, + "ṏ", + None, + "ṑ", + None, + "ṓ", + None, + "ṕ", + None, + "ṗ", + None, + "ṙ", + None, + "ṛ", + None, + "ṝ", + None, + "ṟ", + None, + "ṡ", + None, + "ṣ", + None, + "ṥ", + None, + "ṧ", + None, + "ṩ", + None, + "ṫ", + None, + "ṭ", + None, + "ṯ", + None, + "ṱ", + None, + "ṳ", + None, + "ṵ", + None, + "ṷ", + None, + "ṹ", + None, + "ṻ", + None, + "ṽ", + None, + "ṿ", + None, + "ẁ", + None, + "ẃ", + None, + "ẅ", + None, + "ẇ", + None, + "ẉ", + None, + "ẋ", + None, + "ẍ", + None, + "ẏ", + None, + "ẑ", + None, + "ẓ", + None, + "ẕ", + None, + "aʾ", + "ṡ", + None, + "ß", + None, + "ạ", + None, + "ả", + None, + "ấ", + None, + "ầ", + None, + "ẩ", + None, + "ẫ", + None, + "ậ", + None, + "ắ", + None, + "ằ", + None, + "ẳ", + None, + "ẵ", + None, + "ặ", + None, + "ẹ", + None, + "ẻ", + None, + "ẽ", + None, + "ế", + None, + "ề", + None, + "ể", + None, + "ễ", + None, + "ệ", + None, + "ỉ", + None, + "ị", + None, + "ọ", + None, + "ỏ", + None, + "ố", + None, + "ồ", + None, + "ổ", + None, + "ỗ", + None, + "ộ", + None, + "ớ", + None, + "ờ", + None, + "ở", + None, + "ỡ", + None, + "ợ", + None, + "ụ", + None, + "ủ", + None, + "ứ", + None, + "ừ", + None, + "ử", + None, + "ữ", + None, + "ự", + None, + "ỳ", + None, + "ỵ", + None, + "ỷ", + None, + "ỹ", + None, + "ỻ", + None, + "ỽ", + None, + "ỿ", + None, + "ἀ", + "ἁ", + "ἂ", + "ἃ", + "ἄ", + "ἅ", + "ἆ", + "ἇ", + None, + None, + "ἐ", + "ἑ", + "ἒ", + "ἓ", + "ἔ", + "ἕ", + None, + None, + "ἠ", + "ἡ", + "ἢ", + "ἣ", + "ἤ", + "ἥ", + "ἦ", + "ἧ", + None, + "ἰ", + "ἱ", + "ἲ", + "ἳ", + "ἴ", + "ἵ", + "ἶ", + "ἷ", + None, + None, + "ὀ", + "ὁ", + "ὂ", + "ὃ", + "ὄ", + "ὅ", + None, + None, + None, + "ὑ", + None, + "ὓ", + None, + "ὕ", + None, + "ὗ", + None, + "ὠ", + "ὡ", + "ὢ", + "ὣ", + "ὤ", + "ὥ", + "ὦ", + "ὧ", + None, + "ά", + None, + "έ", + None, + "ή", + None, + "ί", + None, + "ό", + None, + "ύ", + None, + "ώ", + None, + "ἀι", + "ἁι", + "ἂι", + "ἃι", + "ἄι", + "ἅι", + "ἆι", + "ἇι", + "ἀι", + "ἁι", + "ἂι", + "ἃι", + "ἄι", + "ἅι", + "ἆι", + "ἇι", + "ἠι", + "ἡι", + "ἢι", + "ἣι", + "ἤι", + "ἥι", + "ἦι", + "ἧι", + "ἠι", + "ἡι", + "ἢι", + "ἣι", + "ἤι", + "ἥι", + "ἦι", + "ἧι", + "ὠι", + "ὡι", + "ὢι", + "ὣι", + "ὤι", + "ὥι", + "ὦι", + "ὧι", + "ὠι", + "ὡι", + "ὢι", + "ὣι", + "ὤι", + "ὥι", + "ὦι", + "ὧι", + None, + "ὰι", + "αι", + "άι", + None, + None, + "ᾶι", + "ᾰ", + "ᾱ", + "ὰ", + "ά", + "αι", + " ̓", + "ι", + " ̓", + " ͂", + " ̈͂", + "ὴι", + "ηι", + "ήι", + None, + None, + "ῆι", + "ὲ", + "έ", + "ὴ", + "ή", + "ηι", + " ̓̀", + " ̓́", + " ̓͂", + None, + "ΐ", + None, + None, + "ῐ", + "ῑ", + "ὶ", + "ί", + None, + " ̔̀", + " ̔́", + " ̔͂", + None, + "ΰ", + None, + "ῠ", + "ῡ", + "ὺ", + "ύ", + "ῥ", + " ̈̀", + " ̈́", + "`", + None, + "ὼι", + "ωι", + "ώι", + None, + None, + "ῶι", + "ὸ", + "ό", + "ὼ", + "ώ", + "ωι", + " ́", + " ̔", + None, + " ", + None, + "", + None, + None, + "‐", + None, + " ̳", + None, + None, + None, + None, + " ", + None, + "′′", + "′′′", + None, + "‵‵", + "‵‵‵", + None, + "!!", + None, + " ̅", + None, + "??", + "?!", + "!?", + None, + "′′′′", + None, + " ", + None, + None, + None, + "0", + "i", + None, + "4", + "5", + "6", + "7", + "8", + "9", + "+", + "−", + "=", + "(", + ")", + "n", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "+", + "−", + "=", + "(", + ")", + None, + "a", + "e", + "o", + "x", + "ə", + "h", + "k", + "l", + "m", + "n", + "p", + "s", + "t", + None, + None, + "rs", + None, + None, + None, + None, + "a/c", + "a/s", + "c", + "°c", + None, + "c/o", + "c/u", + "ɛ", + None, + "°f", + "g", + "h", + "ħ", + "i", + "l", + None, + "n", + "no", + None, + "p", + "q", + "r", + None, + "sm", + "tel", + "tm", + None, + "z", + None, + "ω", + None, + "z", + None, + "k", + "å", + "b", + "c", + None, + "e", + "f", + "ⅎ", + "m", + "o", + "א", + "ב", + "ג", + "ד", + "i", + None, + "fax", + "π", + "γ", + "π", + "∑", + None, + "d", + "e", + "i", + "j", + None, + "1⁄7", + "1⁄9", + "1⁄10", + "1⁄3", + "2⁄3", + "1⁄5", + "2⁄5", + "3⁄5", + "4⁄5", + "1⁄6", + "5⁄6", + "1⁄8", + "3⁄8", + "5⁄8", + "7⁄8", + "1⁄", + "i", + "ii", + "iii", + "iv", + "v", + "vi", + "vii", + "viii", + "ix", + "x", + "xi", + "xii", + "l", + "c", + "d", + "m", + "i", + "ii", + "iii", + "iv", + "v", + "vi", + "vii", + "viii", + "ix", + "x", + "xi", + "xii", + "l", + "c", + "d", + "m", + None, + "ↄ", + None, + "0⁄3", + None, + None, + None, + "∫∫", + "∫∫∫", + None, + "∮∮", + "∮∮∮", + None, + "〈", + "〉", + None, + None, + None, + None, + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "(1)", + "(2)", + "(3)", + "(4)", + "(5)", + "(6)", + "(7)", + "(8)", + "(9)", + "(10)", + "(11)", + "(12)", + "(13)", + "(14)", + "(15)", + "(16)", + "(17)", + "(18)", + "(19)", + "(20)", + None, + "(a)", + "(b)", + "(c)", + "(d)", + "(e)", + "(f)", + "(g)", + "(h)", + "(i)", + "(j)", + "(k)", + "(l)", + "(m)", + "(n)", + "(o)", + "(p)", + "(q)", + "(r)", + "(s)", + "(t)", + "(u)", + "(v)", + "(w)", + "(x)", + "(y)", + "(z)", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "0", + None, + "∫∫∫∫", + None, + "::=", + "==", + "===", + None, + "⫝̸", + None, + None, + None, + "ⰰ", + "ⰱ", + "ⰲ", + "ⰳ", + "ⰴ", + "ⰵ", + "ⰶ", + "ⰷ", + "ⰸ", + "ⰹ", + "ⰺ", + "ⰻ", + "ⰼ", + "ⰽ", + "ⰾ", + "ⰿ", + "ⱀ", + "ⱁ", + "ⱂ", + "ⱃ", + "ⱄ", + "ⱅ", + "ⱆ", + "ⱇ", + "ⱈ", + "ⱉ", + "ⱊ", + "ⱋ", + "ⱌ", + "ⱍ", + "ⱎ", + "ⱏ", + "ⱐ", + "ⱑ", + "ⱒ", + "ⱓ", + "ⱔ", + "ⱕ", + "ⱖ", + "ⱗ", + "ⱘ", + "ⱙ", + "ⱚ", + "ⱛ", + "ⱜ", + "ⱝ", + "ⱞ", + "ⱟ", + None, + "ⱡ", + None, + "ɫ", + "ᵽ", + "ɽ", + None, + "ⱨ", + None, + "ⱪ", + None, + "ⱬ", + None, + "ɑ", + "ɱ", + "ɐ", + "ɒ", + None, + "ⱳ", + None, + "ⱶ", + None, + "j", + "v", + "ȿ", + "ɀ", + "ⲁ", + None, + "ⲃ", + None, + "ⲅ", + None, + "ⲇ", + None, + "ⲉ", + None, + "ⲋ", + None, + "ⲍ", + None, + "ⲏ", + None, + "ⲑ", + None, + "ⲓ", + None, + "ⲕ", + None, + "ⲗ", + None, + "ⲙ", + None, + "ⲛ", + None, + "ⲝ", + None, + "ⲟ", + None, + "ⲡ", + None, + "ⲣ", + None, + "ⲥ", + None, + "ⲧ", + None, + "ⲩ", + None, + "ⲫ", + None, + "ⲭ", + None, + "ⲯ", + None, + "ⲱ", + None, + "ⲳ", + None, + "ⲵ", + None, + "ⲷ", + None, + "ⲹ", + None, + "ⲻ", + None, + "ⲽ", + None, + "ⲿ", + None, + "ⳁ", + None, + "ⳃ", + None, + "ⳅ", + None, + "ⳇ", + None, + "ⳉ", + None, + "ⳋ", + None, + "ⳍ", + None, + "ⳏ", + None, + "ⳑ", + None, + "ⳓ", + None, + "ⳕ", + None, + "ⳗ", + None, + "ⳙ", + None, + "ⳛ", + None, + "ⳝ", + None, + "ⳟ", + None, + "ⳡ", + None, + "ⳣ", + None, + "ⳬ", + None, + "ⳮ", + None, + "ⳳ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ⵡ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "母", + None, + "龟", + None, + "一", + "丨", + "丶", + "丿", + "乙", + "亅", + "二", + "亠", + "人", + "儿", + "入", + "八", + "冂", + "冖", + "冫", + "几", + "凵", + "刀", + "力", + "勹", + "匕", + "匚", + "匸", + "十", + "卜", + "卩", + "厂", + "厶", + "又", + "口", + "囗", + "土", + "士", + "夂", + "夊", + "夕", + "大", + "女", + "子", + "宀", + "寸", + "小", + "尢", + "尸", + "屮", + "山", + "巛", + "工", + "己", + "巾", + "干", + "幺", + "广", + "廴", + "廾", + "弋", + "弓", + "彐", + "彡", + "彳", + "心", + "戈", + "戶", + "手", + "支", + "攴", + "文", + "斗", + "斤", + "方", + "无", + "日", + "曰", + "月", + "木", + "欠", + "止", + "歹", + "殳", + "毋", + "比", + "毛", + "氏", + "气", + "水", + "火", + "爪", + "父", + "爻", + "爿", + "片", + "牙", + "牛", + "犬", + "玄", + "玉", + "瓜", + "瓦", + "甘", + "生", + "用", + "田", + "疋", + "疒", + "癶", + "白", + "皮", + "皿", + "目", + "矛", + "矢", + "石", + "示", + "禸", + "禾", + "穴", + "立", + "竹", + "米", + "糸", + "缶", + "网", + "羊", + "羽", + "老", + "而", + "耒", + "耳", + "聿", + "肉", + "臣", + "自", + "至", + "臼", + "舌", + "舛", + "舟", + "艮", + "色", + "艸", + "虍", + "虫", + "血", + "行", + "衣", + "襾", + "見", + "角", + "言", + "谷", + "豆", + "豕", + "豸", + "貝", + "赤", + "走", + "足", + "身", + "車", + "辛", + "辰", + "辵", + "邑", + "酉", + "釆", + "里", + "金", + "長", + "門", + "阜", + "隶", + "隹", + "雨", + "靑", + "非", + "面", + "革", + "韋", + "韭", + "音", + "頁", + "風", + "飛", + "食", + "首", + "香", + "馬", + "骨", + "高", + "髟", + "鬥", + "鬯", + "鬲", + "鬼", + "魚", + "鳥", + "鹵", + "鹿", + "麥", + "麻", + "黃", + "黍", + "黑", + "黹", + "黽", + "鼎", + "鼓", + "鼠", + "鼻", + "齊", + "齒", + "龍", + "龜", + "龠", + None, + " ", + None, + ".", + None, + "〒", + None, + "十", + "卄", + "卅", + None, + None, + None, + None, + None, + " ゙", + " ゚", + None, + "より", + None, + "コト", + None, + None, + None, + "ᄀ", + "ᄁ", + "ᆪ", + "ᄂ", + "ᆬ", + "ᆭ", + "ᄃ", + "ᄄ", + "ᄅ", + "ᆰ", + "ᆱ", + "ᆲ", + "ᆳ", + "ᆴ", + "ᆵ", + "ᄚ", + "ᄆ", + "ᄇ", + "ᄈ", + "ᄡ", + "ᄉ", + "ᄊ", + "ᄋ", + "ᄌ", + "ᄍ", + "ᄎ", + "ᄏ", + "ᄐ", + "ᄑ", + "ᄒ", + "ᅡ", + "ᅢ", + "ᅣ", + "ᅤ", + "ᅥ", + "ᅦ", + "ᅧ", + "ᅨ", + "ᅩ", + "ᅪ", + "ᅫ", + "ᅬ", + "ᅭ", + "ᅮ", + "ᅯ", + "ᅰ", + "ᅱ", + "ᅲ", + "ᅳ", + "ᅴ", + "ᅵ", + None, + "ᄔ", + "ᄕ", + "ᇇ", + "ᇈ", + "ᇌ", + "ᇎ", + "ᇓ", + "ᇗ", + "ᇙ", + "ᄜ", + "ᇝ", + "ᇟ", + "ᄝ", + "ᄞ", + "ᄠ", + "ᄢ", + "ᄣ", + "ᄧ", + "ᄩ", + "ᄫ", + "ᄬ", + "ᄭ", + "ᄮ", + "ᄯ", + "ᄲ", + "ᄶ", + "ᅀ", + "ᅇ", + "ᅌ", + "ᇱ", + "ᇲ", + "ᅗ", + "ᅘ", + "ᅙ", + "ᆄ", + "ᆅ", + "ᆈ", + "ᆑ", + "ᆒ", + "ᆔ", + "ᆞ", + "ᆡ", + None, + None, + "一", + "二", + "三", + "四", + "上", + "中", + "下", + "甲", + "乙", + "丙", + "丁", + "天", + "地", + "人", + None, + None, + None, + "(ᄀ)", + "(ᄂ)", + "(ᄃ)", + "(ᄅ)", + "(ᄆ)", + "(ᄇ)", + "(ᄉ)", + "(ᄋ)", + "(ᄌ)", + "(ᄎ)", + "(ᄏ)", + "(ᄐ)", + "(ᄑ)", + "(ᄒ)", + "(가)", + "(나)", + "(다)", + "(라)", + "(마)", + "(바)", + "(사)", + "(아)", + "(자)", + "(차)", + "(카)", + "(타)", + "(파)", + "(하)", + "(주)", + "(오전)", + "(오후)", + None, + "(一)", + "(二)", + "(三)", + "(四)", + "(五)", + "(六)", + "(七)", + "(八)", + "(九)", + "(十)", + "(月)", + "(火)", + "(水)", + "(木)", + "(金)", + "(土)", + "(日)", + "(株)", + "(有)", + "(社)", + "(名)", + "(特)", + "(財)", + "(祝)", + "(労)", + "(代)", + "(呼)", + "(学)", + "(監)", + "(企)", + "(資)", + "(協)", + "(祭)", + "(休)", + "(自)", + "(至)", + "問", + "幼", + "文", + "箏", + None, + "pte", + "21", + "22", + "23", + "24", + "25", + "26", + "27", + "28", + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "ᄀ", + "ᄂ", + "ᄃ", + "ᄅ", + "ᄆ", + "ᄇ", + "ᄉ", + "ᄋ", + "ᄌ", + "ᄎ", + "ᄏ", + "ᄐ", + "ᄑ", + "ᄒ", + "가", + "나", + "다", + "라", + "마", + "바", + "사", + "아", + "자", + "차", + "카", + "타", + "파", + "하", + "참고", + "주의", + "우", + None, + "一", + "二", + "三", + "四", + "五", + "六", + "七", + "八", + "九", + "十", + "月", + "火", + "水", + "木", + "金", + "土", + "日", + "株", + "有", + "社", + "名", + "特", + "財", + "祝", + "労", + "秘", + "男", + "女", + "適", + "優", + "印", + "注", + "項", + "休", + "写", + "正", + "上", + "中", + "下", + "左", + "右", + "医", + "宗", + "学", + "監", + "企", + "資", + "協", + "夜", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "43", + "44", + "45", + "46", + "47", + "48", + "49", + "50", + "1月", + "2月", + "3月", + "4月", + "5月", + "6月", + "7月", + "8月", + "9月", + "10月", + "11月", + "12月", + "hg", + "erg", + "ev", + "ltd", + "ア", + "イ", + "ウ", + "エ", + "オ", + "カ", + "キ", + "ク", + "ケ", + "コ", + "サ", + "シ", + "ス", + "セ", + "ソ", + "タ", + "チ", + "ツ", + "テ", + "ト", + "ナ", + "ニ", + "ヌ", + "ネ", + "ノ", + "ハ", + "ヒ", + "フ", + "ヘ", + "ホ", + "マ", + "ミ", + "ム", + "メ", + "モ", + "ヤ", + "ユ", + "ヨ", + "ラ", + "リ", + "ル", + "レ", + "ロ", + "ワ", + "ヰ", + "ヱ", + "ヲ", + "令和", + "アパート", + "アルファ", + "アンペア", + "アール", + "イニング", + "インチ", + "ウォン", + "エスクード", + "エーカー", + "オンス", + "オーム", + "カイリ", + "カラット", + "カロリー", + "ガロン", + "ガンマ", + "ギガ", + "ギニー", + "キュリー", + "ギルダー", + "キロ", + "キログラム", + "キロメートル", + "キロワット", + "グラム", + "グラムトン", + "クルゼイロ", + "クローネ", + "ケース", + "コルナ", + "コーポ", + "サイクル", + "サンチーム", + "シリング", + "センチ", + "セント", + "ダース", + "デシ", + "ドル", + "トン", + "ナノ", + "ノット", + "ハイツ", + "パーセント", + "パーツ", + "バーレル", + "ピアストル", + "ピクル", + "ピコ", + "ビル", + "ファラッド", + "フィート", + "ブッシェル", + "フラン", + "ヘクタール", + "ペソ", + "ペニヒ", + "ヘルツ", + "ペンス", + "ページ", + "ベータ", + "ポイント", + "ボルト", + "ホン", + "ポンド", + "ホール", + "ホーン", + "マイクロ", + "マイル", + "マッハ", + "マルク", + "マンション", + "ミクロン", + "ミリ", + "ミリバール", + "メガ", + "メガトン", + "メートル", + "ヤード", + "ヤール", + "ユアン", + "リットル", + "リラ", + "ルピー", + "ルーブル", + "レム", + "レントゲン", + "ワット", + "0点", + "1点", + "2点", + "3点", + "4点", + "5点", + "6点", + "7点", + "8点", + "9点", + "10点", + "11点", + "12点", + "13点", + "14点", + "15点", + "16点", + "17点", + "18点", + "19点", + "20点", + "21点", + "22点", + "23点", + "24点", + "hpa", + "da", + "au", + "bar", + "ov", + "pc", + "dm", + "dm2", + "dm3", + "iu", + "平成", + "昭和", + "大正", + "明治", + "株式会社", + "pa", + "na", + "μa", + "ma", + "ka", + "kb", + "mb", + "gb", + "cal", + "kcal", + "pf", + "nf", + "μf", + "μg", + "mg", + "kg", + "hz", + "khz", + "mhz", + "ghz", + "thz", + "μl", + "ml", + "dl", + "kl", + "fm", + "nm", + "μm", + "mm", + "cm", + "km", + "mm2", + "cm2", + "m2", + "km2", + "mm3", + "cm3", + "m3", + "km3", + "m∕s", + "m∕s2", + "pa", + "kpa", + "mpa", + "gpa", + "rad", + "rad∕s", + "rad∕s2", + "ps", + "ns", + "μs", + "ms", + "pv", + "nv", + "μv", + "mv", + "kv", + "mv", + "pw", + "nw", + "μw", + "mw", + "kw", + "mw", + "kω", + "mω", + None, + "bq", + "cc", + "cd", + "c∕kg", + None, + "db", + "gy", + "ha", + "hp", + "in", + "kk", + "km", + "kt", + "lm", + "ln", + "log", + "lx", + "mb", + "mil", + "mol", + "ph", + None, + "ppm", + "pr", + "sr", + "sv", + "wb", + "v∕m", + "a∕m", + "1日", + "2日", + "3日", + "4日", + "5日", + "6日", + "7日", + "8日", + "9日", + "10日", + "11日", + "12日", + "13日", + "14日", + "15日", + "16日", + "17日", + "18日", + "19日", + "20日", + "21日", + "22日", + "23日", + "24日", + "25日", + "26日", + "27日", + "28日", + "29日", + "30日", + "31日", + "gal", + None, + None, + None, + None, + None, + None, + "ꙁ", + None, + "ꙃ", + None, + "ꙅ", + None, + "ꙇ", + None, + "ꙉ", + None, + "ꙋ", + None, + "ꙍ", + None, + "ꙏ", + None, + "ꙑ", + None, + "ꙓ", + None, + "ꙕ", + None, + "ꙗ", + None, + "ꙙ", + None, + "ꙛ", + None, + "ꙝ", + None, + "ꙟ", + None, + "ꙡ", + None, + "ꙣ", + None, + "ꙥ", + None, + "ꙧ", + None, + "ꙩ", + None, + "ꙫ", + None, + "ꙭ", + None, + "ꚁ", + None, + "ꚃ", + None, + "ꚅ", + None, + "ꚇ", + None, + "ꚉ", + None, + "ꚋ", + None, + "ꚍ", + None, + "ꚏ", + None, + "ꚑ", + None, + "ꚓ", + None, + "ꚕ", + None, + "ꚗ", + None, + "ꚙ", + None, + "ꚛ", + None, + "ъ", + "ь", + None, + None, + None, + "ꜣ", + None, + "ꜥ", + None, + "ꜧ", + None, + "ꜩ", + None, + "ꜫ", + None, + "ꜭ", + None, + "ꜯ", + None, + "ꜳ", + None, + "ꜵ", + None, + "ꜷ", + None, + "ꜹ", + None, + "ꜻ", + None, + "ꜽ", + None, + "ꜿ", + None, + "ꝁ", + None, + "ꝃ", + None, + "ꝅ", + None, + "ꝇ", + None, + "ꝉ", + None, + "ꝋ", + None, + "ꝍ", + None, + "ꝏ", + None, + "ꝑ", + None, + "ꝓ", + None, + "ꝕ", + None, + "ꝗ", + None, + "ꝙ", + None, + "ꝛ", + None, + "ꝝ", + None, + "ꝟ", + None, + "ꝡ", + None, + "ꝣ", + None, + "ꝥ", + None, + "ꝧ", + None, + "ꝩ", + None, + "ꝫ", + None, + "ꝭ", + None, + "ꝯ", + None, + "ꝯ", + None, + "ꝺ", + None, + "ꝼ", + None, + "ᵹ", + "ꝿ", + None, + "ꞁ", + None, + "ꞃ", + None, + "ꞅ", + None, + "ꞇ", + None, + "ꞌ", + None, + "ɥ", + None, + "ꞑ", + None, + "ꞓ", + None, + "ꞗ", + None, + "ꞙ", + None, + "ꞛ", + None, + "ꞝ", + None, + "ꞟ", + None, + "ꞡ", + None, + "ꞣ", + None, + "ꞥ", + None, + "ꞧ", + None, + "ꞩ", + None, + "ɦ", + "ɜ", + "ɡ", + "ɬ", + "ɪ", + None, + "ʞ", + "ʇ", + "ʝ", + "ꭓ", + "ꞵ", + None, + "ꞷ", + None, + "ꞹ", + None, + "ꞻ", + None, + "ꞽ", + None, + "ꞿ", + None, + "ꟁ", + None, + "ꟃ", + None, + "ꞔ", + "ʂ", + "ᶎ", + "ꟈ", + None, + "ꟊ", + None, + "ɤ", + "\ua7cd", + None, + "\ua7cf", + None, + "ꟑ", + None, + "ꟓ", + None, + "ꟕ", + None, + "ꟗ", + None, + "ꟙ", + None, + "\ua7db", + None, + "ƛ", + None, + "s", + "c", + "f", + "q", + "ꟶ", + None, + "ħ", + "œ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ꜧ", + "ꬷ", + "ɫ", + "ꭒ", + None, + "ʍ", + None, + None, + "Ꭰ", + "Ꭱ", + "Ꭲ", + "Ꭳ", + "Ꭴ", + "Ꭵ", + "Ꭶ", + "Ꭷ", + "Ꭸ", + "Ꭹ", + "Ꭺ", + "Ꭻ", + "Ꭼ", + "Ꭽ", + "Ꭾ", + "Ꭿ", + "Ꮀ", + "Ꮁ", + "Ꮂ", + "Ꮃ", + "Ꮄ", + "Ꮅ", + "Ꮆ", + "Ꮇ", + "Ꮈ", + "Ꮉ", + "Ꮊ", + "Ꮋ", + "Ꮌ", + "Ꮍ", + "Ꮎ", + "Ꮏ", + "Ꮐ", + "Ꮑ", + "Ꮒ", + "Ꮓ", + "Ꮔ", + "Ꮕ", + "Ꮖ", + "Ꮗ", + "Ꮘ", + "Ꮙ", + "Ꮚ", + "Ꮛ", + "Ꮜ", + "Ꮝ", + "Ꮞ", + "Ꮟ", + "Ꮠ", + "Ꮡ", + "Ꮢ", + "Ꮣ", + "Ꮤ", + "Ꮥ", + "Ꮦ", + "Ꮧ", + "Ꮨ", + "Ꮩ", + "Ꮪ", + "Ꮫ", + "Ꮬ", + "Ꮭ", + "Ꮮ", + "Ꮯ", + "Ꮰ", + "Ꮱ", + "Ꮲ", + "Ꮳ", + "Ꮴ", + "Ꮵ", + "Ꮶ", + "Ꮷ", + "Ꮸ", + "Ꮹ", + "Ꮺ", + "Ꮻ", + "Ꮼ", + "Ꮽ", + "Ꮾ", + "Ꮿ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "豈", + "更", + "車", + "賈", + "滑", + "串", + "句", + "龜", + "契", + "金", + "喇", + "奈", + "懶", + "癩", + "羅", + "蘿", + "螺", + "裸", + "邏", + "樂", + "洛", + "烙", + "珞", + "落", + "酪", + "駱", + "亂", + "卵", + "欄", + "爛", + "蘭", + "鸞", + "嵐", + "濫", + "藍", + "襤", + "拉", + "臘", + "蠟", + "廊", + "朗", + "浪", + "狼", + "郎", + "來", + "冷", + "勞", + "擄", + "櫓", + "爐", + "盧", + "老", + "蘆", + "虜", + "路", + "露", + "魯", + "鷺", + "碌", + "祿", + "綠", + "菉", + "錄", + "鹿", + "論", + "壟", + "弄", + "籠", + "聾", + "牢", + "磊", + "賂", + "雷", + "壘", + "屢", + "樓", + "淚", + "漏", + "累", + "縷", + "陋", + "勒", + "肋", + "凜", + "凌", + "稜", + "綾", + "菱", + "陵", + "讀", + "拏", + "樂", + "諾", + "丹", + "寧", + "怒", + "率", + "異", + "北", + "磻", + "便", + "復", + "不", + "泌", + "數", + "索", + "參", + "塞", + "省", + "葉", + "說", + "殺", + "辰", + "沈", + "拾", + "若", + "掠", + "略", + "亮", + "兩", + "凉", + "梁", + "糧", + "良", + "諒", + "量", + "勵", + "呂", + "女", + "廬", + "旅", + "濾", + "礪", + "閭", + "驪", + "麗", + "黎", + "力", + "曆", + "歷", + "轢", + "年", + "憐", + "戀", + "撚", + "漣", + "煉", + "璉", + "秊", + "練", + "聯", + "輦", + "蓮", + "連", + "鍊", + "列", + "劣", + "咽", + "烈", + "裂", + "說", + "廉", + "念", + "捻", + "殮", + "簾", + "獵", + "令", + "囹", + "寧", + "嶺", + "怜", + "玲", + "瑩", + "羚", + "聆", + "鈴", + "零", + "靈", + "領", + "例", + "禮", + "醴", + "隸", + "惡", + "了", + "僚", + "寮", + "尿", + "料", + "樂", + "燎", + "療", + "蓼", + "遼", + "龍", + "暈", + "阮", + "劉", + "杻", + "柳", + "流", + "溜", + "琉", + "留", + "硫", + "紐", + "類", + "六", + "戮", + "陸", + "倫", + "崙", + "淪", + "輪", + "律", + "慄", + "栗", + "率", + "隆", + "利", + "吏", + "履", + "易", + "李", + "梨", + "泥", + "理", + "痢", + "罹", + "裏", + "裡", + "里", + "離", + "匿", + "溺", + "吝", + "燐", + "璘", + "藺", + "隣", + "鱗", + "麟", + "林", + "淋", + "臨", + "立", + "笠", + "粒", + "狀", + "炙", + "識", + "什", + "茶", + "刺", + "切", + "度", + "拓", + "糖", + "宅", + "洞", + "暴", + "輻", + "行", + "降", + "見", + "廓", + "兀", + "嗀", + None, + "塚", + None, + "晴", + None, + "凞", + "猪", + "益", + "礼", + "神", + "祥", + "福", + "靖", + "精", + "羽", + None, + "蘒", + None, + "諸", + None, + "逸", + "都", + None, + "飯", + "飼", + "館", + "鶴", + "郞", + "隷", + "侮", + "僧", + "免", + "勉", + "勤", + "卑", + "喝", + "嘆", + "器", + "塀", + "墨", + "層", + "屮", + "悔", + "慨", + "憎", + "懲", + "敏", + "既", + "暑", + "梅", + "海", + "渚", + "漢", + "煮", + "爫", + "琢", + "碑", + "社", + "祉", + "祈", + "祐", + "祖", + "祝", + "禍", + "禎", + "穀", + "突", + "節", + "練", + "縉", + "繁", + "署", + "者", + "臭", + "艹", + "著", + "褐", + "視", + "謁", + "謹", + "賓", + "贈", + "辶", + "逸", + "難", + "響", + "頻", + "恵", + "𤋮", + "舘", + None, + "並", + "况", + "全", + "侀", + "充", + "冀", + "勇", + "勺", + "喝", + "啕", + "喙", + "嗢", + "塚", + "墳", + "奄", + "奔", + "婢", + "嬨", + "廒", + "廙", + "彩", + "徭", + "惘", + "慎", + "愈", + "憎", + "慠", + "懲", + "戴", + "揄", + "搜", + "摒", + "敖", + "晴", + "朗", + "望", + "杖", + "歹", + "殺", + "流", + "滛", + "滋", + "漢", + "瀞", + "煮", + "瞧", + "爵", + "犯", + "猪", + "瑱", + "甆", + "画", + "瘝", + "瘟", + "益", + "盛", + "直", + "睊", + "着", + "磌", + "窱", + "節", + "类", + "絛", + "練", + "缾", + "者", + "荒", + "華", + "蝹", + "襁", + "覆", + "視", + "調", + "諸", + "請", + "謁", + "諾", + "諭", + "謹", + "變", + "贈", + "輸", + "遲", + "醙", + "鉶", + "陼", + "難", + "靖", + "韛", + "響", + "頋", + "頻", + "鬒", + "龜", + "𢡊", + "𢡄", + "𣏕", + "㮝", + "䀘", + "䀹", + "𥉉", + "𥳐", + "𧻓", + "齃", + "龎", + None, + "ff", + "fi", + "fl", + "ffi", + "ffl", + "st", + None, + "մն", + "մե", + "մի", + "վն", + "մխ", + None, + "יִ", + None, + "ײַ", + "ע", + "א", + "ד", + "ה", + "כ", + "ל", + "ם", + "ר", + "ת", + "+", + "שׁ", + "שׂ", + "שּׁ", + "שּׂ", + "אַ", + "אָ", + "אּ", + "בּ", + "גּ", + "דּ", + "הּ", + "וּ", + "זּ", + None, + "טּ", + "יּ", + "ךּ", + "כּ", + "לּ", + None, + "מּ", + None, + "נּ", + "סּ", + None, + "ףּ", + "פּ", + None, + "צּ", + "קּ", + "רּ", + "שּ", + "תּ", + "וֹ", + "בֿ", + "כֿ", + "פֿ", + "אל", + "ٱ", + "ٻ", + "پ", + "ڀ", + "ٺ", + "ٿ", + "ٹ", + "ڤ", + "ڦ", + "ڄ", + "ڃ", + "چ", + "ڇ", + "ڍ", + "ڌ", + "ڎ", + "ڈ", + "ژ", + "ڑ", + "ک", + "گ", + "ڳ", + "ڱ", + "ں", + "ڻ", + "ۀ", + "ہ", + "ھ", + "ے", + "ۓ", + None, + "ڭ", + "ۇ", + "ۆ", + "ۈ", + "ۇٴ", + "ۋ", + "ۅ", + "ۉ", + "ې", + "ى", + "ئا", + "ئە", + "ئو", + "ئۇ", + "ئۆ", + "ئۈ", + "ئې", + "ئى", + "ی", + "ئج", + "ئح", + "ئم", + "ئى", + "ئي", + "بج", + "بح", + "بخ", + "بم", + "بى", + "بي", + "تج", + "تح", + "تخ", + "تم", + "تى", + "تي", + "ثج", + "ثم", + "ثى", + "ثي", + "جح", + "جم", + "حج", + "حم", + "خج", + "خح", + "خم", + "سج", + "سح", + "سخ", + "سم", + "صح", + "صم", + "ضج", + "ضح", + "ضخ", + "ضم", + "طح", + "طم", + "ظم", + "عج", + "عم", + "غج", + "غم", + "فج", + "فح", + "فخ", + "فم", + "فى", + "في", + "قح", + "قم", + "قى", + "قي", + "كا", + "كج", + "كح", + "كخ", + "كل", + "كم", + "كى", + "كي", + "لج", + "لح", + "لخ", + "لم", + "لى", + "لي", + "مج", + "مح", + "مخ", + "مم", + "مى", + "مي", + "نج", + "نح", + "نخ", + "نم", + "نى", + "ني", + "هج", + "هم", + "هى", + "هي", + "يج", + "يح", + "يخ", + "يم", + "يى", + "يي", + "ذٰ", + "رٰ", + "ىٰ", + " ٌّ", + " ٍّ", + " َّ", + " ُّ", + " ِّ", + " ّٰ", + "ئر", + "ئز", + "ئم", + "ئن", + "ئى", + "ئي", + "بر", + "بز", + "بم", + "بن", + "بى", + "بي", + "تر", + "تز", + "تم", + "تن", + "تى", + "تي", + "ثر", + "ثز", + "ثم", + "ثن", + "ثى", + "ثي", + "فى", + "في", + "قى", + "قي", + "كا", + "كل", + "كم", + "كى", + "كي", + "لم", + "لى", + "لي", + "ما", + "مم", + "نر", + "نز", + "نم", + "نن", + "نى", + "ني", + "ىٰ", + "ير", + "يز", + "يم", + "ين", + "يى", + "يي", + "ئج", + "ئح", + "ئخ", + "ئم", + "ئه", + "بج", + "بح", + "بخ", + "بم", + "به", + "تج", + "تح", + "تخ", + "تم", + "ته", + "ثم", + "جح", + "جم", + "حج", + "حم", + "خج", + "خم", + "سج", + "سح", + "سخ", + "سم", + "صح", + "صخ", + "صم", + "ضج", + "ضح", + "ضخ", + "ضم", + "طح", + "ظم", + "عج", + "عم", + "غج", + "غم", + "فج", + "فح", + "فخ", + "فم", + "قح", + "قم", + "كج", + "كح", + "كخ", + "كل", + "كم", + "لج", + "لح", + "لخ", + "لم", + "له", + "مج", + "مح", + "مخ", + "مم", + "نج", + "نح", + "نخ", + "نم", + "نه", + "هج", + "هم", + "هٰ", + "يج", + "يح", + "يخ", + "يم", + "يه", + "ئم", + "ئه", + "بم", + "به", + "تم", + "ته", + "ثم", + "ثه", + "سم", + "سه", + "شم", + "شه", + "كل", + "كم", + "لم", + "نم", + "نه", + "يم", + "يه", + "ـَّ", + "ـُّ", + "ـِّ", + "طى", + "طي", + "عى", + "عي", + "غى", + "غي", + "سى", + "سي", + "شى", + "شي", + "حى", + "حي", + "جى", + "جي", + "خى", + "خي", + "صى", + "صي", + "ضى", + "ضي", + "شج", + "شح", + "شخ", + "شم", + "شر", + "سر", + "صر", + "ضر", + "طى", + "طي", + "عى", + "عي", + "غى", + "غي", + "سى", + "سي", + "شى", + "شي", + "حى", + "حي", + "جى", + "جي", + "خى", + "خي", + "صى", + "صي", + "ضى", + "ضي", + "شج", + "شح", + "شخ", + "شم", + "شر", + "سر", + "صر", + "ضر", + "شج", + "شح", + "شخ", + "شم", + "سه", + "شه", + "طم", + "سج", + "سح", + "سخ", + "شج", + "شح", + "شخ", + "طم", + "ظم", + "اً", + None, + "تجم", + "تحج", + "تحم", + "تخم", + "تمج", + "تمح", + "تمخ", + "جمح", + "حمي", + "حمى", + "سحج", + "سجح", + "سجى", + "سمح", + "سمج", + "سمم", + "صحح", + "صمم", + "شحم", + "شجي", + "شمخ", + "شمم", + "ضحى", + "ضخم", + "طمح", + "طمم", + "طمي", + "عجم", + "عمم", + "عمى", + "غمم", + "غمي", + "غمى", + "فخم", + "قمح", + "قمم", + "لحم", + "لحي", + "لحى", + "لجج", + "لخم", + "لمح", + "محج", + "محم", + "محي", + "مجح", + "مجم", + "مخج", + "مخم", + None, + "مجخ", + "همج", + "همم", + "نحم", + "نحى", + "نجم", + "نجى", + "نمي", + "نمى", + "يمم", + "بخي", + "تجي", + "تجى", + "تخي", + "تخى", + "تمي", + "تمى", + "جمي", + "جحى", + "جمى", + "سخى", + "صحي", + "شحي", + "ضحي", + "لجي", + "لمي", + "يحي", + "يجي", + "يمي", + "ممي", + "قمي", + "نحي", + "قمح", + "لحم", + "عمي", + "كمي", + "نجح", + "مخي", + "لجم", + "كمم", + "لجم", + "نجح", + "جحي", + "حجي", + "مجي", + "فمي", + "بحي", + "كمم", + "عجم", + "صمم", + "سخي", + "نجي", + None, + None, + "صلے", + "قلے", + "الله", + "اكبر", + "محمد", + "صلعم", + "رسول", + "عليه", + "وسلم", + "صلى", + "صلى الله عليه وسلم", + "جل جلاله", + "ریال", + None, + None, + ",", + "、", + None, + ":", + ";", + "!", + "?", + "〖", + "〗", + None, + None, + None, + "—", + "–", + "_", + "(", + ")", + "{", + "}", + "〔", + "〕", + "【", + "】", + "《", + "》", + "〈", + "〉", + "「", + "」", + "『", + "』", + None, + "[", + "]", + " ̅", + "_", + ",", + "、", + None, + ";", + ":", + "?", + "!", + "—", + "(", + ")", + "{", + "}", + "〔", + "〕", + "#", + "&", + "*", + "+", + "-", + "<", + ">", + "=", + None, + "\\", + "$", + "%", + "@", + None, + " ً", + "ـً", + " ٌ", + None, + " ٍ", + None, + " َ", + "ـَ", + " ُ", + "ـُ", + " ِ", + "ـِ", + " ّ", + "ـّ", + " ْ", + "ـْ", + "ء", + "آ", + "أ", + "ؤ", + "إ", + "ئ", + "ا", + "ب", + "ة", + "ت", + "ث", + "ج", + "ح", + "خ", + "د", + "ذ", + "ر", + "ز", + "س", + "ش", + "ص", + "ض", + "ط", + "ظ", + "ع", + "غ", + "ف", + "ق", + "ك", + "ل", + "م", + "ن", + "ه", + "و", + "ى", + "ي", + "لآ", + "لأ", + "لإ", + "لا", + None, + None, + None, + "!", + '"', + "#", + "$", + "%", + "&", + "'", + "(", + ")", + "*", + "+", + ",", + "-", + ".", + "/", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ":", + ";", + "<", + "=", + ">", + "?", + "@", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "[", + "\\", + "]", + "^", + "_", + "`", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "{", + "|", + "}", + "~", + "⦅", + "⦆", + ".", + "「", + "」", + "、", + "・", + "ヲ", + "ァ", + "ィ", + "ゥ", + "ェ", + "ォ", + "ャ", + "ュ", + "ョ", + "ッ", + "ー", + "ア", + "イ", + "ウ", + "エ", + "オ", + "カ", + "キ", + "ク", + "ケ", + "コ", + "サ", + "シ", + "ス", + "セ", + "ソ", + "タ", + "チ", + "ツ", + "テ", + "ト", + "ナ", + "ニ", + "ヌ", + "ネ", + "ノ", + "ハ", + "ヒ", + "フ", + "ヘ", + "ホ", + "マ", + "ミ", + "ム", + "メ", + "モ", + "ヤ", + "ユ", + "ヨ", + "ラ", + "リ", + "ル", + "レ", + "ロ", + "ワ", + "ン", + "゙", + "゚", + None, + "ᄀ", + "ᄁ", + "ᆪ", + "ᄂ", + "ᆬ", + "ᆭ", + "ᄃ", + "ᄄ", + "ᄅ", + "ᆰ", + "ᆱ", + "ᆲ", + "ᆳ", + "ᆴ", + "ᆵ", + "ᄚ", + "ᄆ", + "ᄇ", + "ᄈ", + "ᄡ", + "ᄉ", + "ᄊ", + "ᄋ", + "ᄌ", + "ᄍ", + "ᄎ", + "ᄏ", + "ᄐ", + "ᄑ", + "ᄒ", + None, + "ᅡ", + "ᅢ", + "ᅣ", + "ᅤ", + "ᅥ", + "ᅦ", + None, + "ᅧ", + "ᅨ", + "ᅩ", + "ᅪ", + "ᅫ", + "ᅬ", + None, + "ᅭ", + "ᅮ", + "ᅯ", + "ᅰ", + "ᅱ", + "ᅲ", + None, + "ᅳ", + "ᅴ", + "ᅵ", + None, + "¢", + "£", + "¬", + " ̄", + "¦", + "¥", + "₩", + None, + "│", + "←", + "↑", + "→", + "↓", + "■", + "○", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𐐨", + "𐐩", + "𐐪", + "𐐫", + "𐐬", + "𐐭", + "𐐮", + "𐐯", + "𐐰", + "𐐱", + "𐐲", + "𐐳", + "𐐴", + "𐐵", + "𐐶", + "𐐷", + "𐐸", + "𐐹", + "𐐺", + "𐐻", + "𐐼", + "𐐽", + "𐐾", + "𐐿", + "𐑀", + "𐑁", + "𐑂", + "𐑃", + "𐑄", + "𐑅", + "𐑆", + "𐑇", + "𐑈", + "𐑉", + "𐑊", + "𐑋", + "𐑌", + "𐑍", + "𐑎", + "𐑏", + None, + None, + None, + None, + "𐓘", + "𐓙", + "𐓚", + "𐓛", + "𐓜", + "𐓝", + "𐓞", + "𐓟", + "𐓠", + "𐓡", + "𐓢", + "𐓣", + "𐓤", + "𐓥", + "𐓦", + "𐓧", + "𐓨", + "𐓩", + "𐓪", + "𐓫", + "𐓬", + "𐓭", + "𐓮", + "𐓯", + "𐓰", + "𐓱", + "𐓲", + "𐓳", + "𐓴", + "𐓵", + "𐓶", + "𐓷", + "𐓸", + "𐓹", + "𐓺", + "𐓻", + None, + None, + None, + None, + None, + None, + None, + None, + "𐖗", + "𐖘", + "𐖙", + "𐖚", + "𐖛", + "𐖜", + "𐖝", + "𐖞", + "𐖟", + "𐖠", + "𐖡", + None, + "𐖣", + "𐖤", + "𐖥", + "𐖦", + "𐖧", + "𐖨", + "𐖩", + "𐖪", + "𐖫", + "𐖬", + "𐖭", + "𐖮", + "𐖯", + "𐖰", + "𐖱", + None, + "𐖳", + "𐖴", + "𐖵", + "𐖶", + "𐖷", + "𐖸", + "𐖹", + None, + "𐖻", + "𐖼", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ː", + "ˑ", + "æ", + "ʙ", + "ɓ", + None, + "ʣ", + "ꭦ", + "ʥ", + "ʤ", + "ɖ", + "ɗ", + "ᶑ", + "ɘ", + "ɞ", + "ʩ", + "ɤ", + "ɢ", + "ɠ", + "ʛ", + "ħ", + "ʜ", + "ɧ", + "ʄ", + "ʪ", + "ʫ", + "ɬ", + "𝼄", + "ꞎ", + "ɮ", + "𝼅", + "ʎ", + "𝼆", + "ø", + "ɶ", + "ɷ", + "q", + "ɺ", + "𝼈", + "ɽ", + "ɾ", + "ʀ", + "ʨ", + "ʦ", + "ꭧ", + "ʧ", + "ʈ", + "ⱱ", + None, + "ʏ", + "ʡ", + "ʢ", + "ʘ", + "ǀ", + "ǁ", + "ǂ", + "𝼊", + "𝼞", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𐳀", + "𐳁", + "𐳂", + "𐳃", + "𐳄", + "𐳅", + "𐳆", + "𐳇", + "𐳈", + "𐳉", + "𐳊", + "𐳋", + "𐳌", + "𐳍", + "𐳎", + "𐳏", + "𐳐", + "𐳑", + "𐳒", + "𐳓", + "𐳔", + "𐳕", + "𐳖", + "𐳗", + "𐳘", + "𐳙", + "𐳚", + "𐳛", + "𐳜", + "𐳝", + "𐳞", + "𐳟", + "𐳠", + "𐳡", + "𐳢", + "𐳣", + "𐳤", + "𐳥", + "𐳦", + "𐳧", + "𐳨", + "𐳩", + "𐳪", + "𐳫", + "𐳬", + "𐳭", + "𐳮", + "𐳯", + "𐳰", + "𐳱", + "𐳲", + None, + None, + None, + None, + None, + None, + None, + None, + "\U00010d70", + "\U00010d71", + "\U00010d72", + "\U00010d73", + "\U00010d74", + "\U00010d75", + "\U00010d76", + "\U00010d77", + "\U00010d78", + "\U00010d79", + "\U00010d7a", + "\U00010d7b", + "\U00010d7c", + "\U00010d7d", + "\U00010d7e", + "\U00010d7f", + "\U00010d80", + "\U00010d81", + "\U00010d82", + "\U00010d83", + "\U00010d84", + "\U00010d85", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𑣀", + "𑣁", + "𑣂", + "𑣃", + "𑣄", + "𑣅", + "𑣆", + "𑣇", + "𑣈", + "𑣉", + "𑣊", + "𑣋", + "𑣌", + "𑣍", + "𑣎", + "𑣏", + "𑣐", + "𑣑", + "𑣒", + "𑣓", + "𑣔", + "𑣕", + "𑣖", + "𑣗", + "𑣘", + "𑣙", + "𑣚", + "𑣛", + "𑣜", + "𑣝", + "𑣞", + "𑣟", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𖹠", + "𖹡", + "𖹢", + "𖹣", + "𖹤", + "𖹥", + "𖹦", + "𖹧", + "𖹨", + "𖹩", + "𖹪", + "𖹫", + "𖹬", + "𖹭", + "𖹮", + "𖹯", + "𖹰", + "𖹱", + "𖹲", + "𖹳", + "𖹴", + "𖹵", + "𖹶", + "𖹷", + "𖹸", + "𖹹", + "𖹺", + "𖹻", + "𖹼", + "𖹽", + "𖹾", + "𖹿", + None, + None, + "\U00016ebb", + "\U00016ebc", + "\U00016ebd", + "\U00016ebe", + "\U00016ebf", + "\U00016ec0", + "\U00016ec1", + "\U00016ec2", + "\U00016ec3", + "\U00016ec4", + "\U00016ec5", + "\U00016ec6", + "\U00016ec7", + "\U00016ec8", + "\U00016ec9", + "\U00016eca", + "\U00016ecb", + "\U00016ecc", + "\U00016ecd", + "\U00016ece", + "\U00016ecf", + "\U00016ed0", + "\U00016ed1", + "\U00016ed2", + "\U00016ed3", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𝅗𝅥", + "𝅘𝅥", + "𝅘𝅥𝅮", + "𝅘𝅥𝅯", + "𝅘𝅥𝅰", + "𝅘𝅥𝅱", + "𝅘𝅥𝅲", + None, + None, + None, + "𝆹𝅥", + "𝆺𝅥", + "𝆹𝅥𝅮", + "𝆺𝅥𝅮", + "𝆹𝅥𝅯", + "𝆺𝅥𝅯", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + None, + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + None, + "c", + "d", + None, + "g", + None, + "j", + "k", + None, + "n", + "o", + "p", + "q", + None, + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + None, + "f", + None, + "h", + "i", + "j", + "k", + "l", + "m", + "n", + None, + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + None, + "d", + "e", + "f", + "g", + None, + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + None, + "s", + "t", + "u", + "v", + "w", + "x", + "y", + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + None, + "d", + "e", + "f", + "g", + None, + "i", + "j", + "k", + "l", + "m", + None, + "o", + None, + "s", + "t", + "u", + "v", + "w", + "x", + "y", + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "ı", + "ȷ", + None, + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "θ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∇", + "α", + "β", + "γ", + "δ", + "ε", + "ζ", + "η", + "θ", + "ι", + "κ", + "λ", + "μ", + "ν", + "ξ", + "ο", + "π", + "ρ", + "σ", + "τ", + "υ", + "φ", + "χ", + "ψ", + "ω", + "∂", + "ε", + "θ", + "κ", + "φ", + "ρ", + "π", + "ϝ", + None, + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "а", + "б", + "в", + "г", + "д", + "е", + "ж", + "з", + "и", + "к", + "л", + "м", + "о", + "п", + "р", + "с", + "т", + "у", + "ф", + "х", + "ц", + "ч", + "ш", + "ы", + "э", + "ю", + "ꚉ", + "ә", + "і", + "ј", + "ө", + "ү", + "ӏ", + "а", + "б", + "в", + "г", + "д", + "е", + "ж", + "з", + "и", + "к", + "л", + "о", + "п", + "с", + "у", + "ф", + "х", + "ц", + "ч", + "ш", + "ъ", + "ы", + "ґ", + "і", + "ѕ", + "џ", + "ҫ", + "ꙑ", + "ұ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "𞤢", + "𞤣", + "𞤤", + "𞤥", + "𞤦", + "𞤧", + "𞤨", + "𞤩", + "𞤪", + "𞤫", + "𞤬", + "𞤭", + "𞤮", + "𞤯", + "𞤰", + "𞤱", + "𞤲", + "𞤳", + "𞤴", + "𞤵", + "𞤶", + "𞤷", + "𞤸", + "𞤹", + "𞤺", + "𞤻", + "𞤼", + "𞤽", + "𞤾", + "𞤿", + "𞥀", + "𞥁", + "𞥂", + "𞥃", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "ا", + "ب", + "ج", + "د", + None, + "و", + "ز", + "ح", + "ط", + "ي", + "ك", + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + "ر", + "ش", + "ت", + "ث", + "خ", + "ذ", + "ض", + "ظ", + "غ", + "ٮ", + "ں", + "ڡ", + "ٯ", + None, + "ب", + "ج", + None, + "ه", + None, + "ح", + None, + "ي", + "ك", + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + None, + "ش", + "ت", + "ث", + "خ", + None, + "ض", + None, + "غ", + None, + "ج", + None, + "ح", + None, + "ي", + None, + "ل", + None, + "ن", + "س", + "ع", + None, + "ص", + "ق", + None, + "ش", + None, + "خ", + None, + "ض", + None, + "غ", + None, + "ں", + None, + "ٯ", + None, + "ب", + "ج", + None, + "ه", + None, + "ح", + "ط", + "ي", + "ك", + None, + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + None, + "ش", + "ت", + "ث", + "خ", + None, + "ض", + "ظ", + "غ", + "ٮ", + None, + "ڡ", + None, + "ا", + "ب", + "ج", + "د", + "ه", + "و", + "ز", + "ح", + "ط", + "ي", + None, + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + "ر", + "ش", + "ت", + "ث", + "خ", + "ذ", + "ض", + "ظ", + "غ", + None, + "ب", + "ج", + "د", + None, + "و", + "ز", + "ح", + "ط", + "ي", + None, + "ل", + "م", + "ن", + "س", + "ع", + "ف", + "ص", + "ق", + "ر", + "ش", + "ت", + "ث", + "خ", + "ذ", + "ض", + "ظ", + "غ", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "0,", + "1,", + "2,", + "3,", + "4,", + "5,", + "6,", + "7,", + "8,", + "9,", + None, + "(a)", + "(b)", + "(c)", + "(d)", + "(e)", + "(f)", + "(g)", + "(h)", + "(i)", + "(j)", + "(k)", + "(l)", + "(m)", + "(n)", + "(o)", + "(p)", + "(q)", + "(r)", + "(s)", + "(t)", + "(u)", + "(v)", + "(w)", + "(x)", + "(y)", + "(z)", + "〔s〕", + "c", + "r", + "cd", + "wz", + None, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "hv", + "mv", + "sd", + "ss", + "ppv", + "wc", + None, + "mc", + "md", + "mr", + None, + "dj", + None, + None, + None, + "ほか", + "ココ", + "サ", + None, + "手", + "字", + "双", + "デ", + "二", + "多", + "解", + "天", + "交", + "映", + "無", + "料", + "前", + "後", + "再", + "新", + "初", + "終", + "生", + "販", + "声", + "吹", + "演", + "投", + "捕", + "一", + "三", + "遊", + "左", + "中", + "右", + "指", + "走", + "打", + "禁", + "空", + "合", + "満", + "有", + "月", + "申", + "割", + "営", + "配", + None, + "〔本〕", + "〔三〕", + "〔二〕", + "〔安〕", + "〔点〕", + "〔打〕", + "〔盗〕", + "〔勝〕", + "〔敗〕", + None, + "得", + "可", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + "丽", + "丸", + "乁", + "𠄢", + "你", + "侮", + "侻", + "倂", + "偺", + "備", + "僧", + "像", + "㒞", + "𠘺", + "免", + "兔", + "兤", + "具", + "𠔜", + "㒹", + "內", + "再", + "𠕋", + "冗", + "冤", + "仌", + "冬", + "况", + "𩇟", + "凵", + "刃", + "㓟", + "刻", + "剆", + "割", + "剷", + "㔕", + "勇", + "勉", + "勤", + "勺", + "包", + "匆", + "北", + "卉", + "卑", + "博", + "即", + "卽", + "卿", + "𠨬", + "灰", + "及", + "叟", + "𠭣", + "叫", + "叱", + "吆", + "咞", + "吸", + "呈", + "周", + "咢", + "哶", + "唐", + "啓", + "啣", + "善", + "喙", + "喫", + "喳", + "嗂", + "圖", + "嘆", + "圗", + "噑", + "噴", + "切", + "壮", + "城", + "埴", + "堍", + "型", + "堲", + "報", + "墬", + "𡓤", + "売", + "壷", + "夆", + "多", + "夢", + "奢", + "𡚨", + "𡛪", + "姬", + "娛", + "娧", + "姘", + "婦", + "㛮", + "㛼", + "嬈", + "嬾", + "𡧈", + "寃", + "寘", + "寧", + "寳", + "𡬘", + "寿", + "将", + "当", + "尢", + "㞁", + "屠", + "屮", + "峀", + "岍", + "𡷤", + "嵃", + "𡷦", + "嵮", + "嵫", + "嵼", + "巡", + "巢", + "㠯", + "巽", + "帨", + "帽", + "幩", + "㡢", + "𢆃", + "㡼", + "庰", + "庳", + "庶", + "廊", + "𪎒", + "廾", + "𢌱", + "舁", + "弢", + "㣇", + "𣊸", + "𦇚", + "形", + "彫", + "㣣", + "徚", + "忍", + "志", + "忹", + "悁", + "㤺", + "㤜", + "悔", + "𢛔", + "惇", + "慈", + "慌", + "慎", + "慌", + "慺", + "憎", + "憲", + "憤", + "憯", + "懞", + "懲", + "懶", + "成", + "戛", + "扝", + "抱", + "拔", + "捐", + "𢬌", + "挽", + "拼", + "捨", + "掃", + "揤", + "𢯱", + "搢", + "揅", + "掩", + "㨮", + "摩", + "摾", + "撝", + "摷", + "㩬", + "敏", + "敬", + "𣀊", + "旣", + "書", + "晉", + "㬙", + "暑", + "㬈", + "㫤", + "冒", + "冕", + "最", + "暜", + "肭", + "䏙", + "朗", + "望", + "朡", + "杞", + "杓", + "𣏃", + "㭉", + "柺", + "枅", + "桒", + "梅", + "𣑭", + "梎", + "栟", + "椔", + "㮝", + "楂", + "榣", + "槪", + "檨", + "𣚣", + "櫛", + "㰘", + "次", + "𣢧", + "歔", + "㱎", + "歲", + "殟", + "殺", + "殻", + "𣪍", + "𡴋", + "𣫺", + "汎", + "𣲼", + "沿", + "泍", + "汧", + "洖", + "派", + "海", + "流", + "浩", + "浸", + "涅", + "𣴞", + "洴", + "港", + "湮", + "㴳", + "滋", + "滇", + "𣻑", + "淹", + "潮", + "𣽞", + "𣾎", + "濆", + "瀹", + "瀞", + "瀛", + "㶖", + "灊", + "災", + "灷", + "炭", + "𠔥", + "煅", + "𤉣", + "熜", + "𤎫", + "爨", + "爵", + "牐", + "𤘈", + "犀", + "犕", + "𤜵", + "𤠔", + "獺", + "王", + "㺬", + "玥", + "㺸", + "瑇", + "瑜", + "瑱", + "璅", + "瓊", + "㼛", + "甤", + "𤰶", + "甾", + "𤲒", + "異", + "𢆟", + "瘐", + "𤾡", + "𤾸", + "𥁄", + "㿼", + "䀈", + "直", + "𥃳", + "𥃲", + "𥄙", + "𥄳", + "眞", + "真", + "睊", + "䀹", + "瞋", + "䁆", + "䂖", + "𥐝", + "硎", + "碌", + "磌", + "䃣", + "𥘦", + "祖", + "𥚚", + "𥛅", + "福", + "秫", + "䄯", + "穀", + "穊", + "穏", + "𥥼", + "𥪧", + "竮", + "䈂", + "𥮫", + "篆", + "築", + "䈧", + "𥲀", + "糒", + "䊠", + "糨", + "糣", + "紀", + "𥾆", + "絣", + "䌁", + "緇", + "縂", + "繅", + "䌴", + "𦈨", + "𦉇", + "䍙", + "𦋙", + "罺", + "𦌾", + "羕", + "翺", + "者", + "𦓚", + "𦔣", + "聠", + "𦖨", + "聰", + "𣍟", + "䏕", + "育", + "脃", + "䐋", + "脾", + "媵", + "𦞧", + "𦞵", + "𣎓", + "𣎜", + "舁", + "舄", + "辞", + "䑫", + "芑", + "芋", + "芝", + "劳", + "花", + "芳", + "芽", + "苦", + "𦬼", + "若", + "茝", + "荣", + "莭", + "茣", + "莽", + "菧", + "著", + "荓", + "菊", + "菌", + "菜", + "𦰶", + "𦵫", + "𦳕", + "䔫", + "蓱", + "蓳", + "蔖", + "𧏊", + "蕤", + "𦼬", + "䕝", + "䕡", + "𦾱", + "𧃒", + "䕫", + "虐", + "虜", + "虧", + "虩", + "蚩", + "蚈", + "蜎", + "蛢", + "蝹", + "蜨", + "蝫", + "螆", + "䗗", + "蟡", + "蠁", + "䗹", + "衠", + "衣", + "𧙧", + "裗", + "裞", + "䘵", + "裺", + "㒻", + "𧢮", + "𧥦", + "䚾", + "䛇", + "誠", + "諭", + "變", + "豕", + "𧲨", + "貫", + "賁", + "贛", + "起", + "𧼯", + "𠠄", + "跋", + "趼", + "跰", + "𠣞", + "軔", + "輸", + "𨗒", + "𨗭", + "邔", + "郱", + "鄑", + "𨜮", + "鄛", + "鈸", + "鋗", + "鋘", + "鉼", + "鏹", + "鐕", + "𨯺", + "開", + "䦕", + "閷", + "𨵷", + "䧦", + "雃", + "嶲", + "霣", + "𩅅", + "𩈚", + "䩮", + "䩶", + "韠", + "𩐊", + "䪲", + "𩒖", + "頋", + "頩", + "𩖶", + "飢", + "䬳", + "餩", + "馧", + "駂", + "駾", + "䯎", + "𩬰", + "鬒", + "鱀", + "鳽", + "䳎", + "䳭", + "鵧", + "𪃎", + "䳸", + "𪄅", + "𪈎", + "𪊑", + "麻", + "䵖", + "黹", + "黾", + "鼅", + "鼏", + "鼖", + "鼻", + "𪘀", + None, + None, + None, + None, + None, + None, + None, ) diff --git a/contrib/python/idna/py3/patches/01-fix-tests.patch b/contrib/python/idna/py3/patches/01-fix-tests.patch new file mode 100644 index 00000000000..ebd7c115b2e --- /dev/null +++ b/contrib/python/idna/py3/patches/01-fix-tests.patch @@ -0,0 +1,26 @@ +--- contrib/python/idna/py3/tests/test_idna_cli.py (index) ++++ contrib/python/idna/py3/tests/test_idna_cli.py (working tree) +@@ -278,6 +278,7 @@ class CLIModuleEntryTests(unittest.TestCase): + check=False, + capture_output=True, + text=True, ++ env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout.strip(), "テスト") +@@ -291,6 +292,7 @@ class CLIModuleEntryTests(unittest.TestCase): + check=False, + capture_output=True, + text=True, ++ env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual( +@@ -305,6 +307,7 @@ class CLIModuleEntryTests(unittest.TestCase): + check=False, + capture_output=True, + text=True, ++ env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(result.stdout, "") diff --git a/contrib/python/idna/py3/tests/test_idna.py b/contrib/python/idna/py3/tests/test_idna.py index 65eecfd88ee..4a31866b626 100644 --- a/contrib/python/idna/py3/tests/test_idna.py +++ b/contrib/python/idna/py3/tests/test_idna.py @@ -2,14 +2,13 @@ import unittest import warnings -from typing import List, Tuple import idna class IDNATests(unittest.TestCase): def setUp(self): - self.tld_strings: List[Tuple[str, bytes]] = [ + self.tld_strings: list[tuple[str, bytes]] = [ ("\u6d4b\u8bd5", b"xn--0zwm56d"), ("\u092a\u0930\u0940\u0915\u094d\u0937\u093e", b"xn--11b5bs3a9aj6g"), ("\ud55c\uad6d", b"xn--3e0b707e"), @@ -104,7 +103,7 @@ class IDNATests(unittest.TestCase): import codecs import time - import idna.codec # noqa: F401 (register the idna2008 codec) + import idna.codec # register the idna2008 codec payload = "・" * 8000 + "漢" start = time.perf_counter() @@ -350,6 +349,61 @@ class IDNATests(unittest.TestCase): self.assertRaises(idna.IDNAError, idna.encode, value) self.assertRaises(idna.IDNAError, idna.decode, value) + def test_decode_display(self): + # A label whose Punycode decode succeeds but contains disallowed + # codepoints — under display decoding, the original A-label is kept. + self.assertRaises(idna.IDNAError, idna.decode, "a.b.c.xn--pokxncvks") + self.assertEqual( + idna.decode("a.b.c.xn--pokxncvks", display=True), + "a.b.c.xn--pokxncvks", + ) + + # Mixed valid/invalid labels: the valid label still decodes, the + # invalid xn-- label is preserved verbatim. + self.assertEqual( + idna.decode("xn--zckzah.xn--pokxncvks", display=True), + "テスト.xn--pokxncvks", + ) + + # A label whose Punycode itself is malformed. + self.assertEqual( + idna.decode("xn--.example", display=True), + "xn--.example", + ) + + # Uppercase A-label prefix: the kept label is lowercased to match + # what a successful ulabel() call would have returned. + self.assertEqual( + idna.decode("XN--POKXNCVKS.example", display=True), + "xn--pokxncvks.example", + ) + + # display must not swallow errors for non-xn-- labels. + self.assertRaises( + idna.IDNAError, + idna.decode, + "-bad.example", + display=True, + ) + + # display should be a no-op for fully valid input. + self.assertEqual( + idna.decode("xn--zckzah.xn--zckzah", display=True), + "テスト.テスト", + ) + + # Trailing dot preserved under display recovery. + self.assertEqual( + idna.decode("xn--pokxncvks.", display=True), + "xn--pokxncvks.", + ) + + # Bytes input is supported, matching decode()'s normal contract. + self.assertEqual( + idna.decode(b"a.b.c.xn--pokxncvks", display=True), + "a.b.c.xn--pokxncvks", + ) + if __name__ == "__main__": unittest.main() diff --git a/contrib/python/idna/py3/tests/test_idna_cli.py b/contrib/python/idna/py3/tests/test_idna_cli.py new file mode 100644 index 00000000000..fa79aa66b9a --- /dev/null +++ b/contrib/python/idna/py3/tests/test_idna_cli.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python + +import io +import subprocess +import sys +import unittest +from contextlib import contextmanager, redirect_stderr, redirect_stdout +from unittest import mock + +from idna import cli +from idna.package_data import __version__ + + +def run_cli(*argv, stdin=None): + """Invoke ``cli.main`` with ``argv`` and capture (rc, stdout, stderr). + + Pass ``stdin`` as a string to simulate a piped stdin; omit it to + simulate an interactive terminal. + """ + out = io.StringIO() + err = io.StringIO() + with redirect_stdout(out), redirect_stderr(err), _patched_stdin(stdin): + rc = cli.main(list(argv)) + return rc, out.getvalue(), err.getvalue() + + +class _FakeStdin(io.StringIO): + """``io.StringIO`` with a configurable ``isatty`` result.""" + + def __init__(self, data: str, is_tty: bool) -> None: + super().__init__(data) + self._is_tty = is_tty + + def isatty(self) -> bool: + return self._is_tty + + +@contextmanager +def _patched_stdin(data): + """Replace ``sys.stdin`` with a fake stream and isatty() result.""" + stream = _FakeStdin("" if data is None else data, is_tty=data is None) + with mock.patch.object(sys, "stdin", stream): + yield + + +class CLIAutoDetectTests(unittest.TestCase): + def test_autodetect_encodes_unicode_input(self): + rc, out, err = run_cli("пример.рф") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--e1afmkfd.xn--p1ai") + self.assertEqual(err, "") + + def test_autodetect_decodes_alabel(self): + rc, out, _ = run_cli("xn--e1afmkfd.xn--p1ai") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "пример.рф") + + def test_autodetect_decodes_when_any_label_is_alabel(self): + # Mixed label: at least one xn-- label triggers decode mode. + rc, out, _ = run_cli("xn--11b5bs3a9aj6g.example") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "परीक्षा.example") + + def test_autodetect_encodes_plain_ascii_input(self): + rc, out, _ = run_cli("example.com") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "example.com") + + def test_looks_like_alabel_is_case_insensitive(self): + self.assertTrue(cli._looks_like_alabel("XN--ZCKZAH")) + self.assertTrue(cli._looks_like_alabel("foo.Xn--ZcKzAh")) + self.assertFalse(cli._looks_like_alabel("example.com")) + self.assertFalse(cli._looks_like_alabel("xnfoo.example")) + + def test_looks_like_alabel_handles_unicode_dots(self): + # IDEOGRAPHIC FULL STOP should split labels for the purpose of + # spotting an A-label, matching the behavior of idna.encode/decode. + self.assertTrue(cli._looks_like_alabel("foo。xn--zckzah")) + + +class CLIExplicitModeTests(unittest.TestCase): + def test_encode_flag_forces_encode(self): + rc, out, _ = run_cli("-e", "παράδειγμα") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--hxajbheg2az3al") + + def test_long_encode_flag_forces_encode(self): + rc, out, _ = run_cli("--encode", "한국") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--3e0b707e") + + def test_decode_flag_forces_decode(self): + rc, out, _ = run_cli("-d", "xn--hxajbheg2az3al") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "παράδειγμα") + + def test_long_decode_flag_forces_decode(self): + rc, out, _ = run_cli("--decode", "xn--3e0b707e") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "한국") + + def test_encode_and_decode_are_mutually_exclusive(self): + # argparse exits via SystemExit (code 2) for mutually-exclusive errors. + with self.assertRaises(SystemExit) as ctx, redirect_stderr(io.StringIO()): + cli.main(["-e", "-d", "example.com"]) + self.assertEqual(ctx.exception.code, 2) + + +class CLIUTS46Tests(unittest.TestCase): + def test_uts46_is_applied_by_default(self): + # Uppercase Greek is not PVALID under IDNA 2008 alone; UTS #46 + # case-folds it so encoding succeeds. + rc, out, _ = run_cli("ΠΑΡΆΔΕΙΓΜΑ.ΕΛ") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--hxajbheg2az3al.xn--qxam") + + def test_strict_disables_uts46(self): + rc, out, err = run_cli("--strict", "ΠΑΡΆΔΕΙΓΜΑ.ΕΛ") + self.assertEqual(rc, 1) + self.assertEqual(out, "") + self.assertIn("encode failed", err) + self.assertIn("U+03A0", err) + + def test_strict_does_not_affect_valid_inputs(self): + rc, out, _ = run_cli("--strict", "пример.рф") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--e1afmkfd.xn--p1ai") + + def test_uts46_applies_to_explicit_decode(self): + # Trailing FULLWIDTH FULL STOP (U+FF0E) is mapped to "." by UTS #46 + # so the input still decodes as a single A-label. + rc, out, _ = run_cli("-d", "xn--zckzah．") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "テスト.") + + +class CLIErrorHandlingTests(unittest.TestCase): + def test_invalid_input_returns_nonzero_and_writes_to_stderr(self): + rc, out, err = run_cli("foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out, "") + self.assertTrue(err.startswith("idna: encode failed for 'foo_bar':")) + + def test_invalid_alabel_decode_returns_nonzero(self): + # Trailing hyphen after the xn-- prefix is rejected by ulabel(). + rc, out, err = run_cli("-d", "xn--") + self.assertEqual(rc, 1) + self.assertEqual(out, "") + self.assertIn("decode failed", err) + + def test_missing_argument_with_tty_stdin_is_an_argparse_error(self): + # Interactive shell (no piped stdin) and no domain arg → argparse exits 2. + with self.assertRaises(SystemExit) as ctx, redirect_stderr(io.StringIO()), _patched_stdin(None): + cli.main([]) + self.assertEqual(ctx.exception.code, 2) + + +class CLIVersionTests(unittest.TestCase): + def test_version_flag(self): + buf = io.StringIO() + with self.assertRaises(SystemExit) as ctx, redirect_stdout(buf): + cli.main(["--version"]) + self.assertEqual(ctx.exception.code, 0) + self.assertIn(__version__, buf.getvalue()) + + +class CLIMultipleDomainsTests(unittest.TestCase): + def test_multiple_unicode_positional_domains_are_all_encoded(self): + rc, out, _ = run_cli("пример.рф", "παράδειγμα", "한국") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al", "xn--3e0b707e"], + ) + + def test_multiple_alabel_positional_domains_are_all_decoded(self): + rc, out, _ = run_cli("xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al", "xn--3e0b707e") + self.assertEqual(rc, 0) + self.assertEqual(out.splitlines(), ["пример.рф", "παράδειγμα", "한국"]) + + def test_mode_flag_applies_to_every_positional_domain(self): + rc, out, _ = run_cli("-e", "한국", "παράδειγμα") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--3e0b707e", "xn--hxajbheg2az3al"], + ) + + def test_failures_are_reported_per_domain_and_processing_continues(self): + rc, out, err = run_cli("пример.рф", "foo_bar", "παράδειγμα") + self.assertEqual(rc, 1) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al"], + ) + self.assertIn("encode failed for 'foo_bar'", err) + + +class CLIModeLockTests(unittest.TestCase): + """The first input picks the mode; the rest of the stream follows.""" + + def test_first_unicode_input_locks_subsequent_inputs_to_encode(self): + # 'foo_bar' is invalid in either direction; the stderr mode word + # reveals which mode the second input was processed in. + rc, out, err = run_cli("пример.рф", "foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["xn--e1afmkfd.xn--p1ai"]) + self.assertIn("encode failed for 'foo_bar'", err) + + def test_first_alabel_input_locks_subsequent_inputs_to_decode(self): + rc, out, err = run_cli("xn--zckzah", "foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["テスト"]) + self.assertIn("decode failed for 'foo_bar'", err) + + def test_explicit_mode_flag_overrides_first_input_heuristic(self): + # First input looks like an A-label but -e forces encode for everything. + rc, out, err = run_cli("-e", "xn--zckzah", "foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["xn--zckzah"]) + self.assertIn("encode failed for 'foo_bar'", err) + + def test_stdin_first_line_locks_mode_for_the_stream(self): + rc, out, err = run_cli(stdin="xn--zckzah\nfoo_bar\n") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["テスト"]) + self.assertIn("decode failed for 'foo_bar'", err) + + +class CLIStdinTests(unittest.TestCase): + def test_reads_one_domain_per_line_from_piped_stdin(self): + rc, out, _ = run_cli(stdin="пример.рф\nπαράδειγμα\n한국\n") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al", "xn--3e0b707e"], + ) + + def test_blank_lines_in_stdin_are_skipped(self): + rc, out, _ = run_cli(stdin="\nпример.рф\n\n \nπαράδειγμα\n") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al"], + ) + + def test_empty_stdin_exits_cleanly_with_no_output(self): + rc, out, err = run_cli(stdin="") + self.assertEqual(rc, 0) + self.assertEqual(out, "") + self.assertEqual(err, "") + + def test_positional_domains_take_precedence_over_piped_stdin(self): + rc, out, _ = run_cli("example.com", stdin="ignored.example\n") + self.assertEqual(rc, 0) + self.assertEqual(out.splitlines(), ["example.com"]) + + def test_stdin_errors_continue_processing_and_set_exit_code(self): + rc, out, err = run_cli(stdin="пример.рф\nfoo_bar\nπαράδειγμα\n") + self.assertEqual(rc, 1) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al"], + ) + self.assertIn("encode failed for 'foo_bar'", err) + + def test_decode_flag_applies_to_every_stdin_line(self): + rc, out, _ = run_cli("-d", stdin="xn--hxajbheg2az3al\nxn--3e0b707e\n") + self.assertEqual(rc, 0) + self.assertEqual(out.splitlines(), ["παράδειγμα", "한국"]) + + +class CLIModuleEntryTests(unittest.TestCase): + def test_python_dash_m_idna_runs_cli(self): + # End-to-end: spawn ``python -m idna`` and check the round trip. + result = subprocess.run( + [sys.executable, "-m", "idna", "xn--zckzah"], + check=False, + capture_output=True, + text=True, + env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout.strip(), "テスト") + self.assertEqual(result.stderr, "") + + def test_python_dash_m_idna_reads_piped_stdin(self): + # End-to-end: pipe a list of A-labels and confirm they all decode. + result = subprocess.run( + [sys.executable, "-m", "idna"], + input="xn--e1afmkfd.xn--p1ai\nxn--zckzah\n", + check=False, + capture_output=True, + text=True, + env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual( + result.stdout.splitlines(), + ["пример.рф", "テスト"], + ) + self.assertEqual(result.stderr, "") + + def test_python_dash_m_idna_reports_errors(self): + result = subprocess.run( + [sys.executable, "-m", "idna", "foo_bar"], + check=False, + capture_output=True, + text=True, + env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(result.stdout, "") + self.assertIn("encode failed", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/python/idna/py3/ya.make b/contrib/python/idna/py3/ya.make index 6f0b1006c52..4e3ded2a921 100644 --- a/contrib/python/idna/py3/ya.make +++ b/contrib/python/idna/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.15) +VERSION(3.18) LICENSE(BSD-3-Clause) @@ -11,6 +11,8 @@ NO_LINT() PY_SRCS( TOP_LEVEL idna/__init__.py + idna/__main__.py + idna/cli.py idna/codec.py idna/compat.py idna/core.py @@ -23,6 +25,7 @@ PY_SRCS( RESOURCE_FILES( PREFIX contrib/python/idna/py3/ .dist-info/METADATA + .dist-info/entry_points.txt .dist-info/top_level.txt idna/py.typed ) diff --git a/contrib/python/propcache/.dist-info/METADATA b/contrib/python/propcache/.dist-info/METADATA new file mode 100644 index 00000000000..37cb00053d8 --- /dev/null +++ b/contrib/python/propcache/.dist-info/METADATA @@ -0,0 +1,539 @@ +Metadata-Version: 2.4 +Name: propcache +Version: 0.5.2 +Summary: Accelerated property cache +Home-page: https://github.com/aio-libs/propcache +Author: Andrew Svetlov +Author-email: [email protected] +Maintainer: aiohttp team <[email protected]> +Maintainer-email: [email protected] +License: Apache-2.0 +Project-URL: Chat: Matrix, https://matrix.to/#/#aio-libs:matrix.org +Project-URL: Chat: Matrix Space, https://matrix.to/#/#aio-libs-space:matrix.org +Project-URL: CI: GitHub Workflows, https://github.com/aio-libs/propcache/actions?query=branch:master +Project-URL: Code of Conduct, https://github.com/aio-libs/.github/blob/master/CODE_OF_CONDUCT.md +Project-URL: Coverage: codecov, https://codecov.io/github/aio-libs/propcache +Project-URL: Docs: Changelog, https://propcache.readthedocs.io/en/latest/changes/ +Project-URL: Docs: RTD, https://propcache.readthedocs.io +Project-URL: GitHub: issues, https://github.com/aio-libs/propcache/issues +Project-URL: GitHub: repo, https://github.com/aio-libs/propcache +Keywords: cython,cext,propcache +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: Cython +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Topic :: Internet :: WWW/HTTP +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Requires-Python: >=3.10 +Description-Content-Type: text/x-rst +License-File: LICENSE +License-File: NOTICE +Dynamic: license-file + +propcache +========= + +The module provides a fast implementation of cached properties for Python 3.10+. + +.. image:: https://github.com/aio-libs/propcache/actions/workflows/ci-cd.yml/badge.svg + :target: https://github.com/aio-libs/propcache/actions?query=workflow%3ACI + :align: right + +.. image:: https://codecov.io/gh/aio-libs/propcache/branch/master/graph/badge.svg + :target: https://codecov.io/gh/aio-libs/propcache + +.. image:: https://badge.fury.io/py/propcache.svg + :target: https://badge.fury.io/py/propcache + + +.. image:: https://readthedocs.org/projects/propcache/badge/?version=latest + :target: https://propcache.readthedocs.io + + +.. image:: https://img.shields.io/pypi/pyversions/propcache.svg + :target: https://pypi.python.org/pypi/propcache + +.. image:: https://img.shields.io/matrix/aio-libs:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat + :target: https://matrix.to/#/%23aio-libs:matrix.org + :alt: Matrix Room — #aio-libs:matrix.org + +.. image:: https://img.shields.io/matrix/aio-libs-space:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs-space%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat + :target: https://matrix.to/#/%23aio-libs-space:matrix.org + :alt: Matrix Space — #aio-libs-space:matrix.org + +Introduction +------------ + +The API is designed to be nearly identical to the built-in ``functools.cached_property`` class, +except for the additional ``under_cached_property`` class which uses ``self._cache`` +instead of ``self.__dict__`` to store the cached values and prevents ``__set__`` from being called. + +For full documentation please read https://propcache.readthedocs.io. + +Installation +------------ + +:: + + $ pip install propcache + +The library is Python 3 only! + +PyPI contains binary wheels for Linux, Windows and MacOS. If you want to install +``propcache`` on another operating system where wheels are not provided, +the the tarball will be used to compile the library from +the source code. It requires a C compiler and and Python headers installed. + +To skip the compilation you must explicitly opt-in by using a PEP 517 +configuration setting ``pure-python``, or setting the ``PROPCACHE_NO_EXTENSIONS`` +environment variable to a non-empty value, e.g.: + +.. code-block:: console + + $ pip install propcache --config-settings=pure-python=false + +Please note that the pure-Python (uncompiled) version is much slower. However, +PyPy always uses a pure-Python implementation, and, as such, it is unaffected +by this variable. + + +API documentation +------------------ + +The documentation is located at https://propcache.readthedocs.io. + +Source code +----------- + +The project is hosted on GitHub_ + +Please file an issue on the `bug tracker +<https://github.com/aio-libs/propcache/issues>`_ if you have found a bug +or have some suggestion in order to improve the library. + +Discussion list +--------------- + +*aio-libs* google group: https://groups.google.com/forum/#!forum/aio-libs + +Feel free to post your questions and ideas here. + + +Authors and License +------------------- + +The ``propcache`` package is derived from ``yarl`` which is written by Andrew Svetlov. + +It's *Apache 2* licensed and freely available. + + +.. _GitHub: https://github.com/aio-libs/propcache + +========= +Changelog +========= + +.. + You should *NOT* be adding new change log entries to this file, this + file is managed by towncrier. You *may* edit previous change logs to + fix problems like typo corrections or such. + To add a new change log entry, please see + https://pip.pypa.io/en/latest/development/#adding-a-news-entry + we named the news folder "changes". + + WARNING: Don't drop the next directive! + +.. towncrier release notes start + +0.5.2 +===== + +*(2026-05-08)* + + +No significant changes. + + +---- + + +0.5.1 +===== + +*(2026-05-08)* + + +No significant changes. + + +---- + + +0.5.0 +===== + +*(2026-05-08)* + + +Features +-------- + +- Added support for newer type hints and remove ``Optional`` and ``Union`` from all annotations + -- by `@Vizonex <https://github.com/sponsors/Vizonex>`__ + + *Related issues and pull requests on GitHub:* + `#193 <https://github.com/aio-libs/propcache/issues/193>`__. + + +Removals and backward incompatible breaking changes +--------------------------------------------------- + +- Dropped support for Python 3.9 as it has reached end of life. + + *Related issues and pull requests on GitHub:* + `#216 <https://github.com/aio-libs/propcache/issues/216>`__. + + +Packaging updates and notes for downstreams +------------------------------------------- + +- Changed the Cython build dependency from ``~= 3.1.0`` to ``>= 3.2.0``, + removing the upper version bound to avoid conflicts for downstream packagers + -- by `@jameshilliard <https://github.com/sponsors/jameshilliard>`__ and `@gundalow <https://github.com/sponsors/gundalow>`__. + + The upstream Cython version is pinned to 3.2.4 in the CI/CD environment. + + *Related issues and pull requests on GitHub:* + `#184 <https://github.com/aio-libs/propcache/issues/184>`__, `#188 <https://github.com/aio-libs/propcache/issues/188>`__, `#214 <https://github.com/aio-libs/propcache/issues/214>`__. + +- Start building and shipping riscv64 wheels + -- by `@justeph <https://github.com/sponsors/justeph>`__. + + *Related issues and pull requests on GitHub:* + `#194 <https://github.com/aio-libs/propcache/issues/194>`__. + +- The `PEP 517 <https://peps.python.org/pep-517>`__ build backend now supports a new ``build-inplace`` + config setting (and ``PROPCACHE_BUILD_INPLACE`` environment variable) + for controlling whether to build the project in-tree or in a + temporary directory. It only affects wheels and is set up to build + in a temporary directory by default. It does not affect editable + wheel builds; they will keep being built in-tree regardless. + + Here's an example of using this setting: + + .. code-block:: console + + $ python -m build --config-setting=build-inplace=true + + Additionally, when building wheels in an automatically created + temporary directory, the build backend now normalizes the + respective file system path to a deterministic source checkout + directory by injecting the ``-ffile-prefix-map`` compiler option + into the ``CFLAGS`` environment variable, as suggested by known + `reproducible build practices + <https://reproducible-builds.org/docs/build-path/>`__. + + The effect is that downstreams will get reproducible build results. + + *Related issues and pull requests on GitHub:* + `#218 <https://github.com/aio-libs/propcache/issues/218>`__. + + +---- + + +0.4.1 +===== + +*(2025-10-08)* + + +Bug fixes +--------- + +- Fixed reference leak caused by ``Py_INCREF`` because Cython has its own reference counter systems -- by `@Vizonex <https://github.com/sponsors/Vizonex>`__. + + *Related issues and pull requests on GitHub:* + `#162 <https://github.com/aio-libs/propcache/issues/162>`__. + + +Contributor-facing changes +-------------------------- + +- Fixes the default value for the ``os`` + parameter in ``reusable-build-wheel.yml`` + to be ``ubuntu-latest`` instead of + ``ubuntu``. + + *Related issues and pull requests on GitHub:* + `#155 <https://github.com/aio-libs/propcache/issues/155>`__. + + +---- + + +0.4.0 +===== + +*(2025-10-04)* + + +Features +-------- + +- Optimized propcache by replacing sentinel ``object`` for checking if + the ``object`` is ``NULL`` and changed ``dict`` API for + Python C-API -- by `@Vizonex <https://github.com/sponsors/Vizonex>`__. + + *Related issues and pull requests on GitHub:* + `#121 <https://github.com/aio-libs/propcache/issues/121>`__. + + +Contributor-facing changes +-------------------------- + +- Builds have been added for arm64 Windows + wheels and the ``reusable-build-wheel.yml`` + workflow has been modified to allow for + an OS value (``windows-11-arm``) which + does not include the ``-latest`` postfix + -- by `@finnagin <https://github.com/sponsors/finnagin>`__. + + *Related issues and pull requests on GitHub:* + `#133 <https://github.com/aio-libs/propcache/issues/133>`__. + +- Added CI for CPython 3.14 -- by `@kumaraditya303 <https://github.com/sponsors/kumaraditya303>`__. + + *Related issues and pull requests on GitHub:* + `#140 <https://github.com/aio-libs/propcache/issues/140>`__. + + +---- + + +0.3.2 +===== + +*(2025-06-09)* + + +Improved documentation +---------------------- + +- Fixed incorrect decorator usage in the ``~propcache.api.under_cached_property`` example code -- by `@meanmail <https://github.com/sponsors/meanmail>`__. + + *Related issues and pull requests on GitHub:* + `#109 <https://github.com/aio-libs/propcache/issues/109>`__. + + +Packaging updates and notes for downstreams +------------------------------------------- + +- Updated to use Cython 3.1 universally across the build path -- by `@lysnikolaou <https://github.com/sponsors/lysnikolaou>`__. + + *Related issues and pull requests on GitHub:* + `#117 <https://github.com/aio-libs/propcache/issues/117>`__. + +- Made Cython line tracing opt-in via the ``with-cython-tracing`` build config setting -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + Previously, line tracing was enabled by default in ``pyproject.toml``, which caused build issues for some users and made wheels nearly twice as slow. + + Now line tracing is only enabled when explicitly requested via ``pip install . --config-setting=with-cython-tracing=true`` or by setting the ``PROPCACHE_CYTHON_TRACING`` environment variable. + + *Related issues and pull requests on GitHub:* + `#118 <https://github.com/aio-libs/propcache/issues/118>`__. + + +---- + + +0.3.1 +===== + +*(2025-03-25)* + + +Bug fixes +--------- + +- Improved typing annotations, fixing some type errors under correct usage + and improving typing robustness generally -- by `@Dreamsorcerer <https://github.com/sponsors/Dreamsorcerer>`__. + + *Related issues and pull requests on GitHub:* + `#103 <https://github.com/aio-libs/propcache/issues/103>`__. + + +---- + + +0.3.0 +===== + +*(2025-02-20)* + + +Features +-------- + +- Implemented support for the free-threaded build of CPython 3.13 -- by `@lysnikolaou <https://github.com/sponsors/lysnikolaou>`__. + + *Related issues and pull requests on GitHub:* + `#84 <https://github.com/aio-libs/propcache/issues/84>`__. + + +Packaging updates and notes for downstreams +------------------------------------------- + +- Started building wheels for the free-threaded build of CPython 3.13 -- by `@lysnikolaou <https://github.com/sponsors/lysnikolaou>`__. + + *Related issues and pull requests on GitHub:* + `#84 <https://github.com/aio-libs/propcache/issues/84>`__. + + +Contributor-facing changes +-------------------------- + +- GitHub Actions CI/CD is now configured to manage caching pip-ecosystem + dependencies using `re-actors/cache-python-deps`_ -- an action by + `@webknjaz <https://github.com/sponsors/webknjaz>`__ that takes into account ABI stability and the exact + version of Python runtime. + + .. _`re-actors/cache-python-deps`: + https://github.com/marketplace/actions/cache-python-deps + + *Related issues and pull requests on GitHub:* + `#93 <https://github.com/aio-libs/propcache/issues/93>`__. + + +---- + + +0.2.1 +===== + +*(2024-12-01)* + + +Bug fixes +--------- + +- Stopped implicitly allowing the use of Cython pre-release versions when + building the distribution package -- by `@ajsanchezsanz <https://github.com/sponsors/ajsanchezsanz>`__ and + `@markgreene74 <https://github.com/sponsors/markgreene74>`__. + + *Related commits on GitHub:* + `64df0a6 <https://github.com/aio-libs/propcache/commit/64df0a6>`__. + +- Fixed ``wrapped`` and ``func`` not being accessible in the Cython versions of ``propcache.api.cached_property`` and ``propcache.api.under_cached_property`` decorators -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#72 <https://github.com/aio-libs/propcache/issues/72>`__. + + +Removals and backward incompatible breaking changes +--------------------------------------------------- + +- Removed support for Python 3.8 as it has reached end of life -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#57 <https://github.com/aio-libs/propcache/issues/57>`__. + + +Packaging updates and notes for downstreams +------------------------------------------- + +- Stopped implicitly allowing the use of Cython pre-release versions when + building the distribution package -- by `@ajsanchezsanz <https://github.com/sponsors/ajsanchezsanz>`__ and + `@markgreene74 <https://github.com/sponsors/markgreene74>`__. + + *Related commits on GitHub:* + `64df0a6 <https://github.com/aio-libs/propcache/commit/64df0a6>`__. + + +---- + + +0.2.0 +===== + +*(2024-10-07)* + + +Bug fixes +--------- + +- Fixed loading the C-extensions on Python 3.8 -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#26 <https://github.com/aio-libs/propcache/issues/26>`__. + + +Features +-------- + +- Improved typing for the ``propcache.api.under_cached_property`` decorator -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#38 <https://github.com/aio-libs/propcache/issues/38>`__. + + +Improved documentation +---------------------- + +- Added API documentation for the ``propcache.api.cached_property`` and ``propcache.api.under_cached_property`` decorators -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#16 <https://github.com/aio-libs/propcache/issues/16>`__. + + +Packaging updates and notes for downstreams +------------------------------------------- + +- Moved ``propcache.api.under_cached_property`` and ``propcache.api.cached_property`` to `propcache.api` -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + Both decorators remain importable from the top-level package, however importing from `propcache.api` is now the recommended way to use them. + + *Related issues and pull requests on GitHub:* + `#19 <https://github.com/aio-libs/propcache/issues/19>`__, `#24 <https://github.com/aio-libs/propcache/issues/24>`__, `#32 <https://github.com/aio-libs/propcache/issues/32>`__. + +- Converted project to use a src layout -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#22 <https://github.com/aio-libs/propcache/issues/22>`__, `#29 <https://github.com/aio-libs/propcache/issues/29>`__, `#37 <https://github.com/aio-libs/propcache/issues/37>`__. + + +---- + + +0.1.0 +===== + +*(2024-10-03)* + + +Features +-------- + +- Added ``armv7l`` wheels -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#5 <https://github.com/aio-libs/propcache/issues/5>`__. + + +---- + + +0.0.0 +===== + +*(2024-10-02)* + + +- Initial release. diff --git a/contrib/python/propcache/.dist-info/top_level.txt b/contrib/python/propcache/.dist-info/top_level.txt new file mode 100644 index 00000000000..8c9accf6226 --- /dev/null +++ b/contrib/python/propcache/.dist-info/top_level.txt @@ -0,0 +1 @@ +propcache diff --git a/contrib/python/propcache/.yandex_meta/yamaker.yaml b/contrib/python/propcache/.yandex_meta/yamaker.yaml new file mode 100644 index 00000000000..4daee40a45c --- /dev/null +++ b/contrib/python/propcache/.yandex_meta/yamaker.yaml @@ -0,0 +1,4 @@ +mark_as_cython_c: + - propcache/_helpers_c.pyx +exclude: + - tests/test_benchmarks.py diff --git a/contrib/python/propcache/LICENSE b/contrib/python/propcache/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/contrib/python/propcache/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/contrib/python/propcache/NOTICE b/contrib/python/propcache/NOTICE new file mode 100644 index 00000000000..fa53b2b138d --- /dev/null +++ b/contrib/python/propcache/NOTICE @@ -0,0 +1,13 @@ + Copyright 2016-2021, Andrew Svetlov and aio-libs team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/contrib/python/propcache/README.rst b/contrib/python/propcache/README.rst new file mode 100644 index 00000000000..ce10fbf8037 --- /dev/null +++ b/contrib/python/propcache/README.rst @@ -0,0 +1,98 @@ +propcache +========= + +The module provides a fast implementation of cached properties for Python 3.10+. + +.. image:: https://github.com/aio-libs/propcache/actions/workflows/ci-cd.yml/badge.svg + :target: https://github.com/aio-libs/propcache/actions?query=workflow%3ACI + :align: right + +.. image:: https://codecov.io/gh/aio-libs/propcache/branch/master/graph/badge.svg + :target: https://codecov.io/gh/aio-libs/propcache + +.. image:: https://badge.fury.io/py/propcache.svg + :target: https://badge.fury.io/py/propcache + + +.. image:: https://readthedocs.org/projects/propcache/badge/?version=latest + :target: https://propcache.readthedocs.io + + +.. image:: https://img.shields.io/pypi/pyversions/propcache.svg + :target: https://pypi.python.org/pypi/propcache + +.. image:: https://img.shields.io/matrix/aio-libs:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat + :target: https://matrix.to/#/%23aio-libs:matrix.org + :alt: Matrix Room — #aio-libs:matrix.org + +.. image:: https://img.shields.io/matrix/aio-libs-space:matrix.org?label=Discuss%20on%20Matrix%20at%20%23aio-libs-space%3Amatrix.org&logo=matrix&server_fqdn=matrix.org&style=flat + :target: https://matrix.to/#/%23aio-libs-space:matrix.org + :alt: Matrix Space — #aio-libs-space:matrix.org + +Introduction +------------ + +The API is designed to be nearly identical to the built-in ``functools.cached_property`` class, +except for the additional ``under_cached_property`` class which uses ``self._cache`` +instead of ``self.__dict__`` to store the cached values and prevents ``__set__`` from being called. + +For full documentation please read https://propcache.readthedocs.io. + +Installation +------------ + +:: + + $ pip install propcache + +The library is Python 3 only! + +PyPI contains binary wheels for Linux, Windows and MacOS. If you want to install +``propcache`` on another operating system where wheels are not provided, +the the tarball will be used to compile the library from +the source code. It requires a C compiler and and Python headers installed. + +To skip the compilation you must explicitly opt-in by using a PEP 517 +configuration setting ``pure-python``, or setting the ``PROPCACHE_NO_EXTENSIONS`` +environment variable to a non-empty value, e.g.: + +.. code-block:: console + + $ pip install propcache --config-settings=pure-python=false + +Please note that the pure-Python (uncompiled) version is much slower. However, +PyPy always uses a pure-Python implementation, and, as such, it is unaffected +by this variable. + + +API documentation +------------------ + +The documentation is located at https://propcache.readthedocs.io. + +Source code +----------- + +The project is hosted on GitHub_ + +Please file an issue on the `bug tracker +<https://github.com/aio-libs/propcache/issues>`_ if you have found a bug +or have some suggestion in order to improve the library. + +Discussion list +--------------- + +*aio-libs* google group: https://groups.google.com/forum/#!forum/aio-libs + +Feel free to post your questions and ideas here. + + +Authors and License +------------------- + +The ``propcache`` package is derived from ``yarl`` which is written by Andrew Svetlov. + +It's *Apache 2* licensed and freely available. + + +.. _GitHub: https://github.com/aio-libs/propcache diff --git a/contrib/python/propcache/propcache/__init__.py b/contrib/python/propcache/propcache/__init__.py new file mode 100644 index 00000000000..bbfd78a3628 --- /dev/null +++ b/contrib/python/propcache/propcache/__init__.py @@ -0,0 +1,32 @@ +"""propcache: An accelerated property cache for Python classes.""" + +from typing import TYPE_CHECKING + +_PUBLIC_API = ("cached_property", "under_cached_property") + +__version__ = "0.5.2" +__all__ = () + +# Imports have moved to `propcache.api` in 0.2.0+. +# This module is now a facade for the API. +if TYPE_CHECKING: + from .api import cached_property as cached_property # noqa: F401 + from .api import under_cached_property as under_cached_property # noqa: F401 + + +def _import_facade(attr: str) -> object: + """Import the public API from the `api` module.""" + if attr in _PUBLIC_API: + from . import api # pylint: disable=import-outside-toplevel + + return getattr(api, attr) + raise AttributeError(f"module '{__package__}' has no attribute '{attr}'") + + +def _dir_facade() -> list[str]: + """Include the public API in the module's dir() output.""" + return [*_PUBLIC_API, *globals().keys()] + + +__getattr__ = _import_facade +__dir__ = _dir_facade diff --git a/contrib/python/yarl/yarl/_helpers.py b/contrib/python/propcache/propcache/_helpers.py index ac01158c7f5..1e52895c151 100644 --- a/contrib/python/yarl/yarl/_helpers.py +++ b/contrib/python/propcache/propcache/_helpers.py @@ -2,10 +2,10 @@ import os import sys from typing import TYPE_CHECKING -__all__ = ("cached_property",) +__all__ = ("cached_property", "under_cached_property") -NO_EXTENSIONS = bool(os.environ.get("YARL_NO_EXTENSIONS")) # type: bool +NO_EXTENSIONS = bool(os.environ.get("PROPCACHE_NO_EXTENSIONS")) # type: bool if sys.implementation.name != "cpython": NO_EXTENSIONS = True @@ -13,19 +13,27 @@ if sys.implementation.name != "cpython": # isort: off if TYPE_CHECKING: from ._helpers_py import cached_property as cached_property_py + from ._helpers_py import under_cached_property as under_cached_property_py cached_property = cached_property_py + under_cached_property = under_cached_property_py elif not NO_EXTENSIONS: # pragma: no branch try: - from ._helpers_c import cached_property as cached_property_c # type: ignore[attr-defined, unused-ignore] # noqa: E501 + from ._helpers_c import cached_property as cached_property_c # type: ignore[attr-defined, unused-ignore] + from ._helpers_c import under_cached_property as under_cached_property_c # type: ignore[attr-defined, unused-ignore] cached_property = cached_property_c + under_cached_property = under_cached_property_c except ImportError: # pragma: no cover from ._helpers_py import cached_property as cached_property_py + from ._helpers_py import under_cached_property as under_cached_property_py cached_property = cached_property_py # type: ignore[assignment, misc] + under_cached_property = under_cached_property_py else: from ._helpers_py import cached_property as cached_property_py + from ._helpers_py import under_cached_property as under_cached_property_py cached_property = cached_property_py # type: ignore[assignment, misc] + under_cached_property = under_cached_property_py # isort: on diff --git a/contrib/python/propcache/propcache/_helpers_c.pyx b/contrib/python/propcache/propcache/_helpers_c.pyx new file mode 100644 index 00000000000..9e9e5589b95 --- /dev/null +++ b/contrib/python/propcache/propcache/_helpers_c.pyx @@ -0,0 +1,103 @@ +# cython: language_level=3, freethreading_compatible=True +from types import GenericAlias + +from cpython.dict cimport PyDict_GetItem +from cpython.object cimport PyObject + + +cdef extern from "Python.h": + # Call a callable Python object callable with exactly + # 1 positional argument arg and no keyword arguments. + # Return the result of the call on success, or raise + # an exception and return NULL on failure. + PyObject* PyObject_CallOneArg( + object callable, object arg + ) except NULL + int PyDict_SetItem( + object dict, object key, PyObject* value + ) except -1 + void Py_DECREF(PyObject*) + + +cdef class under_cached_property: + """Use as a class method decorator. It operates almost exactly like + the Python `@property` decorator, but it puts the result of the + method it decorates into the instance dict after the first call, + effectively replacing the function it decorates with an instance + variable. It is, in Python parlance, a data descriptor. + + """ + + cdef readonly object wrapped + cdef object name + + def __init__(self, object wrapped): + self.wrapped = wrapped + self.name = wrapped.__name__ + + @property + def __doc__(self): + return self.wrapped.__doc__ + + def __get__(self, object inst, owner): + if inst is None: + return self + cdef dict cache = inst._cache + cdef PyObject* val = PyDict_GetItem(cache, self.name) + if val == NULL: + val = PyObject_CallOneArg(self.wrapped, inst) + PyDict_SetItem(cache, self.name, val) + Py_DECREF(val) + return <object>val + + def __set__(self, inst, value): + raise AttributeError("cached property is read-only") + + __class_getitem__ = classmethod(GenericAlias) + + +cdef class cached_property: + """Use as a class method decorator. It operates almost exactly like + the Python `@property` decorator, but it puts the result of the + method it decorates into the instance dict after the first call, + effectively replacing the function it decorates with an instance + variable. It is, in Python parlance, a data descriptor. + + """ + + cdef readonly object func + cdef object name + + def __init__(self, func): + self.func = func + self.name = None + + @property + def __doc__(self): + return self.func.__doc__ + + def __set_name__(self, owner, object name): + if self.name is None: + self.name = name + elif name != self.name: + raise TypeError( + "Cannot assign the same cached_property to two different names " + f"({self.name!r} and {name!r})." + ) + + def __get__(self, inst, owner): + if inst is None: + return self + if self.name is None: + raise TypeError( + "Cannot use cached_property instance" + " without calling __set_name__ on it.") + cdef dict cache = inst.__dict__ + cdef PyObject* val = PyDict_GetItem(cache, self.name) + if val is NULL: + val = PyObject_CallOneArg(self.func, inst) + PyDict_SetItem(cache, self.name, val) + Py_DECREF(val) + return <object>val + + __class_getitem__ = classmethod(GenericAlias) diff --git a/contrib/python/propcache/propcache/_helpers_py.py b/contrib/python/propcache/propcache/_helpers_py.py new file mode 100644 index 00000000000..db5b8d6378f --- /dev/null +++ b/contrib/python/propcache/propcache/_helpers_py.py @@ -0,0 +1,64 @@ +"""Various helper functions.""" + +from __future__ import annotations + +import sys +from collections.abc import Callable, Mapping +from functools import cached_property +from typing import Any, Generic, Protocol, TypeVar, overload + +__all__ = ("under_cached_property", "cached_property") + + +if sys.version_info >= (3, 11): + from typing import Self +else: + Self = Any + +_T = TypeVar("_T") +# We use Mapping to make it possible to use TypedDict, but this isn't +# technically type safe as we need to assign into the dict. +_Cache = TypeVar("_Cache", bound=Mapping[str, Any]) + + +class _CacheImpl(Protocol[_Cache]): + _cache: _Cache + + +class under_cached_property(Generic[_T]): + """Use as a class method decorator. + + It operates almost exactly like + the Python `@property` decorator, but it puts the result of the + method it decorates into the instance dict after the first call, + effectively replacing the function it decorates with an instance + variable. It is, in Python parlance, a data descriptor. + """ + + def __init__(self, wrapped: Callable[[Any], _T]) -> None: + self.wrapped = wrapped + self.__doc__ = wrapped.__doc__ + self.name = wrapped.__name__ + + @overload + def __get__(self, inst: None, owner: type[object] | None = None) -> Self: ... + + @overload + def __get__( + self, inst: _CacheImpl[Any], owner: type[object] | None = None + ) -> _T: ... + + def __get__( + self, inst: _CacheImpl[Any] | None, owner: type[object] | None = None + ) -> _T | Self: + if inst is None: + return self + try: + return inst._cache[self.name] # type: ignore[no-any-return] + except KeyError: + val = self.wrapped(inst) + inst._cache[self.name] = val + return val + + def __set__(self, inst: _CacheImpl[Any], value: _T) -> None: + raise AttributeError("cached property is read-only") diff --git a/contrib/python/propcache/propcache/api.py b/contrib/python/propcache/propcache/api.py new file mode 100644 index 00000000000..22389e6337f --- /dev/null +++ b/contrib/python/propcache/propcache/api.py @@ -0,0 +1,8 @@ +"""Public API of the property caching library.""" + +from ._helpers import cached_property, under_cached_property + +__all__ = ( + "cached_property", + "under_cached_property", +) diff --git a/contrib/python/propcache/propcache/py.typed b/contrib/python/propcache/propcache/py.typed new file mode 100644 index 00000000000..dcf2c804da5 --- /dev/null +++ b/contrib/python/propcache/propcache/py.typed @@ -0,0 +1 @@ +# Placeholder diff --git a/contrib/python/propcache/tests/conftest.py b/contrib/python/propcache/tests/conftest.py new file mode 100644 index 00000000000..ee43f339231 --- /dev/null +++ b/contrib/python/propcache/tests/conftest.py @@ -0,0 +1,115 @@ +import argparse +from dataclasses import dataclass +from functools import cached_property +from importlib import import_module +from types import ModuleType + +import pytest + +C_EXT_MARK = pytest.mark.c_extension + + +@dataclass(frozen=True) +class PropcacheImplementation: + """A facade for accessing importable propcache module variants. + + An instance essentially represents a c-extension or a pure-python module. + The actual underlying module is accessed dynamically through a property and + is cached. + + It also has a text tag depending on what variant it is, and a string + representation suitable for use in Pytest's test IDs via parametrization. + """ + + is_pure_python: bool + """A flag showing whether this is a pure-python module or a C-extension.""" + + @cached_property + def tag(self) -> str: + """Return a text representation of the pure-python attribute.""" + return "pure-python" if self.is_pure_python else "c-extension" + + @cached_property + def imported_module(self) -> ModuleType: + """Return a loaded importable containing a propcache variant.""" + importable_module = "_helpers_py" if self.is_pure_python else "_helpers_c" + return import_module(f"propcache.{importable_module}") + + def __str__(self) -> str: + """Render the implementation facade instance as a string.""" + return f"{self.tag}-module" + + + scope="session", + params=( + pytest.param( + PropcacheImplementation(is_pure_python=False), + marks=C_EXT_MARK, + ), + PropcacheImplementation(is_pure_python=True), + ), + ids=str, +) +def propcache_implementation(request: pytest.FixtureRequest) -> PropcacheImplementation: + """Return a propcache variant facade.""" + return request.param # type: ignore[no-any-return] + + [email protected](scope="session") +def propcache_module( + propcache_implementation: PropcacheImplementation, +) -> ModuleType: + """Return a pre-imported module containing a propcache variant.""" + return propcache_implementation.imported_module + + +def pytest_addoption( + parser: pytest.Parser, + pluginmanager: pytest.PytestPluginManager, +) -> None: + """Define a new ``--c-extensions`` flag. + + This lets the callers deselect tests executed against the C-extension + version of the ``propcache`` implementation. + """ + del pluginmanager + parser.addoption( + "--c-extensions", # disabled with `--no-c-extensions` + action=argparse.BooleanOptionalAction, + default=True, + dest="c_extensions", + help="Test C-extensions (on by default)", + ) + + +def pytest_collection_modifyitems( + session: pytest.Session, + config: pytest.Config, + items: list[pytest.Item], +) -> None: + """Deselect tests against C-extensions when requested via CLI.""" + test_c_extensions = config.getoption("--c-extensions") is True + + if test_c_extensions: + return + + selected_tests: list[pytest.Item] = [] + deselected_tests: list[pytest.Item] = [] + + for item in items: + c_ext = item.get_closest_marker(C_EXT_MARK.name) is not None + + target_items_list = deselected_tests if c_ext else selected_tests + target_items_list.append(item) + + config.hook.pytest_deselected(items=deselected_tests) + items[:] = selected_tests + + +def pytest_configure(config: pytest.Config) -> None: + """Declare the C-extension marker in config.""" + config.addinivalue_line( + "markers", + f"{C_EXT_MARK.name}: tests running against the C-extension implementation.", + ) diff --git a/contrib/python/propcache/tests/test_api.py b/contrib/python/propcache/tests/test_api.py new file mode 100644 index 00000000000..85beec3606a --- /dev/null +++ b/contrib/python/propcache/tests/test_api.py @@ -0,0 +1,11 @@ +"""Test we do not break the public API.""" + +from propcache import _helpers, api + + +def test_api() -> None: + """Verify the public API is accessible.""" + assert api.cached_property is not None + assert api.under_cached_property is not None + assert api.cached_property is _helpers.cached_property + assert api.under_cached_property is _helpers.under_cached_property diff --git a/contrib/python/propcache/tests/test_cached_property.py b/contrib/python/propcache/tests/test_cached_property.py new file mode 100644 index 00000000000..a46c715c31b --- /dev/null +++ b/contrib/python/propcache/tests/test_cached_property.py @@ -0,0 +1,226 @@ +import gc +import sys +from collections.abc import Callable +from operator import not_ +from typing import TYPE_CHECKING, Any, Protocol, TypeVar + +import pytest + +from propcache.api import cached_property + +IS_PYPY = hasattr(sys, "pypy_version_info") + +if sys.version_info >= (3, 11): + from typing import assert_type + +_T_co = TypeVar("_T_co", covariant=True) + + +class APIProtocol(Protocol): + def cached_property( + self, func: Callable[[Any], _T_co] + ) -> cached_property[_T_co]: ... + + +def test_cached_property(propcache_module: APIProtocol) -> None: + class A: + def __init__(self) -> None: + """Init.""" + + @propcache_module.cached_property + def prop(self) -> int: + return 1 + + a = A() + if sys.version_info >= (3, 11): + assert_type(a.prop, int) + assert a.prop == 1 + + +def test_cached_property_without_cache(propcache_module: APIProtocol) -> None: + class A: + + __slots__ = () + + def __init__(self) -> None: + pass + + @propcache_module.cached_property + def prop(self) -> None: + """Mock property.""" + + a = A() + + with pytest.raises(AttributeError): + a.prop = 123 # type: ignore[assignment] + + +def test_cached_property_check_without_cache(propcache_module: APIProtocol) -> None: + class A: + + __slots__ = () + + def __init__(self) -> None: + """Init.""" + + @propcache_module.cached_property + def prop(self) -> None: + """Mock property.""" + + a = A() + with pytest.raises((TypeError, AttributeError)): + assert a.prop == 1 + + +def test_cached_property_caching(propcache_module: APIProtocol) -> None: + class A: + def __init__(self) -> None: + """Init.""" + + @propcache_module.cached_property + def prop(self) -> int: + """Docstring.""" + return 1 + + a = A() + assert a.prop == 1 + + +def test_cached_property_class_docstring(propcache_module: APIProtocol) -> None: + class A: + def __init__(self) -> None: + """Init.""" + + @propcache_module.cached_property + def prop(self) -> None: + """Docstring.""" + + if TYPE_CHECKING: + assert isinstance(A.prop, cached_property) + else: + assert isinstance(A.prop, propcache_module.cached_property) + assert "Docstring." == A.prop.__doc__ + + +def test_set_name(propcache_module: APIProtocol) -> None: + """Test that the __set_name__ method is called and checked.""" + + class A: + + @propcache_module.cached_property + def prop(self) -> None: + """Docstring.""" + + A.prop.__set_name__(A, "prop") + + match = r"Cannot assign the same cached_property to two " + with pytest.raises(TypeError, match=match): + A.prop.__set_name__(A, "something_else") + + +def test_get_without_set_name(propcache_module: APIProtocol) -> None: + """Test that get without __set_name__ fails.""" + cp = propcache_module.cached_property(not_) + + class A: + """A class.""" + + A.cp = cp # type: ignore[attr-defined] + match = r"Cannot use cached_property instance " + with pytest.raises(TypeError, match=match): + _ = A().cp # type: ignore[attr-defined] + + +def test_ensured_wrapped_function_is_accessible(propcache_module: APIProtocol) -> None: + """Test that the wrapped function can be accessed from python.""" + + class A: + def __init__(self) -> None: + """Init.""" + + @propcache_module.cached_property + def prop(self) -> int: + """Docstring.""" + return 1 + + a = A() + assert A.prop.func(a) == 1 + + [email protected]_extension [email protected](IS_PYPY, reason="PyPy has no C extension") +def test_cached_property_no_refcount_leak(propcache_module: APIProtocol) -> None: + """Test that cached_property does not leak references.""" + + class CachedPropertySentinel: + """A unique object we can track.""" + + def count_sentinels() -> int: + """Count the number of CachedPropertySentinel instances in gc.""" + gc.collect() + return sum( + 1 for obj in gc.get_objects() if isinstance(obj, CachedPropertySentinel) + ) + + class A: + def __init__(self) -> None: + """Init.""" + + @propcache_module.cached_property + def prop(self) -> CachedPropertySentinel: + """Return a sentinel object.""" + return CachedPropertySentinel() + + initial_sentinel_count = count_sentinels() + + a = A() + + # First access - creates and caches the object + result = a.prop + # sys.getrefcount returns 1 higher than actual (the temp ref from the call) + # After first access: result owns 1, __dict__ owns 1, getrefcount call owns 1 = 3 + initial_refcount = sys.getrefcount(result) + + # Should have exactly 1 Sentinel instance now + assert count_sentinels() == initial_sentinel_count + 1 + + # Second access - should return the cached object without creating new refs + result2 = a.prop + assert result is result2 + # After second access: result owns 1, result2 owns 1, __dict__ owns 1, getrefcount call owns 1 = 4 + # Only result2 should add 1 + second_refcount = sys.getrefcount(result) + assert second_refcount == initial_refcount + 1 + + # Still should have exactly 1 Sentinel instance + assert count_sentinels() == initial_sentinel_count + 1 + + # Third access + result3 = a.prop + assert result is result3 + # result2 and result3 each add 1 + third_refcount = sys.getrefcount(result) + assert third_refcount == initial_refcount + 2 + + # Clean up local refs - should be back to just result and __dict__ + del result2 + del result3 + after_cleanup_refcount = sys.getrefcount(result) + assert after_cleanup_refcount == initial_refcount + + # Clear the cache and verify no leak when re-fetching + # After clearing: only result owns it + del a.__dict__["prop"] + cleared_refcount = sys.getrefcount(result) + assert cleared_refcount == initial_refcount - 1 # No longer in __dict__ + + # Re-fetch - this should create a new object, not affect old one + result4 = a.prop + assert result4 is not result # Should be a new object + refetch_refcount = sys.getrefcount(result) + assert refetch_refcount == cleared_refcount # Original object refcount unchanged + + # Now we should have 2 Sentinel instances: + # - original in `result` + # - new one in `result4` + assert count_sentinels() == initial_sentinel_count + 2 diff --git a/contrib/python/propcache/tests/test_init.py b/contrib/python/propcache/tests/test_init.py new file mode 100644 index 00000000000..6bcffe7faa6 --- /dev/null +++ b/contrib/python/propcache/tests/test_init.py @@ -0,0 +1,43 @@ +"""Test imports can happen from top-level.""" + +import pytest + +import propcache +from propcache import _helpers + + +def test_api_at_top_level() -> None: + """Verify the public API is accessible at top-level.""" + assert propcache.cached_property is not None + assert propcache.under_cached_property is not None + assert propcache.cached_property is _helpers.cached_property + assert propcache.under_cached_property is _helpers.under_cached_property + + + "prop_name", + ("cached_property", "under_cached_property"), +) +def test_public_api_is_discoverable_in_dir(prop_name: str) -> None: + """Verify the public API is discoverable programmatically.""" + assert prop_name in dir(propcache) + + +def test_importing_invalid_attr_raises() -> None: + """Verify importing an invalid attribute raises an AttributeError.""" + match = r"^module 'propcache' has no attribute 'invalid_attr'$" + with pytest.raises(AttributeError, match=match): + propcache.invalid_attr + + +def test_import_error_invalid_attr() -> None: + """Verify importing an invalid attribute raises an ImportError.""" + # No match here because the error is raised by the import system + # and may vary between Python versions. + with pytest.raises(ImportError): + from propcache import invalid_attr # noqa: F401 + + +def test_no_wildcard_imports() -> None: + """Verify wildcard imports are prohibited.""" + assert not propcache.__all__ diff --git a/contrib/python/propcache/tests/test_under_cached_property.py b/contrib/python/propcache/tests/test_under_cached_property.py new file mode 100644 index 00000000000..9c0dc304f65 --- /dev/null +++ b/contrib/python/propcache/tests/test_under_cached_property.py @@ -0,0 +1,252 @@ +import gc +import sys +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Protocol, TypedDict, TypeVar + +import pytest + +from propcache.api import under_cached_property + +IS_PYPY = hasattr(sys, "pypy_version_info") + +if sys.version_info >= (3, 11): + from typing import assert_type + +_T_co = TypeVar("_T_co", covariant=True) + + +class APIProtocol(Protocol): + def under_cached_property( + self, func: Callable[[Any], _T_co] + ) -> under_cached_property[_T_co]: ... + + +def test_under_cached_property(propcache_module: APIProtocol) -> None: + class A: + def __init__(self) -> None: + self._cache: dict[str, int] = {} + + @propcache_module.under_cached_property + def prop(self) -> int: + return 1 + + @propcache_module.under_cached_property + def prop2(self) -> str: + return "foo" + + a = A() + if sys.version_info >= (3, 11): + assert_type(a.prop, int) + assert a.prop == 1 + if sys.version_info >= (3, 11): + assert_type(a.prop2, str) + assert a.prop2 == "foo" + + +def test_under_cached_property_typeddict(propcache_module: APIProtocol) -> None: + """Test static typing passes with TypedDict.""" + + class _Cache(TypedDict, total=False): + prop: int + prop2: str + + class A: + def __init__(self) -> None: + self._cache: _Cache = {} + + @propcache_module.under_cached_property + def prop(self) -> int: + return 1 + + @propcache_module.under_cached_property + def prop2(self) -> str: + return "foo" + + a = A() + if sys.version_info >= (3, 11): + assert_type(a.prop, int) + assert a.prop == 1 + if sys.version_info >= (3, 11): + assert_type(a.prop2, str) + assert a.prop2 == "foo" + + +def test_under_cached_property_assignment(propcache_module: APIProtocol) -> None: + class A: + def __init__(self) -> None: + self._cache: dict[str, Any] = {} + + @propcache_module.under_cached_property + def prop(self) -> None: + """Mock property.""" + + a = A() + + with pytest.raises(AttributeError): + a.prop = 123 # type: ignore[assignment] + + +def test_under_cached_property_without_cache(propcache_module: APIProtocol) -> None: + class A: + def __init__(self) -> None: + """Init.""" + self._cache: dict[str, int] = {} + + @propcache_module.under_cached_property + def prop(self) -> None: + """Mock property.""" + + a = A() + + with pytest.raises(AttributeError): + a.prop = 123 # type: ignore[assignment] + + +def test_under_cached_property_check_without_cache( + propcache_module: APIProtocol, +) -> None: + class A: + def __init__(self) -> None: + """Init.""" + # Note that self._cache is intentionally missing + # here to verify AttributeError + + @propcache_module.under_cached_property + def prop(self) -> None: + """Mock property.""" + + a = A() + with pytest.raises(AttributeError): + _ = a.prop # type: ignore[call-overload] + + +def test_under_cached_property_caching(propcache_module: APIProtocol) -> None: + class A: + def __init__(self) -> None: + self._cache: dict[str, int] = {} + + @propcache_module.under_cached_property + def prop(self) -> int: + """Docstring.""" + return 1 + + a = A() + assert a.prop == 1 + + +def test_under_cached_property_class_docstring(propcache_module: APIProtocol) -> None: + class A: + def __init__(self) -> None: + """Init.""" + + @propcache_module.under_cached_property + def prop(self) -> None: + """Docstring.""" + + if TYPE_CHECKING: + # At type checking, the fixture doesn't represent the real module, so + # we use the global-level imported module to verify the isinstance() check here + # matches the behaviour users would see in real code. + assert isinstance(A.prop, under_cached_property) + else: + assert isinstance(A.prop, propcache_module.under_cached_property) + assert "Docstring." == A.prop.__doc__ + + +def test_ensured_wrapped_function_is_accessible(propcache_module: APIProtocol) -> None: + """Test that the wrapped function can be accessed from python.""" + + class A: + def __init__(self) -> None: + """Init.""" + self._cache: dict[str, int] = {} + + @propcache_module.under_cached_property + def prop(self) -> int: + """Docstring.""" + return 1 + + a = A() + assert A.prop.wrapped(a) == 1 + + [email protected]_extension [email protected](IS_PYPY, reason="PyPy has no C extension") +def test_under_cached_property_no_refcount_leak(propcache_module: APIProtocol) -> None: + """Test that under_cached_property does not leak references.""" + + class UnderCachedPropertySentinel: + """A unique object we can track.""" + + def count_sentinels() -> int: + """Count the number of UnderCachedPropertySentinel instances in gc.""" + gc.collect() + return sum( + 1 + for obj in gc.get_objects() + if isinstance(obj, UnderCachedPropertySentinel) + ) + + class A: + def __init__(self) -> None: + """Init.""" + self._cache: dict[str, Any] = {} + + @propcache_module.under_cached_property + def prop(self) -> UnderCachedPropertySentinel: + """Return a sentinel object.""" + return UnderCachedPropertySentinel() + + initial_sentinel_count = count_sentinels() + + a = A() + + # First access - creates and caches the object + result = a.prop + # sys.getrefcount returns 1 higher than actual (the temp ref from the call) + # After first access: result owns 1, _cache owns 1, getrefcount call owns 1 = 3 + initial_refcount = sys.getrefcount(result) + + # Should have exactly 1 Sentinel instance now + assert count_sentinels() == initial_sentinel_count + 1 + + # Second access - should return the cached object without creating new refs + result2 = a.prop + assert result is result2 + # After second access: result owns 1, result2 owns 1, _cache owns 1, getrefcount call owns 1 = 4 + # Only result2 should add 1 + second_refcount = sys.getrefcount(result) + assert second_refcount == initial_refcount + 1 + + # Still should have exactly 1 Sentinel instance + assert count_sentinels() == initial_sentinel_count + 1 + + # Third access + result3 = a.prop + assert result is result3 + # result2 and result3 each add 1 + third_refcount = sys.getrefcount(result) + assert third_refcount == initial_refcount + 2 + + # Clean up local refs - should be back to just result and _cache + del result2 + del result3 + after_cleanup_refcount = sys.getrefcount(result) + assert after_cleanup_refcount == initial_refcount + + # Clear the cache and verify no leak when re-fetching + # After clearing: only result owns it + del a._cache["prop"] + cleared_refcount = sys.getrefcount(result) + assert cleared_refcount == initial_refcount - 1 # No longer in _cache + + # Re-fetch - this should create a new object, not affect old one + result4 = a.prop + assert result4 is not result # Should be a new object + refetch_refcount = sys.getrefcount(result) + assert refetch_refcount == cleared_refcount # Original object refcount unchanged + + # Now we should have 2 Sentinel instances: + # - original in `result` + # - new one in `result4` + assert count_sentinels() == initial_sentinel_count + 2 diff --git a/contrib/python/propcache/tests/ya.make b/contrib/python/propcache/tests/ya.make new file mode 100644 index 00000000000..53173b0a58f --- /dev/null +++ b/contrib/python/propcache/tests/ya.make @@ -0,0 +1,11 @@ +PY3TEST() + +PEERDIR( + contrib/python/propcache +) + +ALL_PYTEST_SRCS() + +NO_LINT() + +END() diff --git a/contrib/python/propcache/ya.make b/contrib/python/propcache/ya.make new file mode 100644 index 00000000000..dea81d7aa7f --- /dev/null +++ b/contrib/python/propcache/ya.make @@ -0,0 +1,34 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(0.5.2) + +LICENSE(Apache-2.0) + +NO_COMPILER_WARNINGS() + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + propcache/__init__.py + propcache/_helpers.py + propcache/_helpers_py.py + propcache/api.py + CYTHON_C + propcache/_helpers_c.pyx +) + +RESOURCE_FILES( + PREFIX contrib/python/propcache/ + .dist-info/METADATA + .dist-info/top_level.txt + propcache/py.typed +) + +END() + +RECURSE_FOR_TESTS( + tests +) diff --git a/contrib/python/yarl/.dist-info/METADATA b/contrib/python/yarl/.dist-info/METADATA index 9870bc4c565..2995555026d 100644 --- a/contrib/python/yarl/.dist-info/METADATA +++ b/contrib/python/yarl/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: yarl -Version: 1.12.1 +Version: 1.14.0 Summary: Yet another URL library Home-page: https://github.com/aio-libs/yarl Author: Andrew Svetlov @@ -38,6 +38,7 @@ License-File: LICENSE License-File: NOTICE Requires-Dist: idna >=2.0 Requires-Dist: multidict >=4.0 +Requires-Dist: propcache >=0.2.0 yarl ==== @@ -150,9 +151,8 @@ Installation The library is Python 3 only! PyPI contains binary wheels for Linux, Windows and MacOS. If you want to install -``yarl`` on another operating system (like *Alpine Linux*, which is not -manylinux-compliant because of the missing glibc and therefore, cannot be -used with our wheels) the the tarball will be used to compile the library from +``yarl`` on another operating system where wheels are not provided, +the the tarball will be used to compile the library from the source code. It requires a C compiler and and Python headers installed. To skip the compilation you must explicitly opt-in by using a PEP 517 @@ -170,7 +170,7 @@ by this variable. Dependencies ------------ -YARL requires multidict_ library. +YARL requires multidict_ and propcache_ libraries. API documentation @@ -246,6 +246,8 @@ It's *Apache 2* licensed and freely available. .. _multidict: https://github.com/aio-libs/multidict +.. _propcache: https://github.com/aio-libs/propcache + ========= Changelog ========= @@ -262,6 +264,118 @@ Changelog .. towncrier release notes start +1.14.0 +====== + +*(2024-10-08)* + + +Packaging updates and notes for downstreams +------------------------------------------- + +- Switched to using the ``propcache`` package for property caching + -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + The ``propcache`` package is derived from the property caching + code in ``yarl`` and has been broken out to avoid maintaining it for multiple + projects. + + *Related issues and pull requests on GitHub:* + `#1169 <https://github.com/aio-libs/yarl/issues/1169>`__. + + +Contributor-facing changes +-------------------------- + +- Started testing with Hypothesis -- by `@webknjaz <https://github.com/sponsors/webknjaz>`__ and `@bdraco <https://github.com/sponsors/bdraco>`__. + + Special thanks to `@Zac-HD <https://github.com/sponsors/Zac-HD>`__ for helping us get started with this framework. + + *Related issues and pull requests on GitHub:* + `#860 <https://github.com/aio-libs/yarl/issues/860>`__. + + +Miscellaneous internal changes +------------------------------ + +- Improved performance of ``yarl.URL.is_default_port()`` when no explicit port is set -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1168 <https://github.com/aio-libs/yarl/issues/1168>`__. + +- Improved performance of converting ``~yarl.URL`` to a string when no explicit port is set -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1170 <https://github.com/aio-libs/yarl/issues/1170>`__. + +- Improved performance of the ``yarl.URL.origin()`` method -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1175 <https://github.com/aio-libs/yarl/issues/1175>`__. + +- Improved performance of encoding hosts -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1176 <https://github.com/aio-libs/yarl/issues/1176>`__. + + +---- + + +1.13.1 +====== + +*(2024-09-27)* + + +Miscellaneous internal changes +------------------------------ + +- Improved performance of calling ``yarl.URL.build()`` with ``authority`` -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1163 <https://github.com/aio-libs/yarl/issues/1163>`__. + + +---- + + +1.13.0 +====== + +*(2024-09-26)* + + +Bug fixes +--------- + +- Started rejecting ASCII hostnames with invalid characters. For host strings that + look like authority strings, the exception message includes advice on what to do + instead -- by `@mjpieters <https://github.com/sponsors/mjpieters>`__. + + *Related issues and pull requests on GitHub:* + `#880 <https://github.com/aio-libs/yarl/issues/880>`__, `#954 <https://github.com/aio-libs/yarl/issues/954>`__. + +- Fixed IPv6 addresses missing brackets when the ``~yarl.URL`` was converted to a string -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1157 <https://github.com/aio-libs/yarl/issues/1157>`__, `#1158 <https://github.com/aio-libs/yarl/issues/1158>`__. + + +Features +-------- + +- Added ``~yarl.URL.host_subcomponent`` which returns the ``3986#section-3.2.2`` host subcomponent -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + The only current practical difference between ``~yarl.URL.raw_host`` and ``~yarl.URL.host_subcomponent`` is that IPv6 addresses are returned bracketed. + + *Related issues and pull requests on GitHub:* + `#1159 <https://github.com/aio-libs/yarl/issues/1159>`__. + + +---- + + 1.12.1 ====== diff --git a/contrib/python/yarl/README.rst b/contrib/python/yarl/README.rst index fe9c856551d..c7a286e6212 100644 --- a/contrib/python/yarl/README.rst +++ b/contrib/python/yarl/README.rst @@ -109,9 +109,8 @@ Installation The library is Python 3 only! PyPI contains binary wheels for Linux, Windows and MacOS. If you want to install -``yarl`` on another operating system (like *Alpine Linux*, which is not -manylinux-compliant because of the missing glibc and therefore, cannot be -used with our wheels) the the tarball will be used to compile the library from +``yarl`` on another operating system where wheels are not provided, +the the tarball will be used to compile the library from the source code. It requires a C compiler and and Python headers installed. To skip the compilation you must explicitly opt-in by using a PEP 517 @@ -129,7 +128,7 @@ by this variable. Dependencies ------------ -YARL requires multidict_ library. +YARL requires multidict_ and propcache_ libraries. API documentation @@ -204,3 +203,5 @@ It's *Apache 2* licensed and freely available. .. _GitHub: https://github.com/aio-libs/yarl .. _multidict: https://github.com/aio-libs/multidict + +.. _propcache: https://github.com/aio-libs/propcache diff --git a/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch b/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch index 79c9ccc2d3e..49ecf81c0ff 100644 --- a/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch +++ b/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch @@ -8,28 +8,6 @@ def test_is_default_port_for_absolute_url_with_default_port(): url = URL("http://example.com:80") assert url.is_default_port() ---- contrib/python/yarl/tests/test_url_build.py (index) -+++ contrib/python/yarl/tests/test_url_build.py (working tree) -@@ -15,16 +15,19 @@ def test_build_simple(): - assert str(u) == "http://127.0.0.1" - - - def test_url_build_ipv6(): - u = URL.build(scheme="http", host="::1") - assert str(u) == "http://::1" - - - def test_url_build_ipv6_brackets(): - u = URL.build(scheme="http", host="[::1]") - assert str(u) == "http://::1" - - - def test_url_ipv4_in_ipv6(): - u = URL.build(scheme="http", host="2001:db8:122:344::192.0.2.33") - assert str(u) == "http://2001:db8:122:344::c000:221" --- contrib/python/yarl/tests/test_url_update_netloc.py (index) +++ contrib/python/yarl/tests/test_url_update_netloc.py (working tree) @@ -196,6 +196,7 @@ def test_with_port(): diff --git a/contrib/python/yarl/tests/test_cache.py b/contrib/python/yarl/tests/test_cache.py index 46d5b01f159..9b0e75a3551 100644 --- a/contrib/python/yarl/tests/test_cache.py +++ b/contrib/python/yarl/tests/test_cache.py @@ -13,7 +13,7 @@ def test_cache_clear() -> None: def test_cache_info() -> None: info = yarl.cache_info() - assert info.keys() == {"idna_encode", "idna_decode", "ip_address"} + assert info.keys() == {"idna_encode", "idna_decode", "ip_address", "host_validate"} def test_cache_configure_default() -> None: @@ -22,11 +22,17 @@ def test_cache_configure_default() -> None: def test_cache_configure_None() -> None: yarl.cache_configure( - idna_encode_size=None, idna_decode_size=None, ip_address_size=None + idna_encode_size=None, + idna_decode_size=None, + ip_address_size=None, + host_validate_size=None, ) def test_cache_configure_explicit() -> None: yarl.cache_configure( - idna_encode_size=128, idna_decode_size=128, ip_address_size=128 + idna_encode_size=128, + idna_decode_size=128, + ip_address_size=128, + host_validate_size=128, ) diff --git a/contrib/python/yarl/tests/test_helpers.py b/contrib/python/yarl/tests/test_helpers.py deleted file mode 100644 index cdfff129e6d..00000000000 --- a/contrib/python/yarl/tests/test_helpers.py +++ /dev/null @@ -1,91 +0,0 @@ -import platform - -import pytest - -from yarl import _helpers, _helpers_py - -IS_PYPY = platform.python_implementation() == "PyPy" - - -class CachedPropertyMixin: - cached_property = NotImplemented - - def test_cached_property(self) -> None: - class A: - def __init__(self): - self._cache = {} - - @self.cached_property # type: ignore[misc] - def prop(self): - return 1 - - a = A() - assert a.prop == 1 - - def test_cached_property_class(self) -> None: - class A: - def __init__(self): - """Init.""" - # self._cache not set because its never accessed in this test - - @self.cached_property # type: ignore[misc] - def prop(self): - """Docstring.""" - - assert isinstance(A.prop, self.cached_property) - assert A.prop.__doc__ == "Docstring." - - def test_cached_property_assignment(self) -> None: - class A: - def __init__(self): - self._cache = {} - - @self.cached_property # type: ignore[misc] - def prop(self): - """Mock property.""" - - a = A() - - with pytest.raises(AttributeError): - a.prop = 123 - - def test_cached_property_without_cache(self) -> None: - class A: - def __init__(self): - pass - - @self.cached_property # type: ignore[misc] - def prop(self): - """Mock property.""" - - a = A() - - with pytest.raises(AttributeError): - a.prop = 123 - - def test_cached_property_check_without_cache(self) -> None: - class A: - def __init__(self): - pass - - @self.cached_property # type: ignore[misc] - def prop(self): - """Mock property.""" - - a = A() - with pytest.raises(AttributeError): - assert a.prop == 1 - - -class TestPyCachedProperty(CachedPropertyMixin): - cached_property = _helpers_py.cached_property # type: ignore[assignment] - - -if ( - not _helpers.NO_EXTENSIONS - and not IS_PYPY - and hasattr(_helpers, "cached_property_c") -): - - class TestCCachedProperty(CachedPropertyMixin): - cached_property = _helpers.cached_property_c # type: ignore[assignment, attr-defined, unused-ignore] # noqa: E501 diff --git a/contrib/python/yarl/tests/test_quoting.py b/contrib/python/yarl/tests/test_quoting.py index d9b6ae8e4bf..1f9cba6aa40 100644 --- a/contrib/python/yarl/tests/test_quoting.py +++ b/contrib/python/yarl/tests/test_quoting.py @@ -1,4 +1,8 @@ +from typing import Type + import pytest +from hypothesis import assume, example, given, note +from hypothesis import strategies as st from yarl._quoting import NO_EXTENSIONS from yarl._quoting_py import _Quoter as _PyQuoter @@ -16,6 +20,10 @@ if not NO_EXTENSIONS: def unquoter(request): return request.param + quoters = [_PyQuoter, _CQuoter] + quoter_ids = ["PyQuoter", "CQuoter"] + unquoters = [_PyUnquoter, _CUnquoter] + unquoter_ids = ["PyUnquoter", "CUnquoter"] else: @pytest.fixture(params=[_PyQuoter], ids=["py_quoter"]) @@ -26,6 +34,11 @@ else: def unquoter(request): return request.param + quoters = [_PyQuoter] + quoter_ids = ["PyQuoter"] + unquoters = [_PyUnquoter] + unquoter_ids = ["PyUnquoter"] + def hexescape(char): """Escape char as RFC 2396 specifies""" @@ -455,3 +468,79 @@ def test_quoter_path_with_plus(quoter): def test_unquoter_path_with_plus(unquoter): s = "/test/x+y%2Bz/:+%2B/" assert "/test/x+y+z/:++/" == unquoter(unsafe="+")(s) + + +@given(safe=st.text(), protected=st.text(), qs=st.booleans(), requote=st.booleans()) +def test_fuzz__PyQuoter(safe, protected, qs, requote): + """Verify that _PyQuoter can be instantiated with any valid arguments.""" + assert _PyQuoter(safe=safe, protected=protected, qs=qs, requote=requote) + + +@given(ignore=st.text(), unsafe=st.text(), qs=st.booleans()) +def test_fuzz__PyUnquoter(ignore, unsafe, qs): + """Verify that _PyUnquoter can be instantiated with any valid arguments.""" + assert _PyUnquoter(ignore=ignore, unsafe=unsafe, qs=qs) + + +@example(text_input="0") +@given( + text_input=st.text( + alphabet=st.characters(max_codepoint=127, blacklist_characters="%") + ), +) [email protected]("quoter", quoters, ids=quoter_ids) [email protected]("unquoter", unquoters, ids=unquoter_ids) +def test_quote_unquote_parameter( + quoter: Type[_PyQuoter], + unquoter: Type[_PyUnquoter], + text_input: str, +) -> None: + quote = quoter() + unquote = unquoter() + text_quoted = quote(text_input) + note(f"text_quoted={text_quoted!r}") + text_output = unquote(text_quoted) + assert text_input == text_output + + +@example(text_input="0") +@given( + text_input=st.text( + alphabet=st.characters(max_codepoint=127, blacklist_characters="%") + ), +) [email protected]("quoter", quoters, ids=quoter_ids) [email protected]("unquoter", unquoters, ids=unquoter_ids) +def test_quote_unquote_parameter_requote( + quoter: Type[_PyQuoter], + unquoter: Type[_PyUnquoter], + text_input: str, +) -> None: + quote = quoter(requote=True) + unquote = unquoter() + text_quoted = quote(text_input) + note(f"text_quoted={text_quoted!r}") + text_output = unquote(text_quoted) + assert text_input == text_output + + +@example(text_input="0") +@given( + text_input=st.text( + alphabet=st.characters(max_codepoint=127, blacklist_characters="%") + ), +) [email protected]("quoter", quoters, ids=quoter_ids) [email protected]("unquoter", unquoters, ids=unquoter_ids) +def test_quote_unquote_parameter_path_safe( + quoter: Type[_PyQuoter], + unquoter: Type[_PyUnquoter], + text_input: str, +) -> None: + quote = quoter() + unquote = unquoter(ignore="/%", unsafe="+") + assume("+" not in text_input and "/" not in text_input) + text_quoted = quote(text_input) + note(f"text_quoted={text_quoted!r}") + text_output = unquote(text_quoted) + assert text_input == text_output diff --git a/contrib/python/yarl/tests/test_url.py b/contrib/python/yarl/tests/test_url.py index 1dd98197fe1..193f1b584c8 100644 --- a/contrib/python/yarl/tests/test_url.py +++ b/contrib/python/yarl/tests/test_url.py @@ -63,6 +63,11 @@ def test_origin(): assert URL("http://example.com:8888") == url.origin() +def test_origin_with_no_auth(): + url = URL("http://example.com:8888/path/to?a=1&b=2") + assert URL("http://example.com:8888") == url.origin() + + def test_origin_nonascii(): url = URL("http://user:password@оун-упа.укр:8888/path/to?a=1&b=2") assert str(url.origin()) == "http://xn----8sb1bdhvc.xn--j1amh:8888" @@ -176,6 +181,24 @@ def test_raw_host(): assert url.raw_host == url._val.hostname + ("host"), + [ + ("example.com"), + ("[::1]"), + ("xn--gnter-4ya.com"), + ], +) +def test_host_subcomponent(host: str): + url = URL(f"http://{host}") + assert url.host_subcomponent == host + + +def test_host_subcomponent_return_idna_encoded_host(): + url = URL("http://оун-упа.укр") + assert url.host_subcomponent == "xn----8sb1bdhvc.xn--j1amh" + + def test_raw_host_non_ascii(): url = URL("http://оун-упа.укр") assert "xn----8sb1bdhvc.xn--j1amh" == url.raw_host @@ -286,35 +309,43 @@ def test_port_for_implicit_port(): def test_port_for_relative_url(): url = URL("/path/to") assert url.port is None + assert url._port_not_default is None + assert url.explicit_port is None def test_port_for_unknown_scheme(): url = URL("unknown://example.com") assert url.port is None + assert url._port_not_default is None + assert url.explicit_port is None def test_explicit_port_for_explicit_port(): url = URL("http://example.com:8888") assert 8888 == url.explicit_port assert url.explicit_port == url._val.port + assert url._port_not_default == 8888 def test_explicit_port_for_implicit_port(): url = URL("http://example.com") assert url.explicit_port is None assert url.explicit_port == url._val.port + assert url._port_not_default is None def test_explicit_port_for_relative_url(): url = URL("/path/to") assert url.explicit_port is None assert url.explicit_port == url._val.port + assert url._port_not_default is None def test_explicit_port_for_unknown_scheme(): url = URL("unknown://example.com") assert url.explicit_port is None assert url.explicit_port == url._val.port + assert url._port_not_default is None def test_raw_path_string_empty(): @@ -2041,3 +2072,14 @@ def test_parsing_populates_cache(): assert url._cache["raw_query_string"] == "a=b" assert url._cache["raw_fragment"] == "frag" assert url._cache["scheme"] == "http" + + + ("host", "is_authority"), + [ + *(("other_gen_delim_" + c, False) for c in "[]"), + ], +) +def test_build_with_invalid_ipv6_host(host: str, is_authority: bool): + with pytest.raises(ValueError, match="Invalid IPv6 URL"): + URL(f"http://{host}/") diff --git a/contrib/python/yarl/tests/test_url_build.py b/contrib/python/yarl/tests/test_url_build.py index a0ab77b2d4e..cc9b15f7e4f 100644 --- a/contrib/python/yarl/tests/test_url_build.py +++ b/contrib/python/yarl/tests/test_url_build.py @@ -15,22 +15,24 @@ def test_build_simple(): assert str(u) == "http://127.0.0.1" def test_url_build_ipv6(): u = URL.build(scheme="http", host="::1") - assert str(u) == "http://::1" + assert str(u) == "http://[::1]" -def test_url_build_ipv6_brackets(): - u = URL.build(scheme="http", host="[::1]") - assert str(u) == "http://::1" +def test_url_build_ipv6_brackets_encoded(): + u = URL.build(scheme="http", host="[::1]", encoded=True) + assert str(u) == "http://[::1]" + + +def test_url_build_ipv6_brackets_not_encoded(): + u = URL.build(scheme="http", host="::1", encoded=False) + assert str(u) == "http://[::1]" def test_url_ipv4_in_ipv6(): u = URL.build(scheme="http", host="2001:db8:122:344::192.0.2.33") - assert str(u) == "http://2001:db8:122:344::c000:221" + assert str(u) == "http://[2001:db8:122:344::c000:221]" def test_build_with_scheme(): @@ -118,6 +120,25 @@ def test_build_with_authority_and_host(): URL.build(authority="host.com", host="example.com") + ("host", "is_authority"), + [ + ("user:[email protected]", True), + ("[email protected]", True), + ("host:com", False), + ("not_percent_encoded%Zf", False), + ("still_not_percent_encoded%fZ", False), + *(("other_gen_delim_" + c, False) for c in "/?#[]"), + ], +) +def test_build_with_invalid_host(host: str, is_authority: bool): + match = r"Host '[^']+' cannot contain '[^']+' \(at position \d+\)" + if is_authority: + match += ", if .* use 'authority' instead of 'host'" + with pytest.raises(ValueError, match=f"{match}$"): + URL.build(host=host) + + def test_build_with_authority(): url = URL.build(scheme="http", authority="степан:[email protected]:8000", path="path") assert ( @@ -132,6 +153,31 @@ def test_build_with_authority_without_encoding(): assert str(url) == "http://foo:[email protected]:8000/path" +def test_build_with_authority_empty_host_no_scheme(): + url = URL.build(authority="", path="path") + assert str(url) == "path" + + +def test_build_with_authority_and_only_user(): + url = URL.build(scheme="https", authority="user:@foo.com", path="/path") + assert str(url) == "https://user:@foo.com/path" + + +def test_build_with_authority_with_port(): + url = URL.build(scheme="https", authority="foo.com:8080", path="/path") + assert str(url) == "https://foo.com:8080/path" + + +def test_build_with_authority_with_ipv6(): + url = URL.build(scheme="https", authority="[::1]", path="/path") + assert str(url) == "https://[::1]/path" + + +def test_build_with_authority_with_ipv6_and_port(): + url = URL.build(scheme="https", authority="[::1]:81", path="/path") + assert str(url) == "https://[::1]:81/path" + + def test_query_str(): u = URL.build(scheme="http", host="127.0.0.1", path="/", query_string="arg=value1") assert str(u) == "http://127.0.0.1/?arg=value1" diff --git a/contrib/python/yarl/tests/test_url_parsing.py b/contrib/python/yarl/tests/test_url_parsing.py index 4fe95185e71..0d58c9d09bb 100644 --- a/contrib/python/yarl/tests/test_url_parsing.py +++ b/contrib/python/yarl/tests/test_url_parsing.py @@ -604,3 +604,28 @@ def test_schemes_that_require_host(scheme: str) -> None: ) with pytest.raises(ValueError, match=expect): URL(f"{scheme}://:1") + + + ("url", "hostname", "hostname_without_brackets"), + [ + ("http://[::1]", "[::1]", "::1"), + ("http://[::1]:8080", "[::1]", "::1"), + ("http://127.0.0.1:8080", "127.0.0.1", "127.0.0.1"), + ( + "http://xn--jxagkqfkduily1i.eu", + "xn--jxagkqfkduily1i.eu", + "xn--jxagkqfkduily1i.eu", + ), + ], +) +def test_url_round_trips( + url: str, hostname: str, hostname_without_brackets: str +) -> None: + """Verify that URLs round-trip correctly.""" + parsed = URL(url) + assert parsed._val.hostname == hostname_without_brackets + assert parsed.raw_host == hostname_without_brackets + assert parsed.host_subcomponent == hostname + assert str(parsed) == url + assert str(URL(str(parsed))) == url diff --git a/contrib/python/yarl/tests/test_url_update_netloc.py b/contrib/python/yarl/tests/test_url_update_netloc.py index cfe6ca7d457..a8ff1f9f8b3 100644 --- a/contrib/python/yarl/tests/test_url_update_netloc.py +++ b/contrib/python/yarl/tests/test_url_update_netloc.py @@ -171,6 +171,26 @@ def test_with_host_non_ascii(): assert url2.authority == "оун-упа.укр:123" + ("host", "is_authority"), + [ + ("user:[email protected]", True), + ("[email protected]", True), + ("host:com", False), + ("not_percent_encoded%Zf", False), + ("still_not_percent_encoded%fZ", False), + *(("other_gen_delim_" + c, False) for c in "/?#[]"), + ], +) +def test_with_invalid_host(host: str, is_authority: bool): + url = URL("http://example.com:123") + match = r"Host '[^']+' cannot contain '[^']+' \(at position \d+\)" + if is_authority: + match += ", if .* use 'authority' instead of 'host'" + with pytest.raises(ValueError, match=f"{match}$"): + url.with_host(host=host) + + def test_with_host_percent_encoded(): url = URL("http://%25cf%2580%cf%80:%25cf%2580%cf%[email protected]:123") url2 = url.with_host("%cf%80.org") diff --git a/contrib/python/yarl/tests/ya.make b/contrib/python/yarl/tests/ya.make index b93020851f0..73d45a2785a 100644 --- a/contrib/python/yarl/tests/ya.make +++ b/contrib/python/yarl/tests/ya.make @@ -1,6 +1,7 @@ PY3TEST() PEERDIR( + contrib/python/hypothesis contrib/python/yarl ) diff --git a/contrib/python/yarl/ya.make b/contrib/python/yarl/ya.make index 43d5345771f..f8435827974 100644 --- a/contrib/python/yarl/ya.make +++ b/contrib/python/yarl/ya.make @@ -2,13 +2,14 @@ PY3_LIBRARY() -VERSION(1.12.1) +VERSION(1.14.0) LICENSE(Apache-2.0) PEERDIR( contrib/python/idna contrib/python/multidict + contrib/python/propcache ) NO_COMPILER_WARNINGS() @@ -18,15 +19,11 @@ NO_LINT() PY_SRCS( TOP_LEVEL yarl/__init__.py - yarl/_helpers.py - yarl/_helpers_c.pyi - yarl/_helpers_py.py yarl/_quoting.py yarl/_quoting_c.pyi yarl/_quoting_py.py yarl/_url.py CYTHON_C - yarl/_helpers_c.pyx yarl/_quoting_c.pyx ) diff --git a/contrib/python/yarl/yarl/__init__.py b/contrib/python/yarl/yarl/__init__.py index 50d24c0e778..d9703013c6e 100644 --- a/contrib/python/yarl/yarl/__init__.py +++ b/contrib/python/yarl/yarl/__init__.py @@ -8,7 +8,7 @@ from ._url import ( cache_info, ) -__version__ = "1.12.1" +__version__ = "1.14.0" __all__ = ( "URL", diff --git a/contrib/python/yarl/yarl/_helpers_c.pyi b/contrib/python/yarl/yarl/_helpers_c.pyi deleted file mode 100644 index 69034921219..00000000000 --- a/contrib/python/yarl/yarl/_helpers_c.pyi +++ /dev/null @@ -1,6 +0,0 @@ -from typing import Any - -class cached_property: - def __init__(self, wrapped: Any) -> None: ... - def __get__(self, inst: Any, owner: Any) -> Any: ... - def __set__(self, inst: Any, value: Any) -> None: ... diff --git a/contrib/python/yarl/yarl/_helpers_c.pyx b/contrib/python/yarl/yarl/_helpers_c.pyx deleted file mode 100644 index e6eec375435..00000000000 --- a/contrib/python/yarl/yarl/_helpers_c.pyx +++ /dev/null @@ -1,36 +0,0 @@ -# cython: language_level=3 - -cdef _sentinel = object() - -cdef class cached_property: - """Use as a class method decorator. It operates almost exactly like - the Python `@property` decorator, but it puts the result of the - method it decorates into the instance dict after the first call, - effectively replacing the function it decorates with an instance - variable. It is, in Python parlance, a data descriptor. - - """ - - cdef object wrapped - cdef object name - - def __init__(self, wrapped): - self.wrapped = wrapped - self.name = wrapped.__name__ - - @property - def __doc__(self): - return self.wrapped.__doc__ - - def __get__(self, inst, owner): - if inst is None: - return self - cdef dict cache = inst._cache - val = cache.get(self.name, _sentinel) - if val is _sentinel: - val = self.wrapped(inst) - cache[self.name] = val - return val - - def __set__(self, inst, value): - raise AttributeError("cached property is read-only") diff --git a/contrib/python/yarl/yarl/_helpers_py.py b/contrib/python/yarl/yarl/_helpers_py.py deleted file mode 100644 index 5a18afb5c39..00000000000 --- a/contrib/python/yarl/yarl/_helpers_py.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Various helper functions.""" - -from typing import Any, Callable, Dict, Generic, Optional, Protocol, Type, TypeVar - -_T = TypeVar("_T") - - -class _TSelf(Protocol, Generic[_T]): - _cache: Dict[str, _T] - - -class cached_property(Generic[_T]): - """Use as a class method decorator. - - It operates almost exactly like - the Python `@property` decorator, but it puts the result of the - method it decorates into the instance dict after the first call, - effectively replacing the function it decorates with an instance - variable. It is, in Python parlance, a data descriptor. - """ - - def __init__(self, wrapped: Callable[..., _T]) -> None: - self.wrapped = wrapped - self.__doc__ = wrapped.__doc__ - self.name = wrapped.__name__ - - def __get__(self, inst: _TSelf[_T], owner: Optional[Type[Any]] = None) -> _T: - try: - try: - return inst._cache[self.name] - except KeyError: - val = self.wrapped(inst) - inst._cache[self.name] = val - return val - except AttributeError: - if inst is None: - return self - raise - - def __set__(self, inst: _TSelf[_T], value: _T) -> None: - raise AttributeError("cached property is read-only") diff --git a/contrib/python/yarl/yarl/_url.py b/contrib/python/yarl/yarl/_url.py index 8a48628fcd9..eff4dc3fbd6 100644 --- a/contrib/python/yarl/yarl/_url.py +++ b/contrib/python/yarl/yarl/_url.py @@ -1,4 +1,5 @@ import math +import re import sys import warnings from collections.abc import Mapping, Sequence @@ -31,8 +32,8 @@ from urllib.parse import ( import idna from multidict import MultiDict, MultiDictProxy, istr +from propcache.api import under_cached_property as cached_property -from ._helpers import cached_property from ._quoting import _Quoter, _Unquoter DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443, "ftp": 21} @@ -45,6 +46,22 @@ SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp")) sentinel = object() +# reg-name: unreserved / pct-encoded / sub-delims +# this pattern matches anything that is *not* in those classes. and is only used +# on lower-cased ASCII values. +_not_reg_name = re.compile( + r""" + # any character not in the unreserved or sub-delims sets, plus % + # (validated with the additional check for pct-encoded sequences below) + [^a-z0-9\-._~!$&'()*+,;=%] + | + # % only allowed if it is part of a pct-encoded + # sequence of 2 hex digits. + %(?![0-9a-f]{2}) + """, + re.VERBOSE, +) + SimpleQuery = Union[str, int, float] QueryVariable = Union[SimpleQuery, "Sequence[SimpleQuery]"] Query = Union[ @@ -64,6 +81,7 @@ class CacheInfo(TypedDict): idna_encode: _CacheInfo idna_decode: _CacheInfo ip_address: _CacheInfo + host_validate: _CacheInfo class _SplitResultDict(TypedDict, total=False): @@ -77,6 +95,7 @@ class _SplitResultDict(TypedDict, total=False): class _InternalURLCache(TypedDict, total=False): + _origin: "URL" absolute: bool scheme: str raw_authority: str @@ -269,7 +288,7 @@ class URL: raise ValueError(msg) else: host = "" - host = cls._encode_host(host) + host = cls._encode_host(host, validate_host=False) raw_user = None if username is None else cls._REQUOTER(username) raw_password = None if password is None else cls._REQUOTER(password) netloc = cls._make_netloc( @@ -350,10 +369,15 @@ class URL: if encoded: netloc = authority else: - tmp = SplitResult("", authority, "", "", "") - port = None if tmp.port == DEFAULT_PORTS.get(scheme) else tmp.port + _user, _password, _host, _port = cls._split_netloc(authority) + port = None if _port == DEFAULT_PORTS.get(scheme) else _port + _host = ( + "" + if _host is None + else cls._encode_host(_host, validate_host=False) + ) netloc = cls._make_netloc( - tmp.username, tmp.password, tmp.hostname, port, encode=True + _user, _password, _host, port, encode=True, encode_host=False ) elif not user and not password and not host and not port: netloc = "" @@ -395,7 +419,7 @@ class URL: netloc=self._make_netloc( self.raw_user, self.raw_password, - self.raw_host, + self.host_subcomponent, port, encode_host=False, ) @@ -506,12 +530,12 @@ class URL: Return False for relative URLs. """ - default = self._default_port - explicit = self.explicit_port - if explicit is None: - # A relative URL does not have an implicit port / default port - return default is not None - return explicit == default + if (explicit := self.explicit_port) is None: + # If the explicit port is None, then the URL must be + # using the default port unless its a relative URL + # which does not have an implicit port / default port + return self.absolute + return explicit == self._default_port def origin(self) -> "URL": """Return an URL with scheme, host and port parts only. @@ -520,13 +544,24 @@ class URL: """ # TODO: add a keyword-only option for keeping user/pass maybe? - if not self.absolute: + return self._origin + + @cached_property + def _origin(self) -> "URL": + """Return an URL with scheme, host and port parts only. + + user, password, path, query and fragment are removed. + """ + v = self._val + if not v.netloc: raise ValueError("URL should be absolute") - if not self._val.scheme: + if not v.scheme: raise ValueError("URL should have scheme") - v = self._val - netloc = self._make_netloc(None, None, v.hostname, v.port) - val = v._replace(netloc=netloc, path="", query="", fragment="") + if "@" not in v.netloc: + val = v._replace(path="", query="", fragment="") + else: + netloc = self._make_netloc(None, None, v.hostname, v.port) + val = v._replace(netloc=netloc, path="", query="", fragment="") return URL(val, encoded=True) def relative(self) -> "URL": @@ -575,15 +610,15 @@ class URL: @cached_property def _default_port(self) -> Union[int, None]: """Default port for the scheme or None if not known.""" - return DEFAULT_PORTS.get(self.scheme) + return DEFAULT_PORTS.get(self._val.scheme) @cached_property def _port_not_default(self) -> Union[int, None]: """The port part of URL normalized to None if its the default port.""" - port = self.port - if self._default_port == port: + explicit_port = self.explicit_port + if explicit_port is None or explicit_port == self._default_port: return None - return port + return explicit_port @cached_property def authority(self) -> str: @@ -647,6 +682,8 @@ class URL: None for relative URLs. + When working with IPv6 addresses, use the `host_subcomponent` property instead + as it will return the host subcomponent with brackets. """ # Use host instead of hostname for sake of shortness # May add .hostname prop later @@ -660,17 +697,36 @@ class URL: None for relative URLs. """ - raw = self.raw_host - if raw is None: + if (raw := self.raw_host) is None: return None - if "%" in raw: - # Hack for scoped IPv6 addresses like - # fe80::2%Перевірка - # presence of '%' sign means only IPv6 address, so idna is useless. + if raw and raw[-1].isdigit() or ":" in raw: + # IP addresses are never IDNA encoded return raw return _idna_decode(raw) @cached_property + def host_subcomponent(self) -> Union[str, None]: + """Return the host subcomponent part of URL. + + None for relative URLs. + + https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 + + `IP-literal = "[" ( IPv6address / IPvFuture ) "]"` + + Examples: + - `http://example.com:8080` -> `example.com` + - `http://example.com:80` -> `example.com` + - `https://127.0.0.1:8443` -> `127.0.0.1` + - `https://[::1]:8443` -> `[::1]` + - `http://[::1]` -> `[::1]` + + """ + if (raw := self.raw_host) is None: + return None + return f"[{raw}]" if ":" in raw else raw + + @cached_property def port(self) -> Union[int, None]: """Port part of URL, with scheme-based fallback. @@ -725,7 +781,7 @@ class URL: @cached_property def _parsed_query(self) -> List[Tuple[str, str]]: """Parse query part of URL.""" - return parse_qsl(self.raw_query_string, keep_blank_values=True) + return parse_qsl(self._val.query, keep_blank_values=True) @cached_property def query(self) -> "MultiDictProxy[str]": @@ -753,7 +809,7 @@ class URL: Empty string if query is missing. """ - return self._QS_UNQUOTER(self.raw_query_string) + return self._QS_UNQUOTER(self._val.query) @cached_property def path_qs(self) -> str: @@ -765,9 +821,9 @@ class URL: @cached_property def raw_path_qs(self) -> str: """Encoded path of URL with query.""" - if not self.raw_query_string: + if not self._val.query: return self.raw_path - return f"{self.raw_path}?{self.raw_query_string}" + return f"{self.raw_path}?{self._val.query}" @cached_property def raw_fragment(self) -> str: @@ -785,7 +841,7 @@ class URL: Empty string if fragment is missing. """ - return self._UNQUOTER(self.raw_fragment) + return self._UNQUOTER(self._val.fragment) @cached_property def raw_parts(self) -> Tuple[str, ...]: @@ -824,7 +880,7 @@ class URL: """ path = self.raw_path if not path or path == "/": - if self.raw_fragment or self.raw_query_string: + if self._val.fragment or self._val.query: return URL(self._val._replace(query="", fragment=""), encoded=True) return self parts = path.split("/") @@ -938,14 +994,21 @@ class URL: return prefix + "/".join(_normalize_path_segments(segments)) @classmethod - def _encode_host(cls, host: str, human: bool = False) -> str: - if "%" in host: - raw_ip, sep, zone = host.partition("%") - else: - raw_ip = host - sep = zone = "" - - if raw_ip and raw_ip[-1].isdigit() or ":" in raw_ip: + def _encode_host( + cls, host: str, human: bool = False, validate_host: bool = True + ) -> str: + if host and host[-1].isdigit() or ":" in host: + # If the host ends with a digit or contains a colon, its likely + # an IP address. So we check with _ip_compressed_version + # and fall-through if its not an IP address. This is a performance + # optimization to avoid parsing IP addresses as much as possible + # because it is orders of magnitude slower than almost any other + # operation this library does. + if "%" in host: + raw_ip, sep, zone = host.partition("%") + else: + raw_ip = host + sep = zone = "" # Might be an IP address, check it # # IP Addresses can look like: @@ -953,14 +1016,11 @@ class URL: # - 127.0.0.1 (last character is a digit) # - 2001:db8::ff00:42:8329 (contains a colon) # - 2001:db8::ff00:42:8329%eth0 (contains a colon) - # - [2001:db8::ff00:42:8329] (contains a colon) + # - [2001:db8::ff00:42:8329] (contains a colon -- brackets should + # have been removed before it gets here) # Rare IP Address formats are not supported per: # https://datatracker.ietf.org/doc/html/rfc3986#section-7.4 # - # We try to avoid parsing IP addresses as much as possible - # since its orders of magnitude slower than almost any other operation - # this library does. - # # IP parsing is slow, so its wrapped in an LRU try: ip_compressed_version = _ip_compressed_version(raw_ip) @@ -970,18 +1030,23 @@ class URL: # These checks should not happen in the # LRU to keep the cache size small host, version = ip_compressed_version - if sep: - host += "%" + zone if version == 6: - return f"[{host}]" - return host + return f"[{host}%{zone}]" if sep else f"[{host}]" + return f"{host}%{zone}" if sep else host host = host.lower() + if human: + return host + # IDNA encoding is slow, # skip it for ASCII-only strings # Don't move the check into _idna_encode() helper # to reduce the cache size - if human or host.isascii(): + if host.isascii(): + # Check for invalid characters explicitly; _idna_encode() does this + # for non-ascii host names. + if validate_host: + _host_validate(host) return host return _idna_encode(host) @@ -998,26 +1063,24 @@ class URL: ) -> str: if host is None: return "" - quoter = cls._REQUOTER if requote else cls._QUOTER - if encode_host: - ret = cls._encode_host(host) - else: - ret = host + ret = cls._encode_host(host) if encode_host else host if port is not None: ret = f"{ret}:{port}" + if user is None and password is None: + return ret + quoter = cls._REQUOTER if requote else cls._QUOTER if password is not None: if not user: user = "" - else: - if encode: - user = quoter(user) + elif encode: + user = quoter(user) if encode: password = quoter(password) - user = user + ":" + password + user = f"{user}:{password}" elif user and encode: user = quoter(user) if user: - ret = user + "@" + ret + ret = f"{user}@{ret}" return ret @classmethod @@ -1298,7 +1361,7 @@ class URL: new_query_string = self._get_str_query(*args, **kwargs) if not new_query_string: return self - if current_query := self.raw_query_string: + if current_query := self._val.query: # both strings are already encoded so we can use a simple # string join if current_query[-1] == "&": @@ -1362,7 +1425,7 @@ class URL: raise TypeError("Invalid fragment type") else: raw_fragment = self._FRAGMENT_QUOTER(fragment) - if self.raw_fragment == raw_fragment: + if self._val.fragment == raw_fragment: return self return URL(self._val._replace(fragment=raw_fragment), encoded=True) @@ -1496,7 +1559,7 @@ class URL: netloc = self._make_netloc( user, password, host, self.explicit_port, encode_host=False ) - val = SplitResult(self.scheme, netloc, path, query_string, fragment) + val = SplitResult(self._val.scheme, netloc, path, query_string, fragment) return urlunsplit(val) @@ -1537,12 +1600,31 @@ def _ip_compressed_version(raw_ip: str) -> Tuple[str, int]: return ip.compressed, ip.version +@lru_cache(_MAXCACHE) +def _host_validate(host: str) -> None: + """Validate an ascii host name.""" + invalid = _not_reg_name.search(host) + if invalid is None: + return + value, pos, extra = invalid.group(), invalid.start(), "" + if value == "@" or (value == ":" and "@" in host[pos:]): + # this looks like an authority string + extra = ( + ", if the value includes a username or password, " + "use 'authority' instead of 'host'" + ) + raise ValueError( + f"Host {host!r} cannot contain {value!r} (at position " f"{pos}){extra}" + ) from None + + @rewrite_module def cache_clear() -> None: """Clear all LRU caches.""" _idna_decode.cache_clear() _idna_encode.cache_clear() _ip_compressed_version.cache_clear() + _host_validate.cache_clear() @rewrite_module @@ -1552,6 +1634,7 @@ def cache_info() -> CacheInfo: "idna_encode": _idna_encode.cache_info(), "idna_decode": _idna_decode.cache_info(), "ip_address": _ip_compressed_version.cache_info(), + "host_validate": _host_validate.cache_info(), } @@ -1561,12 +1644,14 @@ def cache_configure( idna_encode_size: Union[int, None] = _MAXCACHE, idna_decode_size: Union[int, None] = _MAXCACHE, ip_address_size: Union[int, None] = _MAXCACHE, + host_validate_size: Union[int, None] = _MAXCACHE, ) -> None: """Configure LRU cache sizes.""" - global _idna_decode, _idna_encode, _ip_compressed_version + global _idna_decode, _idna_encode, _ip_compressed_version, _host_validate _idna_encode = lru_cache(idna_encode_size)(_idna_encode.__wrapped__) _idna_decode = lru_cache(idna_decode_size)(_idna_decode.__wrapped__) _ip_compressed_version = lru_cache(ip_address_size)( _ip_compressed_version.__wrapped__ ) + _host_validate = lru_cache(host_validate_size)(_host_validate.__wrapped__) diff --git a/library/cpp/yt/error/error_code.cpp b/library/cpp/yt/error/error_code.cpp index d82d03214d4..ef9ce0f33c1 100644 --- a/library/cpp/yt/error/error_code.cpp +++ b/library/cpp/yt/error/error_code.cpp @@ -62,11 +62,10 @@ void TErrorCodeRegistry::RegisterErrorCode(int code, const TErrorCodeInfo& error if (code == 119) { return; } - YT_LOG_FATAL( - "Duplicate error code (Code: %v, StoredCodeInfo: %v, NewCodeInfo: %v)", - code, - CodeToInfo_[code], - errorCodeInfo); + YT_TLOG_FATAL("Duplicate error code") + .With("Code", code) + .With("StoredCodeInfo", CodeToInfo_[code]) + .With("NewCodeInfo", errorCodeInfo); } } @@ -91,11 +90,11 @@ void TErrorCodeRegistry::RegisterErrorCodeRange(int from, int to, std::string na TErrorCodeRangeInfo newRange{from, to, std::move(namespaceName), std::move(formatter)}; for (const auto& range : ErrorCodeRanges_) { - YT_LOG_FATAL_IF( + YT_TLOG_FATAL_IF( range.Intersects(newRange), - "Intersecting error code ranges registered (FirstRange: %v, SecondRange: %v)", - range, - newRange); + "Intersecting error code ranges registered") + .With("FirstRange", range) + .With("SecondRange", newRange); } ErrorCodeRanges_.push_back(std::move(newRange)); CheckCodesAgainstRanges(); @@ -105,14 +104,13 @@ void TErrorCodeRegistry::CheckCodesAgainstRanges() const { for (const auto& [code, info] : CodeToInfo_) { for (const auto& range : ErrorCodeRanges_) { - YT_LOG_FATAL_IF( + YT_TLOG_FATAL_IF( range.Contains(code), - "Error code range contains another registered code " - "(Range: %v, Code: %v, RangeCodeInfo: %v, StandaloneCodeInfo: %v)", - range, - code, - range.Get(code), - info); + "Error code range contains another registered code") + .With("Range", range) + .With("Code", code) + .With("RangeCodeInfo", range.Get(code)) + .With("StandaloneCodeInfo", info); } } } diff --git a/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp b/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp index cb3e244e3b9..95b2ade9b65 100644 --- a/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp +++ b/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp @@ -20,7 +20,7 @@ TEST(TStreamLogManagerTest, Simple) TStringOutput output(str); auto logManager = CreateStreamLogManager(&output); TLogger Logger(logManager.get(), "Test"); - YT_LOG_INFO("Hello world"); + YT_TLOG_INFO("Hello world"); } TVector<TStringBuf> tokens; diff --git a/library/cpp/yt/logging/logger-inl.h b/library/cpp/yt/logging/logger-inl.h index ff4d6d81bbe..d6c4e4c6775 100644 --- a/library/cpp/yt/logging/logger-inl.h +++ b/library/cpp/yt/logging/logger-inl.h @@ -5,6 +5,7 @@ #endif #include "tagged_payload.h" +#include "tag.h" #include <library/cpp/yt/yson_string/convert.h> #include <library/cpp/yt/yson_string/string.h> @@ -343,35 +344,6 @@ struct TStaticAnchorRef std::atomic<bool>* Registered; }; -//! Wraps the format spec passed to |TTaggedLoggingGuard::With| and validates at compile -//! time that it is a |%|-prefixed string literal (e.g. |"%v"|, |"%08x"|). The stored -//! spec has the leading |%| stripped, as expected by |FormatValue|. -class TLoggingTagSpec -{ -public: - template <size_t N> - consteval TLoggingTagSpec(const char (&spec)[N]) - : Spec_(spec + 1, N - 2) - { - static_assert(N >= 2, "Logging tag format spec must be a non-empty string literal"); - if (spec[0] != '%') { - TheLoggingTagFormatSpecMustStartWithPercentSign(); - } - } - - TStringBuf Get() const - { - return Spec_; - } - -private: - const TStringBuf Spec_; - - // Undefined on purpose: calling it from the |consteval| ctor turns a missing - // leading |%| into a compile error that names the violated rule. - static void TheLoggingTagFormatSpecMustStartWithPercentSign(); -}; - class TWellKnownTaggedLoggingGuard; //! Accumulates a tagged log message via a fluent |.With| chain and emits the event in @@ -408,6 +380,15 @@ public: return Enabled_; } + //! The fluent macros end their expansion in a call to this instead of naming the guard + //! directly. A tag-less |YT_TLOG_INFO("Message");| would otherwise expand to a discarded + //! id-expression, and -Wunused-value fires on those whenever the call is spelled inside + //! a macro argument (e.g. within |BIND(...)|) rather than a macro body. + TTaggedLoggingGuard& Self() & + { + return *this; + } + template <class TValue> TTaggedLoggingGuard& With(TStringBuf tag, const TValue& value) & { @@ -420,6 +401,23 @@ public: return DoWith(tag, value, spec.Get()); } + //! Attaches a keyed tag composed from several values, e.g. |.WithFormat("Method", "%v.%v", service, method)|. + template <class... TArgs> + TTaggedLoggingGuard& WithFormat(TStringBuf tag, TFormatString<TArgs...> format, TArgs&&... args) & + { + Format(Writer_.BeginTag(tag), format, std::forward<TArgs>(args)...); + Writer_.EndTag(); + return *this; + } + + //! Splices a pre-built list of keyed tags, preserving them as individual tags. Chosen + //! over the well-known single-argument |With| below by exact match. + TTaggedLoggingGuard& With(const TLoggingTagList& tags) & + { + Writer_.AppendTags(tags.GetPayload()); + return *this; + } + //! Attaches a well-known tag whose key is resolved from #value's type via the //! |GetWellKnownLoggingTag| ADL point (the type must opt in, e.g. errors). //! @@ -618,6 +616,12 @@ public: { return *this; } + + template <class... TArgs> + TNullTaggedLoggingGuard& WithFormat(TArgs&&...) + { + return *this; + } }; template <class TMessage> diff --git a/library/cpp/yt/logging/logger.h b/library/cpp/yt/logging/logger.h index 2e832995016..68529760453 100644 --- a/library/cpp/yt/logging/logger.h +++ b/library/cpp/yt/logging/logger.h @@ -468,13 +468,19 @@ void LogStructuredEvent( // Tags are supplied via a fluent |.With(name, value)| (or |.With(name, value, "%spec")|) // chain; they are carried as structured key/value pairs in the event payload. A single- // argument |.With(value)| attaches the value under a statically known key resolved by ADL -// (e.g. |.With(error)| under the "Error" key): +// (e.g. |.With(error)| under the "Error" key). |.WithFormat(name, format, args...)| composes +// one tag out of several values: // // YT_TLOG_INFO("Message") // .With("Key", value) // .With("Count", count, "%08x") +// .WithFormat("Method", "%v.%v", service, method) // .With(error); // +// |.With(tagList)| splices a #TLoggingTagList -- a set of keyed tags a component builds +// once (with the same fluent API) and attaches to many events, without collapsing them +// into a single tag. +// // If the message is not logged then the |.With| chain is not evaluated, so tag value // expressions cost nothing. @@ -498,7 +504,7 @@ void LogStructuredEvent( (message)); \ !loggingGuard__.IsEnabled()) \ { } else \ - loggingGuard__ + loggingGuard__.Self() #ifdef YT_ENABLE_TRACE_LOGGING #define YT_TLOG_TRACE(message) YT_TLOG_EVENT_FLUENT(Logger, ::NYT::NLogging::ELogLevel::Trace, message) @@ -545,7 +551,7 @@ void LogStructuredEvent( (message)); \ loggingGuard__.TryEnter(); \ loggingGuard__.Commit()) \ - loggingGuard__ + loggingGuard__.Self() #define YT_TLOG_FATAL_IF(condition, message) if (condition) [[unlikely]] YT_TLOG_FATAL(message) #define YT_TLOG_FATAL_UNLESS(condition, message) if (!(condition)) [[unlikely]] YT_TLOG_FATAL(message) @@ -564,7 +570,7 @@ void LogStructuredEvent( ::NYT::EErrorCode::Fatal, \ "Malformed request or incorrect state detected") \ << ::NYT::TErrorAttribute("message", loggingGuard__.Commit())) \ - loggingGuard__ + loggingGuard__.Self() #define YT_TLOG_ALERT_AND_THROW_IF(condition, message) if (condition) [[unlikely]] YT_TLOG_ALERT_AND_THROW(message) #define YT_TLOG_ALERT_AND_THROW_UNLESS(condition, message) if (!(condition)) [[unlikely]] YT_TLOG_ALERT_AND_THROW(message) diff --git a/library/cpp/yt/logging/tag-inl.h b/library/cpp/yt/logging/tag-inl.h new file mode 100644 index 00000000000..72a352ed7b8 --- /dev/null +++ b/library/cpp/yt/logging/tag-inl.h @@ -0,0 +1,114 @@ +#ifndef TAG_INL_H_ +#error "Direct inclusion of this file is not allowed, include tag.h" +// For the sake of sane code completion. +#include "tag.h" +#endif + +#include <library/cpp/yt/string/string_builder.h> + +#include <utility> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +class TLoggingTagSpec +{ +public: + template <size_t N> + consteval TLoggingTagSpec(const char (&spec)[N]) + : Spec_(spec + 1, N - 2) + { + static_assert(N >= 2, "Logging tag format spec must be a non-empty string literal"); + if (spec[0] != '%') { + TheLoggingTagFormatSpecMustStartWithPercentSign(); + } + } + + TStringBuf Get() const + { + return Spec_; + } + +private: + const TStringBuf Spec_; + + // Undefined on purpose: calling it from the |consteval| ctor turns a missing + // leading |%| into a compile error that names the violated rule. + static void TheLoggingTagFormatSpecMustStartWithPercentSign(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class TValue> +TLoggingTagList& TLoggingTagList::With(TStringBuf key, const TValue& value) & +{ + DoWith(key, value, "v"_sb); + return *this; +} + +template <class TValue> +TLoggingTagList& TLoggingTagList::With(TStringBuf key, const TValue& value, TLoggingTagSpec spec) & +{ + DoWith(key, value, spec.Get()); + return *this; +} + +template <class... TArgs> +TLoggingTagList& TLoggingTagList::WithFormat(TStringBuf key, TFormatString<TArgs...> format, TArgs&&... args) & +{ + TStringBuilder builder; + Format(&builder, format, std::forward<TArgs>(args)...); + AppendTag(key, builder.GetBuffer()); + return *this; +} + +template <class TValue> +TLoggingTagList&& TLoggingTagList::With(TStringBuf key, const TValue& value) && +{ + DoWith(key, value, "v"_sb); + return std::move(*this); +} + +template <class TValue> +TLoggingTagList&& TLoggingTagList::With(TStringBuf key, const TValue& value, TLoggingTagSpec spec) && +{ + DoWith(key, value, spec.Get()); + return std::move(*this); +} + +template <class... TArgs> +TLoggingTagList&& TLoggingTagList::WithFormat(TStringBuf key, TFormatString<TArgs...> format, TArgs&&... args) && +{ + TStringBuilder builder; + Format(&builder, format, std::forward<TArgs>(args)...); + AppendTag(key, builder.GetBuffer()); + return std::move(*this); +} + +inline bool TLoggingTagList::IsEmpty() const +{ + return Payload_.empty(); +} + +inline TStringBuf TLoggingTagList::GetPayload() const +{ + return Payload_; +} + +template <class TValue> +void TLoggingTagList::DoWith(TStringBuf key, const TValue& value, TStringBuf spec) +{ + TStringBuilder builder; + FormatValue(&builder, value, spec); + AppendTag(key, builder.GetBuffer()); +} + +inline void TLoggingTagList::AppendTag(TStringBuf key, TStringBuf value) +{ + TTaggedPayloadWriter::AppendTag(&Payload_, key, value); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging diff --git a/library/cpp/yt/logging/tag.h b/library/cpp/yt/logging/tag.h new file mode 100644 index 00000000000..9c49e9b17d0 --- /dev/null +++ b/library/cpp/yt/logging/tag.h @@ -0,0 +1,60 @@ +#pragma once + +#include "tagged_payload.h" + +#include <library/cpp/yt/string/format.h> + +#include <util/generic/strbuf.h> + +#include <string> + +namespace NYT::NLogging { + +//////////////////////////////////////////////////////////////////////////////// + +//! Wraps the format spec passed to a tag-appending |With| and validates at compile time +//! that it is a |%|-prefixed string literal (e.g. |"%v"|, |"%08x"|). +class TLoggingTagSpec; + +//////////////////////////////////////////////////////////////////////////////// + +//! An opaque, pre-serialized list of logging tags. +class TLoggingTagList +{ +public: + TLoggingTagList() = default; + + template <class TValue> + TLoggingTagList& With(TStringBuf key, const TValue& value) &; + template <class TValue> + TLoggingTagList& With(TStringBuf key, const TValue& value, TLoggingTagSpec spec) &; + template <class... TArgs> + TLoggingTagList& WithFormat(TStringBuf key, TFormatString<TArgs...> format, TArgs&&... args) &; + + template <class TValue> + TLoggingTagList&& With(TStringBuf key, const TValue& value) &&; + template <class TValue> + TLoggingTagList&& With(TStringBuf key, const TValue& value, TLoggingTagSpec spec) &&; + template <class... TArgs> + TLoggingTagList&& WithFormat(TStringBuf key, TFormatString<TArgs...> format, TArgs&&... args) &&; + + bool IsEmpty() const; + + //! The serialized tag section, spliced verbatim by #TTaggedPayloadWriter::AppendTags. + TStringBuf GetPayload() const; + +private: + std::string Payload_; + + template <class TValue> + void DoWith(TStringBuf key, const TValue& value, TStringBuf spec); + void AppendTag(TStringBuf key, TStringBuf value); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NLogging + +#define TAG_INL_H_ +#include "tag-inl.h" +#undef TAG_INL_H_ diff --git a/library/cpp/yt/logging/tagged_payload-inl.h b/library/cpp/yt/logging/tagged_payload-inl.h index 4b71a5bebef..5cb7f1ded8c 100644 --- a/library/cpp/yt/logging/tagged_payload-inl.h +++ b/library/cpp/yt/logging/tagged_payload-inl.h @@ -85,6 +85,36 @@ inline TTaggedPayloadWriter& TTaggedPayloadWriter::EndTag() & return *this; } +inline TTaggedPayloadWriter& TTaggedPayloadWriter::AppendTags(TStringBuf tags) & +{ + YT_ASSERT(MessageEnded_ && !InTag_); + Builder_.AppendString(tags); + return *this; +} + +inline void TTaggedPayloadWriter::AppendTag(std::string* payload, TStringBuf key, TStringBuf value) +{ + // High bit reserved for the well-known flag. + YT_ASSERT(key.size() < WellKnownTagFlag); + + auto keySize = static_cast<ui32>(key.size()); + auto valueSize = static_cast<ui32>(value.size()); + + // Every appended byte is overwritten below, so skip zero-filling them. + auto offset = payload->size(); + ResizeUninitialized(*payload, offset + 2 * sizeof(ui32) + key.size() + value.size()); + + char* ptr = payload->data() + offset; + auto write = [&] (const void* data, size_t size) { + ::memcpy(ptr, data, size); + ptr += size; + }; + write(&keySize, sizeof(keySize)); + write(key.data(), key.size()); + write(&valueSize, sizeof(valueSize)); + write(value.data(), value.size()); +} + inline TTaggedLogEventPayload TTaggedPayloadWriter::Finish() & { YT_ASSERT(MessageEnded_ && !InTag_); diff --git a/library/cpp/yt/logging/tagged_payload.h b/library/cpp/yt/logging/tagged_payload.h index da46f30f42a..67c8dc1f854 100644 --- a/library/cpp/yt/logging/tagged_payload.h +++ b/library/cpp/yt/logging/tagged_payload.h @@ -114,6 +114,16 @@ public: //! Ends the current tag, filling in the value's reserved length prefix. TTaggedPayloadWriter& EndTag() &; + //! Splices an already-serialized tag section (see #TLoggingTagList::GetPayload) + //! verbatim. Must follow #EndMessage, must not interrupt a tag, and must precede + //! any well-known tag, which the layout requires to come last. + TTaggedPayloadWriter& AppendTags(TStringBuf tags) &; + + //! Appends a single framed keyed tag to #payload, for producers that hold an + //! already-formatted value and accumulate a tag section of their own (see + //! #TLoggingTagList). + static void AppendTag(std::string* payload, TStringBuf key, TStringBuf value); + //! Returns the serialized payload. Must follow #EndMessage. TTaggedLogEventPayload Finish() &; diff --git a/library/cpp/yt/logging/unittests/logger_ut.cpp b/library/cpp/yt/logging/unittests/logger_ut.cpp index db176ca5394..3c194a5d4b8 100644 --- a/library/cpp/yt/logging/unittests/logger_ut.cpp +++ b/library/cpp/yt/logging/unittests/logger_ut.cpp @@ -192,6 +192,131 @@ TEST(TTaggedApiTest, CustomSpec) EXPECT_EQ(decoded.Tags[0], std::pair(std::string("Arg1"), std::string("100"))); } +TEST(TTaggedApiTest, WithFormat) +{ + TMockLogManager manager; + TLogger Logger(&manager, "Test"); + YT_TLOG_INFO("Message") + .WithFormat("Method", "%v.%v", "MyService", "MyMethod") + .With("Arg1", 123); + + auto decoded = DecodeSingleEvent(manager); + EXPECT_EQ(decoded.Message, "Message"); + ASSERT_EQ(decoded.Tags.size(), 2u); + EXPECT_EQ(decoded.Tags[0], std::pair(std::string("Method"), std::string("MyService.MyMethod"))); + EXPECT_EQ(decoded.Tags[1], std::pair(std::string("Arg1"), std::string("123"))); +} + +TEST(TTaggedApiTest, TagList) +{ + TMockLogManager manager; + TLogger Logger(&manager, "Test"); + + auto tags = TLoggingTagList() + .With("Address", "localhost:1234") + .With("ConnectionId", 42) + .With("Flags", 256, "%x") + .WithFormat("Method", "%v.%v", "MyService", "MyMethod"); + + YT_TLOG_INFO("Message") + .With("Before", 1) + .With(tags) + .With("After", 2); + + auto decoded = DecodeSingleEvent(manager); + EXPECT_EQ(decoded.Message, "Message"); + ASSERT_EQ(decoded.Tags.size(), 6u); + EXPECT_EQ(decoded.Tags[0], std::pair(std::string("Before"), std::string("1"))); + EXPECT_EQ(decoded.Tags[1], std::pair(std::string("Address"), std::string("localhost:1234"))); + EXPECT_EQ(decoded.Tags[2], std::pair(std::string("ConnectionId"), std::string("42"))); + EXPECT_EQ(decoded.Tags[3], std::pair(std::string("Flags"), std::string("100"))); + EXPECT_EQ(decoded.Tags[4], std::pair(std::string("Method"), std::string("MyService.MyMethod"))); + EXPECT_EQ(decoded.Tags[5], std::pair(std::string("After"), std::string("2"))); +} + +TEST(TTaggedApiTest, EmptyTagList) +{ + TMockLogManager manager; + TLogger Logger(&manager, "Test"); + + TLoggingTagList tags; + EXPECT_TRUE(tags.IsEmpty()); + + YT_TLOG_INFO("Message") + .With(tags); + + auto decoded = DecodeSingleEvent(manager); + EXPECT_EQ(decoded.Message, "Message"); + EXPECT_TRUE(decoded.Tags.empty()); +} + +TEST(TTaggedApiTest, TagListReusedAcrossEvents) +{ + TMockLogManager manager; + TLogger Logger(&manager, "Test"); + + TLoggingTagList tags; + tags.With("Shared", "yes"); + EXPECT_FALSE(tags.IsEmpty()); + + YT_TLOG_INFO("First").With(tags); + YT_TLOG_INFO("Second").With(tags); + + ASSERT_EQ(manager.GetEvents().size(), 2u); + for (const auto& event : manager.GetEvents()) { + auto decoded = DecodeEvent(event); + ASSERT_EQ(decoded.Tags.size(), 1u); + EXPECT_EQ(decoded.Tags[0], std::pair(std::string("Shared"), std::string("yes"))); + } +} + +TEST(TTaggedApiTest, TagListBeforeWellKnownTag) +{ + TMockLogManager manager; + TLogger Logger(&manager, "Test"); + + TLoggingTagList tags; + tags.With("Shared", "yes"); + auto error = TError("boom"); + + YT_TLOG_INFO("Message") + .With(tags) + .With(error); + + ASSERT_EQ(manager.GetEvents().size(), 1u); + TTaggedPayloadReader reader(std::get<TTaggedLogEventPayload>(manager.GetEvents()[0].Payload)); + EXPECT_EQ(reader.ReadMessage(), "Message"); + + auto shared = reader.TryReadTag(); + ASSERT_TRUE(shared); + EXPECT_EQ(shared->Key, "Shared"); + EXPECT_FALSE(shared->IsWellKnown); + + auto errorTag = reader.TryReadTag(); + ASSERT_TRUE(errorTag); + EXPECT_EQ(errorTag->Key, "Error"); + EXPECT_TRUE(errorTag->IsWellKnown); + + EXPECT_FALSE(reader.TryReadTag()); +} + +TEST(TTaggedApiTest, WithFormatDisabledDoesNotEvaluateArgs) +{ + TMockLogManager manager(/*minLevel*/ ELogLevel::Warning); + TLogger Logger(&manager, "Test"); + + int evaluated = 0; + auto evaluate = [&] { + ++evaluated; + return 123; + }; + YT_TLOG_INFO("Message") + .WithFormat("Arg1", "%v-%v", evaluate(), evaluate()); + + EXPECT_EQ(evaluated, 0); + EXPECT_TRUE(manager.GetEvents().empty()); +} + TEST(TTaggedApiTest, LoggerTagFoldedIntoMessage) { TMockLogManager manager; diff --git a/ydb/core/cms/console/console_tenants_manager.cpp b/ydb/core/cms/console/console_tenants_manager.cpp index d64c0ec597a..49594dcc953 100644 --- a/ydb/core/cms/console/console_tenants_manager.cpp +++ b/ydb/core/cms/console/console_tenants_manager.cpp @@ -4136,7 +4136,7 @@ void TTenantsManager::Handle(TEvHive::TEvShrinkStoragePoolDone::TPtr &ev, const return; } auto pool = poolIt->second; - if (pool->State != TStoragePool::EState::SHRINKING) { + if (pool->State != TStoragePool::EState::SHRINKING && pool->State != TStoragePool::EState::NOT_UPDATED) { return; } diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index e85738f0fbf..a9ca1d476b3 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -115,6 +115,7 @@ #include <ydb/core/nbs/cloud/blockstore/config/protos/storage.pb.h> #include <ydb/core/nbs/cloud/blockstore/libs/storage/volume/volume.h> #include <ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct.h> +#include <ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller.h> #endif #include <ydb/core/mon/mon.h> @@ -220,6 +221,7 @@ #include <ydb/core/tx/columnshard/column_fetching/cache_policy.h> #include <ydb/core/tx/general_cache/usage/service.h> #include <ydb/core/tx/priorities/usage/config.h> +#include <ydb/core/tx/priorities/service/service.h> #include <ydb/core/tx/priorities/usage/service.h> #include <ydb/core/tx/limiter/grouped_memory/usage/config.h> @@ -600,10 +602,17 @@ static TInterconnectSettings GetInterconnectSettings(const NKikimrConfig::TInter result.EnableUringSQPOLL = config.GetEnableUringSQPOLL(); } - result.EnableInterconnectSessionV2 = config.GetEnableInterconnectSessionV2(); - result.ChecksumInterconnectSessionV2 = config.GetChecksumInterconnectSessionV2(); - result.EnableSQPOLLv2 = config.GetEnableSQPOLLv2(); - result.EnablePreserializeInV2 = config.GetEnablePreserializeInV2(); + if (config.HasV2Config()) { + const auto& v2 = config.GetV2Config(); + result.V2.Enable = v2.GetEnable(); + result.V2.ChecksumEvents = v2.GetChecksumEvents(); + result.V2.EnableSQPOLL = v2.GetEnableSQPOLL(); + result.V2.EnablePreserializeEvents = v2.GetEnablePreserializeEvents(); + result.V2.UringEngineThreads = v2.GetUringEngineThreads(); + result.V2.UringEngineRingsPerShard = v2.GetUringEngineRingsPerShard(); + result.V2.UringEngineSqThreadIdleMs = v2.GetUringEngineSqThreadIdleMs(); + result.V2.ShareRingsAmongThreads = v2.GetShareRingsAmongThreads(); + } return result; } @@ -765,14 +774,17 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s icCommon->ChannelsConfig = channels; icCommon->Settings = settings; - if (settings.EnableInterconnectSessionV2) { + if (settings.V2.Enable) { // Create the shared v2 io_uring data-plane engine once, at startup, and publish it in Common. // The actor system does not exist yet, so the engine is bound to it later (once it is up, // TInterconnectProxyTCP::Registered calls SetActorSystem). CreateUringEngine returns null when // io_uring is unavailable, in which case v2 is simply never negotiated during the handshake. - icCommon->UringEngineV2 = CreateUringEngine(icConfig.GetUringEngineThreadsV2(), + icCommon->UringEngineV2 = CreateUringEngine(settings.V2.UringEngineThreads, interconectCounters->GetSubgroup("subsystem", "uring"), - settings.EnableSQPOLLv2); + settings.V2.EnableSQPOLL, + settings.V2.UringEngineRingsPerShard, + settings.V2.UringEngineSqThreadIdleMs, + settings.V2.ShareRingsAmongThreads); setup->OnActorSystemCreated.push_back([engine = icCommon->UringEngineV2](TActorSystem *actorSystem) { if (engine) { engine->SetActorSystem(actorSystem); @@ -1360,6 +1372,7 @@ void TLocalServiceInitializer::InitializeServices( #if defined(YDB_EMBEDDED_NBS_ENABLED) addToLocalConfig(TTabletTypes::BlockStoreVolumeDirect, &NYdb::NBS::NStorage::CreateVolumeTablet, TMailboxType::ReadAsFilled, appData->UserPoolId); addToLocalConfig(TTabletTypes::BlockStorePartitionDirect, &NYdb::NBS::NBlockStore::NStorage::NPartitionDirect::CreatePartitionTablet, TMailboxType::ReadAsFilled, appData->UserPoolId); + addToLocalConfig(TTabletTypes::DbsController, &NYdb::NBS::NBlockStore::NStorage::NDbsController::CreateDbsControllerTablet, TMailboxType::ReadAsFilled, appData->UserPoolId); #endif if (Config.GetShutdownConfig().HasDrainTimeoutSeconds()) { @@ -2646,7 +2659,7 @@ void TCompPrioritiesInitializer::InitializeServices(NActors::TActorSystemSetup* TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets"); TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorGroup = tabletGroup->GetSubgroup("type", "TX_COMP_PRIORITIES"); - auto service = NPrioritiesQueue::TCompServiceOperator::CreateService(serviceConfig, conveyorGroup); + auto service = NPrioritiesQueue::CreateService<NPrioritiesQueue::TCompConveyorPolicy>(serviceConfig, conveyorGroup); setup->LocalServices.push_back(std::make_pair( NPrioritiesQueue::TCompServiceOperator::MakeServiceId(NodeId), diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index 3b751848e0b..8b8ffd0fa96 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -122,6 +122,8 @@ PEERDIR( ydb/core/tx/long_tx_service ydb/core/tx/long_tx_service/public ydb/core/tx/mediator + ydb/core/tx/priorities/service + ydb/core/tx/priorities/usage ydb/core/tx/replication/controller ydb/core/tx/replication/service ydb/core/tx/scheme_board @@ -200,6 +202,7 @@ IF (OS_LINUX AND YDB_EMBEDDED_NBS_ENABLED) PEERDIR( ydb/core/nbs/cloud/blockstore/bootstrap ydb/core/nbs/cloud/blockstore/config/protos + ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller ydb/core/nbs/cloud/blockstore/libs/storage/ss_proxy ydb/core/nbs/cloud/blockstore/libs/storage/volume diff --git a/ydb/core/http_proxy/ut/sqs_topic_ut.cpp b/ydb/core/http_proxy/ut/sqs_topic_ut.cpp index e73901b2549..3c486175ca1 100644 --- a/ydb/core/http_proxy/ut/sqs_topic_ut.cpp +++ b/ydb/core/http_proxy/ut/sqs_topic_ut.cpp @@ -2700,7 +2700,7 @@ Y_UNIT_TEST_SUITE(TestSqsTopicHttpProxy) { auto json = CreateQueue({ {"QueueName", queueName}, - {"Attributes", NJson::TJsonMap{{"FifoQueue", "true"}, {"ContentBasedDeduplication", "true"}}} + {"Attributes", NJson::TJsonMap{{"FifoQueue", "true"}}} }); UNIT_ASSERT(!GetByPath<TString>(json, "QueueUrl").empty()); @@ -2713,11 +2713,11 @@ Y_UNIT_TEST_SUITE(TestSqsTopicHttpProxy) { UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteSpeedMessagesPerSecond(), - NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT + NPQ::FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND ); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteBurstMessages(), - NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST + NPQ::FIFO_PARTITION_WRITE_BURST_MESSAGES ); driver.Stop(true); @@ -2747,11 +2747,11 @@ Y_UNIT_TEST_SUITE(TestSqsTopicHttpProxy) { UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteSpeedMessagesPerSecond(), - NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT + NPQ::FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND ); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteBurstMessages(), - NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST + NPQ::FIFO_PARTITION_WRITE_BURST_MESSAGES ); driver.Stop(true); @@ -2781,11 +2781,11 @@ Y_UNIT_TEST_SUITE(TestSqsTopicHttpProxy) { UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteSpeedMessagesPerSecond(), - NPQ::DEFAULT_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND + NPQ::FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND ); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteBurstMessages(), - NPQ::DEFAULT_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND + NPQ::FIFO_PARTITION_WRITE_BURST_MESSAGES ); driver.Stop(true); diff --git a/ydb/core/http_proxy/ut/sqs_topic_xml_ut.cpp b/ydb/core/http_proxy/ut/sqs_topic_xml_ut.cpp index 5b036d41f45..67c7f7a57ac 100644 --- a/ydb/core/http_proxy/ut/sqs_topic_xml_ut.cpp +++ b/ydb/core/http_proxy/ut/sqs_topic_xml_ut.cpp @@ -1336,7 +1336,7 @@ Y_UNIT_TEST_SUITE(TestSqsTopicHttpProxyXml) { auto json = CreateQueueXml({ {"QueueName", queueName}, - {"Attributes", NJson::TJsonMap{{"FifoQueue", "true"}, {"ContentBasedDeduplication", "true"}}} + {"Attributes", NJson::TJsonMap{{"FifoQueue", "true"}}} }); UNIT_ASSERT(!GetByPath<TString>(json, "QueueUrl").empty()); @@ -1349,11 +1349,11 @@ Y_UNIT_TEST_SUITE(TestSqsTopicHttpProxyXml) { UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteSpeedMessagesPerSecond(), - NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT + NPQ::FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND ); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteBurstMessages(), - NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST + NPQ::FIFO_PARTITION_WRITE_BURST_MESSAGES ); driver.Stop(true); @@ -1383,11 +1383,11 @@ Y_UNIT_TEST_SUITE(TestSqsTopicHttpProxyXml) { UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteSpeedMessagesPerSecond(), - NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT + NPQ::FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND ); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteBurstMessages(), - NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST + NPQ::FIFO_PARTITION_WRITE_BURST_MESSAGES ); driver.Stop(true); @@ -1417,11 +1417,11 @@ Y_UNIT_TEST_SUITE(TestSqsTopicHttpProxyXml) { UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteSpeedMessagesPerSecond(), - NPQ::DEFAULT_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND + NPQ::FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND ); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteBurstMessages(), - NPQ::DEFAULT_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND + NPQ::FIFO_PARTITION_WRITE_BURST_MESSAGES ); driver.Stop(true); diff --git a/ydb/core/http_proxy/ut/ymq_ut.cpp b/ydb/core/http_proxy/ut/ymq_ut.cpp index 51947810463..ccf90bb0c25 100644 --- a/ydb/core/http_proxy/ut/ymq_ut.cpp +++ b/ydb/core/http_proxy/ut/ymq_ut.cpp @@ -377,7 +377,7 @@ Y_UNIT_TEST_SUITE(TestYmqHttpProxy) { const TString queueName = "CreateQueueContentBasedDeduplicationRateLimit.fifo"; auto json = CreateQueue({ {"QueueName", queueName}, - {"Attributes", NJson::TJsonMap{{"FifoQueue", "true"}, {"ContentBasedDeduplication", "true"}}} + {"Attributes", NJson::TJsonMap{{"FifoQueue", "true"}}} }); const TString queueUrl = GetByPath<TString>(json, "QueueUrl"); UNIT_ASSERT(!queueUrl.empty()); @@ -385,11 +385,11 @@ Y_UNIT_TEST_SUITE(TestYmqHttpProxy) { const auto description = DescribeMigrationTopicByQueueUrl(*this, queueUrl, queueName); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteSpeedMessagesPerSecond(), - NKikimr::NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT + NKikimr::NPQ::FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND ); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteBurstMessages(), - NKikimr::NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST + NKikimr::NPQ::FIFO_PARTITION_WRITE_BURST_MESSAGES ); } @@ -417,11 +417,11 @@ Y_UNIT_TEST_SUITE(TestYmqHttpProxy) { const auto description = DescribeMigrationTopicByQueueUrl(*this, queueUrl, queueName); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteSpeedMessagesPerSecond(), - NKikimr::NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT + NKikimr::NPQ::FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND ); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteBurstMessages(), - NKikimr::NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST + NKikimr::NPQ::FIFO_PARTITION_WRITE_BURST_MESSAGES ); } @@ -449,11 +449,11 @@ Y_UNIT_TEST_SUITE(TestYmqHttpProxy) { const auto description = DescribeMigrationTopicByQueueUrl(*this, queueUrl, queueName); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteSpeedMessagesPerSecond(), - NKikimr::NPQ::DEFAULT_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND + NKikimr::NPQ::FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND ); UNIT_ASSERT_VALUES_EQUAL( description.GetPartitionWriteBurstMessages(), - NKikimr::NPQ::DEFAULT_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND + NKikimr::NPQ::FIFO_PARTITION_WRITE_BURST_MESSAGES ); } diff --git a/ydb/core/kqp/common/kqp_tx.cpp b/ydb/core/kqp/common/kqp_tx.cpp index 198fa194166..cef634fd781 100644 --- a/ydb/core/kqp/common/kqp_tx.cpp +++ b/ydb/core/kqp/common/kqp_tx.cpp @@ -335,8 +335,7 @@ bool HasUncommittedChangesRead(THashSet<NKikimr::TTableId>& modifiedTables, cons modifiedTables.insert(getTable(index.GetTable())); } - // For plans compatibility with old indexes. Don't need it for new. - if (!settings.GetLookupColumns().empty() && tableModifiedBefore) { + if (settings.GetNeedLookup() && tableModifiedBefore) { AFL_ENSURE(settings.GetType() != NKikimrKqp::TKqpTableSinkSettings::MODE_INSERT); return true; } diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp index adb86b967cb..6ae388f4800 100644 --- a/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/manager.cpp @@ -10,6 +10,7 @@ #include <ydb/library/conclusion/generic/result.h> #include <ydb/library/actors/core/actor.h> #include <ydb/core/external_sources/iceberg_fields.h> +#include <ydb/services/scheme_secret/resolver.h> namespace NKikimr::NKqp { @@ -60,6 +61,18 @@ TString GetSecretName(const NYql::TCreateObjectSettings& settings, const TString return GetOrEmpty(settings, secretKeyPrefix + "_path"); } +[[nodiscard]] TYqlConclusionStatus CheckOldSecretCreationAllowed( + bool disableOldSecretCreation, + const TString& secretName) +{ + if (disableOldSecretCreation && secretName && !NSecret::IsSchemeSecret(secretName)) { + return TYqlConclusionStatus::Fail( + NYql::TIssuesIds::KIKIMR_BAD_REQUEST, + "Old secrets are disabled for creating new objects. Please use new secrets"); + } + return TYqlConclusionStatus::Success(); +} + [[nodiscard]] TYqlConclusionStatus FillCreateExternalDataSourceDesc( NKikimrSchemeOp::TExternalDataSourceDescription& externalDataSourceDesc, const TString& name, @@ -71,6 +84,9 @@ TString GetSecretName(const NYql::TCreateObjectSettings& settings, const TString externalDataSourceDesc.SetLocation(GetOrEmpty(settings, "location")); externalDataSourceDesc.SetInstallation(GetOrEmpty(settings, "installation")); + const bool disableOldSecretCreation = actorSystem && + AppData(actorSystem)->FeatureFlags.GetDisableOldSecretCreation(); + const TString& authMethod = GetOrEmpty(settings, "auth_method"); if (authMethod == "NONE") { externalDataSourceDesc.MutableAuth()->MutableNone(); @@ -78,30 +94,78 @@ TString GetSecretName(const NYql::TCreateObjectSettings& settings, const TString auto& sa = *externalDataSourceDesc.MutableAuth()->MutableServiceAccount(); sa.SetId(GetOrEmpty(settings, "service_account_id")); sa.SetSecretName(GetSecretName(settings, "service_account_secret")); + if (const auto status = + CheckOldSecretCreationAllowed( + disableOldSecretCreation, sa.GetSecretName()); status.IsFail()) + { + return status; + } } else if (authMethod == "BASIC") { auto& basic = *externalDataSourceDesc.MutableAuth()->MutableBasic(); basic.SetLogin(GetOrEmpty(settings, "login")); basic.SetPasswordSecretName(GetSecretName(settings, "password_secret")); + if (const auto status = + CheckOldSecretCreationAllowed( + disableOldSecretCreation, basic.GetPasswordSecretName()); status.IsFail()) + { + return status; + } } else if (authMethod == "MDB_BASIC") { auto& mdbBasic = *externalDataSourceDesc.MutableAuth()->MutableMdbBasic(); mdbBasic.SetServiceAccountId(GetOrEmpty(settings, "service_account_id")); mdbBasic.SetServiceAccountSecretName(GetSecretName(settings, "service_account_secret")); mdbBasic.SetLogin(GetOrEmpty(settings, "login")); mdbBasic.SetPasswordSecretName(GetSecretName(settings, "password_secret")); + if (const auto status = + CheckOldSecretCreationAllowed( + disableOldSecretCreation, mdbBasic.GetServiceAccountSecretName()); status.IsFail()) + { + return status; + } + if (const auto status = + CheckOldSecretCreationAllowed( + disableOldSecretCreation, mdbBasic.GetPasswordSecretName()); status.IsFail()) + { + return status; + } } else if (authMethod == "AWS") { auto& aws = *externalDataSourceDesc.MutableAuth()->MutableAws(); aws.SetAwsAccessKeyIdSecretName(GetSecretName(settings, "aws_access_key_id_secret")); aws.SetAwsSecretAccessKeySecretName(GetSecretName(settings, "aws_secret_access_key_secret")); aws.SetAwsRegion(GetOrEmpty(settings, "aws_region")); + if (const auto status = + CheckOldSecretCreationAllowed( + disableOldSecretCreation, aws.GetAwsAccessKeyIdSecretName()); status.IsFail()) + { + return status; + } + if (const auto status = + CheckOldSecretCreationAllowed( + disableOldSecretCreation, aws.GetAwsSecretAccessKeySecretName()); status.IsFail()) + { + return status; + } } else if (authMethod == "TOKEN") { auto& token = *externalDataSourceDesc.MutableAuth()->MutableToken(); token.SetTokenSecretName(GetSecretName(settings, "token_secret")); + if (const auto status = + CheckOldSecretCreationAllowed( + disableOldSecretCreation, token.GetTokenSecretName()); status.IsFail()) + { + return status; + } } else if (authMethod == "IAM") { auto& iam = *externalDataSourceDesc.MutableAuth()->MutableIam(); iam.SetServiceAccountId(GetOrEmpty(settings, "service_account_id")); iam.SetInitialTokenSecretName(GetSecretName(settings, "initial_token_secret")); // Note: user must not be allowed to specify resource_id; // database authorization relies on resource_id lookup; + if (const auto status = + CheckOldSecretCreationAllowed( + disableOldSecretCreation, iam.GetInitialTokenSecretName()); status.IsFail()) + { + return status; + } } else { return TYqlConclusionStatus::Fail(NYql::TIssuesIds::KIKIMR_INTERNAL_ERROR, TStringBuilder() << "Internal error. Unknown auth method: " << authMethod); } diff --git a/ydb/core/kqp/gateway/behaviour/external_data_source/ya.make b/ydb/core/kqp/gateway/behaviour/external_data_source/ya.make index 7e5663a694b..03ad2926194 100644 --- a/ydb/core/kqp/gateway/behaviour/external_data_source/ya.make +++ b/ydb/core/kqp/gateway/behaviour/external_data_source/ya.make @@ -15,6 +15,7 @@ PEERDIR( ydb/services/metadata/abstract ydb/services/metadata/initializer ydb/services/metadata/secret + ydb/services/scheme_secret ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp index cfc69d14f9f..dd0dbee62b6 100644 --- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp @@ -990,7 +990,7 @@ namespace { bool ParseAsyncReplicationSettingsBase( TReplicationSettingsBase& dstSettings, const TCoNameValueTupleList& srcSettings, TExprContext& ctx, TPositionHandle pos, - const TString& objectName = "replication" + bool disableOldSecretCreation, const TString& objectName = "replication" ) { for (auto setting : srcSettings) { auto name = setting.Name().Value(); @@ -1089,14 +1089,40 @@ namespace { return false; } + if (disableOldSecretCreation) { + auto checkSecret = [&](const TString& secretName) { + if (secretName && !secretName.StartsWith('/')) { + ctx.AddError( + TIssue( + ctx.GetPosition(pos), + "Old secrets are disabled for creating new objects. Please use new secrets" + ) + ); + return false; + } + return true; + }; + + if (const auto& x = dstSettings.OAuthToken; x && !checkSecret(x->TokenSecretName)) { + return false; + } + if (const auto& x = dstSettings.StaticCredentials; x && !checkSecret(x->PasswordSecretName)) { + return false; + } + if (const auto& x = dstSettings.IamCredentials; x && !checkSecret(x->InitialToken.TokenSecretName)) { + return false; + } + } + return true; } bool ParseAsyncReplicationSettings( - TReplicationSettings& dstSettings, const TCoNameValueTupleList& srcSettings, TExprContext& ctx, TPositionHandle pos + TReplicationSettings& dstSettings, const TCoNameValueTupleList& srcSettings, TExprContext& ctx, TPositionHandle pos, + bool disableOldSecretCreation ) { - if (!ParseAsyncReplicationSettingsBase(dstSettings, srcSettings, ctx, pos)) { + if (!ParseAsyncReplicationSettingsBase(dstSettings, srcSettings, ctx, pos, disableOldSecretCreation)) { return false; } @@ -1138,9 +1164,10 @@ namespace { } bool ParseTransferSettings( - TTransferSettings& dstSettings, const TCoNameValueTupleList& srcSettings, TExprContext& ctx, TPositionHandle pos + TTransferSettings& dstSettings, const TCoNameValueTupleList& srcSettings, TExprContext& ctx, TPositionHandle pos, + bool disableOldSecretCreation ) { - if (!ParseAsyncReplicationSettingsBase(dstSettings, srcSettings, ctx, pos, "transfer")) { + if (!ParseAsyncReplicationSettingsBase(dstSettings, srcSettings, ctx, pos, disableOldSecretCreation, "transfer")) { return false; } @@ -3386,7 +3413,13 @@ public: ); } - if (!ParseAsyncReplicationSettings(settings.Settings, createReplication.ReplicationSettings(), ctx, createReplication.Pos())) { + if (!ParseAsyncReplicationSettings( + settings.Settings, + createReplication.ReplicationSettings(), + ctx, + createReplication.Pos(), + SessionCtx->Config().FeatureFlags.GetDisableOldSecretCreation()) + ) { return SyncError(); } @@ -3442,7 +3475,10 @@ public: TAlterReplicationSettings settings; settings.Name = TString(alterReplication.Replication()); - if (!ParseAsyncReplicationSettings(settings.Settings, alterReplication.ReplicationSettings(), ctx, alterReplication.Pos())) { + if (!ParseAsyncReplicationSettings( + settings.Settings, alterReplication.ReplicationSettings(), ctx, alterReplication.Pos(), + SessionCtx->Config().FeatureFlags.GetDisableOldSecretCreation()) + ) { return SyncError(); } @@ -3497,7 +3533,13 @@ public: createTransfer.TransformLambda() }; - if (!ParseTransferSettings(settings.Settings, createTransfer.TransferSettings(), ctx, createTransfer.Pos())) { + if (!ParseTransferSettings( + settings.Settings, + createTransfer.TransferSettings(), + ctx, + createTransfer.Pos(), + SessionCtx->Config().FeatureFlags.GetDisableOldSecretCreation()) + ) { return SyncError(); } @@ -3554,7 +3596,9 @@ public: settings.Name = TString(alterTransfer.Transfer()); settings.TranformLambda = alterTransfer.TransformLambda(); - if (!ParseTransferSettings(settings.Settings, alterTransfer.TransferSettings(), ctx, alterTransfer.Pos())) { + if (!ParseTransferSettings(settings.Settings, alterTransfer.TransferSettings(), ctx, alterTransfer.Pos(), + SessionCtx->Config().FeatureFlags.GetDisableOldSecretCreation()) + ) { return SyncError(); } diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp index a3a598cd427..38e8b2f456d 100644 --- a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp @@ -486,7 +486,25 @@ bool ExploreNode(TExprBase node, TExprContext& ctx, const TKiDataSink& dataSink, } else { const auto& tableData = tablesData->ExistingTable(cluster, table); YQL_ENSURE(tableData.Metadata); - txRes.AddWriteOpToQueryBlock(node, tableData.Metadata->Name, tableData.Metadata->Indexes, tableOp & KikimrReadOps(), false, {}); + + bool needMainTableRead = bool(tableOp & KikimrReadOps()); + if (!needMainTableRead && tableOp != TYdbOperation::Replace && !write.ReturningColumns().Empty()) { + auto inputColumnsSetting = GetSetting(write.Settings().Ref(), "input_columns"); + YQL_ENSURE(inputColumnsSetting); + auto inputColumns = TCoNameValueTuple(inputColumnsSetting).Value().Cast<TCoAtomList>(); + THashSet<TStringBuf> inputColumnsSet; + for (const auto& col : inputColumns) { + inputColumnsSet.insert(col.Value()); + } + for (const auto& returnCol : write.ReturningColumns().Cast<TCoAtomList>()) { + if (!inputColumnsSet.contains(returnCol.Value())) { + needMainTableRead = true; + break; + } + } + } + + txRes.AddWriteOpToQueryBlock(node, tableData.Metadata->Name, tableData.Metadata->Indexes, needMainTableRead, false, {}); } if (!write.ReturningColumns().Empty()) { @@ -577,7 +595,10 @@ bool ExploreNode(TExprBase node, TExprContext& ctx, const TKiDataSink& dataSink, txRes.PrepareForResult(); } - txRes.AddWriteOpToQueryBlock(node, tableData.Metadata->Name, tableData.Metadata->Indexes, tableOp & KikimrReadOps(), false, {}); + const bool needMainTableRead = bool(tableOp & KikimrReadOps()) + || !del.ReturningColumns().Empty(); // For RETURNING row existence must be checked. + + txRes.AddWriteOpToQueryBlock(node, tableData.Metadata->Name, tableData.Metadata->Indexes, needMainTableRead, false, {}); if (!del.ReturningColumns().Empty()) { txRes.AddResult( Build<TResWrite>(ctx, del.Pos()) diff --git a/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp index 7d367abc7d4..a20e45cd0b4 100644 --- a/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp +++ b/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp @@ -1,5 +1,7 @@ +#include <ydb/core/base/counters.h> #include <ydb/core/base/tablet_pipecache.h> +#include <ydb/core/formats/arrow/serializer/native.h> #include <ydb/core/kqp/ut/common/olap_indexes_enums.h> #include <ydb/core/kqp/ut/olap/combinatory/variator.h> #include <ydb/core/kqp/ut/olap/helpers/local.h> @@ -10,15 +12,19 @@ #include <ydb/core/tx/columnshard/engines/changes/with_appended.h> #include <ydb/core/local_indexes/bloom/const.h> #include <ydb/core/tx/columnshard/hooks/testing/controller.h> +#include <ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h> #include <ydb/core/tx/columnshard/test_helper/controllers.h> #include <ydb/core/tx/columnshard/test_helper/test_combinator.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/status_codes.h> +#include <library/cpp/string_utils/base64/base64.h> #include <library/cpp/testing/unittest/registar.h> #include <util/datetime/base.h> #include <util/generic/serialized_enum.h> +#include <format> + namespace NKikimr::NKqp { static void ExecQuery(TKikimrRunner& kikimr, bool useQueryService, const TString& query) { if (useQueryService) { @@ -701,7 +707,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { UNIT_ASSERT_C(!status.IsSuccess(), status.GetIssues().ToString()); } - Y_UNIT_TEST(MinMaxIndexUsedInQueries, EUseQueryService, ELocalIndexAsSchemeObject) { + Y_UNIT_TEST(LocalIndexesUsedInQueries, EUseQueryService, ELocalIndexAsSchemeObject) { const bool UseQueryService = (Arg<0>() == EUseQueryService::QueryService); const bool LocalIndexAsSchemeObject = (Arg<1>() == ELocalIndexAsSchemeObject::SchemeObjectEnabled); auto settings = TKikimrSettings() @@ -745,34 +751,47 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { }; assertDDLQueryOk(R"( - CREATE TABLE `/Root/minmax_test_applied_applied` ( + CREATE TABLE `/Root/olap_indexes_applied_test` ( `key` Int32 NOT NULL, `value` String NOT NULL, + `value_bloom` String NOT NULL, PRIMARY KEY (`key`) ) PARTITION BY HASH (`key`) WITH ( STORE = COLUMN ); - ALTER TABLE `/Root/minmax_test_applied_applied` ADD INDEX `value_mm` LOCAL USING min_max ON(`value`); + ALTER TABLE `/Root/olap_indexes_applied_test` + ADD INDEX idx_ngram LOCAL USING bloom_ngram_filter + ON (`value_bloom`) + ; + + ALTER TABLE `/Root/olap_indexes_applied_test` ADD INDEX `value_mm` LOCAL USING min_max ON(`value`); )"); runDMLQuery(R"( $data1 = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); - UPSERT INTO `/Root/minmax_test_applied_applied` (`key`, `value`) - SELECT CAST(item AS Int32) AS `key`, "Value_" || CAST(item+1 AS String) AS `value` FROM AS_TABLE($data1); + UPSERT INTO `/Root/olap_indexes_applied_test` (`key`, `value`, `value_bloom`) SELECT + CAST(item AS Int32) AS `key`, + "Value_" || CAST(item AS String) AS `value`, + "Value_" || CAST(item AS String) AS `value_bloom`, + FROM AS_TABLE($data1); )"); runDMLQuery(R"( $data2 = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); - UPSERT INTO `/Root/minmax_test_applied_applied` (`key`, `value`) - SELECT CAST(item AS Int32) AS `key`, "Value_" || CAST(item AS String) AS `value` FROM AS_TABLE($data2); + UPSERT INTO `/Root/olap_indexes_applied_test` (`key`, `value`, `value_bloom`) SELECT + CAST(item AS Int32) AS `key`, + "Value_" || CAST(item+1 AS String) AS `value`, + "Value_" || CAST(item+1 AS String) AS `value_bloom`, + FROM AS_TABLE($data2); )"); + csController->WaitCompactions(TDuration::Seconds(5)); struct TQueryResult { ui64 CountResult; - bool MinMaxIndexUsed; + bool LocalIndexUsed; }; auto runQuery = [&](TString text) -> TQueryResult { @@ -782,69 +801,75 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { return TQueryResult { .CountResult = NYT::NodeFromYsonString(ysonArrayWithOneInteger).AsList()[0].AsList()[0].AsUint64(), - .MinMaxIndexUsed = skippedAndApprovedBeforeQuery < skippedAndApprovedAfterQuery + .LocalIndexUsed = skippedAndApprovedBeforeQuery < skippedAndApprovedAfterQuery }; }; TQueryResult resLess = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` < "Value_500000"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` < "Value_500000"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resLess.CountResult, 944450, "incorrect result for query with '<' filter over min_max-indexed column"); - UNIT_ASSERT_C(resLess.MinMaxIndexUsed, "query with '<' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resLess.LocalIndexUsed, "query with '<' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resGreater = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` > "Value_500000"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` > "Value_500000"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resGreater.CountResult, 555549, "incorrect result for query with '>' filter over min_max-indexed column"); - UNIT_ASSERT_C(resGreater.MinMaxIndexUsed, "query with '>' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resGreater.LocalIndexUsed, "query with '>' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resLessOrEqual = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` <= "Value_500000"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` <= "Value_500000"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resLessOrEqual.CountResult, 944451, "incorrect result for query with '<=' filter over min_max-indexed column"); - UNIT_ASSERT_C(resLessOrEqual.MinMaxIndexUsed, "query with '<=' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resLessOrEqual.LocalIndexUsed, "query with '<=' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resGreaterOrEqual = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` >= "Value_500000"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` >= "Value_500000"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resGreaterOrEqual.CountResult, 555550, "incorrect result for query with '>=' filter over min_max-indexed column"); - UNIT_ASSERT_C(resGreaterOrEqual.MinMaxIndexUsed, "query with '>=' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resGreaterOrEqual.LocalIndexUsed, "query with '>=' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resBetween = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` BETWEEN "Value_500000" AND "Value_500001"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` BETWEEN "Value_500000" AND "Value_500001"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resBetween.CountResult, 2, "incorrect result for query with 'BETWEEN' filter over min_max-indexed column"); - UNIT_ASSERT_C(resBetween.MinMaxIndexUsed, "query with 'BETWEEN' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resBetween.LocalIndexUsed, "query with 'BETWEEN' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resAnd = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` >= "Value_500000" AND `value` <= "Value_500001"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` >= "Value_500000" AND `value` <= "Value_500001"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resAnd.CountResult, 2, "incorrect result for query with '`col` >= ... AND `col` <= ...' filter over min_max-indexed column"); - UNIT_ASSERT_C(resAnd.MinMaxIndexUsed, "query with '`col` >= ... AND `col` <= ...' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resAnd.LocalIndexUsed, "query with '`col` >= ... AND `col` <= ...' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resOr = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` >= "Value_500000" OR `value` <= "Value_500001"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` >= "Value_500000" OR `value` <= "Value_500001"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resOr.CountResult, 1500000, "incorrect result for query with '`col` >= ... OR `col` <= ...' filter over min_max-indexed column"); - UNIT_ASSERT_C(resOr.MinMaxIndexUsed, "query with '`col` >= ... OR `col` <= ...' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resOr.LocalIndexUsed, "query with '`col` >= ... OR `col` <= ...' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resNeq = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` != "Value_500000"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` != "Value_500000"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resNeq.CountResult, 1499999, "incorrect result for query with '!=' filter over min_max-indexed column"); - UNIT_ASSERT_C(!resNeq.MinMaxIndexUsed, "query with '!=' filter over min_max-indexed column use min_max index, but it shouldn't"); + UNIT_ASSERT_C(!resNeq.LocalIndexUsed, "query with '!=' filter over min_max-indexed column use min_max index, but it shouldn't"); TQueryResult resIsNotNull = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` IS NOT NULL; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` IS NOT NULL; )"); UNIT_ASSERT_VALUES_EQUAL_C(resIsNotNull.CountResult, 1500000, "incorrect result for query with 'IS NOT NULL' filter over min_max-indexed column"); - UNIT_ASSERT_C(!resIsNotNull.MinMaxIndexUsed, "query with 'IS NOT NULL' filter over min_max-indexed column use min_max index, but it shouldn't(will use in future, see https://github.com/ydb-platform/ydb/issues/38574)"); + UNIT_ASSERT_C(!resIsNotNull.LocalIndexUsed, "query with 'IS NOT NULL' filter over min_max-indexed column use min_max index, but it shouldn't(will use in future, see https://github.com/ydb-platform/ydb/issues/38574)"); TQueryResult resDistinctFromNull = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` IS DISTINCT FROM NULL; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` IS DISTINCT FROM NULL; )"); UNIT_ASSERT_VALUES_EQUAL_C(resDistinctFromNull.CountResult, 1500000, "incorrect result for query with 'IS DISCTINCT FROM NULL' filter over min_max-indexed column"); - UNIT_ASSERT_C(!resDistinctFromNull.MinMaxIndexUsed, "query with 'IS DISCTINCT FROM NULL' filter over min_max-indexed column use min_max index, but it shouldn't(will use in future, see https://github.com/ydb-platform/ydb/issues/38574)"); + UNIT_ASSERT_C(!resDistinctFromNull.LocalIndexUsed, "query with 'IS DISCTINCT FROM NULL' filter over min_max-indexed column use min_max index, but it shouldn't(will use in future, see https://github.com/ydb-platform/ydb/issues/38574)"); + + TQueryResult resBloom = runQuery(R"( + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value_bloom` LIKE '%alue?_50000%' ESCAPE '?'; + )"); + UNIT_ASSERT_VALUES_EQUAL_C(resBloom.CountResult, 11, "incorrect result for query with 'LIKE' filter over bloom_ngram_filter-indexed column"); + UNIT_ASSERT_C(resBloom.LocalIndexUsed, "query with 'LIKE' filter over bloom_ngram_filter-indexed column doesn't use bloom_ngram_filter index)"); } Y_UNIT_TEST(MinMaxIndexStoredInBSForStringsAndInLocalDBOtherwise, EUseQueryService, ELocalIndexAsSchemeObject) { @@ -1155,6 +1180,113 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { ExecQuery(kikimr, UseQueryService, "ALTER TABLE `/Root/olapTable` DROP INDEX idx_ngram;"); } + // Reproduction of https://github.com/ydb-platform/ydb/issues/45739: + // adding a bloom ngram index without a WITH clause must fall back to defaults + // instead of failing with an internal error. + Y_UNIT_TEST(AddLocalBloomNgramIndexWithoutWithClause, EUseQueryService, ELocalIndexAsSchemeObject) { + const bool UseQueryService = (Arg<0>() == EUseQueryService::QueryService); + const bool LocalIndexAsSchemeObject = (Arg<1>() == ELocalIndexAsSchemeObject::SchemeObjectEnabled); + TKikimrSettings settings{}; + settings.AppConfig.MutableFeatureFlags()->SetEnableLocalIndexAsSchemeObject(LocalIndexAsSchemeObject); + TKikimrRunner kikimr(settings); + + ExecQuery(kikimr, UseQueryService, R"( + CREATE TABLE `/Root/olapBloomWithoutWithClause` ( + `key` Int32 NOT NULL, + `value1` String NOT NULL, + `value2` String NOT NULL, + INDEX idx_ngram1 LOCAL USING bloom_ngram_filter ON (`value1`), + PRIMARY KEY (`key`) + ) + PARTITION BY HASH (`key`) + WITH (STORE = COLUMN); + )"); + + ExecQuery(kikimr, UseQueryService, R"( + --!syntax_v1 + ALTER TABLE `/Root/olapBloomWithoutWithClause` + ADD INDEX idx_ngram2 LOCAL USING bloom_ngram_filter + ON (`value2`); + )"); + } + + // Reproduction of https://github.com/ydb-platform/ydb/issues/45738: + // a bloom ngram index must be allowed on a String column (documented as supported), + // not rejected as an inappropriate column type. + Y_UNIT_TEST(AddLocalBloomNgramIndexOnStringColumn, EUseQueryService, ELocalIndexAsSchemeObject) { + const bool UseQueryService = (Arg<0>() == EUseQueryService::QueryService); + const bool LocalIndexAsSchemeObject = (Arg<1>() == ELocalIndexAsSchemeObject::SchemeObjectEnabled); + auto settings = MakeLocalIndexOnInsertTestSettings(); + settings.AppConfig.MutableFeatureFlags()->SetEnableLocalIndexAsSchemeObject(LocalIndexAsSchemeObject); + settings.AppConfig.MutableColumnShardConfig()->SetReaderClassName("SIMPLE"); + TKikimrRunner kikimr(settings); + + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>(); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverrideMemoryLimitForPortionReading(1e+10); + csController->SetOverrideBlobSplitSettings(NOlap::NSplitter::TSplitSettings()); + + ExecQuery(kikimr, UseQueryService, R"( + CREATE TABLE `/Root/olapStringNgram` ( + `key` Int32 NOT NULL, + `value` String NOT NULL, + PRIMARY KEY (`key`) + ) + PARTITION BY HASH (`key`) + WITH (STORE = COLUMN); + )"); + + ExecQuery(kikimr, UseQueryService, R"( + ALTER TABLE `/Root/olapStringNgram` + ADD INDEX idx_ngram LOCAL USING bloom_ngram_filter + ON (`value`) + WITH (ngram_size = 3, false_positive_probability = 0.01, case_sensitive = true); + )"); + + // Data (insert) goes through the query client; index probing uses scan queries, which push + // the LIKE filter down to the shard where the ngram index checker runs. + auto runDML = [&](const TString& query) { + auto result = kikimr.GetQueryClient().ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + }; + auto runScan = [&](const TString& query) -> TString { + auto it = kikimr.GetTableClient().StreamExecuteScanQuery(query).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + return StreamResultToYson(it); + }; + + // Two overlapping writes so each shard accumulates several portions and compaction merges + // them into ngram-indexed portions the reader can skip over. + runDML(R"( + $rows = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); + UPSERT INTO `/Root/olapStringNgram` (`key`, `value`) + SELECT CAST(item AS Int32) AS `key`, "value_" || CAST(item + 1 AS String) AS `value` FROM AS_TABLE($rows); + )"); + runDML(R"( + $rows = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); + UPSERT INTO `/Root/olapStringNgram` (`key`, `value`) + SELECT CAST(item AS Int32) AS `key`, "value_" || CAST(item AS String) AS `value` FROM AS_TABLE($rows); + )"); + csController->WaitCompactions(TDuration::Seconds(5)); + + // Correctness: the LIKE select returns the expected row. + CompareYson(R"([["value_1234567"]])", + runScan(R"(SELECT `value` FROM `/Root/olapStringNgram` WHERE `value` LIKE "%value_1234567%";)"), + "unexpected LIKE result over ngram-indexed column"); + + // The ngram index must be consulted for a LIKE filter: a substring absent from every row + // must let the index skip the indexed portions instead of scanning them. + const ui64 skipBefore = csController->GetIndexesSkippingOnSelect().Val(); + const ui64 approveBefore = csController->GetIndexesApprovedOnSelect().Val(); + CompareYson(R"([[0u]])", + runScan(R"(SELECT COUNT(*) FROM `/Root/olapStringNgram` WHERE `value` LIKE "%nosuchsubstring%";)"), + "unexpected count for absent-substring LIKE"); + const ui64 consulted = (csController->GetIndexesSkippingOnSelect().Val() - skipBefore) + + (csController->GetIndexesApprovedOnSelect().Val() - approveBefore); + UNIT_ASSERT_C(consulted > 0, "ngram index idx_ngram was not used for the LIKE select"); + } + Y_UNIT_TEST(CreateTableWithLocalBloomFilterIndexAndDropIsCorrect, EUseQueryService, ELocalIndexAsSchemeObject) { const bool UseQueryService = (Arg<0>() == EUseQueryService::QueryService); const bool LocalIndexAsSchemeObject = (Arg<1>() == ELocalIndexAsSchemeObject::SchemeObjectEnabled); @@ -2777,6 +2909,129 @@ Y_UNIT_TEST(RenameLocalBloomIndex, EUseQueryService) { TTestIndexesScenario().SetStorageId("__DEFAULT").Initialize().ExecuteAddColumnWithIndexesScenario(); } + // https://github.com/ydb-platform/ydb/issues/26854: general compaction crashed on + // verification=checkRecordsCount == recordsCount because the ngramm index got no data for its column. + // The zero level compacts by portions_count_available and a 1s portions_live_duration instead of the + // ticket's 180s, which a test cannot wait for. `message` is added after the data is written: bulk upsert + // (that is what `ydb import file json` does) demands every column, so a column with no data in the + // merged portions can only appear through a schema change. + TString scriptCompactionWithoutIndexColumnDataHead = R"( + STOP_COMPACTION + ------ + SCHEMA: + CREATE TABLE `/Root/test_table_hang` ( + timestamp Timestamp NOT NULL, + resource_type Utf8 NOT NULL, + resource_id Utf8 NOT NULL, + stream_name Utf8 NOT NULL, + partition Uint32 NOT NULL, + offset Uint64 NOT NULL, + index Uint32 NOT NULL, + level Int32, + json_payload JsonDocument, + ingested_at Timestamp, + saved_at Timestamp, + request_id Utf8, + PRIMARY KEY (timestamp, resource_type, resource_id, stream_name, partition, offset, index) + ) PARTITION BY HASH (timestamp, partition, offset, index) + WITH (STORE = COLUMN, PARTITION_COUNT = 1); + ------ + SCHEMA: + ALTER OBJECT `/Root/test_table_hang` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, + `COMPACTION_PLANNER.CLASS_NAME`=`lc-buckets`, `COMPACTION_PLANNER.FEATURES`=`{"levels": [ + {"class_name": "Zero", "portions_count_limit": 20000000, "expected_blobs_size": 1048576, "portions_live_duration": "1s", "portions_count_available": 1}, + {"class_name": "Zero", "portions_count_limit": 20000000, "expected_blobs_size": 1048576}, + {"class_name": "Zero", "portions_count_limit": 20000000, "expected_blobs_size": 2097152} + ]}`); + ------ + SCHEMA: + ALTER OBJECT `/Root/test_table_hang` (TYPE TABLE) SET (ACTION=ALTER_COLUMN, NAME=json_payload, + `SERIALIZER.CLASS_NAME`=`ARROW_SERIALIZER`, `COMPRESSION.TYPE`=`zstd`, `COMPRESSION.LEVEL`=`4`); + ------ + )"; + + TString scriptCompactionWithoutIndexColumnDataTail = R"( + SCHEMA: + ALTER TABLE `/Root/test_table_hang` ADD COLUMN message Utf8; + ------ + SCHEMA: + ALTER OBJECT `/Root/test_table_hang` (TYPE TABLE) + SET (ACTION=UPSERT_INDEX, NAME=index_ngramm_message, TYPE=BLOOM_NGRAMM_FILTER, + FEATURES=`{"column_name" : "message", "ngramm_size" : 3, "hashes_count" : 2, "filter_size_bytes" : 512, + "records_count" : 3000, "case_sensitive" : false, + "data_extractor" : {"class_name" : "DEFAULT"}, "bits_storage_type": "SIMPLE_STRING"}`); + ------ + SCHEMA: + ALTER OBJECT `/Root/test_table_hang` (TYPE TABLE) SET (ACTION=ALTER_COLUMN, NAME=message, + `SERIALIZER.CLASS_NAME`=`ARROW_SERIALIZER`, `COMPRESSION.TYPE`=`zstd`, `COMPRESSION.LEVEL`=`4`); + ------ + ONE_COMPACTION + ------ + READ: SELECT COUNT(*) FROM `/Root/test_table_hang` WHERE message LIKE '%abc%'; + EXPECTED: [[0u]] + ------ + )"; + + Y_UNIT_TEST(CompactionWithoutIndexColumnData) { + static constexpr ui64 MAX_ROW_CNT = 3; + static constexpr i64 BASE_TIMESTAMP_US = 1760449859000000; // 2025-10-14T13:50:59Z, the ticket's crash time + + const auto buildBatch = [](const ui64 offsetStart) { + NColumnShard::TTableUpdatesBuilder updates(NArrow::MakeArrowSchema( + { { "timestamp", NScheme::TTypeInfo(NScheme::NTypeIds::Timestamp) }, + { "resource_type", NScheme::TTypeInfo(NScheme::NTypeIds::Utf8) }, + { "resource_id", NScheme::TTypeInfo(NScheme::NTypeIds::Utf8) }, + { "stream_name", NScheme::TTypeInfo(NScheme::NTypeIds::Utf8) }, + { "partition", NScheme::TTypeInfo(NScheme::NTypeIds::Uint32) }, + { "offset", NScheme::TTypeInfo(NScheme::NTypeIds::Uint64) }, + { "index", NScheme::TTypeInfo(NScheme::NTypeIds::Uint32) }, + { "level", NScheme::TTypeInfo(NScheme::NTypeIds::Int32) }, + { "json_payload", NScheme::TTypeInfo(NScheme::NTypeIds::Utf8) }, + { "ingested_at", NScheme::TTypeInfo(NScheme::NTypeIds::Timestamp) }, + { "saved_at", NScheme::TTypeInfo(NScheme::NTypeIds::Timestamp) }, + { "request_id", NScheme::TTypeInfo(NScheme::NTypeIds::Utf8) } }, + { "timestamp", "resource_type", "resource_id", "stream_name", "partition", "offset", "index" })); + for (ui64 i = 0; i < MAX_ROW_CNT; ++i) { + const ui64 offset = offsetStart + i; + updates.AddRow() + .Add<i64>(BASE_TIMESTAMP_US + offset) + .Add("resource_type") + .Add("resource_id") + .Add("stream") + .Add<ui32>(0) + .Add<ui64>(offset) + .Add<ui32>(0) + .Add<i32>(1) + .Add(R"({"a" : "b"})") + .Add<i64>(BASE_TIMESTAMP_US) + .Add<i64>(BASE_TIMESTAMP_US) + .Add(std::string("request_") + std::to_string(offset)); + } + return Base64Encode(NArrow::NSerialization::TNativeSerializer().SerializeFull(updates.BuildArrow())); + }; + const TString script = TStringBuilder() << scriptCompactionWithoutIndexColumnDataHead << std::format(R"( + BULK_UPSERT: + /Root/test_table_hang + {} + EXPECT_STATUS:SUCCESS + ------ + BULK_UPSERT: + /Root/test_table_hang + {} + EXPECT_STATUS:SUCCESS + ------ + )", + buildBatch(0).c_str(), buildBatch(MAX_ROW_CNT).c_str()) + << scriptCompactionWithoutIndexColumnDataTail + << std::format(R"( + READ: SELECT COUNT(*) FROM `/Root/test_table_hang`; + EXPECTED: [[{}u]] + )", + 2 * MAX_ROW_CNT); + Variator::ToExecutor(Variator::SingleScript(script)) + .Execute(TKikimrSettings().SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true)); + } + TString scriptDifferentIndexesConfig = R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_ngramm_resource_id, TYPE=BLOOM_NGRAMM_FILTER, FEATURES=`{"column_name" : "resource_id", "ngramm_size" : $$3|8$$, "false_positive_probability" : $$0.01|0.05|0.1$$, "case_sensitive": $$false|true$$, @@ -3868,6 +4123,38 @@ Y_UNIT_TEST(RenameLocalBloomIndex, EUseQueryService) { SELECT COUNT(*) FROM `/Root/olapTableBloomWithDict` WHERE resource_id LIKE "alp%"; )"), "[[2u]]"); } + + Y_UNIT_TEST(DataAndIndexBytesCounters) { + auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true); + TKikimrRunner kikimr(settings); + + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>(); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + + auto helper = TLocalHelper(kikimr); + helper.CreateTestOlapTable(); + + // A MIN_MAX index gives compacted portions index blobs, so IndexBytes becomes non-zero. + ExecQuery(kikimr, false, R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_INDEX, NAME=index_uid, TYPE=MIN_MAX, + FEATURES=`{"column_name" : "uid"}`);)"); + + for (ui32 i = 0; i < 5; ++i) { + WriteTestData(kikimr, "/Root/olapStore/olapTable", 1000000 + i * 100000, 300000000 + i * 100000, 10000); + } + + auto* runtime = kikimr.GetTestServer().GetRuntime(); + auto appCounters = GetServiceCounters(runtime->GetAppData().Counters, "tablets") + ->GetSubgroup("type", "ColumnShard") + ->GetSubgroup("category", "app"); + auto dataBytes = appCounters->GetCounter("SUM(ColumnShard/DataBytes)", false); + auto indexBytes = appCounters->GetCounter("SUM(ColumnShard/IndexBytes)", false); + + // Index blobs are produced by async background compaction, and each shard's counters roll up into + // the SUM(...) aggregate sensor only periodically, so poll until both surface. + csController->WaitCondition(TDuration::Seconds(30), [&]() { return dataBytes->Val() > 0 && indexBytes->Val() > 0; }); + UNIT_ASSERT_GT(dataBytes->Val(), 0); + UNIT_ASSERT_GT(indexBytes->Val(), 0); + } } } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/opt/kqp_concurrent_results_ut.cpp b/ydb/core/kqp/ut/opt/kqp_concurrent_results_ut.cpp new file mode 100644 index 00000000000..31729e9fe71 --- /dev/null +++ b/ydb/core/kqp/ut/opt/kqp_concurrent_results_ut.cpp @@ -0,0 +1,571 @@ +#include <ydb/core/kqp/ut/common/kqp_ut_common.h> + +#include <ydb/public/lib/yson_value/ydb_yson_value.h> + +namespace NKikimr::NKqp { + +using namespace NYdb; +using namespace NYdb::NQuery; + +inline NKikimrConfig::TAppConfig GetAppConfig(bool enableIndexStreamWrite) { + auto app = NKikimrConfig::TAppConfig(); + app.MutableTableServiceConfig()->SetEnableIndexStreamWrite(enableIndexStreamWrite); + return app; +} + +static TExecuteQuerySettings ConcurrentStreamSettings() { + return TExecuteQuerySettings() + .ConcurrentResultSets(true); +} + +static TMap<ui64, TString> CollectConcurrentResults(TExecuteQueryIterator& it) { + TMap<ui64, TString> result; + TMap<ui64, TStringStream> streams; + + for (;;) { + auto streamPart = it.ReadNext().GetValueSync(); + if (!streamPart.IsSuccess()) { + UNIT_ASSERT_C(streamPart.EOS(), streamPart.GetIssues().ToString()); + break; + } + + if (streamPart.HasResultSet()) { + auto idx = streamPart.GetResultSetIndex(); + auto resultSet = streamPart.ExtractResultSet(); + NYson::TYsonWriter writer(&streams[idx], NYson::EYsonFormat::Text, ::NYson::EYsonType::Node, true); + NYdb::FormatResultSetYson(resultSet, writer); + } + } + + for (auto& [idx, stream] : streams) { + result[idx] = stream.Str(); + } + + return result; +} + +static void AssertConcurrentResult(const TMap<ui64, TString>& results, ui64 index, const TString& expectedYson) { + auto it = results.find(index); + UNIT_ASSERT_C(it != results.end(), TStringBuilder() << "result set index " << index << " not found"); + CompareYson(expectedYson, it->second); +} + +static void AssertConcurrentResultUnordered(const TMap<ui64, TString>& results, ui64 index, const TString& expectedYson) { + auto it = results.find(index); + UNIT_ASSERT_C(it != results.end(), TStringBuilder() << "result set index " << index << " not found"); + CompareYsonUnordered(expectedYson, it->second); +} + +static TString ReadTableViaQuery(TSession& session, const TString& table, const TString& columns, const TString& orderByColumn) { + auto query = TStringBuilder() << "SELECT " << columns << " FROM " << table << " ORDER BY " << orderByColumn << ";"; + auto result = session.ExecuteQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + return FormatResultSetYson(result.GetResultSet(0)); +} + +static void ExecuteSchemeQuery(TSession& session, const TString& query) { + auto result = session.ExecuteQuery(query, TTxControl::NoTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); +} + +static void ExecuteDataQuery(TSession& session, const TString& query) { + auto result = session.ExecuteQuery(query, TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); +} + +Y_UNIT_TEST_SUITE(KqpConcurrentResults) { + +Y_UNIT_TEST_TWIN(ConcurrentStreamSelectBetweenReturnings, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t1 (key Int32, val String, PRIMARY KEY(key)); + CREATE TABLE t2 (key Int32, val String, PRIMARY KEY(key)); + CREATE TABLE t3 (key Int32, val String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t1 (key, val) VALUES (1, "a"); + INSERT INTO t2 (key, val) VALUES (10, "x"); + INSERT INTO t3 (key, val) VALUES (20, "y"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t2 (key) VALUES (10) RETURNING key, val; + SELECT key, val FROM t1 ORDER BY key; + UPSERT INTO t3 (key) VALUES (20) RETURNING key, val; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 3u); + AssertConcurrentResult(results, 0, R"([[[10];["x"]]])"); + AssertConcurrentResult(results, 1, R"([[[1];["a"]]])"); + AssertConcurrentResult(results, 2, R"([[[20];["y"]]])"); + } +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamMultipleSelectsAndReturnings, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t1 (key Int32, val String, PRIMARY KEY(key)); + CREATE TABLE t2 (key Int32, val String, PRIMARY KEY(key)); + CREATE TABLE t3 (key Int32, val String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t1 (key, val) VALUES (10, "x"); + INSERT INTO t2 (key, val) VALUES (20, "y"); + INSERT INTO t3 (key, val) VALUES (1, "a"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + SELECT * FROM t3; + UPSERT INTO t1 (key) VALUES (10) RETURNING key, val; + SELECT * FROM t3; + UPSERT INTO t2 (key) VALUES (20) RETURNING key, val; + SELECT * FROM t3; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 5u); + AssertConcurrentResult(results, 0, R"([[[1];["a"]]])"); + AssertConcurrentResult(results, 1, R"([[[10];["x"]]])"); + AssertConcurrentResult(results, 2, R"([[[1];["a"]]])"); + AssertConcurrentResult(results, 3, R"([[[20];["y"]]])"); + AssertConcurrentResult(results, 4, R"([[[1];["a"]]])"); + } +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamSameTableTwoReturnings, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t (key Int32, version Int32, val String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t (key, version, val) VALUES (1, 1, "first"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t (key, version) VALUES (1, 2) RETURNING key, version, val; + UPSERT INTO t (key, version) VALUES (1, 3) RETURNING key, version, val; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 2u); + AssertConcurrentResult(results, 0, R"([[[1];[2];["first"]]])"); + AssertConcurrentResult(results, 1, R"([[[1];[3];["first"]]])"); + } + + CompareYson(R"([[[1];[3];["first"]]])", ReadTableViaQuery(session, "t", "key, version, val", "key")); +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamSameTableBlindWriteThenReturning, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t (key Int32, val String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t (key, val) VALUES (1, "existing"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t (key, val) VALUES (1, "overwritten"); + UPSERT INTO t (key) VALUES (1) RETURNING key, val; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 1u); + AssertConcurrentResult(results, 0, R"([[[1];["overwritten"]]])"); + } + + CompareYson(R"([[[1];["overwritten"]]])", ReadTableViaQuery(session, "t", "key, val", "key")); +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamReturningThenReadSameTable, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t1 (key Int32, val String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t1 (key, val) VALUES (1, "original"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t1 (key, val) VALUES (1, "updated") RETURNING key, val; + SELECT val FROM t1 WHERE key = 1; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 2u); + AssertConcurrentResult(results, 0, R"([[[1];["updated"]]])"); + AssertConcurrentResult(results, 1, R"([[["updated"]]])"); + } +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamReturningWithIndex, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t (c0 Int64, c1 Int64, c2 Int64, PRIMARY KEY(c0), + INDEX idx GLOBAL SYNC ON (c2)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t (c0, c1, c2) VALUES (1, 10, 20); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t (c0) VALUES (1) RETURNING c0, c1, c2; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 1u); + AssertConcurrentResult(results, 0, R"([[[1];[10];[20]]])"); + } + + CompareYson(R"([[[1];[10];[20]]])", ReadTableViaQuery(session, "t", "c0, c1, c2", "c0")); + + ExecuteDataQuery(session, R"( + UPSERT INTO t (c0, c1, c2) VALUES (1, 100, 200); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + $data = SELECT c0, c1, c2 FROM t WHERE c2 = 200 ORDER BY c0; + UPSERT INTO t SELECT c0, (c1 + 1) AS c1, c2 FROM $data RETURNING c0, c1, c2; + SELECT c0, c1, c2 FROM $data; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 2u); + AssertConcurrentResult(results, 0, R"([[[1];[101];[200]]])"); + AssertConcurrentResult(results, 1, R"([[[1];[100];[200]]])"); + } +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamNamedExprSharedByWriteAndSelect, EnableIndexStreamWrite) { + NKikimrConfig::TAppConfig app; + app.MutableTableServiceConfig()->SetEnableIndexStreamWrite(EnableIndexStreamWrite); + auto settings = TKikimrSettings(app).SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE Source (Key String, Value String, PRIMARY KEY(Key)); + CREATE TABLE Dest (Key String, Value String, PRIMARY KEY(Key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO Source (Key, Value) VALUES ("1", "a"), ("2", "b"); + INSERT INTO Dest (Key, Value) VALUES ("1", "c"), ("2", "d"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + $rows = SELECT Key, Value FROM Source ORDER BY Key; + UPSERT INTO Dest (Key) SELECT Key FROM $rows RETURNING Key, Value; + SELECT Key, Value FROM $rows; + SELECT Key, Value FROM Dest; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 3u); + AssertConcurrentResultUnordered(results, 0, R"([[["1"];["c"]];[["2"];["d"]]])"); + AssertConcurrentResult(results, 1, R"([[["1"];["a"]];[["2"];["b"]]])"); + AssertConcurrentResult(results, 2, R"([[["1"];["c"]];[["2"];["d"]]])"); + } +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamNamedExprRandomConsistency, EnableIndexStreamWrite) { + NKikimrConfig::TAppConfig app; + app.MutableTableServiceConfig()->SetEnableIndexStreamWrite(EnableIndexStreamWrite); + auto settings = TKikimrSettings(app).SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE Source (Key String, Value String, PRIMARY KEY(Key)); + CREATE TABLE Dest1 (Key String, Value String, PRIMARY KEY(Key)); + CREATE TABLE Dest2 (Key String, Value String, PRIMARY KEY(Key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO Source (Key, Value) VALUES ("1", ""); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + $rows = SELECT Key, CAST(RandomUuid(Key) AS String) AS Value FROM Source; + UPSERT INTO Dest1 SELECT * FROM $rows RETURNING Value; + UPSERT INTO Dest2 SELECT * FROM $rows RETURNING Value; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(results.at(0), results.at(1)); + } +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamSameTableBlindWriteThenReturningUpsert, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t (key Int32, val String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t (key, val) VALUES (1, "existing"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t (key, val) VALUES (1, "overwritten"); + UPSERT INTO t (key) VALUES (1) RETURNING key, val; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 1u); + AssertConcurrentResult(results, 0, R"([[[1];["overwritten"]]])"); + } + + CompareYson(R"([[[1];["overwritten"]]])", ReadTableViaQuery(session, "t", "key, val", "key")); +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamSameTableBlindWriteThenReturningReplace, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t (key Int32, val String, val2 String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t (key, val, val2) VALUES (1, "first", "second"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + REPLACE INTO t (key, val) VALUES (1, "updated"); + REPLACE INTO t (key, val) VALUES (1, "final") RETURNING key, val, val2; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 1u); + AssertConcurrentResult(results, 0, R"([[[1];["final"];#]])"); + } + + CompareYson(R"([[[1];["final"];#]])", ReadTableViaQuery(session, "t", "key, val, val2", "key")); +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamSameTableBlindWriteThenReturningUpdate, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t (key Int32, val String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t (key, val) VALUES (1, "existing"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t (key, val) VALUES (1, "overwritten"); + UPDATE t SET val = "updated" WHERE key = 1 RETURNING key, val; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 1u); + AssertConcurrentResult(results, 0, R"([[[1];["updated"]]])"); + } + + CompareYson(R"([[[1];["updated"]]])", ReadTableViaQuery(session, "t", "key, val", "key")); +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamSameTableBlindWriteThenReturningDelete, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t (key Int32, val String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t (key, val) VALUES (1, "existing"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t (key, val) VALUES (1, "overwritten"); + DELETE FROM t WHERE key = 1 RETURNING key, val; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 1u); + AssertConcurrentResult(results, 0, R"([[[1];["overwritten"]]])"); + } + + CompareYson(R"([])", ReadTableViaQuery(session, "t", "key, val", "key")); +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamSameTableBlindWriteThenReturningUpdateOn, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t (key Int32, val String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t (key, val) VALUES (1, "existing"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t (key, val) VALUES (1, "overwritten"); + UPDATE t ON (key, val) VALUES (1, "updated") RETURNING key, val; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 1u); + AssertConcurrentResult(results, 0, R"([[[1];["updated"]]])"); + } + + CompareYson(R"([[[1];["updated"]]])", ReadTableViaQuery(session, "t", "key, val", "key")); +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamSameTableBlindWriteThenReturningAllColumnsInInput, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t (key Int32, val String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t (key, val) VALUES (1, "existing"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t (key, val) VALUES (1, "first"); + UPSERT INTO t (key, val) VALUES (1, "overwritten") RETURNING key, val; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 1u); + AssertConcurrentResult(results, 0, R"([[[1];["overwritten"]]])"); + } + + CompareYson(R"([[[1];["overwritten"]]])", ReadTableViaQuery(session, "t", "key, val", "key")); +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamSameTableBlindWriteThenReturningDeleteOn, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t (key Int32, val String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t (key, val) VALUES (1, "existing"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t (key, val) VALUES (1, "overwritten"); + DELETE FROM t ON (key) VALUES (1) RETURNING key, val; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 1u); + AssertConcurrentResult(results, 0, R"([[[1];["overwritten"]]])"); + } + + CompareYson(R"([])", ReadTableViaQuery(session, "t", "key, val", "key")); +} + +Y_UNIT_TEST_TWIN(ConcurrentStreamSameTableBlindWriteThenReturningInsert, EnableIndexStreamWrite) { + auto kikimr = DefaultKikimrRunner({}, GetAppConfig(EnableIndexStreamWrite)); + auto db = kikimr.GetQueryClient(); + auto session = db.GetSession().GetValueSync().GetSession(); + + ExecuteSchemeQuery(session, R"( + CREATE TABLE t (key Int32, val String, val2 String, PRIMARY KEY(key)); + )"); + + ExecuteDataQuery(session, R"( + INSERT INTO t (key, val, val2) VALUES (1, "existing", "other"); + )"); + + { + auto it = db.StreamExecuteQuery(R"( + UPSERT INTO t (key, val, val2) VALUES (1, "overwritten", "other2"); + INSERT INTO t (key, val) VALUES (2, "new") RETURNING key, val, val2; + )", TTxControl::BeginTx().CommitTx(), ConcurrentStreamSettings()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(it.GetStatus(), EStatus::SUCCESS, it.GetIssues().ToString()); + + auto results = CollectConcurrentResults(it); + UNIT_ASSERT_VALUES_EQUAL(results.size(), 1u); + AssertConcurrentResult(results, 0, R"([[[2];["new"];#]])"); + } + + CompareYsonUnordered(R"([[[1];["overwritten"];["other2"]];[[2];["new"];#]])", ReadTableViaQuery(session, "t", "key, val, val2", "key")); +} + +} // Y_UNIT_TEST_SUITE(KqpConcurrentResults) + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/opt/ya.make b/ydb/core/kqp/ut/opt/ya.make index 384afc9f7e5..f8af28139ef 100644 --- a/ydb/core/kqp/ut/opt/ya.make +++ b/ydb/core/kqp/ut/opt/ya.make @@ -8,6 +8,7 @@ SIZE(MEDIUM) SRCS( kqp_agg_ut.cpp + kqp_concurrent_results_ut.cpp kqp_extract_predicate_unpack_ut.cpp kqp_hash_combine_ut.cpp kqp_kv_ut.cpp diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 6d6f6fcd192..420a6111383 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -12940,7 +12940,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { RANK );)").GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR); - UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), "The rank could not be set automatically, the maximum rank of the resource pool classifier is too high: 9223372036854775807", result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS_C(result.GetIssues().ToString(), "Cannot reset property rank", result.GetIssues().ToString()); } TString FetchResourcePoolClassifiers(TTestActorRuntime& runtime, ui32 nodeIndex) { @@ -13117,11 +13117,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) { { auto query = R"( ALTER RESOURCE POOL CLASSIFIER MyResourcePoolClassifier - RESET (RANK, MEMBER_NAME); + RESET (MEMBER_NAME); )"; auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - UNIT_ASSERT_VALUES_EQUAL(FetchResourcePoolClassifiers(kikimr), "{\"resource_pool_classifiers\":[{\"rank\":1042,\"name\":\"MyResourcePoolClassifier\",\"config\":{\"member_name\":\"\",\"resource_pool\":\"test_pool\"},\"database\":\"\\/Root\"},{\"rank\":42,\"name\":\"AnotherResourcePoolClassifier\",\"config\":{\"resource_pool\":\"test_pool\"},\"database\":\"\\/Root\"}]}"); + UNIT_ASSERT_VALUES_EQUAL(FetchResourcePoolClassifiers(kikimr), "{\"resource_pool_classifiers\":[{\"rank\":20,\"name\":\"MyResourcePoolClassifier\",\"config\":{\"member_name\":\"\",\"resource_pool\":\"test_pool\"},\"database\":\"\\/Root\"},{\"rank\":42,\"name\":\"AnotherResourcePoolClassifier\",\"config\":{\"resource_pool\":\"test_pool\"},\"database\":\"\\/Root\"}]}"); } } @@ -15065,6 +15065,37 @@ END DO)", } } + Y_UNIT_TEST(CreateExternalDataSourceWithOldSecretDisabled) { + NKikimrConfig::TFeatureFlags featureFlags; + featureFlags.SetEnableExternalDataSources(true); + featureFlags.SetDisableOldSecretCreation(true); + featureFlags.SetDisableOldSecrets(true); + + NKqp::TKikimrSettings settings; + settings.SetFeatureFlags(featureFlags); + settings.AppConfig.MutableQueryServiceConfig()->AddAvailableExternalDataSources("ObjectStorage"); + TKikimrRunner kikimr(settings); + + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + static const auto query = R"sql( + CREATE EXTERNAL DATA SOURCE `/Root/ExternalDataSource` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="SERVICE_ACCOUNT", + SERVICE_ACCOUNT_ID="mysa", + SERVICE_ACCOUNT_SECRET_NAME="OldSecret" + ); + )sql"; + const auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::BAD_REQUEST, result.GetIssues().ToString()); + UNIT_ASSERT_STRING_CONTAINS_C( + result.GetIssues().ToString(), + "Old secrets are disabled for creating new objects. Please use new secrets", + result.GetIssues().ToString()); + } + Y_UNIT_TEST(SimpleTruncateTableFullPathTableClient) { TestTruncateTable("`/Root/TestTable`", false); } diff --git a/ydb/core/mind/hive/hive.cpp b/ydb/core/mind/hive/hive.cpp index 900efc3527c..a7b2a9b161e 100644 --- a/ydb/core/mind/hive/hive.cpp +++ b/ydb/core/mind/hive/hive.cpp @@ -207,5 +207,9 @@ const std::unordered_map<TTabletTypes::EType, TString> TABLET_TYPE_SHORT_NAMES = const std::unordered_map<TString, TTabletTypes::EType> TABLET_TYPE_BY_SHORT_NAME = MakeReverseMap(TABLET_TYPE_SHORT_NAMES); +TFullTabletId ToFullTabletId(TTabletId tabletId) { + return {tabletId, 0}; +} + } // NHive } // NKikimr diff --git a/ydb/core/mind/hive/hive.h b/ydb/core/mind/hive/hive.h index ad72e02e678..8c4fab48f50 100644 --- a/ydb/core/mind/hive/hive.h +++ b/ydb/core/mind/hive/hive.h @@ -435,6 +435,8 @@ struct TReassignOperation { } }; +TFullTabletId ToFullTabletId(TTabletId tabletId); + } // NHive } // NKikimr diff --git a/ydb/core/mind/hive/hive_events.h b/ydb/core/mind/hive/hive_events.h index 340944c393f..f95e0405ba4 100644 --- a/ydb/core/mind/hive/hive_events.h +++ b/ydb/core/mind/hive/hive_events.h @@ -41,7 +41,7 @@ struct TEvPrivate { EvUpdateBalanceCounters, EvProcessTabletMetrics, EvReassignInactiveGroupsComplete, - EvCompactComplete, + EvMoveDataComplete, EvEnd }; @@ -157,11 +157,11 @@ struct TEvPrivate { TEvReassignInactiveGroupsComplete(const TString& poolName) : PoolName(poolName) {}; }; - struct TEvCompactComplete : TEventLocal<TEvCompactComplete, EvCompactComplete> { + struct TEvMoveDataComplete : TEventLocal<TEvMoveDataComplete, EvMoveDataComplete> { TString PoolName; bool Success; - TEvCompactComplete(const TString& poolName, bool success) : PoolName(poolName), Success(success) {} + TEvMoveDataComplete(const TString& poolName, bool success) : PoolName(poolName), Success(success) {} }; }; diff --git a/ydb/core/mind/hive/hive_impl.cpp b/ydb/core/mind/hive/hive_impl.cpp index e652fe28b8c..e09b79afad3 100644 --- a/ydb/core/mind/hive/hive_impl.cpp +++ b/ydb/core/mind/hive/hive_impl.cpp @@ -551,6 +551,9 @@ TVector<TTabletId> THive::UpdateStoragePools(const google::protobuf::RepeatedPtr for (TTabletId tabletId : tabletsWaiting) { tabletsToUpdate.emplace_back(tabletId); } + if (storagePool.ShrinkRequest) { + Execute(CreateShrinkPool(std::move(storagePool.ShrinkRequest))); + } } return tabletsToUpdate; } @@ -3697,7 +3700,7 @@ void THive::ProcessEvent(std::unique_ptr<IEventHandle> event) { hFunc(TEvHive::TEvShrinkStoragePoolReply, Handle); hFunc(TEvHive::TEvShrinkStoragePoolDone, Handle); hFunc(TEvPrivate::TEvReassignInactiveGroupsComplete, Handle); - hFunc(TEvPrivate::TEvCompactComplete, Handle); + hFunc(TEvPrivate::TEvMoveDataComplete, Handle); } } @@ -3817,7 +3820,7 @@ STFUNC(THive::StateWork) { fFunc(TEvHive::TEvShrinkStoragePoolReply::EventType, EnqueueIncomingEvent); fFunc(TEvHive::TEvShrinkStoragePoolDone::EventType, EnqueueIncomingEvent); fFunc(TEvPrivate::TEvReassignInactiveGroupsComplete::EventType, EnqueueIncomingEvent); - fFunc(TEvPrivate::TEvCompactComplete::EventType, EnqueueIncomingEvent); + fFunc(TEvPrivate::TEvMoveDataComplete::EventType, EnqueueIncomingEvent); hFunc(TEvPrivate::TEvProcessIncomingEvent, Handle); default: if (!HandleDefaultEvents(ev, SelfId())) { @@ -4265,7 +4268,15 @@ void THive::Handle(TEvHive::TEvShrinkStoragePool::TPtr& ev) { } } - Execute(CreateShrinkPool(std::move(ev))); + if (pool.SetShrinkRequest(std::move(ev))) { + THolder<NKikimrBlobStorage::TEvControllerSelectGroups::TGroupParameters> item = pool.BuildRefreshRequest(); + ++pool.RefreshRequestInFlight; + THolder<TEvBlobStorage::TEvControllerSelectGroups> request = MakeHolder<TEvBlobStorage::TEvControllerSelectGroups>(); + NKikimrBlobStorage::TEvControllerSelectGroups& selectRecord = request->Record; + selectRecord.SetReturnAllMatchingGroups(true); + selectRecord.MutableGroupParameters()->AddAllocated(std::move(item).Release()); + SendToBSControllerPipe(request.Release()); + } } void THive::Handle(TEvHive::TEvShrinkStoragePoolReply::TPtr& ev) { @@ -4274,17 +4285,17 @@ void THive::Handle(TEvHive::TEvShrinkStoragePoolReply::TPtr& ev) { void THive::Handle(TEvPrivate::TEvReassignInactiveGroupsComplete::TPtr& ev) { auto& pool = GetStoragePool(ev->Get()->PoolName); - if (!CompactInactiveGroups(pool)) { + if (!MoveDataInactiveGroups(pool)) { CheckRemainingHistory(pool); } } -void THive::Handle(TEvPrivate::TEvCompactComplete::TPtr& ev) { +void THive::Handle(TEvPrivate::TEvMoveDataComplete::TPtr& ev) { auto& pool = GetStoragePool(ev->Get()->PoolName); if (ev->Get()->Success) { CheckRemainingHistory(pool); } else { - if (!CompactInactiveGroups(pool)) { + if (!MoveDataInactiveGroups(pool)) { CheckRemainingHistory(pool); } } @@ -4455,7 +4466,7 @@ void THive::StartShrinkPool(TStoragePoolInfo& pool) { if (ReassignInactiveGroups(pool)) { return; } - if (CompactInactiveGroups(pool)) { + if (MoveDataInactiveGroups(pool)) { return; } CheckRemainingHistory(pool); @@ -4498,9 +4509,9 @@ bool THive::ReassignInactiveGroups(TStoragePoolInfo& pool) { } } -bool THive::CompactInactiveGroups(TStoragePoolInfo& pool) { +bool THive::MoveDataInactiveGroups(TStoragePoolInfo& pool) { std::unordered_set<TStorageGroupId> inactiveGroups(pool.InactiveGroups.begin(), pool.InactiveGroups.end()); - std::vector<TTabletId> tabletsToCompact; + std::vector<TTabletId> tabletsToMoveData; if (pool.RemainingHistory.empty()) { for (const auto& [tabletId, tablet] : Tablets) { bool foundHistory = false; @@ -4516,21 +4527,21 @@ bool THive::CompactInactiveGroups(TStoragePoolInfo& pool) { } } if (foundHistory) { - tabletsToCompact.push_back(tabletId); + tabletsToMoveData.push_back(tabletId); } } } else { auto tabletsWithHistory = pool.RemainingHistory | std::views::transform(&TStoragePoolInfo::THistoryEntry::Tablet); std::unordered_set<TTabletId> uniqueTablets(tabletsWithHistory.begin(), tabletsWithHistory.end()); - tabletsToCompact.assign(uniqueTablets.begin(), uniqueTablets.end()); + tabletsToMoveData.assign(uniqueTablets.begin(), uniqueTablets.end()); } - if (tabletsToCompact.empty()) { + if (tabletsToMoveData.empty()) { return false; } else { - YDB_LOG_INFO("ShrinkPool: starting compact for tablets", + YDB_LOG_INFO("ShrinkPool: starting move data for tablets", {"logPrefix", GetLogPrefix()}, - {"tabletsToCompactCount", tabletsToCompact.size()}); - StartCompactActor(std::move(tabletsToCompact), pool.InactiveGroups, pool.Name); + {"tabletsToMoveDataCount", tabletsToMoveData.size()}); + StartMoveDataActor(std::move(tabletsToMoveData), pool.InactiveGroups, pool.Name); return true; } } diff --git a/ydb/core/mind/hive/hive_impl.h b/ydb/core/mind/hive/hive_impl.h index 1f6103e6d80..c28bdd5d05c 100644 --- a/ydb/core/mind/hive/hive_impl.h +++ b/ydb/core/mind/hive/hive_impl.h @@ -171,7 +171,7 @@ protected: friend struct TNodeInfo; friend struct TLeaderTabletInfo; friend class TReassignTabletsActor; - friend class TCompactActor; + friend class TMoveDataActor; friend class TTxInitScheme; friend class TTxDeleteBase; @@ -258,7 +258,7 @@ protected: void StartHiveStorageBalancer(TStorageBalancerSettings settings); void StartReassignActor(std::vector<TReassignOperation> operations, const TActorId& source, ui32 maxInFlight, TString description, std::unique_ptr<IReassignCallback> callback); void StartReassignActor(std::vector<TReassignOperation> operations); - void StartCompactActor(std::vector<TTabletId> tablets, const std::vector<TStorageGroupId>& groups, const TString& poolName); + void StartMoveDataActor(std::vector<TTabletId> tablets, const std::vector<TStorageGroupId>& groups, const TString& poolName); void CreateEvMonitoring(NMon::TEvRemoteHttpInfo::TPtr& ev, const TActorContext& ctx); NJson::TJsonValue GetBalancerProgressJson(); ITransaction* CreateDeleteTablet(TEvHive::TEvDeleteTablet::TPtr& ev); @@ -632,7 +632,7 @@ protected: void Handle(TEvHive::TEvShrinkStoragePool::TPtr& ev); void Handle(TEvHive::TEvShrinkStoragePoolReply::TPtr& ev); void Handle(TEvPrivate::TEvReassignInactiveGroupsComplete::TPtr& ev); - void Handle(TEvPrivate::TEvCompactComplete::TPtr& ev); + void Handle(TEvPrivate::TEvMoveDataComplete::TPtr& ev); void Handle(TEvHive::TEvShrinkStoragePoolDone::TPtr& ev); protected: @@ -1149,7 +1149,7 @@ protected: void StartShrinkPool(TStoragePoolInfo& pool); bool ReassignInactiveGroups(TStoragePoolInfo& pool); - bool CompactInactiveGroups(TStoragePoolInfo& pool); + bool MoveDataInactiveGroups(TStoragePoolInfo& pool); void CheckRemainingHistory(TStoragePoolInfo& pool); }; diff --git a/ydb/core/mind/hive/compact_actor.cpp b/ydb/core/mind/hive/move_data_actor.cpp index df49f6368c3..583ac29b3d4 100644 --- a/ydb/core/mind/hive/compact_actor.cpp +++ b/ydb/core/mind/hive/move_data_actor.cpp @@ -6,8 +6,8 @@ namespace NKikimr::NHive { -class TCompactActor - : public TActorBootstrapped<TCompactActor> +class TMoveDataActor + : public TActorBootstrapped<TMoveDataActor> , public ISubActor { public: @@ -21,10 +21,10 @@ public: std::vector<TStorageGroupId> Groups; TString PoolName; std::vector<TPipeClient> PipeClients; - i64 CompactsInFlight = 0; + i64 MoveDataInFlight = 0; THive* Hive; - TCompactActor(std::vector<TTabletId> tablets, const std::vector<TStorageGroupId>& groups, const TString& poolName, ui64 maxInFlight, THive* hive) + TMoveDataActor(std::vector<TTabletId> tablets, const std::vector<TStorageGroupId>& groups, const TString& poolName, ui64 maxInFlight, THive* hive) : Tablets(std::move(tablets)) , NextTablet(Tablets.begin()) , Groups(groups) @@ -48,21 +48,21 @@ public: } TString GetDescription() const override { - return TStringBuilder() << "Compact(" << PoolName << ")"; + return TStringBuilder() << "MoveData(" << PoolName << ")"; } - void SendCompact(size_t index, TTabletId tablet) { + void SendMoveData(size_t index, TTabletId tablet) { NTabletPipe::TClientConfig pipeConfig; pipeConfig.RetryPolicy = {.RetryLimitCount = 13}; pipeConfig.CheckAliveness = true; PipeClients[index] = {Register(NTabletPipe::CreateClient(SelfId(), tablet, pipeConfig)), tablet}; NTabletPipe::SendData(SelfId(), PipeClients[index].Client, new TEvTablet::TEvMoveData(Groups)); - ++CompactsInFlight; + ++MoveDataInFlight; } void CheckCompletion() { - if (CompactsInFlight == 0 && NextTablet == Tablets.end()) { - Send(Hive->SelfId(), new TEvPrivate::TEvCompactComplete(PoolName, true)); + if (MoveDataInFlight == 0 && NextTablet == Tablets.end()) { + Send(Hive->SelfId(), new TEvPrivate::TEvMoveDataComplete(PoolName, true)); return PassAway(); } } @@ -70,7 +70,7 @@ public: void Bootstrap() { Become(&TThis::StateWork); for (size_t i = 0; i < PipeClients.size() && NextTablet != Tablets.end(); ++i, ++NextTablet) { - SendCompact(i, *NextTablet); + SendMoveData(i, *NextTablet); } return CheckCompletion(); } @@ -80,9 +80,10 @@ public: for (size_t i = 0; i < PipeClients.size(); ++i) { if (PipeClients[i].Tablet == tablet) { NTabletPipe::CloseClient(SelfId(), PipeClients[i].Client); - --CompactsInFlight; + --MoveDataInFlight; + Hive->Execute(Hive->CreateRestartTablet(ToFullTabletId(tablet))); if (NextTablet != Tablets.end()) { - SendCompact(i, *(NextTablet++)); + SendMoveData(i, *(NextTablet++)); break; } } @@ -93,7 +94,7 @@ public: void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev) { if (ev->Get()->Status != NKikimrProto::OK) { if (ev->Get()->Dead) { - Send(Hive->SelfId(), new TEvPrivate::TEvCompactComplete(PoolName, false)); + Send(Hive->SelfId(), new TEvPrivate::TEvMoveDataComplete(PoolName, false)); return PassAway(); } else { Retry(ev->Get()->TabletId); @@ -109,8 +110,8 @@ public: for (size_t i = 0; i < PipeClients.size(); ++i) { if (PipeClients[i].Tablet == tablet) { NTabletPipe::CloseClient(SelfId(), PipeClients[i].Client); - --CompactsInFlight; - SendCompact(i, tablet); + --MoveDataInFlight; + SendMoveData(i, tablet); break; } } @@ -126,8 +127,8 @@ public: } }; -void THive::StartCompactActor(std::vector<TTabletId> tablets, const std::vector<TStorageGroupId>& groups, const TString& poolName) { - auto* actor = new TCompactActor(std::move(tablets), groups, poolName, 1, this); +void THive::StartMoveDataActor(std::vector<TTabletId> tablets, const std::vector<TStorageGroupId>& groups, const TString& poolName) { + auto* actor = new TMoveDataActor(std::move(tablets), groups, poolName, 1, this); SubActors.emplace_back(actor); RegisterWithSameMailbox(actor); } diff --git a/ydb/core/mind/hive/storage_pool_info.cpp b/ydb/core/mind/hive/storage_pool_info.cpp index d401454e1dd..a092aafeb83 100644 --- a/ydb/core/mind/hive/storage_pool_info.cpp +++ b/ydb/core/mind/hive/storage_pool_info.cpp @@ -35,7 +35,14 @@ void TStoragePoolInfo::UpdateStorageGroup(TStorageGroupId groupId, const TEvCont } void TStoragePoolInfo::DeleteStorageGroup(TStorageGroupId groupId) { - Groups.erase(groupId); + auto it = Groups.find(groupId); + if (it != Groups.end()) { + if (!it->second.IsActive()) { + auto inactiveIt = std::remove(InactiveGroups.begin(), InactiveGroups.end(), groupId); + InactiveGroups.erase(inactiveIt, InactiveGroups.end()); + } + Groups.erase(it); + } } template <> @@ -174,11 +181,17 @@ THolder<TEvControllerSelectGroups::TGroupParameters> TStoragePoolInfo::BuildRefr } bool TStoragePoolInfo::AddTabletToWait(TTabletId tabletId) { - bool result = TabletsWaiting.empty(); + bool result = TabletsWaiting.empty() && !(ShrinkRequest); TabletsWaiting.emplace_back(tabletId); return result; } +bool TStoragePoolInfo::SetShrinkRequest(TEvHive::TEvShrinkStoragePool::TPtr ev) { + bool result = TabletsWaiting.empty() && !(ShrinkRequest); + ShrinkRequest = std::move(ev); + return result; +} + TVector<TTabletId> TStoragePoolInfo::PullWaitingTablets() { return std::move(TabletsWaiting); } diff --git a/ydb/core/mind/hive/storage_pool_info.h b/ydb/core/mind/hive/storage_pool_info.h index 0c67d9004a0..4acbeac1d61 100644 --- a/ydb/core/mind/hive/storage_pool_info.h +++ b/ydb/core/mind/hive/storage_pool_info.h @@ -74,6 +74,7 @@ struct TStoragePoolInfo { ui64 ConsoleVersion = 0; THashSet<THistoryEntry, THistoryHash> RemainingHistory; bool NeedShrinkFromTenant = false; + TEvHive::TEvShrinkStoragePool::TPtr ShrinkRequest; TStoragePoolInfo(const TString& name, THiveSharedSettings* hive); TStoragePoolInfo(const TStoragePoolInfo&) = delete; @@ -98,6 +99,7 @@ struct TStoragePoolInfo { void Invalidate(); THolder<TEvControllerSelectGroups::TGroupParameters> BuildRefreshRequest() const; bool AddTabletToWait(TTabletId tabletId); + bool SetShrinkRequest(TEvHive::TEvShrinkStoragePool::TPtr ev); TVector<TTabletId> PullWaitingTablets(); template <NKikimrConfig::THiveConfig::EHiveStorageSelectStrategy Strategy> size_t SelectGroup(const TVector<double>& groupCandidateUsages); diff --git a/ydb/core/mind/hive/ya.make b/ydb/core/mind/hive/ya.make index f609cdc1f27..4a9a0f632fd 100644 --- a/ydb/core/mind/hive/ya.make +++ b/ydb/core/mind/hive/ya.make @@ -6,7 +6,6 @@ SRCS( boot_queue.cpp boot_queue.h bridge_pile_info.h - compact_actor.cpp data_center_info.h domain_info.cpp domain_info.h @@ -30,6 +29,7 @@ SRCS( metrics.h monitoring.cpp monitoring.h + move_data_actor.cpp node_info.cpp node_info.h object_distribution.h diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller.cpp new file mode 100644 index 00000000000..ec58390d838 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller.cpp @@ -0,0 +1,24 @@ +#include "dbs_controller.h" + +#include "dbs_controller_actor.h" + +#include <ydb/core/base/appdata_fwd.h> + +#include <ydb/library/actors/core/actor.h> + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +using namespace NKikimr; + +//////////////////////////////////////////////////////////////////////////////// + +IActor* CreateDbsControllerTablet( + const TActorId& tablet, + TTabletStorageInfo* info) +{ + return new TDbsControllerActor(tablet, info); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller.h b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller.h new file mode 100644 index 00000000000..d2498d84b8c --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller.h @@ -0,0 +1,19 @@ +#pragma once + +#include <ydb/library/actors/core/actor.h> + +namespace NKikimr { +class TTabletStorageInfo; +} // namespace NKikimr + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +//////////////////////////////////////////////////////////////////////////////// + +NActors::IActor* CreateDbsControllerTablet( + const NActors::TActorId& tablet, + NKikimr::TTabletStorageInfo* info); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_actor.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_actor.cpp new file mode 100644 index 00000000000..1216c7e4d38 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_actor.cpp @@ -0,0 +1,173 @@ +#include "dbs_controller_actor.h" + +#include "dbs_controller_database.h" + +#include <ydb/core/nbs/cloud/storage/core/libs/actors/helpers.h> + +#include <ydb/core/base/tablet_pipe.h> +#include <ydb/core/node_whiteboard/node_whiteboard.h> + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +using namespace NKikimr; +using namespace NActors; + +//////////////////////////////////////////////////////////////////////////////// + +TDbsControllerActor::TDbsControllerActor( + const TActorId& tablet, + NKikimr::TTabletStorageInfo* info) + : TActor(&TThis::StateInit) + , TTabletBase<TDbsControllerActor>( + tablet, + NKikimr::TTabletStorageInfoPtr(info), + nullptr) +{ + LOG_INFO( + TActivationContext::AsActorContext(), + NKikimrServices::DBS_CONTROLLER, + "[%lu] DbsController initialization started", + TabletID()); +} + +TDbsControllerActor::~TDbsControllerActor() = default; + +void TDbsControllerActor::OnDetach(const TActorContext& ctx) +{ + LOG_INFO( + ctx, + NKikimrServices::DBS_CONTROLLER, + "[%lu] OnDetach", + TabletID()); + Die(ctx); +} + +void TDbsControllerActor::OnTabletDead( + TEvTablet::TEvTabletDead::TPtr& ev, + const TActorContext& ctx) +{ + Y_UNUSED(ev); + + LOG_INFO( + ctx, + NKikimrServices::DBS_CONTROLLER, + "[%lu] OnTabletDead", + TabletID()); + Die(ctx); +} + +void TDbsControllerActor::OnActivateExecutor(const TActorContext& ctx) +{ + Become(&TThis::StateWork); + + LOG_INFO( + ctx, + NKikimrServices::DBS_CONTROLLER, + "[%lu] Started DbsController: actor id %s", + TabletID(), + SelfId().ToString().data()); + + if (!Executor()->GetStats().IsFollower()) { + ExecuteTx(ctx, CreateTx<TInitSchema>()); + } + + // allow pipes to connect + SignalTabletActive(ctx); +} + +void TDbsControllerActor::DefaultSignalTabletActive(const TActorContext& ctx) +{ + Y_UNUSED(ctx); +} + +void TDbsControllerActor::HandleServerConnected( + const TEvTabletPipe::TEvServerConnected::TPtr& ev, + const TActorContext& ctx) +{ + const auto* msg = ev->Get(); + + LOG_DEBUG( + ctx, + NKikimrServices::DBS_CONTROLLER, + "[%lu] Pipe client %s server %s connected", + TabletID(), + ToString(msg->ClientId).c_str(), + ToString(msg->ServerId).c_str()); +} + +void TDbsControllerActor::HandleServerDisconnected( + const TEvTabletPipe::TEvServerDisconnected::TPtr& ev, + const TActorContext& ctx) +{ + const auto* msg = ev->Get(); + + LOG_DEBUG( + ctx, + NKikimrServices::DBS_CONTROLLER, + "[%lu] Pipe client %s server %s disconnected", + TabletID(), + ToString(msg->ClientId).c_str(), + ToString(msg->ServerId).c_str()); +} + +void TDbsControllerActor::HandleServerDestroyed( + const TEvTabletPipe::TEvServerDestroyed::TPtr& ev, + const TActorContext& ctx) +{ + const auto* msg = ev->Get(); + + LOG_INFO( + ctx, + NKikimrServices::DBS_CONTROLLER, + "[%lu] Pipe client %s server %s got destroyed", + TabletID(), + ToString(msg->ClientId).c_str(), + ToString(msg->ServerId).c_str()); +} + +void TDbsControllerActor::ReportTabletState(const TActorContext& ctx) +{ + auto service = + NNodeWhiteboard::MakeNodeWhiteboardServiceId(SelfId().NodeId()); + + auto request = std::make_unique< + NNodeWhiteboard::TEvWhiteboard::TEvWhiteboard::TEvTabletStateUpdate>( + TabletID(), + STATE_WORK); + + NYdb::NBS::Send(ctx, service, std::move(request)); +} + +//////////////////////////////////////////////////////////////////////////////// + +void TDbsControllerActor::StateInit(TAutoPtr<NActors::IEventHandle>& ev) +{ + StateInitImpl(ev, SelfId()); +} + +STFUNC(TDbsControllerActor::StateWork) +{ + switch (ev->GetTypeRewrite()) { + cFunc(TEvents::TEvPoison::EventType, PassAway); + + HFunc(TEvTabletPipe::TEvServerConnected, HandleServerConnected); + HFunc(TEvTabletPipe::TEvServerDisconnected, HandleServerDisconnected); + HFunc(TEvTabletPipe::TEvServerDestroyed, HandleServerDestroyed); + + default: + if (!HandleDefaultEvents(ev, SelfId())) { + LOG_ERROR( + TActivationContext::AsActorContext(), + NKikimrServices::DBS_CONTROLLER, + "[%lu] Unhandled event type: %u event %s", + TabletID(), + ev->GetTypeRewrite(), + ev->ToString().c_str()); + } + break; + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_actor.h b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_actor.h new file mode 100644 index 00000000000..3bccc105100 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_actor.h @@ -0,0 +1,72 @@ +#pragma once + +#include "dbs_controller_counters.h" + +#include <ydb/core/nbs/cloud/blockstore/libs/storage/core/tablet.h> + +#include <ydb/core/base/tablet_pipe.h> +#include <ydb/core/tablet_flat/tablet_flat_executed.h> + +#include <ydb/library/actors/core/actor.h> +#include <ydb/library/services/services.pb.h> + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +//////////////////////////////////////////////////////////////////////////////// + +class TDbsControllerActor + : public NActors::TActor<TDbsControllerActor> + , public TTabletBase<TDbsControllerActor> +{ + enum EState + { + STATE_BOOT, + STATE_INIT, + STATE_WORK, + STATE_ZOMBIE, + STATE_MAX, + }; + +public: + TDbsControllerActor( + const NActors::TActorId& tablet, + NKikimr::TTabletStorageInfo* info); + + ~TDbsControllerActor() override; + + static constexpr ui32 LogComponent = NKikimrServices::DBS_CONTROLLER; + using TCounters = TDbsControllerCounters; + +private: + void StateInit(TAutoPtr<NActors::IEventHandle>& ev); + STFUNC(StateWork); + + void OnDetach(const NActors::TActorContext& ctx) override; + void OnTabletDead( + NKikimr::TEvTablet::TEvTabletDead::TPtr& ev, + const NActors::TActorContext& ctx) override; + void OnActivateExecutor(const NActors::TActorContext& ctx) override; + void DefaultSignalTabletActive(const NActors::TActorContext& ctx) override; + + void HandleServerConnected( + const NKikimr::TEvTabletPipe::TEvServerConnected::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleServerDisconnected( + const NKikimr::TEvTabletPipe::TEvServerDisconnected::TPtr& ev, + const NActors::TActorContext& ctx); + + void HandleServerDestroyed( + const NKikimr::TEvTabletPipe::TEvServerDestroyed::TPtr& ev, + const NActors::TActorContext& ctx); + + void ReportTabletState(const NActors::TActorContext& ctx); + + BLOCKSTORE_DBS_CONTROLLER_TRANSACTIONS( + BLOCKSTORE_IMPLEMENT_TRANSACTION, + TTxDbsController) +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_counters.h b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_counters.h new file mode 100644 index 00000000000..40cbd601d78 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_counters.h @@ -0,0 +1,24 @@ +#pragma once + +#include <ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_tx.h> + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +//////////////////////////////////////////////////////////////////////////////// + +struct TDbsControllerCounters +{ + enum ETransactionType + { +#define BLOCKSTORE_TRANSACTION_TYPE(name, ...) TX_##name, + + BLOCKSTORE_DBS_CONTROLLER_TRANSACTIONS(BLOCKSTORE_TRANSACTION_TYPE) + TX_SIZE + +#undef BLOCKSTORE_TRANSACTION_TYPE + }; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_database.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_database.cpp new file mode 100644 index 00000000000..66be14d7e72 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_database.cpp @@ -0,0 +1,19 @@ +#include "dbs_controller_database.h" + +#include "dbs_controller_schema.h" + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +//////////////////////////////////////////////////////////////////////////////// + +void TDbsControllerDatabase::InitSchema() +{ + Materialize<TDbsControllerSchema>(); + + TSchemaInitializer<TDbsControllerSchema::TTables>::InitStorage( + Database.Alter()); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_database.h b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_database.h new file mode 100644 index 00000000000..f48a05b6858 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_database.h @@ -0,0 +1,26 @@ +#pragma once + +#include <ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/protos/dbs_controller.pb.h> + +#include <ydb/core/tablet_flat/flat_cxx_database.h> + +#include <util/generic/maybe.h> + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +//////////////////////////////////////////////////////////////////////////////// + +class TDbsControllerDatabase: public NKikimr::NIceDb::TNiceDb +{ + using TDbsControllerState = + ::NYdb::NBS::DbsController::NProto::TDbsControllerState; + +public: + TDbsControllerDatabase(NKikimr::NTable::TDatabase& database) + : NKikimr::NIceDb::TNiceDb(database) + {} + + void InitSchema(); +}; + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_schema.h b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_schema.h new file mode 100644 index 00000000000..c6327a4d38d --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_schema.h @@ -0,0 +1,45 @@ +#pragma once + +#include <ydb/core/nbs/cloud/blockstore/libs/storage/core/tablet_schema.h> +#include <ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/protos/dbs_controller.pb.h> + +#include <ydb/core/tablet_flat/flat_cxx_database.h> + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +//////////////////////////////////////////////////////////////////////////////// + +struct TDbsControllerSchema: public NKikimr::NIceDb::Schema +{ + enum EChannels + { + SystemChannel, + LogChannel, + IndexChannel, + }; + + struct Dummy: public TTableSchema<1> + { + struct DummyA: public Column<1, NKikimr::NScheme::NTypeIds::Uint32> + { + }; + + struct DummyB: public Column<2, NKikimr::NScheme::NTypeIds::Uint32> + { + }; + + using TKey = TableKey<DummyA>; + using TColumns = TableColumns<DummyA, DummyB>; + + using StoragePolicy = TStoragePolicy<IndexChannel>; + }; + + using TTables = SchemaTables<Dummy>; + + using TSettings = + SchemaSettings<ExecutorLogBatching<true>, ExecutorLogFlushPeriod<0>>; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_tx.h b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_tx.h new file mode 100644 index 00000000000..ef7b5c7560b --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_tx.h @@ -0,0 +1,46 @@ +#pragma once + +#include <ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/protos/dbs_controller.pb.h> + +#include <util/generic/maybe.h> + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +//////////////////////////////////////////////////////////////////////////////// + +#define BLOCKSTORE_DBS_CONTROLLER_TRANSACTIONS(xxx, ...) \ + xxx(InitSchema, __VA_ARGS__) \ + xxx(LoadState, __VA_ARGS__) + +// BLOCKSTORE_DBS_CONTROLLER_TRANSACTIONS + +//////////////////////////////////////////////////////////////////////////////// + +struct TTxDbsController +{ + // + // InitSchema + // + struct TInitSchema + { + explicit TInitSchema() + {} + + void Clear() + {} + }; + + // + // LoadState + // + struct TLoadState + { + explicit TLoadState() + {} + + void Clear() + {} + }; +}; + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_initschema.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_initschema.cpp new file mode 100644 index 00000000000..d2feda2ffd0 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_initschema.cpp @@ -0,0 +1,47 @@ +#include "dbs_controller_actor.h" +#include "dbs_controller_database.h" + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +using namespace NActors; +using namespace NKikimr; +using namespace NKikimr::NTabletFlatExecutor; + +//////////////////////////////////////////////////////////////////////////////// + +bool TDbsControllerActor::PrepareInitSchema( + const TActorContext& ctx, + TTransactionContext& tx, + TTxDbsController::TInitSchema& args) +{ + Y_UNUSED(ctx); + Y_UNUSED(tx); + Y_UNUSED(args); + + return true; +} + +void TDbsControllerActor::ExecuteInitSchema( + const TActorContext& ctx, + TTransactionContext& tx, + TTxDbsController::TInitSchema& args) +{ + Y_UNUSED(ctx); + Y_UNUSED(args); + + TDbsControllerDatabase db(tx.DB); + db.InitSchema(); +} + +void TDbsControllerActor::CompleteInitSchema( + const TActorContext& ctx, + TTxDbsController::TInitSchema& args) +{ + Y_UNUSED(args); + + ExecuteTx(ctx, CreateTx<TLoadState>()); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_loadstate.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_loadstate.cpp new file mode 100644 index 00000000000..75bf7b577ab --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_loadstate.cpp @@ -0,0 +1,49 @@ +#include "dbs_controller_actor.h" +#include "dbs_controller_database.h" + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +using namespace NActors; +using namespace NKikimr; +using namespace NKikimr::NTabletFlatExecutor; + +//////////////////////////////////////////////////////////////////////////////// + +bool TDbsControllerActor::PrepareLoadState( + const TActorContext& ctx, + TTransactionContext& tx, + TTxDbsController::TLoadState& args) +{ + Y_UNUSED(ctx); + Y_UNUSED(tx); + Y_UNUSED(args); + + return true; +} + +void TDbsControllerActor::ExecuteLoadState( + const TActorContext& ctx, + TTransactionContext& tx, + TTxDbsController::TLoadState& args) +{ + Y_UNUSED(ctx); + Y_UNUSED(tx); + Y_UNUSED(args); +} + +void TDbsControllerActor::CompleteLoadState( + const TActorContext& ctx, + TTxDbsController::TLoadState& args) +{ + Y_UNUSED(args); + + LOG_INFO( + ctx, + NKikimrServices::DBS_CONTROLLER, + "[%lu] State loaded, DbsController is ready", + TabletID()); +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/protos/dbs_controller.proto b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/protos/dbs_controller.proto new file mode 100644 index 00000000000..bcdba308ea1 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/protos/dbs_controller.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package NYdb.NBS.DbsController.NProto; +option java_package = "ru.yandex.ydb.core.nbs.dbs_controller.proto"; + +// Persisted controller state. Placeholder for now - fill with real fields as +// the tablet gains functionality. +message TDbsControllerState { + uint64 Version = 1; +} diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/protos/ya.make b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/protos/ya.make new file mode 100644 index 00000000000..13f41bad6aa --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/protos/ya.make @@ -0,0 +1,14 @@ +PROTO_LIBRARY() + +EXCLUDE_TAGS( + GO_PROTO + JAVA_PROTO +) + +SRCS( + dbs_controller.proto +) + +#CPP_PROTO_PLUGIN0(validation ydb/public/lib/validation) + +END() diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/ut/dbs_controller_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/ut/dbs_controller_ut.cpp new file mode 100644 index 00000000000..447f25fba9c --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/ut/dbs_controller_ut.cpp @@ -0,0 +1,37 @@ +#include <ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/dbs_controller_actor.h> + +#include <ydb/core/testlib/basics/runtime.h> +#include <ydb/core/testlib/tablet_helpers.h> + +#include <library/cpp/testing/unittest/registar.h> + +namespace NYdb::NBS::NBlockStore::NStorage::NDbsController { + +using namespace NKikimr; + +//////////////////////////////////////////////////////////////////////////////// + +Y_UNIT_TEST_SUITE(TDbsControllerTest) +{ + Y_UNIT_TEST(ShouldBoot) + { + TTestBasicRuntime runtime; + SetupTabletServices(runtime); + + const ui64 tabletId = MakeTabletID(0, 0, 1); + + CreateTestBootstrapper( + runtime, + CreateTestTabletInfo(tabletId, TTabletTypes::DbsController), + [](const TActorId& tablet, TTabletStorageInfo* info) -> IActor* + { return new TDbsControllerActor(tablet, info); }); + + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvTablet::EvBoot, 1); + runtime.DispatchEvents(options); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYdb::NBS::NBlockStore::NStorage::NDbsController diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/ut/ya.make b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/ut/ya.make new file mode 100644 index 00000000000..9812a696381 --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/ut/ya.make @@ -0,0 +1,15 @@ +UNITTEST_FOR(ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller) + +SRCS( + dbs_controller_ut.cpp +) + +PEERDIR( + ydb/core/base + ydb/core/protos + ydb/core/testlib + ydb/core/testlib/basics + yql/essentials/sql/pg_dummy +) + +END() diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/ya.make b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/ya.make new file mode 100644 index 00000000000..693a0c5de2f --- /dev/null +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/ya.make @@ -0,0 +1,29 @@ +LIBRARY() + +SRCS( + dbs_controller.cpp + dbs_controller_actor.cpp + dbs_controller_database.cpp + dbs_initschema.cpp + dbs_loadstate.cpp +) + +PEERDIR( + ydb/core/nbs/cloud/blockstore/libs/storage/core + ydb/core/nbs/cloud/blockstore/libs/storage/dbs_controller/protos + ydb/core/base + ydb/core/protos + ydb/core/tablet_flat + ydb/library/actors/core + ydb/library/services +) + +END() + +RECURSE( + protos +) + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/part_monitoring.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/part_monitoring.cpp index 1fe17501514..a835073e5bf 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/part_monitoring.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/part_monitoring.cpp @@ -94,10 +94,16 @@ TTabletInfo TPartitionActor::MakeMonTabletInfo() const }; } -void TPartitionActor::HandleHttpInfo( - NMon::TEvRemoteHttpInfo::TPtr& ev, +bool TPartitionActor::OnRenderAppHtmlPage( + NMon::TEvRemoteHttpInfo::TPtr ev, const TActorContext& ctx) { + if (!ev) { + // Probe from the standard tablet page: report that the App page exists + // so its link is shown. + return true; + } + const auto& cgi = ev->Get()->Cgi(); const EMonPage page = ParsePage(cgi); @@ -115,14 +121,14 @@ void TPartitionActor::HandleHttpInfo( ctx.Send( ev->Sender, new NMon::TEvRemoteHttpInfoRes(RenderMonPage(data))); - return; + return true; } // Local DB page: read the persisted state in a transaction; // CompleteMonitoring renders and replies. if (page == EMonPage::LocalDb) { ExecuteTx(ctx, CreateTx<TMonitoring>(ev->Sender)); - return; + return true; } // VChunk page: no index - just the input form (synchronous); with an @@ -137,7 +143,7 @@ void TPartitionActor::HandleHttpInfo( ctx.Send( ev->Sender, new NMon::TEvRemoteHttpInfoRes(RenderMonPage(data))); - return; + return true; } auto* actorSystem = TActivationContext::ActorSystem(); @@ -160,7 +166,7 @@ void TPartitionActor::HandleHttpInfo( requester, new NMon::TEvRemoteHttpInfoRes(RenderMonPage(data))); }); - return; + return true; } const std::optional<size_t> selectedDbg = ParseSelectedDbg(cgi); @@ -198,7 +204,7 @@ void TPartitionActor::HandleHttpInfo( reply << "<meta http-equiv='refresh' content='0; ?TabletID=" << TabletID() << "&page=dbg&dbg=" << *selectedDbg << "'/>"; ctx.Send(ev->Sender, new NMon::TEvRemoteHttpInfoRes(reply)); - return; + return true; } // DBG page: gather snapshots, then render + reply in the callback. Safe @@ -230,6 +236,7 @@ void TPartitionActor::HandleHttpInfo( requester, new NMon::TEvRemoteHttpInfoRes(RenderMonPage(data))); }); + return true; } //////////////////////////////////////////////////////////////////////////////// diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_actor.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_actor.cpp index 07d185ad5bf..16e32a56a1c 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_actor.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_actor.cpp @@ -640,8 +640,6 @@ STFUNC(TPartitionActor::StateWork) TEvPartitionDirectPrivate::TEvPoison, HandlePoisonByBlockedGeneration); - HFunc(NMon::TEvRemoteHttpInfo, HandleHttpInfo); - default: if (!HandleDefaultEvents(ev, SelfId())) { LOG_ERROR( diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_actor.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_actor.h index ceada066408..6c19978475b 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_actor.h +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_actor.h @@ -82,9 +82,12 @@ private: void StateInit(TAutoPtr<NActors::IEventHandle>& ev); STFUNC(StateWork); - void HandleHttpInfo( - NActors::NMon::TEvRemoteHttpInfo::TPtr& ev, - const NActors::TActorContext& ctx); + // The tablet's own monitoring page, reached via the standard tablet page's + // "App" link. The base class passes a null event to ask whether that link + // should appear - it always should. + bool OnRenderAppHtmlPage( + NActors::NMon::TEvRemoteHttpInfo::TPtr ev, + const NActors::TActorContext& ctx) override; void OnDetach(const NActors::TActorContext& ctx) override; void OnTabletDead( diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_ut.cpp index 18f1b62b974..8c7c8aa1199 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_ut.cpp +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/partition_direct_ut.cpp @@ -1439,6 +1439,46 @@ Y_UNIT_TEST_SUITE(TPartitionDirectTest) UNIT_ASSERT_STRING_CONTAINS(html, "Overview"); } + Y_UNIT_TEST(StandardTabletPageRenders) + { + TEnvironmentSetup env{{ + .NodeCount = 8, + .Erasure = TBlobStorageGroupType::Erasure4Plus2Block, + }}; + auto& runtime = env.Runtime; + + auto scopedService = SetupStorage(env, EWriteMode::DirectWrite); + const ui64 tabletId = CreatePartitionTablet(env); + + const TActorId edge = runtime->AllocateEdgeActor( + env.Settings.ControllerNodeId, + __FILE__, + __LINE__); + + // Empty path (not "/app") renders the standard flat-tablet page. + runtime->SendToPipe( + tabletId, + edge, + new NActors::NMon::TEvRemoteHttpInfo( + "?TabletID=" + ToString(tabletId)), + 0, + TTestActorSystem::GetPipeConfigWithRetries()); + + auto response = + env.WaitForEdgeActorEvent<NActors::NMon::TEvRemoteHttpInfoRes>( + edge); + UNIT_ASSERT(response); + + const TString& html = response->Get()->Html; + // The standard tablet page: the info block, the Restart action, and the + // "App" link to this tablet's own monitoring page. + UNIT_ASSERT_STRING_CONTAINS(html, "Tablet generation"); + UNIT_ASSERT_STRING_CONTAINS( + html, + "RestartTabletID=" + ToString(tabletId)); + UNIT_ASSERT_STRING_CONTAINS(html, "tablets/app?"); + } + Y_UNIT_TEST(ShouldSuicideOnPoisonByBlockedGeneration) { TEnvironmentSetup env{{ diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/protos/ya.make b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/protos/ya.make index 9ecb8949d1e..de0a3186df9 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/protos/ya.make +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/protos/ya.make @@ -11,6 +11,4 @@ PEERDIR( ydb/core/protos ) -#CPP_PROTO_PLUGIN0(validation ydb/public/lib/validation) - END() diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/ya.make b/ydb/core/nbs/cloud/blockstore/libs/storage/ya.make index b9990b4fb2d..d9a1882df0b 100644 --- a/ydb/core/nbs/cloud/blockstore/libs/storage/ya.make +++ b/ydb/core/nbs/cloud/blockstore/libs/storage/ya.make @@ -1,5 +1,6 @@ RECURSE( api + dbs_controller partition_direct storage_transport testlib diff --git a/ydb/core/persqueue/public/constants.h b/ydb/core/persqueue/public/constants.h index 4b330799ed2..f73c64de0a9 100644 --- a/ydb/core/persqueue/public/constants.h +++ b/ydb/core/persqueue/public/constants.h @@ -25,7 +25,11 @@ constexpr i32 MAX_READ_RULES_COUNT = 3000; constexpr i32 MAX_SUPPORTED_CODECS_COUNT = 100; constexpr ui64 DEFAULT_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND = 1024 * 1024; -constexpr ui32 CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT = 1000; -constexpr ui32 CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST = 1000; +// FIFO queues use a lower per-partition message rate because of ordering/dedup work. +constexpr ui32 FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND = 1000; +constexpr ui32 FIFO_PARTITION_WRITE_BURST_MESSAGES = 1000; + +constexpr ui32 CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT = FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND; +constexpr ui32 CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST = FIFO_PARTITION_WRITE_BURST_MESSAGES; } // namespace NKikimr::NPQ diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index c3c767af2d2..69bb64c6929 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -549,13 +549,37 @@ message TInterconnectConfig { optional uint32 MaxRdmaRetryBackoffLevel = 61 [default = 8]; // 5s * 2^8 = 1280s, about 21 minutes by default optional bool UseUring = 62 [default = false]; optional bool EnableUringSQPOLL = 63 [default = false]; // only effective when UseUring is set - optional bool EnableInterconnectSessionV2 = 64 [default = false]; - optional bool ChecksumInterconnectSessionV2 = 65 [default = false]; - optional bool EnableSQPOLLv2 = 66 [default = true]; - optional bool EnablePreserializeInV2 = 67 [default = false]; - optional uint32 UringEngineThreadsV2 = 68 [default = 4]; optional bool EnableRdmaSendReceive = 69 [default = false]; + message TV2Config { + optional bool Enable = 1 [default = false]; + optional bool ChecksumEvents = 2 [default = false]; + + // Use io_uring SQPOLL for v2 data-plane rings. If the kernel poller is saturated while shard workers + // still have CPU headroom, set this to false so submissions run on the worker threads instead. + optional bool EnableSQPOLL = 3 [default = true]; + + // If set, this enables event preserialization in IC actor system pool. + optional bool EnablePreserializeEvents = 4 [default = false]; + + // Number of worker threads of ICv2 thread pool (not counting uring internal ones). + optional uint32 UringEngineThreads = 5 [default = 4]; + + // io_uring rings per v2 shard worker (each ring may own a SQPOLL thread). Default 1 preserves legacy + // 1:1 ring/worker topology. Raise this when poller-bound; raise UringEngineThreads when serializer-bound. + optional uint32 UringEngineRingsPerShard = 6 [default = 1]; + + // SQPOLL kernel-thread idle timeout (ms) for v2 rings before the poller sleeps. Only effective when + // EnableSQPOLLv2 is set. Lower values save idle CPU; higher values keep the poller warm for latency. + optional uint32 UringEngineSqThreadIdleMs = 7 [default = 2000]; + + // If enabled, rings of different threads share same kernel thread pools and SQ pollers (effectively creating + // UringEngineRingsPerShard threads instead of UringEngineRingsPerShard * UringEngineThreads). + optional bool ShareRingsAmongThreads = 8 [default = false]; + } + + optional TV2Config V2Config = 70; + // ballast is added to IC handshake frames to ensure correctness of jumbo frames transmission over network optional uint32 HandshakeBallastSize = 14; diff --git a/ydb/core/protos/counters_columnshard.proto b/ydb/core/protos/counters_columnshard.proto index d2714dbdd29..aee63e7972f 100644 --- a/ydb/core/protos/counters_columnshard.proto +++ b/ydb/core/protos/counters_columnshard.proto @@ -50,6 +50,8 @@ enum ESimpleCounters { COUNTER_EVICTED_RAW_BYTES = 40 [(CounterOpts) = {Name: "Index/EvictedBytesRaw"}]; COUNTER_WRITES_IN_FLY = 41 [(CounterOpts) = {Name: "WritesInFly"}]; COUNTER_TX_COMPLETE_LAG = 42 [(CounterOpts) = {Name: "TxCompleteLag"}]; + COUNTER_DATA_BYTES = 43 [(CounterOpts) = {Name: "DataBytes"}]; + COUNTER_INDEX_BYTES = 44 [(CounterOpts) = {Name: "IndexBytes"}]; } enum ECumulativeCounters { diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 127175149f6..8679a47d7ec 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -356,4 +356,5 @@ message TFeatureFlags { optional bool EnableHasPredicatesInResourcePoolClassifiers = 303 [default = true]; optional bool EnableRejectActionInResourcePoolClassifiers = 304 [default = true]; optional bool EnableMoveWithColumnTableReplace = 305 [default = false]; + optional bool DisableOldSecrets = 306 [default = false]; } diff --git a/ydb/core/protos/tablet.proto b/ydb/core/protos/tablet.proto index 2c725df68be..fb66a804784 100644 --- a/ydb/core/protos/tablet.proto +++ b/ydb/core/protos/tablet.proto @@ -54,14 +54,15 @@ message TTabletTypes { BlockStorePartitionDirect = 43; BlockStoreVolumeDirect = 44; NbsLoadTablet = 45; + DbsController = 46; // when adding a new tablet type and keeping parse compatibility with the old version // rename existing reserved item to desired one, and add new reserved item to // the end of reserved list - Reserved46 = 46; Reserved47 = 47; Reserved48 = 48; Reserved49 = 49; + Reserved50 = 50; UserTypeStart = 255; TypeInvalid = -1; diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index aee2907175d..7dd06028b22 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -135,6 +135,7 @@ #include <ydb/core/tx/conveyor/usage/service.h> #include <ydb/core/tx/conveyor_composite/usage/service.h> #include <ydb/core/tx/conveyor_composite/service/service.h> +#include <ydb/core/tx/priorities/service/service.h> #include <ydb/core/tx/priorities/usage/service.h> #include <ydb/core/tx/limiter/grouped_memory/usage/service.h> #include <ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h> @@ -1268,7 +1269,7 @@ namespace Tests { Runtime->RegisterService(NOlap::NGroupedMemoryManager::TDeduplicationMemoryLimiterOperator::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx); } { - auto* actor = NPrioritiesQueue::TCompServiceOperator::CreateService(NPrioritiesQueue::TConfig(), appData.Counters); + auto* actor = NPrioritiesQueue::CreateService<NPrioritiesQueue::TCompConveyorPolicy>(NPrioritiesQueue::TConfig(), appData.Counters); const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::Revolving, 0); Runtime->RegisterService(NPrioritiesQueue::TCompServiceOperator::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx); } diff --git a/ydb/core/testlib/ya.make b/ydb/core/testlib/ya.make index 7b25d62c931..e4e3ba89767 100644 --- a/ydb/core/testlib/ya.make +++ b/ydb/core/testlib/ya.make @@ -113,6 +113,7 @@ PEERDIR( ydb/services/ymq ydb/core/tx/conveyor/service ydb/core/tx/priorities/service + ydb/core/tx/priorities/usage ydb/core/tx/limiter/grouped_memory/usage ydb/services/fq ydb/services/kesus diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 4495181de3a..02cf8bb0bcf 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -341,6 +341,10 @@ void TColumnShard::UpdateIndexCounters() { const std::shared_ptr<const TTabletCountersHandle>& counters = Counters.GetTabletCounters(); counters->SetCounter(COUNTER_INDEX_TABLES, Counters.GetPortionIndexCounters()->GetTablesCount()); + auto diskUsedStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TDiskUsedPortions()); + counters->SetCounter(COUNTER_DATA_BYTES, diskUsedStats.GetDataBlobBytes()); + counters->SetCounter(COUNTER_INDEX_BYTES, diskUsedStats.GetIndexBlobBytes()); + auto insertedStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TPortionsByType<NOlap::NPortion::EProduced::INSERTED>()); counters->SetCounter(COUNTER_INSERTED_PORTIONS, insertedStats.GetCount()); diff --git a/ydb/core/tx/columnshard/counters/portions.cpp b/ydb/core/tx/columnshard/counters/portions.cpp index 31ac9b5eda0..38a0d61a310 100644 --- a/ydb/core/tx/columnshard/counters/portions.cpp +++ b/ydb/core/tx/columnshard/counters/portions.cpp @@ -27,6 +27,7 @@ namespace NKikimr::NOlap { void TSimplePortionsGroupInfo::RemovePortion(const TPortionInfo& p) { BlobBytes.Sub(p.GetTotalBlobBytes()); + IndexBlobBytes.Sub(p.GetIndexBlobBytes()); RawBytes.Sub(p.GetTotalRawBytes()); Count.Sub(1); RecordsCount.Sub(p.GetRecordsCount()); @@ -34,6 +35,7 @@ void TSimplePortionsGroupInfo::RemovePortion(const TPortionInfo& p) { void TSimplePortionsGroupInfo::AddPortion(const TPortionInfo& p) { BlobBytes.Add(p.GetTotalBlobBytes()); + IndexBlobBytes.Add(p.GetIndexBlobBytes()); RawBytes.Add(p.GetTotalRawBytes()); Count.Inc(); RecordsCount.Add(p.GetRecordsCount()); diff --git a/ydb/core/tx/columnshard/counters/portions.h b/ydb/core/tx/columnshard/counters/portions.h index e0acb3e19e7..d7547511158 100644 --- a/ydb/core/tx/columnshard/counters/portions.h +++ b/ydb/core/tx/columnshard/counters/portions.h @@ -19,6 +19,7 @@ class TSimplePortionsGroupInfo { private: using TCountByChannel = THashMap<ui16, i64>; TPositiveControlInteger BlobBytes; + TPositiveControlInteger IndexBlobBytes; TPositiveControlInteger RawBytes; TPositiveControlInteger Count; TPositiveControlInteger RecordsCount; @@ -26,6 +27,7 @@ private: protected: void Add(const TSimplePortionsGroupInfo& item) { BlobBytes.Add(item.BlobBytes); + IndexBlobBytes.Add(item.IndexBlobBytes); RawBytes.Add(item.RawBytes); Count.Add(item.Count); RecordsCount.Add(item.RecordsCount); @@ -44,6 +46,17 @@ public: return BlobBytes.Val(); } + ui64 GetIndexBlobBytes() const { + return IndexBlobBytes.Val(); + } + + ui64 GetDataBlobBytes() const { + const ui64 blob = BlobBytes.Val(); + const ui64 index = IndexBlobBytes.Val(); + AFL_VERIFY(blob >= index)("blob", blob)("index", index); + return blob - index; + } + ui64 GetRawBytes() const { return RawBytes.Val(); } @@ -51,6 +64,7 @@ public: NJson::TJsonValue SerializeToJson() const { NJson::TJsonValue result = NJson::JSON_MAP; result.InsertValue("blob_bytes", BlobBytes.Val()); + result.InsertValue("index_blob_bytes", IndexBlobBytes.Val()); result.InsertValue("raw_bytes", RawBytes.Val()); result.InsertValue("count", Count.Val()); result.InsertValue("records_count", RecordsCount.Val()); @@ -66,8 +80,8 @@ public: } TString DebugString() const { - return TStringBuilder() << "{blob_bytes=" << BlobBytes.Val() << ";raw_bytes=" << RawBytes.Val() << ";count=" << Count.Val() - << ";records=" << RecordsCount.Val() << "}"; + return TStringBuilder() << "{blob_bytes=" << BlobBytes.Val() << ";index_blob_bytes=" << IndexBlobBytes.Val() + << ";raw_bytes=" << RawBytes.Val() << ";count=" << Count.Val() << ";records=" << RecordsCount.Val() << "}"; } TSimplePortionsGroupInfo& operator+=(const TSimplePortionsGroupInfo& item) { @@ -98,6 +112,7 @@ public: bool IsEmpty() const { if (!Count.Val()) { AFL_VERIFY(!BlobBytes.Val())("this", DebugString()); + AFL_VERIFY(!IndexBlobBytes.Val())("this", DebugString()); AFL_VERIFY(!RawBytes.Val())("this", DebugString()); AFL_VERIFY(!RecordsCount.Val())("this", DebugString()); return true; diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/constructor.cpp b/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/constructor.cpp index 02e5e901b4c..ad3de52b88e 100644 --- a/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/constructor.cpp @@ -16,8 +16,9 @@ bool IsSupportedColumnType(const NSchemeShard::TOlapColumnSchema& columnInfo, co const auto extractorProto = dataExtractor.SerializeToProto(); const auto typeId = columnInfo.GetType().GetTypeId(); const bool isUtf8Column = typeId == NScheme::NTypeIds::Utf8; + const bool isStringColumn = typeId == NScheme::NTypeIds::String; const bool isJsonSubColumn = typeId == NScheme::NTypeIds::JsonDocument && extractorProto.HasSubColumn(); - return isUtf8Column || isJsonSubColumn; + return isUtf8Column || isJsonSubColumn || isStringColumn; } } // namespace diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/meta.cpp b/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/meta.cpp index f789d0d1124..640c06b22d5 100644 --- a/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/meta.cpp +++ b/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/meta.cpp @@ -157,7 +157,7 @@ public: template <class TFiller> void FillNGrammHashes(const ui32 nGrammSize, const std::shared_ptr<arrow::Array>& array, TFiller& fillData) { - AFL_VERIFY(array->type_id() == arrow::utf8()->id())("id", array->type()->ToString()); + AFL_VERIFY(array->type_id() == arrow::utf8()->id() || array->type_id() == arrow::binary()->id())("id", array->type()->ToString()); NArrow::SwitchType(array->type_id(), [&](const auto& type) { using TWrap = std::decay_t<decltype(type)>; using T = typename TWrap::T; diff --git a/ydb/core/tx/columnshard/ya.make b/ydb/core/tx/columnshard/ya.make index af63cf4a10c..86f2fae467e 100644 --- a/ydb/core/tx/columnshard/ya.make +++ b/ydb/core/tx/columnshard/ya.make @@ -75,7 +75,7 @@ PEERDIR( ydb/core/tx/conveyor_composite/service ydb/core/tx/general_cache/usage ydb/core/tx/long_tx_service/public - ydb/core/tx/priorities/service + ydb/core/tx/priorities/usage ydb/core/tx/tiering ydb/core/tx/time_cast ydb/core/tx/tracing diff --git a/ydb/core/tx/priorities/service/service.h b/ydb/core/tx/priorities/service/service.h index 663f3519d97..68ac61ed5ae 100644 --- a/ydb/core/tx/priorities/service/service.h +++ b/ydb/core/tx/priorities/service/service.h @@ -3,6 +3,7 @@ #include "manager.h" #include <ydb/core/tx/priorities/usage/events.h> +#include <ydb/core/tx/priorities/usage/service.h> #include <ydb/library/actors/core/actor_bootstrapped.h> #include <ydb/library/actors/core/log.h> @@ -56,4 +57,11 @@ public: void Bootstrap(); }; +template <class TQueuePolicy> +NActors::IActor* CreateService(const TConfig& config, TIntrusivePtr<::NMonitoring::TDynamicCounters> queueSignals) { + using TOperator = TServiceOperatorImpl<TQueuePolicy>; + TOperator::Register(config); + return new TDistributor(config, TOperator::GetQueueName(), queueSignals); +} + } // namespace NKikimr::NPrioritiesQueue diff --git a/ydb/core/tx/priorities/usage/service.h b/ydb/core/tx/priorities/usage/service.h index 79149e918eb..d3831dfddc2 100644 --- a/ydb/core/tx/priorities/usage/service.h +++ b/ydb/core/tx/priorities/usage/service.h @@ -2,7 +2,7 @@ #include "config.h" #include <ydb/library/actors/core/actorid.h> #include <ydb/library/actors/core/actor.h> -#include <ydb/core/tx/priorities/service/service.h> +#include <ydb/library/actors/core/log.h> #include <ydb/core/tx/priorities/usage/events.h> namespace NKikimr::NPrioritiesQueue { @@ -12,6 +12,7 @@ class TServiceOperatorImpl { private: using TSelf = TServiceOperatorImpl<TQueuePolicy>; std::atomic<bool> IsEnabledFlag = false; +public: static void Register(const TConfig& serviceConfig) { Singleton<TSelf>()->IsEnabledFlag = serviceConfig.IsEnabled(); } @@ -19,7 +20,6 @@ private: Y_ABORT_UNLESS(TQueuePolicy::Name.size() == 4); return TQueuePolicy::Name; } -public: [[nodiscard]] static ui64 RegisterClient() { static TAtomicCounter Counter = 0; const ui64 id = Counter.Inc(); @@ -60,11 +60,6 @@ public: static NActors::TActorId MakeServiceId(const ui64 nodeId) { return NActors::TActorId(nodeId, "SrvcPrqe" + GetQueueName()); } - static NActors::IActor* CreateService(const TConfig& config, TIntrusivePtr<::NMonitoring::TDynamicCounters> queueSignals) { - Register(config); - return new TDistributor(config, GetQueueName(), queueSignals); - } - }; class TCompConveyorPolicy { diff --git a/ydb/core/tx/replication/controller/secret_resolver.cpp b/ydb/core/tx/replication/controller/secret_resolver.cpp index 966556d8141..0924762cdd5 100644 --- a/ydb/core/tx/replication/controller/secret_resolver.cpp +++ b/ydb/core/tx/replication/controller/secret_resolver.cpp @@ -60,6 +60,9 @@ class TSecretResolver: public TActorBootstrapped<TSecretResolver> { future.Subscribe([actorSystem, replyActorId](const NThreading::TFuture<NKqp::TEvDescribeSecretsResponse::TDescription>& result) { actorSystem->Send(replyActorId, new NKqp::TEvDescribeSecretsResponse(result.GetValue())); }); + } else if (AppData()->FeatureFlags.GetDisableOldSecrets()) { + // Just in case - when we disable old secrets, we'll make sure they are not needed any more + return Reply(false, "Usage of old secrets is disabled now. Please use new secrets"); } else { Send(NMetadata::NProvider::MakeServiceId(SelfId().NodeId()), new NMetadata::NProvider::TEvAskSnapshot(SnapshotFetcher())); diff --git a/ydb/core/tx/schemeshard/olap/columns/ya.make b/ydb/core/tx/schemeshard/olap/columns/ya.make index 88c4ce195a8..bd72c1a1b6d 100644 --- a/ydb/core/tx/schemeshard/olap/columns/ya.make +++ b/ydb/core/tx/schemeshard/olap/columns/ya.make @@ -7,6 +7,7 @@ SRCS( PEERDIR( ydb/core/protos + ydb/core/scheme_types ydb/core/formats/arrow/dictionary ydb/core/formats/arrow/serializer ydb/core/tx/schemeshard/olap/common diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index a8359d2254b..98563f8dcc4 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3172,6 +3172,10 @@ struct TTableIndexInfo : public TSimpleRefCount<TTableIndexInfo> { return std::visit([]<typename T>(const T& v) { if constexpr (std::is_same_v<std::monostate, T>) { return TString{}; + } else if constexpr (std::is_same_v<NKikimrSchemeOp::TBloomNGrammFilter, T>) { + TString str; + Y_ENSURE(v.SerializeToString(&str)); + return str; } else { TString str{v.SerializeAsString()}; Y_ENSURE(!str.empty()); diff --git a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make index 0fc638a8da1..33b7777de85 100644 --- a/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make +++ b/ydb/core/tx/schemeshard/ut_export_reboots_s3/ya.make @@ -2,7 +2,7 @@ UNITTEST_FOR(ydb/core/tx/schemeshard) FORK_SUBTESTS() -SPLIT_FACTOR(50) +SPLIT_FACTOR(200) REQUIREMENTS(ram:32 cpu:4) diff --git a/ydb/core/ymq/actor/create_topic_tx.cpp b/ydb/core/ymq/actor/create_topic_tx.cpp index 9dff4ec8717..d345845c627 100644 --- a/ydb/core/ymq/actor/create_topic_tx.cpp +++ b/ydb/core/ymq/actor/create_topic_tx.cpp @@ -27,10 +27,10 @@ Ydb::Topic::CreateTopicRequest BuildCreateTopicTx( if (params.HasContentBasedDeduplication) { request.set_content_based_deduplication(params.ContentBasedDeduplication); - if (params.ContentBasedDeduplication) { - request.set_partition_write_speed_messages_per_second(NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT); - request.set_partition_write_burst_messages(NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST); - } + } + if (isFifo) { + request.set_partition_write_speed_messages_per_second(NPQ::FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND); + request.set_partition_write_burst_messages(NPQ::FIFO_PARTITION_WRITE_BURST_MESSAGES); } auto* partitioningSettings = request.mutable_partitioning_settings(); diff --git a/ydb/core/ymq/actor/set_queue_attributes.cpp b/ydb/core/ymq/actor/set_queue_attributes.cpp index 8ff37df5205..317367b7183 100644 --- a/ydb/core/ymq/actor/set_queue_attributes.cpp +++ b/ydb/core/ymq/actor/set_queue_attributes.cpp @@ -5,7 +5,6 @@ #include "params.h" #include "serviceid.h" -#include <ydb/core/persqueue/public/constants.h> #include <ydb/core/persqueue/public/schema/schema.h> #include <ydb/core/ymq/base/limits.h> #include <ydb/core/ymq/base/dlq_helpers.h> @@ -173,13 +172,6 @@ private: if (ValidatedAttributes_.ContentBasedDeduplication) { request.set_set_content_based_deduplication(*ValidatedAttributes_.ContentBasedDeduplication); - if (*ValidatedAttributes_.ContentBasedDeduplication) { - request.set_set_partition_write_speed_messages_per_second(NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT); - request.set_set_partition_write_burst_messages(NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST); - } else { - request.set_set_partition_write_speed_messages_per_second(NPQ::DEFAULT_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND); - request.set_set_partition_write_burst_messages(NPQ::DEFAULT_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND); - } } auto* type = consumer->mutable_alter_shared_consumer_type(); diff --git a/ydb/docs/en/core/_assets/resources_weight.drawio b/ydb/docs/en/core/_assets/resources_weight.drawio index 5e37ed89cbf..6156ef34bda 100644 --- a/ydb/docs/en/core/_assets/resources_weight.drawio +++ b/ydb/docs/en/core/_assets/resources_weight.drawio @@ -40,7 +40,7 @@ <mxPoint x="490" y="230" as="targetPoint" /> </mxGeometry> </mxCell> - <mxCell id="VCuo0HpmyH9Wwde20rOg-12" value="RESOURCES_WEIGHT<div>=100</div>" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="VCuo0HpmyH9Wwde20rOg-11" vertex="1" connectable="0"> + <mxCell id="VCuo0HpmyH9Wwde20rOg-12" value="RESOURCE_WEIGHT<div>=100</div>" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="VCuo0HpmyH9Wwde20rOg-11" vertex="1" connectable="0"> <mxGeometry x="-0.1738" y="6" relative="1" as="geometry"> <mxPoint x="65" y="-12" as="offset" /> </mxGeometry> @@ -146,10 +146,10 @@ <mxPoint x="-97" y="7" as="offset" /> </mxGeometry> </mxCell> - <mxCell id="VCuo0HpmyH9Wwde20rOg-36" value="<div style="text-align: center;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; float: none; display: inline !important;">Old limit (RESOURCES_WEIGHT=100):</span></div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="forced-color-adjust: none; color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5</span><div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;"><br></span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">New limit (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCES_WEIGHT=100)</span><span style="text-wrap: nowrap; background-color: initial;">:</span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 70/3 ~= 2.3 vCPU</span><br style="text-wrap: nowrap;"><span style="text-wrap: nowrap;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 ~= 1.15&nbsp; vCPU</span><span style="text-wrap: nowrap;"><br></span></div>" style="text;whiteSpace=wrap;html=1;" parent="1" vertex="1"> + <mxCell id="VCuo0HpmyH9Wwde20rOg-36" value="<div style="text-align: center;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; float: none; display: inline !important;">Old limit (RESOURCE_WEIGHT=100):</span></div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="forced-color-adjust: none; color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5</span><div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;"><br></span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">New limit (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCE_WEIGHT=100)</span><span style="text-wrap: nowrap; background-color: initial;">:</span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 70/3 ~= 2.3 vCPU</span><br style="text-wrap: nowrap;"><span style="text-wrap: nowrap;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 ~= 1.15&nbsp; vCPU</span><span style="text-wrap: nowrap;"><br></span></div>" style="text;whiteSpace=wrap;html=1;" parent="1" vertex="1"> <mxGeometry x="170.00000000000009" y="820" width="360" height="50" as="geometry" /> </mxCell> - <mxCell id="VCuo0HpmyH9Wwde20rOg-37" value="<div style="text-align: center;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; float: none; display: inline !important;">Old limit (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCES_WEIGHT=200)</span><span style="text-wrap: nowrap;">:</span></div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="forced-color-adjust: none; color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5</span><div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;"><br></span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">New limit (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCES_WEIGHT=200)</span><span style="text-wrap: nowrap; background-color: initial;">:</span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="text-wrap: nowrap;"><span style="text-wrap: nowrap;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5 vCPU</span><span style="text-wrap: nowrap;"><br></span></div>" style="text;whiteSpace=wrap;html=1;" parent="1" vertex="1"> + <mxCell id="VCuo0HpmyH9Wwde20rOg-37" value="<div style="text-align: center;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; float: none; display: inline !important;">Old limit (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCE_WEIGHT=200)</span><span style="text-wrap: nowrap;">:</span></div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="forced-color-adjust: none; color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5</span><div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;"><br></span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">New limit (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCE_WEIGHT=200)</span><span style="text-wrap: nowrap; background-color: initial;">:</span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="text-wrap: nowrap;"><span style="text-wrap: nowrap;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5 vCPU</span><span style="text-wrap: nowrap;"><br></span></div>" style="text;whiteSpace=wrap;html=1;" parent="1" vertex="1"> <mxGeometry x="555" y="820" width="360" height="50" as="geometry" /> </mxCell> </root> diff --git a/ydb/docs/en/core/_assets/resources_weight.png b/ydb/docs/en/core/_assets/resources_weight.png Binary files differindex cb9f2d19145..9853f0606a4 100644 --- a/ydb/docs/en/core/_assets/resources_weight.png +++ b/ydb/docs/en/core/_assets/resources_weight.png diff --git a/ydb/docs/en/core/_includes/olap-data-types.md b/ydb/docs/en/core/_includes/olap-data-types.md deleted file mode 100644 index 0c41154a604..00000000000 --- a/ydb/docs/en/core/_includes/olap-data-types.md +++ /dev/null @@ -1,24 +0,0 @@ -| Data type | Can be used in<br/>column-oriented tables | Can be used<br/>as primary key | ----|---|--- -| `Bool` | ✓ | ✓ | -| `Int8` | ✓ | ☓ | -| `Int16` | ✓ | ☓ | -| `Int32` | ✓ | ✓ | -| `Int64` | ✓ | ✓ | -| `Uint8` | ✓ | ✓ | -| `Uint16` | ✓ | ✓ | -| `Uint32` | ✓ | ✓ | -| `Uint64` | ✓ | ✓ | -| `Float` | ✓ | ☓ | -| `Double` | ✓ | ☓ | -| `Decimal` | ✓ | ☓ | -| `String` | ✓ | ✓ | -| `Utf8` | ✓ | ✓ | -| `Json` | ✓ | ☓ | -| `JsonDocument` | ✓ | ☓ | -| `Yson` | ✓ | ☓ | -| `Uuid` | ☓ | ☓ | -| `Date` | ✓ | ✓ | -| `Datetime` | ✓ | ✓ | -| `Timestamp` | ✓ | ✓ | -| `Interval` | ☓ | ☓ | diff --git a/ydb/docs/en/core/concepts/_includes/secondary_indexes.md b/ydb/docs/en/core/concepts/_includes/secondary_indexes.md index 5ef6631d223..a9259b5abf4 100644 --- a/ydb/docs/en/core/concepts/_includes/secondary_indexes.md +++ b/ydb/docs/en/core/concepts/_includes/secondary_indexes.md @@ -1,90 +1,94 @@ -# Secondary Indexes +# Secondary indexes -{{ ydb-short-name }} automatically creates a primary key index, which is why selection by primary key is always efficient, affecting only the rows needed. Selections by criteria applied to one or more non-key columns typically result in a full table scan. To make these selections efficient, use _secondary indexes_ — global structures backed by a separate index table. +In {{ ydb-short-name }}, an index is automatically created on the primary key, so queries with a condition on the primary key are always executed efficiently, affecting only the required rows. A query with a condition on one or more non-key columns typically results in a full table scan. To make such queries efficient, you should use _secondary indexes_ — global structures with a separate index table. -[Local indexes](../glossary.md#local-index) are a separate kind of auxiliary structure: they are stored with table data and applied in the storage layer on read, without materializing a separate index table (see [Local indexes](#bloom-skip-index) below). +Separately, there are [local indexes](../glossary.md#local-index): auxiliary structures that are stored together with the table data and are used when reading on the storage side, without materializing a separate index table (see the [Local indexes](#bloom-skip-index) section below). -The current version of {{ ydb-short-name }} implements _synchronous_ and _asynchronous_ global secondary indexes. Each index is a hidden table that is updated: +In the current version of {{ ydb-short-name }}, _synchronous_ and _asynchronous_ global secondary indexes are implemented. Each index is a hidden table that is updated: -* For synchronous indexes: Transactionally when the main table changes. -* For asynchronous indexes: In the background while getting the necessary changes from the main table. +* for synchronous indexes — transactionally when the main table is modified; +* for asynchronous indexes — in the background, receiving the necessary changes from the main table. -When a user sends an SQL query to insert, modify, or delete data, the database transparently generates commands to modify the index table. A table may have multiple secondary indexes. An index may include multiple columns, and the sequence of columns in an index matters. A single column may be included in multiple indexes. In addition to the specified columns, every index implicitly stores the table primary key columns to enable navigation from an index record to the table row. +When a user sends an SQL query to insert, modify, or delete data, the database transparently generates commands to modify the index table. A table can have multiple secondary indexes. An index can include multiple columns, and the order of columns in the index is important. A single column can be included in multiple indexes. In addition to the specified columns, the values of the table's primary key columns are always implicitly stored in the index, so that from a found index entry you can navigate to the table entry. -## Synchronous Secondary Index {#sync} +## Synchronous secondary index {#sync} -A synchronous index is updated simultaneously with the table that it indexes. This index ensures [strict consistency](https://en.wikipedia.org/wiki/Consistency_model) through [distributed transactions](../transactions.md#distributed-tx). While reads and blind writes to a table with no index can be performed without a planning stage, significantly reducing delays, such optimization is impossible when writing data to a table with a synchronous index. +A synchronous index is updated simultaneously with the table it indexes. Such an index provides [strong data consistency](https://en.wikipedia.org/wiki/Consistency_model) and uses the [distributed transactions](../transactions.md#distributed-tx) mechanism for this. Thus, while read and blind write operations on a table without an index can be performed without a planning stage, thereby significantly reducing latency, such optimization is not possible for writes to a table with a synchronous index. -## Asynchronous Secondary Index {#async} +## Asynchronous secondary index {#async} -Unlike a synchronous index, an asynchronous index doesn't use distributed transactions. Instead, it receives changes from an indexed table in the background. Write transactions to a table using this index are performed with no planning overheads due to reduced guarantees: an asynchronous index provides [eventual consistency](https://en.wikipedia.org/wiki/Eventual_consistency), but no strict consistency. You can only use asynchronous indexes in read transactions in [Stale Read Only](../transactions.md#modes) mode. +An asynchronous index, unlike a synchronous one, does not use the distributed transaction mechanism, but receives changes from the indexed table in the background. Write transactions to a table with such an index are performed without additional planning overhead, at the cost of reduced guarantees: an asynchronous index provides [eventual data consistency](https://en.wikipedia.org/wiki/Eventual_consistency), but not strong consistency. Using an asynchronous index in read transactions is only possible in [Stale Read Only](../transactions.md#modes) mode. -## Covering Secondary Index {#covering} +## Covering secondary index {#covering} -You can copy the contents of columns into a covering index. This eliminates the need to read data from the main table when performing reads by index and significantly reduces delays. At the same time, such denormalization leads to increased usage of disk space and may slow down inserts and updates due to the need for additional data copying. +It is possible to copy the contents of columns into the index (covering index), thus eliminating the need to read from the main table in index read operations, which significantly reduces latency. At the same time, such denormalization leads to increased disk space consumption and possible slowdown of insert and update operations due to the need for additional data copying. -## Unique Secondary Index {#unique} +## Unique secondary index {#unique} -This type of index enforces unique constraint behavior and, just like a regular secondary index, allows efficient point lookup queries. {{ ydb-short-name }} uses it to perform additional checks, ensuring that each distinct value in the indexed column set appears in the table no more than once. If a modifying query violates the constraint, it is aborted with a `PRECONDITION_FAILED` status. Therefore, client code must be prepared to handle this status. +This type of index implements the semantics of a unique value in a column or set of columns, and, like other indexes, allows efficient point reads on the set of indexed columns. {{ ydb-short-name }} uses it to perform additional checks to ensure that each unique value of the indexed columns appears in the table no more than once. If a modifying query violates this constraint, it is aborted with the status `PRECONDITION_FAILED`. Therefore, user code must be prepared to handle this status. -A unique secondary index is a synchronous index, so from a transactional perspective, the update process is the same as for the [synchronous secondary index](#sync) described above. +A unique secondary index is a synchronous index, so from a transactional perspective, its update process is the same as that of the [synchronous secondary index](#sync) described above. -## Vector Index +## Vector index {#vector} -[Vector Index](../../dev/vector-indexes.md) is a special type of secondary index. +[Vector index](../../dev/vector-indexes.md) is a special type of secondary index. -Unlike traditional secondary indexes, which optimize equality or range searches, vector indexes allow [vector search](../query_execution/vector_search.md) based on distance or similarity functions. +Unlike traditional secondary indexes, which optimize equality or range search, vector indexes allow performing [vector search](../query_execution/vector_search.md) based on distance or similarity functions. -## Fulltext Index +## Full-text index {#fulltext} -[Fulltext index](../../dev/fulltext-indexes.md) is a special type of secondary index. +[Full-text index](../../dev/fulltext-indexes.md) is a special type of secondary index. -Unlike traditional secondary indexes, which optimize equality or range searches, fulltext indexes allow scalable text search by words and phrases (and, with n-grams, by substrings). See also: [Fulltext search](../query_execution/fulltext_search.md). +Unlike traditional secondary indexes, which optimize search by equality or range, full-text indexes allow scalable text search for words and phrases (and when using [N-grams](https://en.wikipedia.org/wiki/N-gram), also for substrings). See also: [Full-text search](../query_execution/fulltext_search.md). -## Local indexes {#bloom-skip-index} +## JSON-index {#json} + +[JSON index](../../dev/json-indexes.md) is a special type of secondary index, like full-text index — both are built on top of an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), but use different tokenizers. -[Local indexes](../query_execution/local_indexes.md) are auxiliary structures stored together with table data and applied while reading in the storage layer. They do not materialize a separate index table. Currently, [Bloom skip indexes](../../dev/bloom-skip-indexes.md) are implemented; other kinds are planned. +JSON indexes allow you to speed up predicates with the [JSON_EXISTS](../../yql/reference/builtins/json.md) and [JSON_VALUE](../../yql/reference/builtins/json.md) functions on the content of a column of type `Json` or `JsonDocument`. The index is built by splitting JSON documents into path tokens and pairs of the form "path + value", which allows you to find matching rows by [JsonPath](../../yql/reference/builtins/json.md#jsonpath) paths without a full table scan. See also: [JSON search](../query_execution/json_search.md). -## Creating a Secondary Index Online {#index-add} +## Local indexes {#bloom-skip-index} -{{ ydb-short-name }} lets you create new and delete existing secondary indexes without stopping the service. For a single table, you can only create one index at a time. +[Local indexes](../query_execution/local_indexes.md) — auxiliary structures stored together with table data and used for reads on the storage side. They do not materialize a separate index table. Currently, [Bloom indexes](../../dev/bloom-skip-indexes.md) are implemented, and other types are planned in the future. -Online index creation consists of the following steps: +## Online creation of a secondary index {#index-add} -1. Taking a snapshot of a data table and creating an index table marked that writes are available. +In {{ ydb-short-name }}, you can create a secondary index and delete an existing secondary index without stopping service. For one table, only one index can be created at a time. - After this step, write transactions are distributed, writing to the main table and the index, respectively. The index is not yet available to the user. +The online index creation operation consists of the following steps: -1. Reading the snapshot of the main table and writing data to the index. +1. Taking a snapshot of the table with data, creating an index table marked as available for writes. - "Writes to the past" are implemented: situations where data updates in step 1 change the data written in step 2 are resolved. + After this step, write transactions become distributed, and data is written to the main table and the index. The index is not yet available to the user. +2. Reading a snapshot of the main table and writing to the index. -1. Publishing the results and deleting the snapshot. + Implements "write to the past": situations are allowed where data updates in step 1 change data written in step 2. +3. Publishing the result, deleting the snapshot. The index is ready to use. Possible impact on user transactions: -* There may be an increase in delays because transactions are now distributed (when creating a synchronous index). -* There may be an enhanced background of `OVERLOADED` errors because index table automatic shard splitting is actively running during data writes. +* Increased latency may be observed because transactions become distributed (when creating a synchronous index). +* An increased background of `OVERLOADED` errors is possible because automatic partitioning of index table shards is actively working during data writes. {% note info %} -The data write rate is chosen to minimize the impact of the write process on user transactions. To control the rate, configure limits for the corresponding queue in the [resource broker](../../reference/configuration/resource_broker_config.md#resource-broker-config). +The data write speed is chosen to minimize the impact of the write process on user transactions. To control the speed, configure limits for the corresponding queue of the [resource broker](../../reference/configuration/resource_broker_config.md#resource-broker-config). {% endnote %} -Creating an index is an asynchronous operation. If the client-server connection is interrupted after the operation has started, index building continues. You can manage asynchronous operations using the {{ ydb-short-name }} CLI. +Creating an index is an asynchronous operation. If a client-server connection break occurs after starting the operation, the index building will continue. You can manage the asynchronous operation via {{ ydb-short-name }} CLI. -## Creating and Deleting Secondary Indexes {#ddl} +## Creating and deleting secondary indexes {#ddl} A secondary index can be: -- Created when creating a table with the YQL [`CREATE TABLE`](../../yql/reference/syntax/create_table/index.md) statement. -- Added to an existing table with the YQL [`ALTER TABLE`](../../yql/reference/syntax/alter_table/index.md) statement or the YDB CLI [`table index add`](../../reference/ydb-cli/commands/secondary_index.md#add) command. -- Deleted from an existing table with the YQL [`ALTER TABLE`](../../yql/reference/syntax/alter_table/index.md) statement or the YDB CLI [`table index drop`](../../reference/ydb-cli/commands/secondary_index.md#drop) command. -- Deleted together with the table using the YQL [`DROP TABLE`](../../yql/reference/syntax/drop_table.md) statement or the YDB CLI `table drop` command. +- Created when creating a table using the YQL [CREATE TABLE](../../yql/reference/syntax/create_table/index.md) command. +- Added to an existing table by the YQL [ALTER TABLE](../../yql/reference/syntax/alter_table/index.md) command or by the {{ ydb-short-name }} CLI [table index add](../../reference/ydb-cli/commands/secondary_index.md#add) command. +- Deleted from an existing table by the YQL command [ALTER TABLE](../../yql/reference/syntax/alter_table/index.md) or by the {{ ydb-short-name }} CLI command [table index drop](../../reference/ydb-cli/commands/secondary_index.md#drop). +- Deleted together with the table by the YQL command [DROP TABLE](../../yql/reference/syntax/drop_table.md) or by the {{ ydb-short-name }} CLI command `table drop`. -## Using Secondary Indexes {#use} +## Using secondary indexes {#use} -For detailed information on using secondary indexes in applications, refer to the [relevant article](../../dev/secondary-indexes.md) in the documentation section for developers. +Detailed information on using secondary indexes in applications is available in [the article about them](../../dev/secondary-indexes.md) in the developer documentation section. diff --git a/ydb/docs/en/core/concepts/glossary.md b/ydb/docs/en/core/concepts/glossary.md index 9a947d1053d..d74ff77d9d7 100644 --- a/ydb/docs/en/core/concepts/glossary.md +++ b/ydb/docs/en/core/concepts/glossary.md @@ -4,13 +4,13 @@ This article provides an overview of terms and definitions used in {{ ydb-short- ## Key terminology {#key-terminology} -This section describes terms that are useful to anyone working with {{ ydb-short-name }}, regardless of their role and use case. +This section describes terms that are useful to anyone working with {{ ydb-short-name }}, regardless of their role or use case. ### Cluster {#cluster} -A **cluster** {{ ydb-short-name }} is a set of interconnected [nodes](#node) {{ ydb-short-name }} that exchange data to execute user queries and reliably store data. These nodes form one of the supported [cluster topologies](#topology), which directly affects its reliability and performance characteristics. +A **cluster** {{ ydb-short-name }} is a set of interconnected [nodes](#node) {{ ydb-short-name }} that exchange data to execute user queries and ensure reliable data storage. These nodes form one of the supported [cluster topologies](#topology), which directly affects its reliability and performance characteristics. -Clusters {{ ydb-short-name }} are multi-tenant and can contain multiple isolated [databases](#database). +Clusters {{ ydb-short-name }} are multi-tenant and can contain several isolated [databases](#database). ### Database {#database} @@ -20,31 +20,31 @@ Another important characteristic of {{ ydb-short-name }} databases is that they ### Node {#node} -{{ ydb-short-name }} **node** is a server process that runs an executable file called `ydbd`. Multiple nodes {{ ydb-short-name }} can run on a single physical server or virtual machine, which is common practice. Thus, in the context of {{ ydb-short-name }}, nodes are **not** synonymous with hosts. +{{ ydb-short-name }} A **node** is a server process that runs an executable called `ydbd`. Multiple nodes {{ ydb-short-name }} can run on a single physical server or virtual machine, which is common practice. Thus, in the context of {{ ydb-short-name }}, nodes are **not** synonymous with hosts. -Since {{ ydb-short-name }} uses a storage and compute separation approach, `ydbd` has several operating modes that determine the node type. The available node types are described below. +Since {{ ydb-short-name }} uses an approach with separate storage and compute layers (storage and compute separation), `ydbd` has several operation modes that define the node type. The available node types are described below. #### Database node {#database-node} -**Database nodes** (also known as **tenant nodes** or **compute nodes**) process user queries addressed to a specific logical [database](#database). Their state resides only in RAM and can be restored from [distributed storage](#distributed-storage). The set of database nodes of a given [cluster {{ ydb-short-name }}](topology.md) can be considered the compute layer of that cluster. Thus, adding database nodes and allocating additional resources (CPU and RAM) to them are the main ways to increase the compute resources of a database. +**Database nodes** (also known as **tenant nodes** or **compute nodes**) process user queries addressed to a specific logical [database](#database). Their state is stored only in RAM and can be restored from [distributed storage](#distributed-storage). The set of database nodes of a given [cluster {{ ydb-short-name }}](topology.md) can be considered the compute layer of that cluster. Thus, adding database nodes and allocating additional resources (CPU and RAM) to them are the main ways to increase the compute resources of a database. The main role of database nodes is to run various [tablets](#tablet) and [actors](#actor), as well as to receive incoming requests over the network. #### Storage node {#storage-node} -**Storage nodes** are stateful nodes responsible for long-term storage of data fragments. The set of storage nodes in a given [cluster {{ ydb-short-name }}](#cluster) is called [distributed storage](#distributed-storage) and can be considered as the storage layer of that cluster. Thus, adding additional storage nodes and their disks is the primary way to increase the cluster's storage capacity and I/O throughput. +**Storage nodes** are stateful nodes responsible for long-term storage of data fragments. The set of storage nodes in a given [cluster {{ ydb-short-name }}](#cluster) is called [distributed storage](#distributed-storage) and can be considered as the storage layer of that cluster. Thus, adding additional storage nodes and their disks is the primary way to increase the storage capacity and I/O throughput of the cluster. #### Hybrid node {#hybrid-mode} -A **hybrid node** is a process that simultaneously performs both the roles of a [database node](#database-node) and a [storage node](#storage-node). Hybrid nodes are often used for development purposes. For example, you can run a container with a full-featured {{ ydb-short-name }} containing only one `ydbd` process in hybrid mode. They are rarely used in production environments. +A **Hybrid node** is a process that simultaneously performs both the roles of a [database node](#database-node) and a [storage node](#storage-node). Hybrid nodes are often used for development purposes. For example, you can run a container with a full-featured {{ ydb-short-name }} containing only one `ydbd` process in hybrid mode. They are rarely used in production environments. #### Static node {#static-node} -**Static nodes** are configured manually during initial cluster initialization or reconfiguration. Typically, they serve as [storage nodes](#storage-node), but technically it is possible to configure them as [database nodes](#database-node). +**Static nodes** are configured manually during initial cluster initialization or reconfiguration. Typically, they serve as [storage nodes](#storage-node), but they can technically be configured as [database nodes](#database-node). #### Dynamic node {#dynamic} -**Dynamic nodes** are added and removed from the cluster on the fly. They can only serve as [database nodes](#database-node). +**Dynamic nodes** are added and removed from the cluster on the fly. They can only act as [database nodes](#database-node). ### Distributed storage {#distributed-storage} @@ -54,45 +54,45 @@ Many terms related to the [implementation of distributed storage](#distributed-s ### Storage group {#storage-group} -A **storage group** is a place for reliable data storage, similar to [RAID](https://en.wikipedia.org/wiki/RAID), but using disks from multiple servers. Depending on the selected [cluster topology](#topology), storage groups use different algorithms to ensure high availability, similar to [standard RAID levels](https://en.wikipedia.org/wiki/Standard_RAID_levels). +**Storage group**, **distributed storage group**, or **Blob storage group** is a place for reliable data storage, similar to [RAID](https://en.wikipedia.org/wiki/RAID), but using disks from multiple servers. Depending on the selected [cluster topology](#topology), storage groups use different algorithms to ensure high availability, similar to [standard RAID levels](https://en.wikipedia.org/wiki/Standard_RAID_levels). [Distributed storage](#distributed-storage) typically manages a large number of relatively small storage groups. Each group can be assigned to a specific [database](#database) to increase the disk space capacity and I/O throughput available to that database. -[Static](#static-group) and [dynamic](#dynamic-group) storage groups are physical, meaning their data is placed directly on [VDisks](#vdisk). +[Static](#static-group) and [dynamic](#dynamic-group) storage groups are physical, meaning their data is placed directly on [VDisk](#vdisk)s. #### Static group {#static-group} -A **static group** is a special [storage group](#storage-group) created during the initial cluster deployment. Its main role is to store data of system [tablets](#tablet), which can be considered as cluster-level metadata. +A **static group** is a special [storage group](#storage-group) created during the initial deployment of the cluster. Its main role is to store data of system [tablets](#tablet), which can be considered as cluster-level metadata. A static group may require special attention during major cluster maintenance, such as decommissioning an [availability zone](#regions-az). #### Dynamic group {#dynamic-group} -Ordinary storage groups that are not [static](#static-group) are called **dynamic groups**. They are called dynamic because they can be created and deleted on the fly while the [cluster](#cluster) is running. +Ordinary storage groups that are not [static](#static-group) are called **dynamic groups**. They are called dynamic because they can be created and removed on the fly while the [cluster](#cluster) is running. #### Virtual storage group {#virtual-storage-groups} -A **virtual storage group** is an entity that is not actually a [storage group](#storage-group) but appears to be one from the outside (provides a similar external interface). It can store its data in other storage groups or in S3. +A **virtual storage group** is an entity that is not actually a [storage group](#storage-group) but appears as one from the outside (provides a similar external interface). It can store its data in other storage groups or in S3. ### Storage pool {#storage-pool} -A **storage pool** is a set of data storage devices with similar characteristics. Each storage pool is assigned a unique name within the {{ ydb-short-name }} cluster. Technically, each storage pool consists of multiple physical disks ( [PDisk](#pdisk)). Each [storage group](#storage-group) is created in a specific storage pool, which determines the performance characteristics of the storage group through the selection of appropriate storage devices. Typically, separate storage pools are created for devices of different types (e.g., NVMe, SSD, and HDD) or for specific models of these devices that have different capacity and access speed. +A **storage pool** is a set of data storage devices with similar characteristics. Each storage pool is assigned a unique name within the {{ ydb-short-name }} cluster. Technically, each storage pool consists of multiple physical disks ( [PDisk](#pdisk)). Each [storage group](#storage-group) is created in a specific storage pool, which determines the performance characteristics of the storage group through the selection of appropriate storage devices. Typically, separate storage pools are created for devices of different types (e.g., NVMe, SSD, and HDD) or for specific models of these devices with different capacity and access speed. ### Actor {#actor} -The [actor model](https://en.wikipedia.org/wiki/Actor_model) is one of the main approaches to execution parallelism used in {{ ydb-short-name }}. In this model, **actors** are lightweight user-space processes that can have and modify their private state, but can only influence each other indirectly through message passing. {{ ydb-short-name }} has its own implementation of this model, which is described [below](#actor-implementation). +The [Actor model](https://en.wikipedia.org/wiki/Actor_model) is one of the main approaches to execution parallelism used in {{ ydb-short-name }}. In this model, **actors** are lightweight user-space processes that can have and modify their private state, but can only influence each other indirectly through message passing. {{ ydb-short-name }} has its own implementation of this model, which is described [below](#actor-implementation). In {{ ydb-short-name }}, actors with reliably persisted state are called [tablets](#tablet). ### Tablet {#tablet} -A **tablet** is one of the main building blocks and abstractions of {{ ydb-short-name }}. It represents an entity responsible for a relatively small segment of user or system data. Typically, a tablet manages up to several gigabytes of data, but some types of tablets can handle larger volumes. +A **tablet** is one of the main building blocks and abstractions of {{ ydb-short-name }}. It represents an entity responsible for a relatively small segment of user or system data. Typically, a tablet manages up to several gigabytes of data, although some types of tablets can handle larger volumes. For example, a [string user table](#row-oriented-table) is managed by one or more tablets of type [DataShard](#data-shard), with each tablet responsible for a continuous range of [primary keys](#primary-key) and their corresponding data. End users who send queries to the {{ ydb-short-name }} cluster for execution do not need to know the details of tablets, their types, or how they work, but this knowledge can be useful, for example, for performance optimization. -Technically, tablets are [actors](#actor) with state reliably stored in [distributed storage](#distributed-storage). This state allows the tablet to continue operating on another [database node](#database-node) if the previous one fails or becomes overloaded. +Technically, tablets are [actors](#actor) with state reliably stored in [distributed storage](#distributed-storage). This state allows a tablet to continue operating on a different [database node](#database-node) if the previous one fails or becomes overloaded. [Tablet implementation details](#tablet-implementation) and related terms, as well as [main tablet types](#tablet-types), are discussed below. @@ -100,7 +100,7 @@ Technically, tablets are [actors](#actor) with state reliably stored in [distrib {{ ydb-short-name }} implements **transactions** at two main levels: -* [Local database](#local-database) and the rest of the [tablet infrastructure](#tablet-implementation) allow [tablets](#tablet) to manipulate their state using **local transactions** with [serializable isolation level](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Serializable_(%D1%83%D0%BF%D0%BE%D1%80%D1%8F%D0%B4%D0%BE%D1%87%D0%B8%D0%B2%D0%B0%D0%B5%D0%BC%D0%BE%D1%81%D1%82%D1%8C)). Technically, they are not local to a single node, since this state is stored remotely in [distributed storage](#distributed-storage). +* [Local database](#local-database) and the rest of the [tablet infrastructure](#tablet-implementation) allow [tablets](#tablet) to manipulate their state using **local transactions** with [serializable isolation level](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Serializable). Technically, they are not local to a single node, since this state is stored remotely in [distributed storage](#distributed-storage). * In the context of {{ ydb-short-name }}, the term **distributed transactions** usually refers to transactions that span multiple tablets. For example, transactions between tables or even rows of the same table are often distributed. * **Single-shard** transactions cover one tablet and execute faster. For example, transactions between rows of the same table partition are often single-shard. @@ -114,79 +114,59 @@ The implementation of distributed transactions is discussed in a separate articl ### Sessions -Logical connections to the database that store the context needed for executing queries and managing transactions. Sessions are described in more detail in the section [{#T}](query_execution/execution_process.md#sessions). +Logical connections to the database that store the context needed for executing queries and managing transactions. Sessions are described in more detail in the section [{#T}](query_execution/index.md#sessions). ### Client-side timeout {#client-timeout} -**Client-side timeout** — a time limit that the application or {{ ydb-short-name }} SDK waits for a database operation to complete (for example, executing a query or receiving a response via a gRPC call). After this time expires, the client usually aborts the wait: closes the connection or data stream, receives a transport or SDK error — before the server has returned an explicit response (see [codes](../reference/ydb-sdk/ydb-status-codes.md) of the {{ ydb-short-name }} server response). +**Client-side timeout** — a time limit that an application or {{ ydb-short-name }} SDK waits for a database operation to complete (for example, executing a query or receiving a response via a gRPC call). After this time expires, the client usually aborts the wait: closes the connection or data stream, receives a transport or SDK error — even before the server has returned an explicit response (see {{ ydb-short-name }} server [response codes](../reference/ydb-sdk/ydb-status-codes.md)). -If the client-side timeout is shorter than the execution time of the query on the {{ ydb-short-name }} side, then due to the specifics of query processing in the cluster, a query aborted on the client may continue to execute on the server for some time. If such a situation occurs on a large scale, the server becomes overloaded with queries for which the client is not waiting for a response. Therefore, frequent retries of the same query immediately after a timeout can exacerbate the overload. For more details, see the articles [{#T}](../troubleshooting/performance/queries/retry-cascade.md) and [{#T}](../troubleshooting/performance/queries/overloaded-errors.md); retry policies in the SDK are described in the section [{#T}](../reference/ydb-sdk/error_handling.md). - -### Transaction retry {#transaction-retry} - -**Transaction retry** — a client practice of re-executing a [transaction](#transactions) entirely from the beginning upon a retryable error (e.g., a temporary network failure or optimistic locking conflict). In {{ ydb-short-name }}, retries should be performed at the transaction level, not at the level of individual queries within it. Built-in retry policies in the {{ ydb-short-name }} SDK and integrations (e.g., [spring-ydb-retry](../integrations/spring/spring-retry.md)) implement this approach. For more details, see [{#T}](../reference/ydb-sdk/error_handling.md). - -### Exponential backoff {#exponential-backoff} - -**Exponential backoff** (also known as **backoff**) — a strategy for pausing between [transaction retry](#transaction-retry) attempts: the wait interval increases exponentially with each attempt, usually with an upper limit. The {{ ydb-short-name }} SDK often uses two levels of backoff — fast and slow — depending on the error type. For more details, see [{#T}](../reference/ydb-sdk/error_handling.md#handling-retryable-errors). - -### Jitter {#jitter} - -**Jitter** is a small random variation added to [exponential backoff](#exponential-backoff) delays. It helps avoid simultaneous retries by many clients after a common failure (a "retry storm") and distributes the load more evenly. - -### Idempotency {#idempotency} - -**Idempotency** is a property of an operation: repeated execution produces the same effect as a single execution (for example, `UPSERT` with a deterministic primary key or read operations). [Transaction retries](#transaction-retry) are safe only for idempotent operations or for retry errors where the server guarantees that the transaction was not committed. SDKs and client libraries {{ ydb-short-name }} can extend the set of retryable status codes if the calling code marks the operation as idempotent. - -### Transaction interceptor {#transaction-interceptor} - -**Transaction interceptor** is a Spring Framework component that wraps methods annotated with `@Transactional` and manages transaction boundaries. Modules like [spring-ydb-retry](../integrations/spring/spring-retry.md) replace the standard Spring interceptor, adding [transaction retry](#transaction-retry) logic around transactional methods. +If the client-side timeout is shorter than the query execution time on the {{ ydb-short-name }} side, then due to the specifics of query processing in the cluster, a query interrupted on the client side may continue to execute on the server for some time. If this situation occurs on a large scale, the server becomes overloaded with queries for which the client is not waiting for a response. Therefore, frequent retries of the same query immediately after a timeout can exacerbate the overload. For more details, see the articles [{#T}](../troubleshooting/performance/queries/retry-cascade.md) and [{#T}](../troubleshooting/performance/queries/overloaded-errors.md); retry policies in the SDK are described in the section [{#T}](../reference/ydb-sdk/error_handling.md). ### Implicit transactions {#implicit-transactions} -**Implicit transaction** is a query execution mode where the [transaction mode](transactions.md#modes) is not specified. In this case, {{ ydb-short-name }} independently determines whether to wrap them in a transaction. This mode is described in more detail in [{#T}](transactions.md#implicit). +**Implicit transaction** is a query execution mode in which the [transaction mode](transactions.md#modes) is not specified. In this case, {{ ydb-short-name }} independently determines whether to wrap them in a transaction. This mode is described in more detail in [{#T}](transactions.md#implicit). ### Multi-version concurrency control {#mvcc} -[**Multi-version concurrency control**](https://en.wikipedia.org/wiki/Multiversion_concurrency_control), also known as **MVCC**, is a method used by {{ ydb-short-name }} to allow multiple concurrent transactions to access the database without interfering with each other. It is described in more detail in a separate article [{#T}](query_execution/mvcc.md). +[**Multi-version concurrency control**](https://en.wikipedia.org/wiki/Multiversion_concurrency_control), **multi-version concurrency control** or **MVCC** is a method used by {{ ydb-short-name }} for concurrent access of multiple parallel transactions to the database without interfering with each other. It is described in more detail in a separate article [{#T}](query_execution/mvcc.md). ### Streaming queries {#streaming-query} -A type of query designed for [stream processing](https://en.wikipedia.org/wiki/Stream_processing) of an unbounded data stream. Unlike regular queries, streaming queries have no execution time limits, automatically restart on errors, and periodically save their state as [checkpoints](#streaming-queries-checkpoints) for fault tolerance. +A type of query designed for [stream processing](https://en.wikipedia.org/wiki/Stream_processing) of an unbounded data stream. Unlike regular queries, streaming queries have no restrictions on execution duration, automatically restart on errors, and periodically save their state as [checkpoints](#streaming-queries-checkpoints) to ensure fault tolerance. Streaming queries are described in more detail in a separate article [{#T}](streaming-query.md). ### Streaming query checkpoints {#streaming-queries-checkpoints} -The periodically saved state of a [streaming query](#streaming-query), necessary for automatically restoring its operation after failures in a distributed system. More about checkpoints in the article [{#T}](../dev/streaming-query/checkpoints.md). +A periodically saved state of a [streaming query](#streaming-query), necessary for automatically restoring its operation after failures in a distributed system. For more details about checkpoints, see the article [{#T}](../dev/streaming-query/checkpoints.md). ### Topology {#topology} -{{ ydb-short-name }} supports several [cluster](#cluster) **topologies** (or **topology**), described in more detail in a separate article [{#T}](topology.md). Below are explanations of several related terms. +{{ ydb-short-name }} supports several **topologies** of a [cluster](#cluster) (or **topology**), described in more detail in a separate article [{#T}](topology.md). Below, several related terms are explained. #### Availability zones and regions {#regions-az} An **availability zone** is a data center or its isolated segment with minimal physical distance between nodes and minimal risk of failure simultaneously with other availability zones. Thus, availability zones should not share common infrastructure such as power supply, cooling, or external network connections. -A **region** is a large geographic area containing multiple availability zones. The distance between availability zones within a region should be about 500 km or less. {{ ydb-short-name }} writes data to each availability zone in the region synchronously, ensuring reasonable latency and uninterrupted operation in case one availability zone fails. +A **region** is a large geographic area containing several availability zones. The distance between availability zones in one region should be about 500 km or less. {{ ydb-short-name }} performs data writes to each availability zone in the region synchronously, ensuring reasonable latency and uninterrupted operation in case of failure of one of the availability zones. #### Rack {#rack} -A **rack**, also known as a **server rack**, is equipment used to organize the placement of multiple servers. Servers in the same rack are more likely to become unavailable simultaneously due to rack-level issues related to power supply, cooling, etc. {{ ydb-short-name }} can take into account which server is in which rack when placing each data fragment in environments based on physical servers. +A **rack** or **server rack** is equipment used to organize the placement of multiple servers. Servers in the same rack are more likely to become unavailable simultaneously due to rack-level issues related to power supply, cooling, etc. {{ ydb-short-name }} can take into account information about which server is in which rack when placing each data fragment in environments based on physical servers. #### Pile {#pile} -A **Pile** is a set of nodes that can fail or be shut down simultaneously while keeping other parts of the cluster operational. A Pile can remain operational when other cluster nodes are shut down. Piles are used in [bridge mode](#bridge) to split the cluster into several parts between which synchronous replication is performed. A Pile can consist of nodes from one or more regions. +A **Pile** is a set of nodes that can fail or be shut down simultaneously while maintaining the operability of other parts of the cluster (pile). A Pile can remain operational when other cluster nodes are shut down. Piles are used in [bridge mode](#bridge) to split the cluster into several parts between which synchronous replication is performed. A Pile can consist of nodes from one or more regions. #### Bridge mode {#bridge} -**Bridge mode** is a special cluster topology in which data is stored with synchronous replication between several [piles](#pile). The features of this mode are described in [{#T}](topology.md#bridge) and [{#T}](bridge.md). +**Bridge mode** is a special cluster topology in which data is stored with synchronous replication between several [piles](#pile). The features of this mode are described in [{#T}](topology.md#bridge) and also in [{#T}](bridge.md). ### Table {#table} A **table** is a structured piece of information organized into rows and columns. Each row represents a single record or item, and each column is a specific attribute or field with a defined data type. -There are two main approaches to representing table data in memory or on disk: [row-oriented (row by row)](#row-oriented-table) and [column-oriented (column by column)](#column-oriented-table). The chosen approach greatly affects the performance characteristics of operations on this data: the former is more suitable for transactional workloads (OLTP), and the latter for analytical workloads (OLAP). {{ ydb-short-name }} supports both approaches. +There are two main approaches to representing tabular data in memory or on disks: [row-based (row by row)](#row-oriented-table) and [column-based (column by column)](#column-oriented-table). The chosen approach greatly affects the performance characteristics of operations on this data: the former is more suitable for transactional workloads (OLTP), and the latter for analytical workloads (OLAP). {{ ydb-short-name }} supports both approaches. #### Row-oriented table {#row-oriented-table} @@ -194,42 +174,42 @@ There are two main approaches to representing table data in memory or on disk: [ #### Column-oriented table {#column-oriented-table} -**Column-oriented tables** (also known as **columnar tables**) store data for each column separately. They are optimized for building aggregates over a small number of columns, but are less suitable for accessing specific rows, as rows need to be reconstructed from their cells on the fly. They are described in more detail in [{#T}](datamodel/table.md#column-oriented-tables). +**Column-oriented tables** or **columnar tables** store data for each column separately. They are optimized for building aggregates over a small number of columns, but are less suitable for accessing specific rows, as rows need to be reconstructed from their cells on the fly. They are described in more detail in [{#T}](datamodel/table.md#column-oriented-tables). #### Primary key {#primary-key} A **primary key** is an ordered list of columns whose values uniquely identify a row. It is used to create the table's [primary index](#primary-index). It is set by the {{ ydb-short-name }} user when [creating a table](../yql/reference/syntax/create_table/index.md) and significantly affects the performance of operations on that table. -A guide on choosing primary keys is provided in [{#T}](../dev/primary-key/index.md). +Guidance on choosing primary keys is provided in [{#T}](../dev/primary-key/index.md). #### Primary index {#primary-index} -A **primary index** (also known as a **primary key index**) is the main data structure used to find rows in a table. It is created based on the chosen [primary key](#primary-key) and determines the physical order of rows in the table; thus, each table can have only one primary index. The primary index is unique. +**Primary index** or **primary key index** is the main data structure used to find rows in a table. It is created based on the selected [primary key](#primary-key) and determines the physical order of rows in the table; thus, each table can have only one primary index. The primary index is unique. #### Secondary index {#secondary-index} -A **secondary index** is an additional data structure used to find rows in a table, typically when this cannot be done efficiently using the [primary index](#primary-index). Unlike the primary index, secondary indexes are managed independently of the table's main data. Thus, a table can have multiple secondary indexes for different scenarios. The capabilities of {{ ydb-short-name }} regarding secondary indexes are described in a separate article [{#T}](query_execution/secondary_indexes.md). A secondary index can be either unique or non-unique. +A **secondary index** is an additional data structure used to find rows in a table, typically when this cannot be done efficiently using the [primary index](#primary-index). Unlike a primary index, secondary indexes are managed independently of the table's main data. Thus, a table can have multiple secondary indexes for different scenarios. The capabilities of {{ ydb-short-name }} regarding secondary indexes are described in a separate article [{#T}](query_execution/secondary_indexes.md). A secondary index can be either unique or non-unique. -Special types of secondary indexes are distinguished separately: [vector index](#vector-index), [full-text index](#fulltext-index), and [JSON index](#json-index). +Special types of secondary indexes include [vector index](#vector-index), [full-text index](#fulltext-index), and [JSON index](#json-index). #### Vector index {#vector-index} -A **vector index** is an additional data structure used to speed up the solution of the [vector search](query_execution/vector_search.md) problem when there is a large amount of data and [exact vector search without an index](../yql/reference/udf/list/knn.md) does not work satisfactorily. +**Vector index** is an additional data structure used to speed up the solution of the [vector search](query_execution/vector_search.md) problem when there is a lot of data and [exact vector search without an index](../yql/reference/udf/list/knn.md) does not work satisfactorily. The capabilities of {{ ydb-short-name }} for approximate nearest neighbor search (ANN search) using vector indexes are described in a separate article [{#T}](../dev/vector-indexes.md). -**Vector index** is a specialized type of [secondary index](#secondary-index) designed for similarity search, unlike traditional secondary indexes optimized for equality or range search. +A **vector index** is a specialized type of [secondary index](#secondary-index) designed for similarity search, as opposed to traditional secondary indexes optimized for equality or range search. #### Full-text index {#fulltext-index} -A **full-text index** is an additional data structure used to speed up text search on a table column (by words and phrases, and when using N-grams, also by substrings). +**Full-text index** is an additional data structure used to speed up text search on a table column (by words and phrases, and when using N-grams, also by substrings). -The full-text search capabilities and index parameters are described in the articles [{#T}](../dev/fulltext-indexes.md) and [{#T}](query_execution/fulltext_search.md). +The features of full-text search and index parameters are described in the articles [{#T}](../dev/fulltext-indexes.md) and [{#T}](query_execution/fulltext_search.md). -#### JSON index {#json-index} +#### JSON-index {#json-index} -**JSON index** is an additional data structure used to speed up predicates with the functions [JSON_EXISTS](../yql/reference/builtins/json.md#json_exists) and [JSON_VALUE](../yql/reference/builtins/json.md#json_value) on a column of type `Json` or `JsonDocument`. Unlike traditional secondary indexes optimized for equality or range searches on individual table columns, the JSON index works with arbitrary [JsonPath](../yql/reference/builtins/json.md#jsonpath) paths within a JSON document. +**JSON index** is an additional data structure used to speed up predicates with the [JSON_EXISTS](../yql/reference/builtins/json.md#json_exists) and [JSON_VALUE](../yql/reference/builtins/json.md#json_value) functions on a column of type `Json` or `JsonDocument`. Unlike traditional secondary indexes, which are optimized for equality or range searches on individual table columns, the JSON index works with arbitrary [JsonPath](../yql/reference/builtins/json.md#jsonpath) paths within a JSON document. -JSON index, like the [full-text index](#fulltext-index), is implemented on top of an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), but uses its own tokenizer for JSON documents. JSON search capabilities are described in the articles {#T} and {#T}. +A JSON index, like a [full-text index](#fulltext-index), is built on top of an [inverted index](https://en.wikipedia.org/wiki/Inverted_index), but uses its own JSON document tokenizer. JSON search capabilities are described in the articles [{#T}](../dev/json-indexes.md) and [{#T}](query_execution/json_search.md). #### Local index {#local-index} @@ -237,106 +217,106 @@ A local index is an auxiliary structure that is stored together with the table d #### Bloom filter {#bloom-filter} -A **Bloom filter** is a [probabilistic data structure](https://en.wikipedia.org/wiki/Bloom_filter) that allows you to quickly check whether an element belongs to a set. False positives are possible, but false negatives are not. +A Bloom filter is a [probabilistic data structure](https://en.wikipedia.org/wiki/Bloom_filter) that allows you to quickly check whether an element belongs to a set. False positives are possible, but false negatives are not. -#### Local Bloom index {#local-bloom-skip-index} +#### Local bloom index {#local-bloom-skip-index} -**Local Bloom index** is a special case of [local index](#local-index): a probabilistic filter over column values based on the [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) that speeds up selective queries by skipping data fragments where the searched value is guaranteed to be absent. For more information, see [Bloom indexes](../dev/bloom-skip-indexes.md), [local indexes](query_execution/local_indexes.md). +A local Bloom index is a special case of a [local index](#local-index): a probabilistic filter on column values based on a [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) that speeds up selective queries by skipping data fragments where the searched value is guaranteed to be absent. For more information, see [Bloom indexes](../dev/bloom-skip-indexes.md) and [local indexes](query_execution/local_indexes.md). #### Column family {#column-family} -**Column family** or **column group** is a feature that allows you to store subsets of columns of a [row table](#row-oriented-table) in a separate family or group. The main use case is storing some columns on other disk types (moving less important columns to HDD) or with different compression settings. If the workload requires many column families, consider using [column-oriented tables](#column-oriented-table). +**Column family** or **column group** is a feature that allows storing subsets of columns of a [row table](#row-oriented-table) separately in a family or group. The main use case is storing some columns on other disk types (moving less important columns to HDD) or with different compression settings. If the workload requires many column families, consider using [column tables](#column-oriented-table). #### Column encoding {#column-encoding} -**Column encoding** is a mechanism for optimizing data storage in table columns, which reduces the amount of disk space used and speeds up the execution of certain operations. +**Column encoding** is a data storage optimization mechanism for table columns that reduces disk space usage and speeds up some operations. -#### Time to Live {#ttl} +#### Time to live {#ttl} **Time to live** or **TTL** is a mechanism for automatically deleting old rows from a table asynchronously in the background. It is described in a separate article [{#T}](ttl.md). ### View {#view} -**view** is a way to save a query and access its results as a real table. The view itself does not store any data except the query text. The query stored in the view is executed each time a SELECT is performed on it, generating the returned result. Any changes to the tables referenced by the view are immediately reflected in the results of reading from it. +**View** is a way to save a query and access its results as if they were a real table. The view itself does not store any data except the query text. The query stored in the view is executed each time a SELECT is performed on it, generating the returned result. Any changes to the tables referenced by the view are immediately reflected in the read results. {% if feature_view %} -Views can be user or system. +Views can be user-defined or system. -#### User views {#user-view} +#### User-defined views {#user-view} -**User views** are created by the user using the [{#T}](../yql/reference/syntax/create-view.md) command. They are described in more detail in [{#T}](../concepts/datamodel/view.md). +**User-defined views** are created by the user using the [{#T}](../yql/reference/syntax/create-view.md) command. They are described in more detail in [{#T}](../concepts/datamodel/view.md). {% endif %} #### System views {#system-view} -**System views** are special views automatically created by the system for monitoring the state of a database and cluster. They are located in a special directory `.sys`, which is in the root folder of each database. System views for databases are described in [{#T}](../dev/system-views.md); system views for the cluster and access management issues are described in [{#T}](../devops/observability/system-views.md). +**System views** are special views automatically created by the system for monitoring the state of the database and cluster. They are located in a special directory `.sys` in the root folder of each database. System views for databases are described in [{#T}](../dev/system-views.md); system views for the cluster and access management issues are described in [{#T}](../devops/observability/system-views.md). ### Topic {#topic} -A **message queue** is used for reliable asynchronous communication between different systems by passing messages. {{ ydb-short-name }} provides infrastructure that ensures exactly-once semantics in such communications. Using it, you can achieve a guarantee of no lost messages and no accidental duplicates. +**Message queue** is used for reliable asynchronous communication between different systems by passing messages. {{ ydb-short-name }} provides infrastructure that ensures exactly-once semantics in such communications. Using it, you can guarantee no lost messages or accidental duplicates. -A **topic** is a named entity in a message queue designed for interaction between [writers](#producer) and [readers](#consumer). +**Topic** is a named entity in a message queue for interaction between [writers](#producer) and [readers](#consumer). -Several terms related to topics are given below. How topics work in {{ ydb-short-name }} is explained in more detail in a separate article [{#T}](datamodel/topic.md). +Several terms related to topics are listed below. How topics work in {{ ydb-short-name }} is explained in more detail in a separate article [{#T}](datamodel/topic.md). #### Partition {#partition} -For horizontal scaling, topics are divided into individual elements called **partitions**. Thus, partitions are the unit of parallelism within a topic. Messages within each partition are ordered. +For horizontal scaling, topics are divided into separate elements called **partitions**. Thus, partitions are the unit of parallelism within a topic. Messages within each partition are ordered. However, subsets of data managed by a single [data shard](#data-shard) or [column shard](#column-shard) may also be called partitions. #### Offset {#offset} -An **offset** is a sequence number that identifies a message within a [partition](#partition). +**Offset** is a sequence number that identifies a message within a [partition](#partition). #### Writer {#producer} -A **producer** (also known as a **writer**) is an entity that writes new messages to a topic. +**Writer** or **producer** is an entity that writes new messages to a topic. #### Reader {#consumer} -A **consumer** (also known as a **reader**) is an entity that reads messages from a topic. +**Reader** or **consumer** is an entity that reads messages from a topic. -### Change Data Capture {#cdc} +### Change data capture {#cdc} -**Change Data Capture** (CDC) is a mechanism that allows you to subscribe to a **change feed** in a specific [table](#table). Technically, it is implemented on top of [topics](#topic). It is described in more detail in a separate article [{#T}](cdc.md). +**Change data capture** or **CDC** is a mechanism that allows subscribing to a **change feed** for a specific [table](#table). Technically, it is implemented on top of [topics](#topic). It is described in more detail in a separate article [{#T}](cdc.md). -#### Change Feed {#changefeed} +#### Change feed {#changefeed} -A **change feed** is an ordered list of changes to a [table](#table) placed in a [topic](#topic). +**Change feed** is an ordered list of changes to a [table](#table) placed in a [topic](#topic). -### Backup Collection {#backup-collection} +### Backup collection {#backup-collection} -A **backup collection** is a [schema object](#scheme-object) that organizes full and incremental [backups](#backup) for selected [row tables](#row-oriented-table). Collections provide [point-in-time recovery](https://en.wikipedia.org/wiki/Point-in-time_recovery) by maintaining [backup chains](#backup-chain) and ensuring consistent recovery of multiple tables. A table can belong to only one backup collection at a time. +**Backup collection** is a [schema object](#scheme-object) that organizes full and incremental [backups](#backup) for selected [row tables](#row-oriented-table). Collections provide [point-in-time recovery](https://en.wikipedia.org/wiki/Point-in-time_recovery) by maintaining [backup chains](#backup-chain) and ensuring consistent recovery of multiple tables. A table can belong to only one backup collection at a time. For more information, see [{#T}](datamodel/backup-collection.md). #### Backup {#backup} -A **backup** is a copy of data at a specific point in time that can be used for data recovery. In the context of [backup collections](#backup-collection), there are two types: +**Backup** is a copy of data at a specific point in time that can be used for data recovery. In the context of [backup collections](#backup-collection), there are two types: -- **Full backup**: A complete snapshot of all data in the collection. Serves as the basis for [backup chains](#backup-chain) and can be restored independently. +- **Full backup**: A complete snapshot of all data in the collection. It serves as the basis for [backup chains](#backup-chain) and can be restored independently. - **Incremental backup**: Captures only changes (inserts, updates, deletes) since the previous backup. Requires the entire backup chain for restoration. -#### Backup Chain {#backup-chain} +#### Backup chain {#backup-chain} -A **backup chain** is an ordered sequence of [backups](#backup) starting with a full backup, followed by zero or more incremental backups. Each incremental backup depends on all previous backups in the chain. Deleting any backup in the chain makes subsequent incremental backups unrecoverable. +**Backup chain** is an ordered sequence of [backups](#backup) starting with a full backup followed by zero or more incremental backups. Each incremental backup depends on all previous backups in the chain. Deleting any backup in the chain makes subsequent incremental backups unrecoverable. {% if feature_async_replication == true %} ### Async replication instance {#async-replication-instance} -An **async replication instance** is a named entity that stores [async replication](async-replication.md) settings (connection settings, list of replicated objects, etc.). It can also be used to obtain information about the state of async replication: [initial scan progress](async-replication.md#initial-scan), [lag](async-replication.md#replication-of-changes), [errors](async-replication.md#error-handling), etc. +**Async replication instance** is a named entity that stores [async replication](async-replication.md) settings (connection settings, list of replicated objects, etc.). It can also be used to obtain information about the async replication status: [initial scan progress](async-replication.md#initial-scan), [lag](async-replication.md#replication-of-changes), [errors](async-replication.md#error-handling), etc. #### Replicated object {#replicated-object} -A **replicated object** is an object (for example, a table) for which asynchronous replication is configured. +**Replicated object** is an object (for example, a table) for which async replication is configured. #### Replica object {#replica-object} -A **replica object** is a mirror copy of the replicated object, automatically created by an asynchronous replication instance. It is typically read-only. +**Replica object** is a mirror copy of the replicated object, automatically created by the async replication instance. Typically, it is read-only. {% endif %} @@ -344,27 +324,27 @@ A **replica object** is a mirror copy of the replicated object, automatically cr ### Transfer instance {#transfer-instance} -A **transfer instance** is a named entity that stores [transfer](transfer.md) settings, including connection settings and data transformation rules. It can also be used to obtain information about the transfer status, such as [errors](transfer.md#error-handling). +**Transfer instance** is a named entity that stores [transfer](transfer.md) settings, including connection settings and data transformation rules. It can also be used to obtain information about the transfer status, for example [errors](transfer.md#error-handling). {% endif %} ### Coordination node {#coordination-node} -A **coordination node** is a schema object that allows client applications to create semaphores to coordinate their actions. Coordination nodes are used to implement distributed locks, service discovery, leader election, and other scenarios. For more information about [coordination nodes](./datamodel/coordination-node.md). +**Coordination node** is a schema object that allows client applications to create semaphores for coordinating their actions. Coordination nodes are used to implement distributed locks, service discovery, leader election, and other scenarios. For more information about [coordination nodes](./datamodel/coordination-node.md). #### Semaphore {#semaphore} -A **semaphore** is an object inside a [coordination node](#coordination-node) that provides a synchronization mechanism for distributed applications. Semaphores can be persistent or temporary and support create, acquire, release, and monitor operations. For more information about [semaphores in {{ ydb-short-name }}](./datamodel/coordination-node.md#semaphore). +**Semaphore** is an object inside a [coordination node](#coordination-node) that provides a synchronization mechanism for distributed applications. Semaphores can be persistent or temporary and support create, acquire, release, and monitor operations. For more information about [semaphores in {{ ydb-short-name }}](./datamodel/coordination-node.md#semaphore). {% if feature_resource_pool == true and feature_resource_pool_classifier == true %} ### Resource pool {#resource-pool} -A **resource pool** is a schema object that describes the limits imposed on resources (CPU, RAM, etc.) available for executing queries in this resource pool. A query is always executed in some resource pool. By `default`, all queries are executed in the resource pool named , which does not impose any limits. For more information about using resource pools, see the article [{#T}](../dev/resource-consumption-management.md). +**Resource pool** is a schema object that describes the limits imposed on resources (CPU, RAM, etc.) available for executing queries in this resource pool. A query is always executed in some resource pool. By default, all queries are executed in a resource pool named `default`, which does not impose any limits. For more information about using resource pools, see [{#T}](../dev/resource-consumption-management.md). ### Resource pool classifier {#resource-pool-classifier} -A **resource pool classifier** is an object designed to manage the distribution of queries among [resource pools](#resource-pool). It describes the rules by which a resource pool is selected for each query. These classifiers are global for the entire [database](#database) and apply to all queries that come into it. For more information about their use, see the article [{#T}](../dev/resource-consumption-management.md). +**Resource pool classifier** is an object designed to manage the distribution of queries among [resource pools](#resource-pool). It describes the rules by which a resource pool is selected for each query. These classifiers are global for the entire [database](#database) and apply to all queries that come into it. For more information about their usage, see [{#T}](../dev/resource-consumption-management.md). {% endif %} @@ -374,25 +354,25 @@ A **resource pool classifier** is an object designed to manage the distribution ### Federated queries {#federated-queries} -**Federated queries** is a feature that allows you to execute queries against data stored in systems external to the {{ ydb-short-name }} cluster. +**Federated queries** is a feature that allows executing queries to data stored in systems external to the {{ ydb-short-name }} cluster. Below are explanations of several terms related to federated queries. How federated queries work in {{ ydb-short-name }} is explained in more detail in a separate article [{#T}](query_execution/federated_query/index.md). #### External data source {#external-data-source} -An **external data source** or **external connection** is metadata that describes how to connect to a supported external system to execute [federated queries](#federated-queries). +**External data source** or **external connection** is metadata describing how to connect to a supported external system to execute [federated queries](#federated-queries). #### External table {#external-table} -An **external table** is metadata that describes a specific data set that can be retrieved from an [external data source](#external-data-source). +**External table** is metadata describing a specific dataset that can be retrieved from an [external data source](#external-data-source). #### Secret {#secret} -A **secret** is confidential metadata that requires special handling. For example, secrets can be used in definitions of [external data sources](#external-data-source) and represent entities such as passwords and tokens. +**Secret** is confidential metadata that requires special handling. For example, secrets can be used in definitions of [external data sources](#external-data-source) and represent entities such as passwords and tokens. ### Authentication token {#auth-token} -An **authentication token** (or **auth token**) is a token used for [authentication](../security/authentication.md) in {{ ydb-short-name }}. +**Auth token** is a token used for [authentication](../security/authentication.md) in {{ ydb-short-name }}. {{ ydb-short-name }} supports [different types of authentication](../security/authentication.md) and various token types. @@ -410,7 +390,7 @@ The **{{ ydb-short-name }} cluster schema** is the hierarchical namespace of the ### Schema root {#scheme-root} -**Cluster schema root** is the root element of the [namespace {{ ydb-short-name }}](datamodel/cluster-namespace.md), whose child elements are [databases](#database). +**Cluster schema root** is the root element of the [{{ ydb-short-name }} namespace](datamodel/cluster-namespace.md), whose child elements are [databases](#database). ### Schema object {#scheme-object} @@ -428,13 +408,13 @@ Folders can contain subfolders, and such nesting can be of arbitrary depth. An **access object** during [authorization](../security/authorization.md) is an entity for which access rights and restrictions are configured. In {{ ydb-short-name }}, access objects are [schema objects](#scheme-object). -Each [schema object](#scheme-object) has an [owner](#access-owner) and an [access control list](#access-control-list) for that object, granted to users and groups ([access subjects](#access-subject)). +Each [schema object](#scheme-object) has an [owner](#access-owner) and an [access control list](#access-control-list) on that object, granted to users and groups ([access subjects](#access-subject)). ### Access subject {#access-subject} An **access subject** is an entity that can access [access objects](#access-object) and perform certain actions in the system. -Obtaining access during these requests and actions depends on the configured [access control lists](#access-control-list) and the subject's [access level](#access-level). +Gaining access during these requests and actions depends on the configured [access control lists](#access-control-list) and the [access level](#access-level) of the subject. An access subject can be a [user](#access-user) or a [group](#access-group). @@ -444,11 +424,11 @@ An **[access right](../security/authorization.md#right)** is an entity that refl ### Access right inheritance {#access-right-inheritance} -**Access right inheritance** is a mechanism where [access rights](#access-right) granted on parent [access objects](#access-object) are inherited by child objects in the hierarchical database structure. This ensures that permissions granted at a higher level of the hierarchy apply to all lower levels, unless they are [explicitly overridden](../reference/ydb-cli/commands/scheme-permissions.md#clear-inheritance). +**Access rights inheritance** is a mechanism where [access rights](#access-right) granted on parent [access objects](#access-object) are inherited by child objects in the hierarchical database structure. This ensures that permissions granted at a higher level of the hierarchy apply to all lower levels unless they are [explicitly overridden](../reference/ydb-cli/commands/scheme-permissions.md#clear-inheritance). -### Access control list {#access-control-list} +### Permissions list {#access-control-list} -An **[access control list](../security/authorization.md#right)** (ACL) is a list of all [rights](#access-right) granted to [access subjects](#access-subject) (users and groups) on a specific [access object](#access-object). +**[access control list](../security/authorization.md#right)** or **ACL** — a list of all [rights](#access-right) granted to [access subjects](#access-subject) (users and groups) on a specific [access object](#access-object). ### Access level {#access-level} @@ -457,32 +437,32 @@ An **access level** provides an [access subject](#access-subject) with additiona - Database - Viewer - Monitoring -- Administration +- Administration. -The access level for a subject is configured using [access level lists](#access-level-list). +The access level for a subject is configured using [access control lists](#access-level-list). ### Access level list {#access-level-list} -An **access level list** or **permission list** is a list of [SID](#access-sid)s of [access subjects](#access-subject) that are allowed a specific [access level](#access-level). +**Access Control List** or **Permission List** — a list of [SID](#access-sid)s of [access subjects](#access-subject) that are allowed a certain [access level](#access-level). In {{ ydb-short-name }}, there are [several such lists](../reference/configuration/security_config.md#security-access-levels) that define who has which [access levels](#access-level). -Detailed information about access level lists, their hierarchy, and how they work is provided in the [Access Level Lists](../security/authorization.md#access-level-lists) section of the authorization documentation. +Detailed information about access control lists, their hierarchy, and operating principles is provided in the [Access control lists](../security/authorization.md#access-level-lists) section of the authorization documentation. ### Owner {#access-owner} -An **[owner](../security/authorization.md#owner)** is an [access subject](#access-subject) ([user](#access-user) or [group](#access-group)) that has full rights to a specific [access object](#access-object). +**[Owner](../security/authorization.md#owner)** — an [access subject](#access-subject) ([user](#access-user) or [group](#access-group)) that has full rights to a specific [access object](#access-object). ### User {#access-user} -A **[user](../security/authorization.md#user)** is a person who uses {{ ydb-short-name }} to perform a specific function. +**[User](../security/authorization.md#user)** — a person who uses {{ ydb-short-name }} to perform a specific function. In {{ ydb-short-name }}, there are different types of users depending on the creation method: -- local users in {{ ydb-short-name }} databases -- external users from third-party directories +- Local users in databases {{ ydb-short-name }}. +- external users from third-party directories. -A user is identified by a [SID](#access-sid). +A user is identified by [SID](#access-sid). #### Local user {#local-user} @@ -490,120 +470,120 @@ A user whose account is created directly in {{ ydb-short-name }} using the YQL c #### External user {#external-user} -A {{ ydb-short-name }} user whose account is created in a third-party directory, for example, in an LDAP directory or IAM system. +A user {{ ydb-short-name }} whose account is created in an external directory, for example, an LDAP directory or IAM system. ### Group {#access-group} -**[Group](../security/authorization.md#group)** or **access group** — a named set of [users](#access-user) and other groups with equal capabilities for their members. +**[Group](../security/authorization.md#group)** or **access group** - a named set of [users](#access-user) and other groups with equal permissions for their members. -A group is identified by a [SID](#access-sid). +A group is identified by [SID](#access-sid). ### Role {#access-role} A role is a named set of [access rights](#access-right) used to assign to [users](#access-user) or [groups](#access-group) of users. -Roles in {{ ydb-short-name }} are implemented using [groups](#access-group), which are created during the initial cluster deployment and assigned a specific [access list](#access-right) on the cluster schema root. For more details about roles, see the article [{#T}](../security/builtin-security.md). +Roles in {{ ydb-short-name }} are implemented using [groups](#access-group), which are created during the initial deployment of the cluster and are assigned a specific [access rights list](#access-right) at the cluster schema root. For more information about roles, see the [{#T}](../security/builtin-security.md) article. ### SID {#access-sid} -**SID** or **security identifier** — a string of the form `<name>` or `<name>@<auth-domain>` that identifies an [access subject](../concepts/glossary.md#access-subject). It is used in [authentication](../security/authentication.md), [authorization](../security/authorization.md), [access lists](#access-control-list), and [access level lists](#access-level-list). +**SID** or **security identifier** — a string of the form `<name>` or `<name>@<auth-domain>` that identifies an [access subject](../concepts/glossary.md#access-subject). It is used in [authentication](../security/authentication.md), [authorization](../security/authorization.md), [access control lists](#access-control-list), and [access level lists](#access-level-list). -A SID identifies an individual [user](#access-user) or [group of users](#access-group). +SID identifies an individual [user](#access-user) or [group of users](#access-group). -The optional suffix `@<auth-domain>` identifies the source of the access subject, i.e., the external directory or system from which it was obtained. For example, users or groups from an LDAP directory may have the suffix `@ldap`. The absence of a suffix means that the user or group is created and exists directly in {{ ydb-short-name }}. +The optional suffix `@<auth-domain>` identifies the source of the access subject, i.e., the external directory or system from which it was obtained. For example, users or groups from an LDAP directory may have the suffix `@ldap`. The absence of a suffix means that the user or group was created and exists directly in {{ ydb-short-name }}. ### Query optimizer {#optimizer} -[**Query optimizer**](https://en.wikipedia.org/wiki/Query_optimization) — a set of {{ ydb-short-name }} components responsible for converting the logical representation of a query into a specific physically executable plan for obtaining the requested result. The main goal of the optimizer is to select, among all possible query execution plans, one that is sufficiently efficient in terms of predicted execution time and cluster resource consumption. It is described in more detail in a separate article [{#T}](query_execution/optimizer.md). +[**Query optimizer**](https://en.wikipedia.org/wiki/Query_optimization) is a set of {{ ydb-short-name }} components responsible for converting the logical representation of a query into a specific physically executable plan for obtaining the requested result. The main goal of the optimizer is to select, among all possible query execution plans, one that is sufficiently efficient in terms of predicted execution time and cluster resource consumption. It is described in more detail in a separate article [{#T}](query_execution/optimizer.md). ### Compilation cache {#compile-cache} -**Compilation cache** or **compile cache** — a cache of compiled queries on each [node](#node) of the cluster. It is used to avoid recompilation: if the query text is already in the node's cache, no additional compilation is performed. For more details, see the section [Query compilation cache](../dev/system-views.md#compile-cache-queries). +**compile cache** is a cache of compiled queries on each [node](#node) of the cluster. It is used to avoid recompilation: if the query text is already in the node's cache, no additional compilation is performed. For more details, see the [Query Compilation Cache](../dev/system-views.md#compile-cache-queries) section. -## Advanced terminology {#advanced-terminology} +## Advanced Terminology {#advanced-terminology} -This section explains terms that are useful for [{{ ydb-short-name }} contributors](../contributor/index.md) and users who want to gain a deeper understanding of what happens inside the system. +This section explains terms that are useful for [{{ ydb-short-name }} contributors](../contributor/index.md) and users who want to understand what happens inside the system more deeply. -### Actor implementation {#actor-implementation} +### Actor Implementation {#actor-implementation} -#### Actor system {#actor-system} +#### Actor System {#actor-system} -**Actor system** — a C++ library with an [implementation](https://github.com/ydb-platform/ydb/tree/main/ydb/library/actors) of the [actor model](https://en.wikipedia.org/wiki/Actor_model) for {{ ydb-short-name }} needs. +**actor system** is a C++ library with an [implementation](https://github.com/ydb-platform/ydb/tree/main/ydb/library/actors) of the [actor model](https://en.wikipedia.org/wiki/Actor_model) for the needs of {{ ydb-short-name }}. -#### Actor service {#actor-service} +#### Actor Service {#actor-service} -**Actor service** — an [actor](#actor) that has a well-known name and typically runs as a single instance on a [node](#node). +**actor service** is an [actor](#actor) that has a well-known name and usually runs as a single instance on a [node](#node). #### ActorId {#actorid} -**ActorId** — a unique identifier of an actor or [tablet](#tablet) in a [cluster](#cluster). +**ActorId** is a unique identifier of an actor or a [tablet](#tablet) in a [cluster](#cluster). -#### Actor system interconnect {#actor-system-interconnect} +#### Actor System Interconnect {#actor-system-interconnect} -**Actor system interconnect** or **interconnect** — the internal network layer of a [cluster](#cluster). All [actors](#actor) communicate with each other in the system via the interconnect. +**actor system interconnect** or **interconnect** is the internal network layer of the [cluster](#cluster). All [actors](#actor) communicate with each other in the system via the interconnect. #### Local {#local} -**Local** — an [actor service](#actor-service) running on each [node](#node). It directly manages [tablets](#tablet) on its node and interacts with [Hive](#hive). It registers with Hive and receives commands to start tablets. +**local** is an [actor service](#actor-service) running on each [node](#node). It directly manages [tablets](#tablet) on its node and interacts with [Hive](#hive). It registers with Hive and receives commands to start tablets. -### Tablet implementation {#tablet-implementation} +### Tablet Implementation {#tablet-implementation} -[**Tablet**](#tablet) — an [actor](#actor) with persistent state. It includes a set of data for which this tablet is responsible and a state machine through which the tablet's data (or state) is modified. The tablet is a fault-tolerant entity because the tablet's data is stored in [distributed storage](#distributed-storage), which survives disk and node failures. The tablet is automatically restarted on another [node](#node) in case of failure or overload of the previous one. Data in the tablet is modified sequentially, as the system infrastructure guarantees that there is no more than one [tablet leader](#tablet-leader) through which tablet data changes are performed. +[**tablet**](#tablet) is an [actor](#actor) with persistent state. It includes a set of data that this tablet is responsible for and a state machine through which the tablet's data (or state) is modified. The tablet is a fault-tolerant entity because the tablet's data is stored in [distributed storage](#distributed-storage), which survives disk and node failures. The tablet is automatically restarted on another [node](#node) in case of failure or overload of the previous one. Data in the tablet is modified sequentially, as the system infrastructure guarantees that there is no more than one [tablet leader](#tablet-leader) through which tablet data changes are performed. -A tablet solves the same problem as the [Paxos](https://en.wikipedia.org/wiki/Paxos_(computer_science)) and [Raft](https://en.wikipedia.org/wiki/Raft_(algorithm)) algorithms in other systems, namely the problem of [distributed consensus](https://en.wikipedia.org/wiki/Consensus_(computer_science)). From a technical standpoint, the implementation of a tablet can be described as a replicated state machine (RSM) on top of a shared log, since the tablet's state is fully described by an ordered log of commands stored in a distributed and fault-tolerant storage. +A tablet solves the same problem as the [Paxos](https://en.wikipedia.org/wiki/Paxos_(computer_science)) and [Raft](https://en.wikipedia.org/wiki/Raft_(algorithm)) algorithms in other systems, namely the problem of [distributed consensus](https://en.wikipedia.org/wiki/Consensus_(computer_science)). From a technical point of view, the tablet implementation can be described as a replicated state machine (RSM) on top of a shared log, since the tablet state is fully described by an ordered log of commands stored in a distributed and fault-tolerant storage. -During execution, the tablet's state machine is managed by three components: +At runtime, the tablet state machine is managed by three components: 1. The common tablet part ensures log consistency and recovery in case of failures. -2. **Executor** is an abstraction of a local database, namely data structures and code that organize work with data stored by the tablet. +2. **executor** is an abstraction of a local database, namely data structures and code that organize work with data stored by the tablet. 3. An actor with user code that implements the specific logic of a particular tablet type. In {{ ydb-short-name }}, there are several types of specialized tablets that store various data for different tasks. Many {{ ydb-short-name }} features, such as [tables](#table) and [topics](#topic), are implemented as different types of tablets. Thus, reusing the tablet infrastructure is one of the key means of extensibility of {{ ydb-short-name }} as a platform. -Typically, in a {{ ydb-short-name }} cluster, there are orders of magnitude more tablets compared to processes or threads that other systems would use for a cluster of similar size. In a {{ ydb-short-name }} cluster, hundreds of thousands or millions of tablets can easily run simultaneously. +Typically, a {{ ydb-short-name }} cluster runs orders of magnitude more tablets compared to the processes or threads that other systems would use for a cluster of similar size. In a {{ ydb-short-name }} cluster, hundreds of thousands or millions of tablets can easily run simultaneously. -Since a tablet stores its state in [distributed storage](#distributed-storage), it can be (re)started on any node of the cluster. Tablets are identified by a [TabletID](#tabletid), a 64-bit number assigned when the tablet is created. +Since a tablet stores its state in [distributed storage](#distributed-storage), it can be (re)started on any node of the cluster. Tablets are identified by [TabletID](#tabletid), a 64-bit number assigned when the tablet is created. -### Tablet leader {#tablet-leader} +### Tablet Leader {#tablet-leader} -A **tablet leader** is the current active leader of a given tablet. The tablet leader accepts commands, assigns them an order, and confirms them to the outside world. It is guaranteed that at any given time there is at most one leader for each tablet. +**Tablet leader** is the current active leader of a given tablet. The tablet leader accepts commands, assigns them an order, and confirms them to the outside world. It is guaranteed that at any given time there is at most one leader for each tablet. ### Tablet candidate {#tablet-candidate} -A **tablet candidate** is one of the election participants that wants to become the [leader](#tablet-leader) of a given tablet. If the candidate wins the election, it becomes the tablet leader. +**Tablet candidate** is one of the election participants that wants to become the [leader](#tablet-leader) of a given tablet. If the candidate wins the election, it becomes the tablet leader. ### Tablet replica {#tablet-follower} -A **tablet follower** or **hot standby** is a copy of the [tablet leader](#tablet-leader) that applies the log of commands accepted by the leader (with some delay). A tablet can have zero or more followers. Followers perform two main functions: +**Tablet follower** or **hot standby** is a copy of the [tablet leader](#tablet-leader) that applies the command log accepted by the leader (with some delay). A tablet can have zero or more replicas. Replicas perform two main functions: -* In case of termination or failure of the leader, followers are preferred [candidates](#tablet-candidate) for the new leader role, as they can become leader much faster than other candidates because they have applied most of the log. -* Followers can respond to read-only requests if the client explicitly chooses an optional relaxed transaction mode that allows stale reads. +* In case of termination or failure of the leader, replicas are preferred [candidates](#tablet-candidate) for the new leader role, as they can become the leader much faster than other candidates because they have applied most of the log. +* Replicas can answer read-only requests if the client explicitly chooses an optional relaxed transaction mode that allows stale reads. ### Tablet generation {#tablet-generation} -A **tablet generation** is a number that identifies the reincarnation of the tablet leader. It changes only when a new leader is elected and always increases. +**Tablet generation** is a number that identifies the reincarnation of the tablet leader. It changes only when a new leader is elected and always increases. ### Tablet local database {#local-database} -A **tablet local database** or **local database** is a set of data structures and associated code that manage the tablet's state and the data it stores. Logically, the state of the local database is represented by a set of tables, very similar to relational tables. Modification of the local database state is performed by local tablet transactions created by the tablet's user actor. +**Tablet local database** or **local database** is a set of data structures and associated code that manage the state of the tablet and the data it stores. Logically, the state of the local database is represented by a set of tables, very similar to relational tables. Modification of the local database state is performed by local tablet transactions created by the user tablet actor. Each table of the local database is stored as an [LSM tree](#lsm-tree). #### Log-structured merge-tree {#lsm-tree} -A **[Log-structured merge-tree](https://en.wikipedia.org/wiki/Log-structured_merge-tree)** or **LSM tree** is a data structure designed to optimize write and read performance in storage systems. It is used in {{ ydb-short-name }} to store [local database](#local-database) tables and [VDisks](#vdisk) data. +**[Log-structured merge-tree](https://en.wikipedia.org/wiki/Log-structured_merge-tree)** is a data structure designed to optimize write and read performance in storage systems. It is used in {{ ydb-short-name }} to store tables of the [local database](#local-database) and data of [VDisks](#vdisk). #### MemTable {#memtable} -All data written to tables of a [local database](#local-database) is initially stored in an in-memory data structure called a **MemTable**. When the MemTable reaches a specified size, it is flushed to disk as an immutable data structure called an [SST](#sst). +All data written to the tables of the [local database](#local-database) is initially stored in an in-memory data structure called **MemTable**. When the MemTable reaches a specified size, it is flushed to disk as an immutable data structure [SST](#sst). #### Sorted string table {#sst} -**Sorted string table** or **SST** is an immutable data structure that stores table rows sorted by key, facilitating efficient key lookup and range scans. Each SST consists of a contiguous series of small data pages, typically about 7 KiB each, which further optimizes disk read operations. An SST is usually part of an [LSM tree](#lsm-tree). +**Sorted string table** is an immutable data structure that stores table rows sorted by key, facilitating efficient key lookup and range scans. Each SST consists of a continuous series of small data pages, typically about 7 KiB each, which further optimizes reading data from disk. An SST is usually part of an [LSM tree](#lsm-tree). #### Tablet pipe {#tablet-pipe} -**Tablet pipe** or **TabletPipe** is a virtual connection that can be established with a tablet. It involves looking up the [tablet leader](#tablet-leader) by [TabletID](#tabletid). This is the recommended way to interact with a tablet. The term **open a pipe to a tablet** describes the process of resolving (finding) a tablet in the cluster and establishing a virtual communication channel with it. +**Tablet pipe** or **TabletPipe** is a virtual connection that can be established with a tablet. It involves finding the [tablet leader](#tablet-leader) by [TabletID](#tabletid). This is the recommended way to work with a tablet. The term **open a pipe to a tablet** describes the process of resolving (finding) a tablet in the cluster and establishing a virtual communication channel with it. #### TabletID {#tabletid} @@ -611,11 +591,11 @@ All data written to tables of a [local database](#local-database) is initially s #### Bootstrapper {#bootstrapper} -**Bootstrapper** is the primary mechanism for starting tablets, used for system tablets (e.g., [Hive](#hive), [DS controller](#ds-controller), the root [SchemeShard](#scheme-shard)). [Hive](#hive) initializes the remaining tablets. +**Bootstrapper** is the main mechanism for starting tablets, used for system tablets (e.g., [Hive](#hive), [DS controller](#ds-controller), root [SchemeShard](#scheme-shard)). [Hive](#hive) initializes the other tablets. ### Shared cache {#shared-cache} -**Shared cache** is an [actor](#actor) that stores data pages recently read from [distributed storage](#distributed-storage). Caching these pages reduces disk I/O operations and speeds up data retrieval, improving overall system performance. +**Shared cache** is an [actor](#actor) that stores data pages recently read from [distributed storage](#distributed-storage). Caching these pages reduces the number of disk I/O operations and speeds up data retrieval, improving overall system performance. ### Memory controller {#memory-controller} @@ -623,23 +603,23 @@ All data written to tables of a [local database](#local-database) is initially s ### Spilling {#spilling} -**Spilling** is a memory management mechanism in {{ ydb-short-name }} that temporarily offloads intermediate query data to external storage when such data exceeds the available RAM of a node. In {{ ydb-short-name }}, disk is currently used for spilling. +**Spilling** is a memory management mechanism in {{ ydb-short-name }} that temporarily offloads intermediate query data to external storage when such data exceeds the available RAM of the node. In {{ ydb-short-name }}, disk is currently used for spilling. For more details about spilling, see [{#T}](query_execution/spilling.md). ### Tablet types {#tablet-types} -[Tablets](#tablet) can be thought of as a framework for building reliable components that operate in a distributed system. Many components of {{ ydb-short-name }} — both system and user-data components — are implemented using this framework; the main ones are listed below. +[Tablets](#tablet) can be considered as a framework for building reliable components operating in a distributed system. Many components of {{ ydb-short-name }} — both system and those working with user data — are implemented using this framework; the main ones are listed below. #### SchemeShard {#scheme-shard} **SchemeShard** or **Scheme shard** is a system tablet that stores the database schema, including metadata of user [tables](#table), [topics](#topic), etc. -Additionally, there is a **root SchemeShard** that stores information about databases created in the cluster. +In addition, there is a **root SchemeShard** that stores information about databases created in the cluster. #### DataShard {#data-shard} -**DataShard** or **Data shard** is a tablet that manages a segment of a [row-based user table](datamodel/table.md#row-oriented-tables). A logical user table is divided into segments by contiguous ranges of the table's primary key. Each such range is handled by a separate DataShard tablet. The range itself is also called a [partition](#partition). The DataShard tablet stores data row by row, which is efficient for OLTP workloads. +**DataShard** or **Data shard** is a tablet that manages a segment of a [row-based user table](datamodel/table.md#row-oriented-tables). A logical user table is divided into segments by continuous ranges of the table's primary key. Each such range is managed by a separate DataShard tablet. The range itself is also called a [partition](#partition). The DataShard tablet stores data row by row, which is efficient for OLTP workloads. #### ColumnShard {#column-shard} @@ -647,7 +627,7 @@ Additionally, there is a **root SchemeShard** that stores information about data #### KeyValue Tablet {#kv-tablet} -**KeyValue** or **KV Tablet** is a tablet that implements a simple key→value mapping, where keys and values are strings. It also has several specific features, such as locks. +**KeyValue**, **KV Tablet** is a tablet that implements a simple key → value mapping, where keys and values are strings. It also has several specific features, such as locks. #### PersQueue Tablet {#pq-tablet} @@ -659,7 +639,7 @@ Additionally, there is a **root SchemeShard** that stores information about data #### Coordinator {#coordinator} -**Coordinator** is a system tablet that ensures the global order of transactions. The coordinator's task is to assign a logical time [PlanStep](#planstep) to each transaction planned through this coordinator. Each transaction is assigned exactly one coordinator, selected by hashing its [TxId](#txid). +**Coordinator** is a system tablet that ensures global ordering of transactions. The coordinator's task is to assign a logical time [PlanStep](#planstep) to each transaction planned through this coordinator. Each transaction is assigned exactly one coordinator, selected by hashing its [TxId](#txid). #### Mediator {#mediator} @@ -667,23 +647,23 @@ Additionally, there is a **root SchemeShard** that stores information about data #### Hive {#hive} -**Hive** is a system tablet responsible for starting and managing other tablets. Its responsibilities include moving tablets between nodes in case of a [node](#node) failure or overload.{% if audience != "corp" %} You can learn more about Hive in a [separate article](../contributor/hive.md).{% endif %} +**Hive** is a system tablet responsible for launching and managing other tablets. Its responsibilities include moving tablets between nodes in case of failure or overload of a [node](#node).{% if audience != "corp" %} For more details about Hive, see the [dedicated article](../contributor/hive.md).{% endif %} #### CMS {#cms} -**CMS** or **cluster management system** is a system tablet responsible for managing information about the current state of the [cluster {{ ydb-short-name }}](#cluster). This information is used to perform gradual cluster restarts without impacting user workloads, maintenance, cluster reconfiguration, etc. +**CMS** or **cluster management system** is a system tablet responsible for managing information about the current state of the [{{ ydb-short-name }} cluster](#cluster). This information is used for performing rolling restarts of the cluster without affecting user workloads, maintenance, cluster reconfiguration, etc. #### NodeBroker {#node-broker} -**NodeBroker** is a system tablet responsible for registering [dynamic nodes](#dynamic) in the cluster. +**NodeBroker** is a system tablet that is responsible for registering [dynamic nodes](#dynamic) in the cluster. #### BSController {#ds-controller} -**BSController**, **blob storage controller** manages the dynamic configuration of distributed storage, including information about [PDisk](#pdisk), [VDisk](#vdisk), and [storage groups](#storage-group). It interacts with [node warden](#node-warden) to start various distributed storage components. It interacts with [Hive](#hive) to allocate [channels](#channel) to tablets. +**BSController** (also known as **blob storage controller**) manages the dynamic configuration of the distributed storage, including information about [PDisk](#pdisk), [VDisk](#vdisk), and [storage groups](#storage-group). It interacts with [node warden](#node-warden) to start various distributed storage components. It interacts with [Hive](#hive) to allocate [channels](#channel) to tablets. #### Console {#console} -**Console** is a system tablet responsible for storing [dynamic configuration](../devops/configuration-management/configuration-v1/dynamic-config.md) and delivering it to cluster nodes. +**Console** is a system tablet responsible for storing the [dynamic configuration](../devops/configuration-management/configuration-v1/dynamic-config.md) and delivering it to cluster nodes. #### Kesus {#kesus} @@ -711,42 +691,36 @@ Additionally, there is a **root SchemeShard** that stores information about data #### StatisticsAggregator {#statistics-aggregator} -**StatisticsAggregator** is a tablet responsible for collecting statistics used in cost-based optimization. +**StatisticsAggregator** is a tablet responsible for collecting statistics used in cost optimization. ### Slot {#slot} **Slot** in {{ ydb-short-name }} can be used in two contexts: -* A **slot** is a portion of server resources allocated to run one {{ ydb-short-name }} [node](#node). The typical slot size is 10 CPU cores and 50 GB of RAM. Slots are used if the {{ ydb-short-name }} cluster is deployed on servers or virtual machines with sufficient resources to host multiple slots. -* A **VDisk slot** or **VSlot** is a portion of a [PDisk](#pdisk) that can be allocated to one of the [VDisk](#vdisk). +* **Slot** is a portion of server resources allocated to run one [node](#node) of {{ ydb-short-name }}. A typical slot size is 10 CPU cores and 50 GB of RAM. Slots are used when the {{ ydb-short-name }} cluster is deployed on servers or virtual machines with sufficient resources to host multiple slots. +* **VDisk slot** or **VSlot** is a portion of a [PDisk](#pdisk) that can be allocated to one of the [VDisk](#vdisk) instances. ### State storage {#state-storage} -**State storage** or **StateStorage** is a distributed service that stores information about tablets, namely: +**State storage** (also known as **StateStorage**) is a distributed service that stores information about tablets, namely: * The current tablet leader or its absence. * Tablet replicas. * Tablet generation and step `(generation:step)`. -State storage is used as a service for tablet name resolution, i.e., to obtain an [ActorId](#actorid) from a [TabletID](#tabletid). StateStorage is also used in the [tablet leader](#tablet-leader) election process. +State storage is used as a service for resolving tablet names, i.e., to obtain an [ActorId](#actorid) from a [TabletID](#tabletid). StateStorage is also used in the [tablet leader](#tablet-leader) election process. -Information in the State Storage is volatile. Thus, it is lost when power is turned off or the process is restarted. Despite its name, this service is not a permanent long-term storage. It only contains information that is easy to recover and that should not be durable. However, the State Storage stores information on multiple nodes to minimize the impact of node failures. This service can also be used to gather a quorum, which is used for tablet leader election. +The information in the state storage is volatile. Thus, it is lost on power failure or process restart. Despite its name, this service is not a permanent long-term storage. It only contains information that is easy to recover and that does not need to be durable. However, the state storage stores information on multiple nodes to minimize the impact of node failures. This service can also be used to gather a quorum, which is used for tablet leader election. -Due to its nature, the State Storage service operates on a best-effort basis. For example, the absence of multiple tablet leaders is guaranteed through a leader election protocol on [Distributed Storage](#distributed-storage), not on the State Storage. - -For more details about the StateStorage and related subsystems, see the Metadata Distribution Services section. +Due to its nature, the state storage service operates on a best-effort basis. For example, the absence of multiple tablet leaders is guaranteed through the leader election protocol on the [distributed storage](#distributed-storage), not on the state storage. ### Board {#board} -**Board** is a distributed service designed for storing metadata as key-value pairs. It is used, among other things, for storing information about [endpoints](../concepts/connect.md#endpoint). - -For more details about the Board and related subsystems, see the Metadata Distribution Services section. +**Board** is a distributed service designed to store metadata as key-value pairs. It is used, among other things, to store information about [endpoints](../concepts/connect.md#endpoint). ### SchemeBoard {#scheme-board} -**SchemeBoard** is a distributed service designed for storing metadata as key-value pairs. It is used, among other things, for storing information about [schemas](#global-schema). - -For more details about the SchemeBoard and related subsystems, see the Metadata Distribution Services section. +**SchemeBoard** is a distributed service designed to store metadata as key-value pairs. It is used, among other things, to store information about [schemas](#global-schema). #### Compaction {#compaction} @@ -754,57 +728,57 @@ For more details about the SchemeBoard and related subsystems, see the Metadata #### gRPC proxy {#grpc-proxy} -**gRPC proxy** is a proxy system for external user requests. Client requests enter the system via the [gRPC](https://grpc.io) protocol, then the proxy component translates them into internal calls to execute these requests, transmitted via [interconnect](#actor-system-interconnect). This proxy provides an interface for both request-response and bidirectional streaming data. +**gRPC proxy** — a proxy system for external user requests. Client requests enter the system via the [gRPC](https://grpc.io) protocol, then the proxy component translates them into internal calls to execute these requests, transmitted via [interconnect](#actor-system-interconnect). This proxy provides an interface for both request-response and bidirectional streaming data. ### Distributed configuration {#distributed-configuration} -**Distributed configuration** or **DistConf** is an internal [configuration](../devops/configuration-management/configuration-v2/config-overview.md) mechanism of the cluster that ensures the startup and configuration of [static nodes](#static-node), automatic management of the [static storage group](#static-group) and [State Storage](../concepts/glossary.md#state-storage). The distributed configuration starts before any [tablets](#tablet), [storage groups](#storage-group), and [State Storage](../concepts/glossary.md#state-storage). +**Distributed configuration** or **DistConf** — an internal mechanism of cluster [configuration](../devops/configuration-management/configuration-v2/config-overview.md) that ensures the startup and configuration of [static nodes](#static-node), automatic management of the [static storage group](#static-group) and [State Storage](../concepts/glossary.md#state-storage). Distributed configuration starts before any [tablets](#tablet), [storage groups](#storage-group), or [State Storage](../concepts/glossary.md#state-storage). For more details about the distributed configuration, see [{#T}](../contributor/configuration-v2.md). -### Distributed Storage Implementation {#distributed-storage-implementation} +### Distributed storage implementation {#distributed-storage-implementation} -**Distributed Storage** is a distributed fault-tolerant data storage layer that stores binary records called [LogoBlob](#logoblob), addressed using a specific type of identifier called [LogoBlobID](#logoblobid). Thus, Distributed Storage is a key-value store that maps a LogoBlobID to a string of up to 10 MB. Distributed Storage consists of many [storage groups](#storage-group), each of which is an independent data repository. +**Distributed storage** — a distributed fault-tolerant data storage layer that stores binary records called [LogoBlob](#logoblob), addressed by a specific type of identifier called [LogoBlobID](#logoblobid). Thus, Distributed storage is a key-value store that maps a LogoBlobID to a string of up to 10 MB. Distributed storage consists of many [storage groups](#storage-group), each of which is an independent data repository. -Distributed storage stores immutable data, with each immutable data block identified by a specific LogoBlobID key. The Distributed storage API is very specific and is intended only for use by [tablets](#tablet) to store their data and change logs. Thus, it is not intended for general-purpose data storage. Data in Distributed storage is deleted using special barrier commands. Due to the absence of mutations in its interface, Distributed storage can be implemented without implementing [distributed consensus](https://en.wikipedia.org/wiki/Consensus_(computer_science)). Distributed storage is just one of the components that tablets use to implement distributed consensus. +Distributed storage stores immutable data, with each immutable data block identified by a specific LogoBlobID key. The Distributed storage API is very specific, intended only for use by [tablets](#tablet) to store their data and change logs. Thus, it is not intended for general-purpose data storage. Data in Distributed storage is deleted using special barrier commands. Due to the absence of mutations in its interface, Distributed storage can be implemented without implementing [distributed consensus](https://en.wikipedia.org/wiki/Consensus_(computer_science)). Distributed storage is just one of the components that tablets use to implement distributed consensus. #### LogoBlob {#logoblob} -**LogoBlob** is a set of binary immutable data identified by a [LogoBlobID](#logoblobid) and stored in [Distributed storage](#distributed-storage). The data block size is limited at the [VDisk](#vdisk) level and above in the stack. Currently, the maximum data block size that VDisks can handle is 10 MB. +**LogoBlob** — a set of binary immutable data, identified by [LogoBlobID](#logoblobid) and stored in [Distributed storage](#distributed-storage). The data block size is limited at the [VDisk](#vdisk) level and above in the stack. Currently, the maximum data block size that a VDisk can handle is 10 MB. #### LogoBlobID {#logoblobid} -**LogoBlobID** is an identifier of a [LogoBlob](#logoblob) in [Distributed storage](#distributed-storage). It has a structure of the form `[TabletID, Generation, Step, Channel, Cookie, BlobSize, PartID]`. The main elements of LogoBlobID are: +**LogoBlobID** — an identifier of a [LogoBlob](#logoblob) in [Distributed storage](#distributed-storage). It has a structure of the form `[TabletID, Generation, Step, Channel, Cookie, BlobSize, PartID]`. The main elements of LogoBlobID: -* `TabletID` is the [ID](#tabletid) of the tablet that owns the LogoBlob. -* `Generation` is the generation of the tablet in which the data block was written. -* `Channel` is the [channel](#channel) of the tablet on which the LogoBlob is written. -* `Step` is an incremental counter, usually within the tablet generation. -* `Cookie` is a unique identifier of the data block within a single `Step`. The Cookie is typically used when writing multiple data blocks to a single `Step`. -* `BlobSize` is the size of the LogoBlob. -* `PartID` is the identifier of the data block part. It is important when the original LogoBlob is split into parts using [error correction coding](#erasure-coding), and the parts are written to the corresponding [VDisk](#vdisk) and [storage groups](#storage-group). +* `TabletID` — the [ID](#tabletid) of the tablet that owns the LogoBlob. +* `Generation` — the generation of the tablet in which the data block was written. +* `Channel` — the [channel](#channel) of the tablet on which the LogoBlob is written. +* `Step` — an incremental counter, usually within the tablet generation. +* `Cookie` — a unique identifier of a data block within a single `Step`. Cookie is typically used when writing multiple data blocks into one `Step`. +* `BlobSize` — the size of the LogoBlob. +* `PartID` — the identifier of a data block part. It is important when the original LogoBlob is split into parts using [erasure coding](#erasure-coding), and the parts are written to the corresponding [VDisk](#vdisk) and [storage groups](#storage-group). #### Replication {#replication} -**Replication** is a process that ensures a sufficient number of copies (replicas) of data to maintain the desired availability characteristics of the {{ ydb-short-name }} cluster. It is typically used in geo-distributed {{ ydb-short-name }} clusters. +**Replication** — a process that ensures a sufficient number of copies (replicas) of data to maintain the desired availability characteristics of the cluster {{ ydb-short-name }}. Typically used in geo-distributed clusters {{ ydb-short-name }}. #### Erasure coding {#erasure-coding} -[**Erasure coding**](https://en.wikipedia.org/wiki/Erasure_code) is a data encoding method in which the original data is supplemented with redundancy and split into multiple fragments, enabling recovery of the original data if one or more fragments are lost. It is widely used in {{ ydb-short-name }} clusters with a single [availability zone](#regions-az), as opposed to [replication](#replication) with 3 replicas. For example, the most popular erasure coding scheme 4+2 provides the same reliability as three replicas, with a space overhead of 1.5 versus 3. +[**Erasure coding**](https://en.wikipedia.org/wiki/Erasure_code) is a data encoding method where the original data is supplemented with redundancy and split into multiple fragments, enabling recovery of the original data if one or more fragments are lost. It is widely used in clusters {{ ydb-short-name }} with a single [availability zone](#regions-az), as opposed to [replication](#replication) with 3 replicas. For example, the most popular erasure coding scheme 4+2 provides the same reliability as three replicas, with a space overhead of 1.5 versus 3. #### PDisk {#pdisk} -**PDisk** or **physical disk** is a component that controls a physical disk drive (block device). In other words, PDisk is a subsystem that implements an abstraction similar to a specialized file system on top of block devices (or files emulating a block device for testing purposes). PDisk provides data integrity control (including [error correction coding](#erasure-coding) of sector groups to recover data on individual damaged sectors, integrity control using checksums), transparent encryption of all data on the disk, and transactional guarantees for disk operations (write confirmation strictly after `fsync`). +**PDisk**, **physical disk** is a component that controls a physical disk drive (block device). In other words, PDisk is a subsystem that implements an abstraction similar to a specialized file system on top of block devices (or files emulating a block device for testing purposes). PDisk provides data integrity control (including [erasure coding](#erasure-coding) of sector groups to recover data on individual damaged sectors, integrity checking using checksums), transparent encryption of all data on the disk, and transactional guarantees for disk operations (write confirmation strictly after `fsync`). -PDisk contains a scheduler that ensures sharing of device bandwidth among multiple clients ( [VDisk](#vdisk)). PDisk divides the block device into blocks called [slots](#slot) (about 128 megabytes in size; smaller blocks are also allowed). At any given time, no more than one VDisk can own each slot. PDisk also maintains a recovery log shared by PDisk service records and all VDisks. +PDisk contains a scheduler that ensures shared use of the device's bandwidth among multiple clients ( [VDisk](#vdisk)). PDisk divides the block device into blocks called [slots](#slot) (about 128 megabytes in size; smaller blocks are also allowed). At any given time, no more than one VDisk can own each slot. PDisk also maintains a recovery log shared by PDisk service records and all VDisks. #### VDisk {#vdisk} -**VDisk** or **virtual disk** is a component that implements data storage of [distributed storage](#distributed-storage) [LogoBlob](#logoblob) on [PDisk](#pdisk). VDisk stores all its data on PDisk. One VDisk corresponds to one PDisk, but typically several VDisks are associated with one PDisk. Unlike PDisk, which hides blocks and journals behind it, VDisk provides an interface at the LogoBlob and [LogoBlobID](#logoblobid) level, for example, writing a LogoBlob, reading LogoBlobID data, and deleting a set of LogoBlobs using a special command. VDisk is a member of a [storage group](#storage-group). The VDisk itself is local, but many VDisks in a given group provide reliable data storage. VDisks in a group synchronize data with each other and replicate data in case of losses. The set of VDisks in a storage group forms a distributed RAID. +**VDisk**, **virtual disk** is a component that implements data storage of [distributed storage](#distributed-storage) [LogoBlob](#logoblob) on [PDisk](#pdisk). VDisk stores all its data on PDisk. One VDisk corresponds to one PDisk, but typically several VDisks are associated with one PDisk. Unlike PDisk, which hides blocks and logs behind it, VDisk provides an interface at the LogoBlob and [LogoBlobID](#logoblobid) level, for example writing a LogoBlob, reading LogoBlobID data, and deleting a set of LogoBlobs using a special command. VDisk is a member of a [storage group](#storage-group). VDisk itself is local, but many VDisks in a given group provide reliable data storage. VDisks in a group synchronize data with each other and replicate data in case of losses. The set of VDisks in a storage group forms a distributed RAID. #### Yard {#yard} -**Yard** is the name of the [PDisk](#pdisk) API. It allows [VDisk](#vdisk) to read and write data to blocks and journals, reserve blocks, delete blocks, and transactionally acquire and release block ownership. In some contexts, Yard can be considered a synonym for PDisk. +**Yard** is the name of the [PDisk](#pdisk) API. It allows [VDisk](#vdisk) to read and write data to blocks and logs, reserve blocks, delete blocks, and transactionally acquire and release ownership of blocks. In some contexts, Yard can be considered a synonym for PDisk. #### Skeleton {#skeleton} @@ -816,58 +790,58 @@ PDisk contains a scheduler that ensures sharing of device bandwidth among multip #### Proxy {#ds-proxy} -**Distributed storage proxy**, **DS-proxy**, or **BS-proxy** acts as a client library for performing operations with [distributed storage](#distributed-storage). The users of the DS-proxy are [tablets](#tablet), which write to and read from distributed storage. The DS-proxy hides the distributed nature of distributed storage from the user. The task of the DS-proxy is to write to a quorum of [VDisks](#vdisk), retry when necessary, and control the write/read flow to prevent VDisk overload. +**Distributed storage proxy**, **DS-proxy**, or **BS-proxy** acts as a client library for performing operations with [distributed storage](#distributed-storage). The users of DS-proxy are [tablets](#tablet) that write to and read from distributed storage. DS-proxy hides the distributed nature of distributed storage from the user. The task of DS-proxy is to write to a quorum of [VDisk](#vdisk), perform retries when necessary, and control the write/read flow to prevent VDisk overload. -Technically, the DS-proxy is implemented as an [actor service](#actor-service) launched by [node warden](#node-warden) on each node for each storage group, handling all requests to the group (writing, reading, and deleting [LogoBlob](#logoblob), group locking). When writing data, the DS-proxy performs [erasure coding](#erasure-coding) of the data, splitting the LogoBlob into parts that are then sent to the corresponding VDisks. The DS-proxy performs the reverse process when reading, receiving parts from VDisks and reconstructing the LogoBlob from them. +Technically, the DS proxy is implemented as an [actor service](#actor-service) launched by [node warden](#node-warden) on each node for each storage group, handling all requests to the group (writing, reading, and deleting [LogoBlob](#logoblob), and group locking). When writing data, the DS proxy performs [erasure coding](#erasure-coding) of the data, splitting the LogoBlob into parts that are then sent to the corresponding VDisks. The DS proxy performs the reverse process when reading, receiving parts from VDisks and reconstructing the LogoBlob from them. #### Node warden {#node-warden} -**Node warden** or `BS_NODE` is an [actor service](#actor-service) on each cluster node that launches [PDisks](#pdisk), [VDisks](#vdisk), and [DS proxies](#ds-proxy) of [static storage groups](#static-group) when the node starts. It also interacts with the [DS controller](#ds-controller) to launch PDisk, VDisk, and DS proxies of [dynamic groups](#dynamic-group). The DS proxy of dynamic groups is launched on demand: node warden processes "undelivered" messages to DS proxies, launches the corresponding DS proxies, and receives group configuration from the DS controller. +**Node warden** or `BS_NODE` is an [actor service](#actor-service) on each cluster node that launches [PDisks](#pdisk), [VDisks](#vdisk), and [DS proxies](#ds-proxy) of [static storage groups](#static-group) when the node starts. It also interacts with the [DS controller](#ds-controller) to launch PDisk, VDisk, and DS proxies of [dynamic groups](#dynamic-group). The DS proxy of dynamic groups is launched on demand: node warden processes undelivered messages to DS proxies, launches the corresponding DS proxies, and receives group configuration from the DS controller. -#### Fail realm {#fail-realm} +#### Failure realm {#fail-realm} -A **fail realm** is a set of [fail domains](#fail-domain) that can fail simultaneously due to a common cause. A correlated failure of two [VDisks](#vdisk) in the same fail realm is more likely than a failure of two VDisks from different fail realms. +A **fail realm** is a set of [failure domains](#fail-domain) that can fail simultaneously due to a common cause. A correlated failure of two [VDisks](#vdisk) in the same fail realm is more likely than a failure of two VDisks from different fail realms. -An example of a fail realm is a set of equipment located in one [data center, or availability zone](#regions-az), which can fail entirely due to a natural disaster, large-scale power outage, or other similar event. +An example of a fail realm is a set of equipment located in a single [data center, or availability zone](#regions-az), which can fail entirely due to a natural disaster, large-scale power outage, or other similar event. #### Fail domain {#fail-domain} A **fail domain** is a set of equipment that can fail simultaneously. A correlated failure of two [VDisk](#vdisk) within the same fail domain is more likely than a failure of two VDisks from different fail domains. In the case of different fail domains, the probability of simultaneous failure also depends on whether the domains in question belong to the same fail realm or different ones. -An example of a fail domain is a set of disks connected to a single server, since all disks of a particular server may become unavailable if the server's power supply or network controller fails. Typically, all servers located in one [server rack](#rack) are considered part of a common fail domain, because power or network issues at the rack level cause all equipment in it to become unavailable. Thus, a typical fail domain corresponds to a server rack (if the [cluster](#cluster) is configured with rack-level topology awareness) or to an individual server. +An example of a fail domain is a set of disks connected to a single server, since all disks of a particular server may become unavailable if the server's power supply or network controller fails. Typically, all servers located in a single [server rack](#rack) are considered a common fail domain, because power or network issues at the rack level cause all equipment in it to become unavailable. Thus, a typical fail domain corresponds to a server rack (if the [cluster](#cluster) is configured with rack-aware topology) or an individual server. -Fail domain-level failures are automatically handled by {{ ydb-short-name }} without stopping the cluster. +Failures at the fail domain level are automatically handled by {{ ydb-short-name }} without stopping the cluster. #### Distributed storage channel {#channel} A **distributed storage channel**, **DS channel**, or **channel** is a logical connection between a [tablet](#tablet) and a [distributed storage](#distributed-storage) group. A tablet can write data to different channels, and each channel maps to a specific [storage group](#storage-group). Having multiple channels allows a tablet to: -* Write more data than a single storage group can contain. -* Store different [LogoBlob](#logoblob) in different storage groups, with different properties such as erasure coding or on different media (HDD, SSD, NVMe). +* Write more data than a single storage group can hold. +* Store different [LogoBlob](#logoblob)s in different storage groups, with different properties, such as erasure coding or on different media (HDD, SSD, NVMe). ### Distributed transaction implementation {#transaction-implementation} -The terms related to the implementation of [distributed transactions](#transactions) are explained below.{% if oss == true %} The implementation itself is described in a separate article [{#T}](../contributor/datashard-distributed-txs.md).{% endif %} +Below are the terms related to the implementation of [distributed transactions](#transactions).{% if oss == true %} The implementation itself is described in a separate article [{#T}](../contributor/datashard-distributed-txs.md).{% endif %} #### Deterministic transactions {#deterministic-transactions} -Distributed transactions {{ ydb-short-name }} are inspired by the research paper [Building Deterministic Transaction Processing Systems without Deterministic Thread Scheduling](http://cs-www.cs.yale.edu/homes/dna/papers/transactions-wodet11.pdf) by Alexander Thomson and Daniel J. Abadi from Yale University. The paper introduces the concept of **deterministic transaction processing**, which allows efficient processing of distributed transactions. The original paper imposed restrictions on the types of operations that can be performed this way. Since these restrictions hindered real user scenarios, {{ ydb-short-name }} developed its own algorithms to execute them, using deterministic transactions as stages of user transaction execution with additional orchestration and locking. +Distributed transactions in {{ ydb-short-name }} are inspired by the research paper [Building Deterministic Transaction Processing Systems without Deterministic Thread Scheduling](http://cs-www.cs.yale.edu/homes/dna/papers/transactions-wodet11.pdf) by Alexander Thomson and Daniel J. Abadi from Yale University. The paper introduced the concept of **deterministic transaction processing**, which allows efficient processing of distributed transactions. The original paper imposed restrictions on the types of operations that could be performed this way. Since these restrictions hindered real-world user scenarios, {{ ydb-short-name }} evolved its algorithms to handle them, using deterministic transactions as stages for executing user transactions with additional orchestration and locking. #### Optimistic locking {#optimistic-locking} -As in many other database management systems, queries {{ ydb-short-name }} can place locks on certain data fragments, such as table rows, to ensure that concurrent changes do not lead to an inconsistent state. However, {{ ydb-short-name }} checks these locks not at the beginning of transactions, but when attempting to commit them. The first approach is called **pessimistic locking** (used, for example, in PostgreSQL), and the second is called **optimistic locking** (used in {{ ydb-short-name }}). +As in many other database management systems, queries in {{ ydb-short-name }} can place locks on certain data fragments, such as table rows, to ensure that concurrent changes do not lead to an inconsistent state. However, {{ ydb-short-name }} checks these locks not at the start of transactions, but when attempting to commit them. The first approach is called **pessimistic locking** (for example, used in PostgreSQL), and the second is called **optimistic locking** (used in {{ ydb-short-name }}). #### Transaction lock invalidation {#tli} -**Transaction Lock Invalidation** (Transaction Lock Invalidation, **TLI**) is the normal behavior of {{ ydb-short-name }} when parallel transactions conflict under [optimistic locking](#optimistic-locking). If one transaction (the violator) writes data and thereby breaks the locks of another transaction (the victim), {{ ydb-short-name }} detects this when the victim commits and rolls it back with error `transaction locks invalidated`. For more details on TLI diagnostics, see [{#T}](../troubleshooting/performance/queries/transaction-lock-invalidation.md). +**Transaction Lock Invalidation** (**TLI**) is the normal behavior of {{ ydb-short-name }} when parallel transactions conflict under [optimistic locks](#optimistic-locking). If one transaction (the violator) writes data and thereby breaks the locks of another transaction (the victim), {{ ydb-short-name }} detects this when the victim commits and aborts it with error `transaction locks invalidated`. For more details on TLI diagnostics, see [{#T}](../troubleshooting/performance/queries/transaction-lock-invalidation.md). #### Preparation phase {#prepare-stage} -**Preparation phase** is a transaction phase during which the transaction body is registered on all participating shards. +**Preparation phase** is the transaction phase during which the transaction body is registered on all participating shards. #### Execution phase {#execute-stage} -**Execution phase** is a transaction phase during which the scheduled transaction is executed and a response is generated. +**Execution phase** is the transaction phase during which the scheduled transaction is executed and a response is generated. In some cases, instead of [preparation](#prepare-stage) and execution, the transaction is executed immediately and a response is generated. For example, this happens for transactions that affect only one shard or for consistent reads from a data snapshot. @@ -877,31 +851,31 @@ In the case of read-only transactions, similar to "read uncommitted" in other da #### Read-write set {#rw-set} -**Read-write set**, **RW-set** is a set of data that will participate in the execution of a [distributed transaction](#transactions). It combines the read set data that will be read and the write set for which modifications will be performed. +**Read-write set** or **RW-set** is a data set that will participate in the execution of a [distributed transaction](#transactions). It combines the read set data that will be read and the write set for which modifications will be performed. #### Read set {#read-set} -**Read set**, **ReadSet data** is what participating shards send during transaction execution. In the case of data transactions, it may contain information about the state of [optimistic locks](#optimistic-locking), shard readiness to commit, or a decision to abort the transaction. +**Read set** or **ReadSet data** is what participating shards send during transaction execution. In the case of data transactions, it may contain information about the state of [optimistic locks](#optimistic-locking), the shard's readiness to commit, or a decision to abort the transaction. #### Transaction proxies {#transaction-proxy} -**Transaction proxy** or `TX_PROXY` is a service that orchestrates the execution of many [distributed transactions](#transactions): sequential phases, phase execution, planning, and result aggregation. In the case of direct orchestration by other actors (e.g., QP data transactions), it is used for caching and allocating unique [TxIDs](#txid). +**Transaction proxy** or `TX_PROXY` is a service that orchestrates the execution of many [distributed transactions](#transactions): sequential phases, phase execution, scheduling, and result aggregation. In the case of direct orchestration by other actors (for example, QP data transactions), it is used for caching and allocating unique [TxIDs](#txid). #### Transaction flags {#txflags} -**Transaction flags** or **TxFlags** is a bitmask of flags that modify the execution of a transaction in some way. +**Transaction flags** or **TxFlags** is a bitmask of flags that somehow modify the execution of a transaction. #### Transaction ID {#txid} -**Transaction ID** or **TxID** is a unique identifier assigned to each transaction when it is accepted by {{ ydb-short-name }}. +**TxID** is a unique identifier assigned to each transaction when it is accepted by {{ ydb-short-name }}. #### Transaction order ID {#transaction-order-id} -**Transaction order ID** is a unique identifier assigned to each transaction during planning. It consists of [PlanStep](#planstep) and [Transaction ID](#txid). +**Transaction order id** is a unique identifier assigned to each transaction during scheduling. It consists of [PlanStep](#planstep) and [Transaction ID](#txid). #### Plan step {#planstep} -**Plan step**, **PlanStep**, **Step** is the logical time at which the execution of a set of transactions is scheduled. +**PlanStep** or **Step** is the logical time at which the execution of a set of transactions is scheduled. #### Mediator time {#mediator-time} @@ -909,18 +883,18 @@ During the execution of distributed transactions, **mediator time** is the logic #### MiniKQL {#minikql} -**MiniKQL** is a language that allows expressing a single [deterministic transaction](#deterministic-transactions) in the system. It is a functional, strongly typed language. Conceptually, the language describes a graph of reading from the database, performing computations on the read data, and writing results to the database and/or to a special document representing the query result (for display to the user). A MiniKQL transaction must explicitly specify its read set (data to be read) and assume deterministic branching (e.g., no randomness). +**MiniKQL** is a language that allows expressing a single [deterministic transaction](#deterministic-transactions) in the system. It is a functional, strongly typed language. Conceptually, the language describes a graph of reading from the database, performing computations on the read data, and writing results to the database and/or to a special document representing the query result (for display to the user). A MiniKQL transaction must explicitly specify its read set (data to be read) and assume deterministic selection of execution branches (for example, no randomness). MiniKQL is a low-level language. End users of the system only see queries in the [YQL](#yql) language, which relies on MiniKQL in its implementation. #### Query Processor {#kqp} -**Query Processor** or **QP** (formerly **KQP**) is a component of {{ ydb-short-name }} responsible for orchestrating the execution of user queries and generating the final response. +**Query Processor** or **QP** (formerly **KQP**) is a {{ ydb-short-name }} component responsible for orchestrating the execution of user queries and generating the final response. ### Global schema {#global-schema} -**Global scheme**, **global schema**, or **database schema** is the schema of all data stored in the [database](#database). It consists of [tables](#table) and other entities such as [topics](#topic). The metadata about these entities is called the global schema. The term is used in contrast to **local schema**, which refers to the data schema inside a [tablet](#tablet). Users of {{ ydb-short-name }} never see the local schema and work only with the global schema. +**Global scheme**, **global schema**, or **database schema** is the schema of all data stored in a [database](#database). It consists of [tables](#table) and other entities such as [topics](#topic). The metadata about these entities is called the global schema. The term is used in contrast to **local schema**, which refers to the data schema inside a [tablet](#tablet). {{ ydb-short-name }} users never see the local schema and work only with the global schema. ### KiKiMR {#kikimr} -**KiKiMR** is the outdated name of {{ ydb-short-name }}, used before it became an [open source product](https://github.com/ydb-platform/ydb). It can still be found in source code, old articles, videos, etc. +**KiKiMR** is the former name of {{ ydb-short-name }}, used before it became an [open source product](https://github.com/ydb-platform/ydb). It can still be found in source code, old articles, videos, etc. diff --git a/ydb/docs/en/core/concepts/query_execution/json_search.md b/ydb/docs/en/core/concepts/query_execution/json_search.md new file mode 100644 index 00000000000..219a4abb4e5 --- /dev/null +++ b/ydb/docs/en/core/concepts/query_execution/json_search.md @@ -0,0 +1,73 @@ +# Searching JSON document contents + +JSON search is a way to find table rows by the content of a JSON document stored in a column of type `Json` or `JsonDocument`: by the existence of a path in the document and by the value located at the specified path. The path is specified using a [JsonPath](../../yql/reference/builtins/json.md#jsonpath) expression, and the checks are performed using the existing functions [JSON_EXISTS](../../yql/reference/builtins/json.md#json_exists) and [JSON_VALUE](../../yql/reference/builtins/json.md#json_value). Typical scenarios: + +* Search for documents with a specific nested field +* Search for documents whose field at a given path equals the required value. +* filtering semi-structured data without a pre-defined schema. + +In {{ ydb-short-name }}, JSON search can be performed in two main ways: + +* Without an index: scanning the table and applying the `JSON_EXISTS` or `JSON_VALUE` functions to each row. The approach is simple but scales poorly: the amount of scanning work grows with the table size. +* with a JSON index — creating a [JSON index](../../dev/json-indexes.md) on a JSON column. This approach is designed for scalable search. + +## JSON search with a JSON index {#json-search-index} + +The key idea of a [JSON index](../../dev/json-indexes.md) is building an [inverted index](https://en.wikipedia.org/wiki/Inverted_index) over paths and "path + value" pairs. Each path of a JSON document (and for equality checks, the scalar value at that path) is encoded into a token. For each token, the index stores a list of primary key values of the corresponding table rows. Queries to the index are reduced to an inverted search over these tokens — following the same scheme as a [full-text index](../../dev/fulltext-indexes.md#basic), but with its own JSON tokenizer. + +The JSON index does not store a copy of the entire JSON document, but decomposes it into individual tokens: + +* For each path in the JSON tree, a token 'path exists' is created. +* if the path leads to a scalar value (string, number, boolean, or `null`), a «path + value» token is additionally created. + +Arrays are transparent in this case: array elements are indexed under the same path as the array itself, so the index responds equally to queries to `$.items`, `$.items[0]`, and `$.items[*]`. + +For example, for a document: + + +```json +{ + "id": 42042, + "name": "Michael", + "email": null, + "items": [null, "str"], + "parts": { + "key": "k1", + "value": false + } +} +``` + + +Conceptually, the following tokens are indexed (without regard to the specifics of the internal representation): + +| Token | What it finds | +| --- | --- | +| `$` | document is not equal to `NULL` | +| `$.id` / `$.id == 42042` | path `id` exists / equals `42042` | +| `$.name` / `$.name == "Michael"` | path `name` exists / equals `"Michael"` | +| `$.email` / `$.email == null` | path `email` exists / equals `null` | +| `$.items` / `$.items == null` / `$.items == "str"` | path `items` exists / contains element `null` / contains element `"str"` | +| `$.parts` | path `parts` exists | +| `$.parts.key` / `$.parts.key == "k1"` | nested path exists / equals `"k1"` | +| `$.parts.value` / `$.parts.value == false` | nested path exists or equals `false` | + +JSON index allows: + +* find rows by path existence using [JSON_EXISTS](../../yql/reference/builtins/json.md#json_exists) +* find rows by a value in a path using [JSON_VALUE](../../yql/reference/builtins/json.md#json_value) or a filter predicate within `JSON_EXISTS`. + +When executing a query, the JSON index can be automatically used by the [optimizer](../glossary.md#optimizer). It is enough to write a regular condition `WHERE` with expressions `JSON_EXISTS` or `JSON_VALUE` on an indexed JSON column. The optimizer recognizes such a predicate and applies reading by the JSON index instead of scanning records of the source table. Additionally, the JSON index, like any other [secondary index](../glossary.md#secondary-index), can be forcefully used by specifying its name in the section `VIEW IndexName`. + +If the predicate cannot be converted into an index access, the behavior of {{ ydb-short-name }} depends on whether the required JSON index was explicitly specified in the query: + +* When the optimizer automatically selects an index, the query is simply executed without index acceleration (the result remains correct). +* When explicitly specifying an index via the `VIEW` expression, an error is returned. + +For more information, see [{#T}](../../yql/reference/syntax/select/json_index.md). + +Additional information: + +* [JSON indexes](../../dev/json-indexes.md) — an overview of supported capabilities, predicates, and limitations. +* [JSON functions](../../yql/reference/builtins/json.md) — reference on `JSON_EXISTS`, `JSON_VALUE`, and the JsonPath language. +* [VIEW (JSON index)](../../yql/reference/syntax/select/json_index.md) — syntax for queries with a JSON index. diff --git a/ydb/docs/en/core/concepts/query_execution/toc_i.yaml b/ydb/docs/en/core/concepts/query_execution/toc_i.yaml index a046a7ca930..42ee92bb1eb 100644 --- a/ydb/docs/en/core/concepts/query_execution/toc_i.yaml +++ b/ydb/docs/en/core/concepts/query_execution/toc_i.yaml @@ -1,8 +1,6 @@ items: - name: Overview href: index.md - - name: Query Execution Process - href: execution_process.md - name: Query optimizer href: optimizer.md - name: Multi-Version Concurrency Control (MVCC) @@ -15,6 +13,8 @@ items: href: fulltext_search.md - name: Hybrid search href: hybrid_search.md + - name: JSON Search + href: json_search.md - name: Local indexes href: local_indexes.md - name: Federated query @@ -25,5 +25,7 @@ items: href: spilling.md - name: Scan queries href: scan_query.md + - name: Query Execution Process + href: execution_process.md - name: YQL queries to topics href: topics.md diff --git a/ydb/docs/en/core/concepts/query_execution/topics.md b/ydb/docs/en/core/concepts/query_execution/topics.md index 5fb695a8bda..eed3a4d8481 100644 --- a/ydb/docs/en/core/concepts/query_execution/topics.md +++ b/ydb/docs/en/core/concepts/query_execution/topics.md @@ -1,14 +1,14 @@ # YQL queries to topics {#yql-syntax} -To read and write messages to [topics](../datamodel/topic.md), familiar YQL constructs are used: [SELECT](../../yql/reference/syntax/select/index.md) for reading and [INSERT](../../yql/reference/syntax/insert_into.md) for writing. +To read and write messages in [topics](../datamodel/topic.md), familiar YQL constructs are used: [SELECT](../../yql/reference/syntax/select/index.md) for reading and [INSERT](../../yql/reference/syntax/insert_into.md) for writing. ## Local and external topics {#local-external-topics} -YQL queries to topics work the same regardless of whether the topic is in the current database or in another {{ ydb-short-name }} database. The source and receiver of messages can be either a topic **in the same database** where the query is executed, or a topic **in another database**. +YQL queries to topics work the same regardless of whether the topic is in the current database or in another {{ ydb-short-name }} database. The source and destination of messages can be either a topic **in the same database** where the query is executed, or a topic **in another database**. ### Local topics {#local-topics} -**Local topics** are topics created in the **same {{ ydb-short-name }} database** as the query being executed. +**Local topics** are topics created in the **same {{ ydb-short-name }} database** as the executed query. In the query text, they are referred to **by a short name** — just like a table in the current database: @@ -29,7 +29,7 @@ INSERT INTO output_topic SELECT ...; Access to them is performed only through a pre-created [external data source](../datamodel/external_data_source.md) with the YDB source type. -After creating a source, for example named `ext_source`, accessing topic `input_topic` in an external database is written as follows: +After creating a source, for example named `ext_source`, accessing the `input_topic` topic in an external database is written as follows: ```yql @@ -41,11 +41,11 @@ The name `ext_source` in the documentation is **conditional** — in your databa ## Reading from a topic {#topic-read} -Reading from a topic can be performed in [table](#table-read) and [streaming](#streaming-read) modes (not to be confused with streaming queries). +Reading from a topic can be limited to only the current data of the topic or work with waiting for new written messages. -### Table reading {#table-read} +### Reading current data {#table-read} -In table mode, reading is performed from the first to the last offset stored in the topic at the time the query is started. If data continues to be written to the topic, the query will stop after reaching the last offset known at startup. Specifying filters on [Service fields](#system-metadata) speeds up reading, as reading occurs only over the specified ranges. +In this mode, reading is performed from the first to the last offset stored in the topic at the time the query is started. If data continues to be written to the topic, the query will stop after reaching the last offset known at startup. Specifying filters on [Service fields](#system-metadata) speeds up reading, as reading occurs only over the specified ranges. ```yql @@ -57,9 +57,9 @@ LIMIT 10; ``` -### Streaming reading {#streaming-read} +### Reading with data waiting {#streaming-read} -To read new messages, use the `WITH (STREAMING = "TRUE")` option — see more in the [Streaming reading of data from a topic](../../yql/reference/syntax/select/streaming.md) section. Reading starts from the current moment and continues until the number of messages specified in the `LIMIT` expression is read. The `LIMIT` parameter is required — without it, the query will not complete, as it will wait for new messages indefinitely. +To wait for new messages, use the `WITH (STREAMING = "TRUE")` option. Reading starts from the current moment and continues until the number of messages specified in the `LIMIT` expression is read. The `LIMIT` parameter is mandatory — without it, the query will not complete, as it will wait for new messages indefinitely. ```yql @@ -148,7 +148,19 @@ FROM ### Service fields {#system-metadata} -When reading, you can request service fields: +When reading, you can request service fields and [user message attributes](../datamodel/topic.md#message): + +| Field | [Type](../../yql/reference/types/index.md) | Description | +| --- | --- | --- | +| `__ydb_create_time` | `Timestamp` | Message creation time | +| `__ydb_write_time` | `Timestamp` | Message write time to topic | +| `__ydb_offset` | `Uint64` | Message offset in partition | +| `__ydb_partition_id` | `Uint64` | Partition number | +| `__ydb_message_group_id` | `String` | Message group ID | +| `__ydb_seq_no` | `Uint64` | Message sequence number within group | +| `__ydb_user_attributes` | `Dict<String,String>` | [User message attributes](../datamodel/topic.md#message) | + +Example of using service fields: ```yql @@ -158,7 +170,7 @@ SELECT __ydb_write_time AS WriteTime, -- message write time __ydb_offset AS Offset, -- message offset in topic __ydb_partition_id AS Partition, -- partition number - __ydb_message_group_id AS MessageGroupId, -- message group identifier + __ydb_message_group_id AS MessageGroupId, -- message group ID __ydb_seq_no AS SeqNo -- sequence number within partition FROM input_topic -- local topic; for external: ext_source.input_topic @@ -166,7 +178,7 @@ LIMIT 10; ``` -Filters on service fields are evaluated before reading data from the topic and significantly reduce the volume of messages read. Supported are comparison operators (`=`, `<>`, `<`, `<=`, `>`, `>=`, `IN`), logical conditions (`AND`, `OR`), and fields `partition_id`, `write_time`, `offset`. Predicates on other service fields do not limit the read volume. +Filters on service fields are evaluated before reading data from the topic and significantly reduce the volume of messages read. Comparison operators (`=`, `<>`, `<`, `<=`, `>`, `>=`, `IN`), logical conditions (`AND`, `OR`), and fields `partition_id`, `write_time`, `offset` are supported. Predicates on other service fields do not limit the read volume. ```yql @@ -182,6 +194,20 @@ WHERE ``` +Example of using custom attributes: + + +```yql +SELECT + COUNT(*) AS ErrorCount +FROM + input_topic -- local topic; for external: ext_source.input_topic +WHERE + __ydb_user_attributes["type"] = "log" + AND __ydb_user_attributes["level"] = "error"; +``` + + ## Writing to a topic {#topic-write} ### Writing a single message {#simple-write} @@ -214,13 +240,13 @@ FROM {% note warning %} -Reading and writing [user attributes](../datamodel/topic.md#message) are not supported. +Writing [custom attributes](../datamodel/topic.md#message) via YQL is not supported. {% endnote %} {% note warning %} -Transactional writes via YQL/`INSERT INTO` are not supported — partial query results may appear in the topic. +Transactional writing via YQL/`INSERT INTO` is not supported — partial query results may appear in the topic. {% endnote %} diff --git a/ydb/docs/en/core/concepts/toc_i.yaml b/ydb/docs/en/core/concepts/toc_i.yaml index b44c1842f22..433520c005d 100644 --- a/ydb/docs/en/core/concepts/toc_i.yaml +++ b/ydb/docs/en/core/concepts/toc_i.yaml @@ -5,11 +5,6 @@ items: include: path: analytics/toc_p.yaml mode: link -- name: Architecture - href: architecture/index.md - include: - path: architecture/toc_p.yaml - mode: link - name: Connecting to a database href: connect.md - name: Schema objects @@ -19,8 +14,6 @@ items: mode: link - name: Cluster topology href: topology.md -- name: Bridge mode - href: bridge.md - name: Query execution href: query_execution/index.md include: @@ -47,3 +40,13 @@ items: when: feature_transfer - name: Streaming queries href: streaming-query.md +- name: Architecture + href: architecture/index.md + include: + path: architecture/toc_p.yaml + mode: link +- name: Architecture + href: architecture/index.md + include: + path: architecture/toc_p.yaml + mode: link diff --git a/ydb/docs/en/core/concepts/topology.md b/ydb/docs/en/core/concepts/topology.md index ffa0406d854..6b6a860054b 100644 --- a/ydb/docs/en/core/concepts/topology.md +++ b/ydb/docs/en/core/concepts/topology.md @@ -33,6 +33,7 @@ Fault-tolerant operation modes of distributed storage require a significant amou | `mirror-3-dc` *(3 nodes)*, can stand a failure of a single server, or a failure of a data center | 3 | 3 | Server | Data center | 3 | Doesn't matter | | `block-4-2`, can stand a failure of 2 racks | 1.5 | 8 ([10 recommended](*recommended-node-count)) | Rack | Data center | 1 | 8 | | `block-4-2` *(reduced)*, can stand a failure of 1 rack | 1.5 | 10 | ½ a rack | Data center | 1 | 5 | +| `block-4-2` *(reduced fault-tolerant)*, can stand a failure of 1 node | 1.5 | 4 | Server | Data center | 1 | Doesn't matter | | `none`, no fault tolerance | 1 | 1 | Node | Node | 1 | 1 | {% note info %} @@ -63,7 +64,11 @@ Cluster response time in bridge mode for most operations is limited by the respo If it is impossible to use the [recommended amount](#cluster-config) of hardware, you can divide servers within a single rack into two dummy fail domains. In this configuration, the failure of one rack results in the failure of two domains instead of just one. In such reduced configurations, {{ ydb-short-name }} will continue to operate if two domains fail. The minimum number of racks in a cluster is five for `block-4-2` mode and two per data center (e.g., six in total) for `mirror-3-dc` mode. -The minimal fault-tolerant configuration of a {{ ydb-short-name }} cluster uses the 3 nodes variant of `mirror-3-dc` operating mode, which requires only three servers with three disks each. In this configuration, each server acts as both a fail domain and a fail realm, and the cluster can withstand the failure of only a single server. Each server must be located in an independent data center to provide reasonable fault tolerance. +There are 2 variants of the minimal fault-tolerant configuration of a {{ ydb-short-name }} cluster: + +- The 3 nodes variant of the `mirror-3-dc` operating mode, which requires only three servers with three disks each. In this configuration, each server acts as both a fail domain and a fail realm, and the cluster can withstand the failure of only a single server. Each server must be located in an independent data center to provide reasonable fault tolerance. + +- The 4 nodes variant of the `block-4-2` operating mode, which requires 4 servers with 2 or more disks each. In this configuration, the disks of each server are manually divided into 2 fail domains using the [`disk_scope`](../reference/configuration/host_configs.md#disk-scope) attribute, which gives a total of 8 fail domains required for the `block-4-2` mode. Such a cluster remains operational when a single server fails. {{ ydb-short-name }} clusters configured with one of these approaches can be used for production environments if they don't require stronger fault tolerance guarantees. diff --git a/ydb/docs/en/core/contributor/hive-booting.md b/ydb/docs/en/core/contributor/hive-booting.md index 7d416de4ea3..40a401f75f6 100644 --- a/ydb/docs/en/core/contributor/hive-booting.md +++ b/ydb/docs/en/core/contributor/hive-booting.md @@ -21,7 +21,7 @@ The boot queue, or *Boot queue*, is stored in Hive's memory and is prioritized. 1. [Resource consumption metrics](hive.md#resources) — tablets with higher consumption have higher priority. 1. Tablets that restart frequently have lower priority. -When processing the queue, a limited number of tablets are processed at once (`max_boot_batch_size` in [configuration](../reference/configuration/hive.md#boot)). This is necessary so that when starting a large number of tablets, Hive does not stop responding to other requests for a long time. +When processing the queue, a limited number of tablets are processed at once (`max_boot_batch_size` in [configuration](../reference/configuration/hive_config.md#boot)). This is necessary so that when starting a large number of tablets, Hive does not stop responding to other requests for a long time. If when processing a tablet it turns out that it cannot be started on any of the nodes, then this tablet is postponed to a separate *Wait queue*. When node availability changes (a new node connects, or a restriction is removed from a node in [Hive UI](../reference/embedded-ui/hive.md)), Hive returns to these tablets and when processing the boot queue alternates tablets from the Boot Queue and tablets from the Wait Queue. @@ -40,7 +40,7 @@ stateDiagram-v2 {% note warning %} -Simultaneous startup of many tablets can create increased load on a node. Therefore, the maximum number of simultaneously starting tablets on one node is limited by the `max_tablets_scheduled` value from [configuration](../reference/configuration/hive.md#boot). At the same time, if one of the nodes hits this limit, Hive stops starting new tablets on other nodes too, so that this does not affect the uniformity of distribution. This behavior can be controlled using the [`boot_strategy`](../reference/configuration/hive.md#boot) parameter. +Simultaneous startup of many tablets can create increased load on a node. Therefore, the maximum number of simultaneously starting tablets on one node is limited by the `max_tablets_scheduled` value from [configuration](../reference/configuration/hive_config.md#boot). At the same time, if one of the nodes hits this limit, Hive stops starting new tablets on other nodes too, so that this does not affect the uniformity of distribution. This behavior can be controlled using the [`boot_strategy`](../reference/configuration/hive_config.md#boot) parameter. {% endnote %} @@ -48,7 +48,7 @@ Simultaneous startup of many tablets can create increased load on a node. Theref There are strict restrictions on which nodes are allowed to start a tablet: not every node can start every type of tablet; tablets of a certain database can only be started on nodes of that database. Additionally, when **moving** tablets, overloaded nodes are not considered. -1. From all suitable nodes, nodes with maximum priority are selected. Priority is determined based on the data center where the node is located. You can explicitly specify data center priorities in the [`default_tablet_preference`](../reference/configuration/hive.md#boot) subsection in the configuration. For [coordinators](../concepts/glossary.md#coordinator) and [mediators](../concepts/glossary.md#mediator), priorities are determined dynamically to maintain them in the same data center when possible. Additionally, if a tablet terminates with an error on a certain node, the priority of that node is lowered for the next start of this tablet. +1. From all suitable nodes, nodes with maximum priority are selected. Priority is determined based on the data center where the node is located. You can explicitly specify data center priorities in the [`default_tablet_preference`](../reference/configuration/hive_config.md#boot) subsection in the configuration. For [coordinators](../concepts/glossary.md#coordinator) and [mediators](../concepts/glossary.md#mediator), priorities are determined dynamically to maintain them in the same data center when possible. Additionally, if a tablet terminates with an error on a certain node, the priority of that node is lowered for the next start of this tablet. 1. For nodes with maximum priority, a target metric is calculated, which almost matches the [Node usage](hive.md#node-usage) metric. It differs in that only those resources consumed by this particular tablet are considered, as well as the presence of a penalty for the number of tablets of the same [schema object](../concepts/glossary.md#schema-object). diff --git a/ydb/docs/en/core/dev/json-indexes.md b/ydb/docs/en/core/dev/json-indexes.md new file mode 100644 index 00000000000..981d907aa35 --- /dev/null +++ b/ydb/docs/en/core/dev/json-indexes.md @@ -0,0 +1,244 @@ +# JSON indexes + +JSON indexes are a type of secondary index implemented on top of an [inverted index](https://en.wikipedia.org/wiki/Inverted_index) that speeds up filtering table rows by conditions imposed on the contents of columns of type `Json` and `JsonDocument`. The index is used if the `WHERE` predicate uses the functions [JSON_EXISTS](../yql/reference/builtins/json.md) and [JSON_VALUE](../yql/reference/builtins/json.md) with [JsonPath](../yql/reference/builtins/json.md#jsonpath) expressions. Unlike traditional secondary indexes optimized for equality or range searches on individual table columns, a JSON index works with arbitrary paths within a JSON document. + +For a general description of JSON search and the structure of an inverted index on JSON document paths, see the [{#T}](../concepts/query_execution/json_search.md) section. + +## Characteristics of JSON indexes {#characteristics} + +JSON indexes in {{ ydb-short-name }} allow: + +* Quickly filter rows by [JSON_EXISTS](#json-exists) and [JSON_VALUE](#json-value) with [JsonPath](../yql/reference/builtins/json.md#jsonpath) expressions +* combine indexed conditions with `AND` and `OR` operators +* use query parameter values passed by the application when processing checked predicates. + +A JSON index is a [global synchronous](../concepts/glossary.md#secondary-index) index — its data is always consistent with the base table. + +When executing a query, a JSON index can be applied: + +- explicitly — via the `<table_name> VIEW <index_name>` operator +- automatically by the [optimizer](../concepts/glossary.md#optimizer), if the predicate matches the formal rules. + +## Syntax of JSON indexes {#syntax} + +Creating a JSON index: + +* when creating a table: [INDEX (CREATE TABLE)](../yql/reference/syntax/create_table/json_index.md) +* Adding to an existing table: [ALTER TABLE](../yql/reference/syntax/alter_table/indexes.md#add-index). + +Deleting JSON indexes is done via [ALTER TABLE](../yql/reference/syntax/alter_table/indexes.md#drop-index): + + +```yql +ALTER TABLE documents DROP INDEX json_idx +``` + + +Query syntax with explicit JSON index specification: + +* [VIEW (JSON index)](../yql/reference/syntax/select/json_index.md). + +Functions and expressions for working with JSON in predicates: + +* [JSON functions](../yql/reference/builtins/json.md) — `JSON_EXISTS` and `JSON_VALUE` +* [JsonPath](../yql/reference/builtins/json.md#jsonpath) — a query language for accessing values inside JSON. + +Ready-made use cases are collected in the [JSON document search recipes](../recipes/json-search/index.md) section. + +## Updating JSON indexes {#update} + +JSON indexes are automatically maintained when data is modified and are updated synchronously together with the main table. Tables with JSON indexes support: + +* `INSERT` +* `UPSERT` +* `REPLACE` +* `UPDATE` +* `DELETE` + +Batch operations (`BATCH UPDATE` and `BATCH DELETE`) are not supported for tables with JSON indexes. When attempting to execute such a query on a table for which a JSON index has been created, the query will be rejected, a corresponding error will be returned to the application, and the data will remain unchanged. + +Additionally, tables with JSON indexes do not support: + +* Bulk data loading via a `BulkUpsert` call — the requested operation will be rejected with a corresponding error message. +* automatic deletion of rows by [TTL](../concepts/ttl.md) — errors are returned when attempting to create a table with both a TTL policy and a JSON index, as well as when trying to retrieve such a combination of properties using the `ALTER TABLE` commands. + +## Supported predicates {#predicates} + +For execution via JSON indexes, only expressions based on the `JSON_EXISTS` and `JSON_VALUE` functions in the `WHERE` block, combined by the `AND` / `OR` operators according to the rules below, are supported. + +### JSON_EXISTS {#json-exists} + +Checking the existence of a path or value inside a JsonPath filter. + +**Allowed:** + + +```yql +-- Document root (value not NULL) +WHERE JSON_EXISTS(doc, '$') + +-- Key chain; array indexes are 'transparent' +WHERE JSON_EXISTS(doc, '$.user.name') +WHERE JSON_EXISTS(doc, '$.items[*].sku') +WHERE JSON_EXISTS(doc, '$.items[0 to last].active') + +-- Filter ? (...) — predicates inside the filter are allowed +WHERE JSON_EXISTS(doc, '$.items ? (@.price == 100)') +WHERE JSON_EXISTS(doc, '$.items ? (@.qty >= 1 && @.qty <= 10)') +WHERE JSON_EXISTS(doc, '$.items ? (@.tag == $t)' PASSING "sale" AS t) + +-- JsonPath methods (path is indexed up to the method; exact check is performed by post-filter) +WHERE JSON_EXISTS(doc, '$.value.type()') +WHERE JSON_EXISTS(doc, '$.arr.size()') + +-- Combinations on one column +WHERE JSON_EXISTS(doc, '$.a') AND JSON_EXISTS(doc, '$.b') +WHERE JSON_EXISTS(doc, '$.a') OR JSON_EXISTS(doc, '$.b') +``` + + +**Prohibited** (error when using the `VIEW` statement or index auto-selection failure): + + +```yql +-- Comparison predicates at the top level of the path (outside ? (...)) +WHERE JSON_EXISTS(doc, '$.key == 10') +WHERE JSON_EXISTS(doc, 'exists($.key)') +WHERE JSON_EXISTS(doc, '$.key starts with "a"') + +-- Negation in JsonPath +WHERE JSON_EXISTS(doc, '!($.key == 10)') + +-- ON ERROR TRUE +WHERE JSON_EXISTS(doc, '$.key' TRUE ON ERROR) + +-- Path without context operator ($) — the passed document is not used +WHERE JSON_EXISTS(doc, '1') +``` + + +{% note info %} + +The `JSON_EXISTS` function returns `true` for any non-empty JsonPath result. The `$.key == 10` predicate specified at the top level would give "path existence" even when the comparison is false, which does not match the expected semantics. Comparisons should be moved into `JSON_VALUE` calls or into a filter of the form `? (...)`. + +{% endnote %} + +### JSON_VALUE {#json-value} + +Extracting a scalar value with a required `RETURNING <type>`. + +To compare a value using `JSON_VALUE`, you must always specify `RETURNING` with the required type. By default, `JSON_VALUE` returns type `Utf8`, which leads to incorrect comparison during query execution — values of different types are compared as strings: + + +```yql +$tmp = Json(@@["1", 1]@@); +SELECT JSON_VALUE($tmp, '$[0]') == "1"; -- true: correct, string compared with string +SELECT JSON_VALUE($tmp, '$[1]') == "1"; -- true: incorrect, number compared with string +``` + + +Supported types for the `RETURNING` section: `Int8` … `Int64`, `Uint8` … `Uint64`, `Float`, `Double`, `Bytes` (`String`), `Text` (`Utf8`), `Bool`. + +**Examples of predicates applied via a JSON index:** + + +```yql +-- Equality (path + value are included in the index) +WHERE JSON_VALUE(doc, '$.user.age' RETURNING Int32) = 25 +WHERE JSON_VALUE(doc, '$.flag' RETURNING Bool) = true +WHERE JSON_VALUE(doc, '$.name' RETURNING Utf8) = "Alice"u + +-- Implicit comparison with true for Bool +WHERE JSON_VALUE(doc, '$.active' RETURNING Bool) + +-- Parameters +WHERE JSON_VALUE(doc, '$.user.id' RETURNING Int64) = $id +WHERE JSON_VALUE(doc, '$.tag' RETURNING Utf8) = $tag + +-- Comparisons (only the path is used in the index, comparison is performed by post-filter) +WHERE JSON_VALUE(doc, '$.score' RETURNING Int64) > 0 +WHERE JSON_VALUE(doc, '$.score' RETURNING Int64) != 100 +WHERE JSON_VALUE(doc, '$.score' RETURNING Int64) BETWEEN 1 AND 10 +WHERE JSON_VALUE(doc, '$.score' RETURNING Int64) NOT BETWEEN 0 AND 5 + +-- IN: list of literals +WHERE JSON_VALUE(doc, '$.status' RETURNING Utf8) IN ("open"u, "pending"u) + +-- IN: specified parameter of type List<Utf8> +WHERE JSON_VALUE(doc, '$.status' RETURNING Utf8) IN $status_list + +-- PASSING for JsonPath variables +WHERE JSON_VALUE(doc, '$.x ? (@.y == $v)' RETURNING Int64 PASSING 42 AS v) = 10 + +-- JsonPath predicates inside the path. +-- Unlike JSON_EXISTS, predicates at the top level are allowed. +WHERE JSON_VALUE(doc, '$.user ? (@.role == "admin")' RETURNING Utf8) = "ok"u +WHERE JSON_VALUE(doc, '$.code starts with "A"' RETURNING String) != "" +WHERE JSON_VALUE(doc, 'exists($.meta)' RETURNING Bool) + +-- AND / OR combinations on one column +WHERE JSON_VALUE(doc, '$.a' RETURNING Int32) = 1 + OR JSON_VALUE(doc, '$.b' RETURNING Int32) = 2 +WHERE JSON_EXISTS(doc, '$.a') AND JSON_VALUE(doc, '$.a' RETURNING Int32) = 10 +``` + + +**Examples of predicates that cannot be applied via a JSON index:** + + +```yql +-- JSON_VALUE call without RETURNING +WHERE JSON_VALUE(doc, '$.key') = "x" + +-- DEFAULT with ON EMPTY / ON ERROR (except NULL) +WHERE JSON_VALUE(doc, '$.k' RETURNING Utf8 DEFAULT "x" ON ERROR) = "y" + +-- Unsupported data type in RETURNING +WHERE JSON_VALUE(doc, '$.ts' RETURNING Timestamp) = ... + +-- RETURNING Bool with comparison operators +WHERE JSON_VALUE(doc, '$.flag' RETURNING Bool) >= true + +-- IS NULL / IS NOT NULL — semantically contradict the 'path existence' index +WHERE JSON_VALUE(doc, '$.k' RETURNING Utf8) IS NULL + +-- Comparison of two JSON_VALUE from different columns +WHERE JSON_VALUE(doc1, '$.k' RETURNING Utf8) = JSON_VALUE(doc2, '$.k' RETURNING Utf8) + +-- Nested JSON_* in arguments +WHERE JSON_VALUE(JSON_QUERY(doc, '$.a'), '$.b' RETURNING Utf8) = "x" +``` + + +{% note info %} + +To check 'value equals `false`' or 'value equals `null`', use a JsonPath filter inside `JSON_EXISTS`, for example `JSON_EXISTS(doc, '$.k ? (@ == false)')` or `JSON_EXISTS(doc, '$.k ? (@ == null)')`, not `JSON_VALUE(...) IS NULL`. + +{% endnote %} + +## Limitations {#limitations} + +* JSON indexes are supported only for [row tables](../concepts/datamodel/table.md#row-oriented-tables). +* The table's primary key must consist of a single column of an integer type (`Uint64`, `Uint32`, `Int64`, or `Int32`). This is a temporary limitation that will be removed in future development. +* A single JSON index indexes exactly one column of type `Json` or `JsonDocument`. +* The `COVER` expression is not supported for JSON indexes. +* A number of data modification operations and mechanisms are [not supported](#update) for tables with JSON indexes. +* The parameter type of a read query from the index cannot be wrapped in `Optional<T>` — optional parameters are not supported. +* Equality comparison with an integer literal whose absolute value exceeds 2⁵³ is not accelerated by the value index (such numbers do not fit into the numeric type used in `Json` and `JsonDocument`) and is reduced to a path existence check. +* Casting floating-point literals (`Float`, `Double`) to integer types during comparison is not performed — such comparison is not accelerated by the index. + +## Recipes {#recipes} + +Ready-made scenarios for working with a JSON index: + +* [{#T}](../recipes/json-search/json-index-quickstart.md) — quick start. +* [{#T}](../recipes/json-search/json-index-catalog.md) — product catalog with nested attributes. +* [{#T}](../recipes/json-search/json-index-parameters.md) — parameterized queries and JsonPath variables. +* [{#T}](../recipes/json-search/json-index-typecheck.md) — field type check and path existence. + +## Related materials {#see-also} + +- [JSON functions](../yql/reference/builtins/json.md) — `JSON_EXISTS`, `JSON_VALUE`, `JSON_QUERY`, JsonPath syntax. +- [Secondary indexes](secondary-indexes.md) — general information about global indexes and `VIEW`. +- [Full-text indexes](fulltext-indexes.md) — a related mechanism built on top of an inverted index of words and phrases. +- [INDEX (CREATE TABLE)](../yql/reference/syntax/create_table/json_index.md) and [VIEW (JSON index)](../yql/reference/syntax/select/json_index.md) — syntax reference. diff --git a/ydb/docs/en/core/dev/resource-consumption-management.md b/ydb/docs/en/core/dev/resource-consumption-management.md index 3490227ea3b..2137b972a9d 100644 --- a/ydb/docs/en/core/dev/resource-consumption-management.md +++ b/ydb/docs/en/core/dev/resource-consumption-management.md @@ -23,14 +23,14 @@ CREATE RESOURCE POOL olap WITH ( CONCURRENT_QUERY_LIMIT=10, QUEUE_SIZE=1000, DATABASE_LOAD_CPU_THRESHOLD=80, - RESOURCES_WEIGHT=100, + RESOURCE_WEIGHT=100, QUERY_CPU_LIMIT_PERCENT_PER_NODE=50, TOTAL_CPU_LIMIT_PERCENT_PER_NODE=70 ) ``` -You can find the full list of resource pool parameters in the [{#T}](../yql/reference/syntax/create-resource-pool.md#parameters) reference. Some parameters are global for the entire database (for example, `CONCURRENT_QUERY_LIMIT`, `QUEUE_SIZE`, `DATABASE_LOAD_CPU_THRESHOLD`), while others apply only to a single compute node (for example, `QUERY_CPU_LIMIT_PERCENT_PER_NODE`, `TOTAL_CPU_LIMIT_PERCENT_PER_NODE`, `TOTAL_MEMORY_LIMIT_PERCENT_PER_NODE`). CPU can be shared among all pools in case of oversubscription on a single compute node using `RESOURCES_WEIGHT`. +You can find the full list of resource pool parameters in the [{#T}](../yql/reference/syntax/create-resource-pool.md#parameters) reference. Some parameters are global for the entire database (for example, `CONCURRENT_QUERY_LIMIT`, `QUEUE_SIZE`, `DATABASE_LOAD_CPU_THRESHOLD`), while others apply only to a single compute node (for example, `QUERY_CPU_LIMIT_PERCENT_PER_NODE`, `TOTAL_CPU_LIMIT_PERCENT_PER_NODE`, `TOTAL_MEMORY_LIMIT_PERCENT_PER_NODE`). CPU can be shared among all pools in case of oversubscription on a single compute node using `RESOURCE_WEIGHT`.  @@ -68,18 +68,18 @@ When a query enters a resource pool for which `DATABASE_LOAD_CPU_THRESHOLD` is s As with `CONCURRENT_QUERY_LIMIT`, when the specified load threshold is exceeded, queries are sent to the waiting queue. -### Resource allocation according to RESOURCES_WEIGHT {#resources_weight} +### Resource allocation according to RESOURCE_WEIGHT {#resources_weight}  -The `RESOURCES_WEIGHT` parameter only takes effect in case of oversubscription and when there is more than one resource pool in the system. In the current implementation, `RESOURCES_WEIGHT` only affects the allocation of `vCPU` resources. When queries appear in a resource pool, it starts participating in resource allocation. For this, the pools recalculate their limits according to the [Max-min fairness](https://en.wikipedia.org/wiki/Max-min_fairness) algorithm. The actual resource redistribution is performed on each compute node individually, as shown in the figure above. +The `RESOURCE_WEIGHT` parameter only takes effect in case of oversubscription and when there is more than one resource pool in the system. In the current implementation, `RESOURCE_WEIGHT` only affects the allocation of `vCPU` resources. When queries appear in a resource pool, it starts participating in resource allocation. For this, the pools recalculate their limits according to the [Max-min fairness](https://en.wikipedia.org/wiki/Max-min_fairness) algorithm. The actual resource redistribution is performed on each compute node individually, as shown in the figure above. Suppose we have a node in the system with $10 vCPU$ available. The following limits are set: - $TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30$, - $QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50$. -In this case, the resource pool will have a limit of $3 vCPU$ per node and $1.5 vCPU$ per query in this pool (figure *a*). If there are 4 such pools in the system and they all try to use maximum resources, this would amount to $12 vCPU$, which exceeds the limit of available resources on the node ($10 vCPU$). In this case, `RESOURCES_WEIGHT` takes effect, and each pool will be allocated $2.5 vCPU$ (figure *b*). +In this case, the resource pool will have a limit of $3 vCPU$ per node and $1.5 vCPU$ per query in this pool (figure *a*). If there are 4 such pools in the system and they all try to use maximum resources, this would amount to $12 vCPU$, which exceeds the limit of available resources on the node ($10 vCPU$). In this case, `RESOURCE_WEIGHT` takes effect, and each pool will be allocated $2.5 vCPU$ (figure *b*). If you need to increase the allocated resources for a specific pool, you can change its weight, for example, to 200. Then this pool will get $3 vCPU$, and the remaining pools will equally share the remaining $7 vCPU$, which amounts to $\frac{7}{3} vCPU$ per pool (figure *c*). @@ -99,7 +99,7 @@ CREATE RESOURCE POOL default WITH ( CONCURRENT_QUERY_LIMIT=-1, QUEUE_SIZE=-1, DATABASE_LOAD_CPU_THRESHOLD=-1, - RESOURCES_WEIGHT=-1, + RESOURCE_WEIGHT=-1, TOTAL_MEMORY_LIMIT_PERCENT_PER_NODE=-1, QUERY_CPU_LIMIT_PERCENT_PER_NODE=-1, TOTAL_CPU_LIMIT_PERCENT_PER_NODE=-1 @@ -200,7 +200,7 @@ CREATE RESOURCE POOL olap WITH ( CONCURRENT_QUERY_LIMIT=20, QUEUE_SIZE=100, DATABASE_LOAD_CPU_THRESHOLD=80, - RESOURCES_WEIGHT=20, + RESOURCE_WEIGHT=20, QUERY_CPU_LIMIT_PERCENT_PER_NODE=80, TOTAL_CPU_LIMIT_PERCENT_PER_NODE=100 ); @@ -208,7 +208,7 @@ CREATE RESOURCE POOL olap WITH ( CREATE RESOURCE POOL the_ceo WITH ( CONCURRENT_QUERY_LIMIT=20, QUEUE_SIZE=100, - RESOURCES_WEIGHT=100, + RESOURCE_WEIGHT=100, QUERY_CPU_LIMIT_PERCENT_PER_NODE=100, TOTAL_CPU_LIMIT_PERCENT_PER_NODE=100 ); @@ -303,11 +303,11 @@ The following query outputs information about all active queries in the system: ```yql select - Query, -- Запрос - WmPoolId, -- Идентификатор пула - WmState, -- Статус запроса в WM - WmEnterTime, -- Время, когда запрос перешел в статус PENDING или DELAYED - WmExitTime -- Время, когда запрос передан на выполнение + Query, -- Query + WmPoolId, -- Pool ID + WmState, -- Query status in WM + WmEnterTime, -- Time when the query transitioned to PENDING or DELAYED status + WmExitTime -- Time when the query was submitted for execution from `.sys/query_sessions` where State = 'EXECUTING' ``` diff --git a/ydb/docs/en/core/dev/toc_p.yaml b/ydb/docs/en/core/dev/toc_p.yaml index a02a2f17fb5..52fddc0e4ed 100644 --- a/ydb/docs/en/core/dev/toc_p.yaml +++ b/ydb/docs/en/core/dev/toc_p.yaml @@ -31,6 +31,8 @@ items: href: fulltext-indexes.md - name: Hybrid search href: hybrid-search.md +- name: JSON indexes + href: json-indexes.md - name: Local indexes href: local-indexes/index.md include: @@ -51,11 +53,11 @@ items: href: system-views.md - name: Change Data Capture href: cdc.md -- name: Structure and limitations of a shared topic reader - href: shared-consumer-internals.md - name: Terraform href: terraform.md - name: Custom attributes href: custom-attributes.md - name: Resource consumption management href: resource-consumption-management.md +- name: Structure and limitations of a shared topic reader + href: shared-consumer-internals.md diff --git a/ydb/docs/en/core/devops/enterprise-manager/ai-assistant.md b/ydb/docs/en/core/devops/enterprise-manager/ai-assistant.md new file mode 100644 index 00000000000..c9fcb5cd8ab --- /dev/null +++ b/ydb/docs/en/core/devops/enterprise-manager/ai-assistant.md @@ -0,0 +1,205 @@ +# Configuring AI assistant in YDB EM + +This guide shows how to enable AI assistant in {{ ydb-short-name }} Enterprise Manager (YDB EM). After the setup, users will see the assistant in the YDB EM web interface. The assistant sends model requests through [Gateway](index.md#architecture) and can use Model Context Protocol (MCP) tools provided by Gateway. + +## Before You Start {#before-start} + +You can use this guide before the first YDB EM deployment or when updating an existing installation. For a new deployment, add the variables to the inventory before running the initial setup playbook. For deployment instructions, see [{#T}](initial-deployment.md). + +Make sure that you have: + +1. Access to the Ansible inventory used to deploy YDB EM. +1. An endpoint of an OpenAI-compatible model that will be reachable from the Gateway host. +1. Permissions to modify the [Gateway token file](#configure-model-access). During setup you will need to write the model access secret into the token file. Gateway reads this file and uses the `Token` value of the entry whose `Name` equals `ydb_em_ai_model_token_name`, for example `model-token`, as the upstream `Authorization` header. +1. If you are updating an existing installation, access to the deployed Gateway host. + +For an existing installation, first confirm that the Gateway host is managed by the same Ansible inventory. Check the active service manager unit, process owner, config path, token file, and listening port before applying the playbook. If the host uses a custom or manual Gateway layout, apply the same settings through that installation's operational procedure instead of running the playbook blindly. + +{% note warning %} + +Do not put model API keys, OAuth tokens, or other secrets into `ydb_em_ai_assistant_client_runtime_config`. Gateway returns this value to the browser from `GET /meta/ai_assistant_client_config`. Keep secrets in the token file. + +{% endnote %} + +## How It Works {#how-it-works} + +The browser works with the assistant through Gateway: + +1. The UI checks `GET /capabilities`. The assistant button is shown when `Settings.Proxy.Model` is `true` and the user setting for AI assistant is enabled. When the backend capability first appears, YDB EM initializes this user setting to enabled. +1. The UI gets runtime settings from `GET /meta/ai_assistant_client_config`. +1. The UI sends model requests to `/proxy/model/...`. +1. Gateway forwards model requests to `ydb_em_ai_model_endpoint` and adds the Authorization header from tokenator. +1. The UI sends MCP requests to `/meta/mcp`. Gateway exposes MCP tools through this endpoint and runs them against YDB EM APIs and cluster proxy endpoints. When documentation search is configured, `search_docs` is exposed through the same MCP endpoint and can be used by the assistant. + +## Configure Model Access {#configure-model-access} + +`ydb_em_ai_model_token_name` is the [tokenator](../../concepts/glossary.md#tokenator) entry name that Gateway uses for model requests. + +Example tokenator entry: + +```textproto +StaticTokenInfo { + Name: "model-token" + Token: "Bearer <model-api-token>" +} +``` + +The token delivery method depends on your deployment process. The requirement is that the Gateway tokenator file contains an entry whose `Name` matches `ydb_em_ai_model_token_name`. + +{% note info %} + +The default YDB EM Ansible token template renders only the `meta` credentials entry. If your installation uses that template unchanged, extend or override the tokenator file through your deployment process so that it also contains `model-token` and, if different, the embeddings token. + +{% endnote %} + +## Configure Ansible Variables {#configure-ansible-variables} + +Add the AI assistant variables to the YDB EM inventory, for example to `examples/inventory/50-inventory.yaml` or another inventory file used by your deployment. + +```yaml +ydb_em_ai_assistant_enabled: true +ydb_em_ai_model_endpoint: "https://llm.example.com" +ydb_em_ai_model_token_name: "model-token" + +ydb_em_ai_assistant_client_runtime_config: + llm: + baseURL: "/proxy/model/v1" + model: "<model-name>" + temperature: 0.2 + mcp: + - id: "ydb-meta" + url: "/meta/mcp" + transport: "streamable" + toolCallTimeoutMs: 600000 +``` + +Parameter | Description +--- | --- +`ydb_em_ai_assistant_enabled` | Enables AI assistant settings in Gateway. +`ydb_em_ai_model_endpoint` | Upstream model API endpoint used by Gateway. +`ydb_em_ai_model_token_name` | Tokenator entry name for model requests. +`ydb_em_ai_assistant_client_runtime_config.llm.baseURL` | Browser-visible model URL. Use Gateway proxy, usually `/proxy/model/v1`. +`ydb_em_ai_assistant_client_runtime_config.llm.model` | Model name sent to the OpenAI-compatible API. +`ydb_em_ai_assistant_client_runtime_config.mcp` | MCP servers available to the assistant. For YDB EM, use `/meta/mcp`. + +Gateway appends the request suffix from `/proxy/model/...` to `ydb_em_ai_model_endpoint`. With `baseURL: "/proxy/model/v1"`, a chat completion request is forwarded to `/v1/chat/completions` on the upstream endpoint. Configure the endpoint so that this path is valid for your model provider. + +## Configure Documentation Search {#configure-docs-search} + +It is recommended to enable documentation search so that the assistant gets the `search_docs` MCP tool. Gateway calls an OpenAI-compatible embeddings endpoint for this tool and appends `/embeddings` to the configured base URL when the suffix is missing. When documentation search is enabled, the assistant gets `search_docs` through the configured `/meta/mcp` server. + +```yaml +ydb_em_docs_search_enabled: true +ydb_em_docs_search_embeddings_upstream_base_url: "https://llm.example.com/v1" +ydb_em_docs_search_embeddings_token_name: "model-token" +ydb_em_docs_search_embeddings_model: "<embeddings-model-name>" +ydb_em_docs_search_vector_size: 0 +ydb_em_docs_search_limit: 6 +ydb_em_docs_search_score: 0.6 +``` + +Parameter | Description +--- | --- +`ydb_em_docs_search_enabled` | Configures the semantic docs search backend and exposes `search_docs` through MCP when all required search settings are valid. +`ydb_em_docs_search_embeddings_upstream_base_url` | OpenAI-compatible embeddings endpoint base URL. +`ydb_em_docs_search_embeddings_token_name` | Tokenator entry name for embeddings requests. +`ydb_em_docs_search_embeddings_model` | Embeddings model name. +`ydb_em_docs_search_vector_size` | Optional embedding vector dimension override. Leave it unset to use the Ansible default `0`. With `0`, Gateway omits the OpenAI-compatible `dimensions` field and skips the response size check. Set a positive value only when the embeddings provider and model support explicit dimensions and the expected size is known. +`ydb_em_docs_search_limit` | Maximum number of documents returned. +`ydb_em_docs_search_score` | Minimum similarity score from `0` to `1`. Documents with a lower score are filtered out. + +## Apply The Configuration {#apply-configuration} + +After updating the inventory, run the YDB EM playbook from the directory with your inventory. The same playbook is used for initial deployment and for applying this change to an existing installation: + +```bash +ansible-playbook ydb_platform.ydb_em.initial_setup +``` + +If your vault file is encrypted, add the vault option used in your deployment: + +```bash +ansible-playbook ydb_platform.ydb_em.initial_setup --ask-vault-pass +``` + +The role renders the Gateway config and the default token file and ensures that the Gateway service is started. If you deliver additional model credentials through an overridden token file or another deployment step, make sure the deployed token file contains the configured entry name. When applying these settings to an already running deployment, restart the Gateway service using your operational procedure if your Ansible run does not restart it after config or token changes. + +## Verify The Setup {#verify} + +Open the YDB EM web interface: + +```text +https://<gateway-host>:8789/ui/clusters +``` + +Check that model proxy is enabled: + +```bash +curl -k https://<gateway-host>:8789/capabilities +``` + +The response should contain: + +```json +{ + "Settings": { + "Proxy": { + "Model": true + } + } +} +``` + +Check the runtime config returned to the UI: + +```bash +curl -k https://<gateway-host>:8789/meta/ai_assistant_client_config +``` + +The response should contain the `llm` block and should not contain model secrets. + +Check the model proxy: + +```bash +curl -k https://<gateway-host>:8789/proxy/model/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"<model-name>","messages":[{"role":"user","content":"ping"}],"max_tokens":1}' +``` + +If your deployment requires user authentication, add the same authentication headers that are used for the YDB EM UI. + +If documentation search is enabled, also check that `/meta/mcp` exposes `search_docs` and run a harmless documentation-search request through your MCP client or assistant tooling. This verifies both the MCP registration and the embeddings and metabase path. + +## Troubleshooting {#troubleshooting} + +### Assistant Button Is Not Shown {#button-not-shown} + +Check that `ydb_em_ai_assistant_enabled` is `true` and that `/capabilities` contains `Settings.Proxy.Model: true`. Also check the user setting for AI assistant in the YDB EM UI: YDB EM initializes it to enabled when the backend capability is present, but a user can turn it off. + +### Runtime Config Is Invalid {#runtime-config-invalid} + +Check `GET /meta/ai_assistant_client_config`. For an enabled assistant, the response must be a JSON object with `llm.baseURL` and `llm.model` string fields. A `null` response means that Gateway did not load an enabled AI assistant client config. + +### Model Proxy Returns An Authorization Error {#model-authorization-error} + +Check that `ydb_em_ai_model_token_name` matches a tokenator entry and that the entry returns the Authorization header expected by the model endpoint. If tokenator cannot find the entry, Gateway sends the upstream request without the expected Authorization header. + +### Model Proxy Calls The Wrong Upstream Path {#wrong-upstream-path} + +Check `ydb_em_ai_model_endpoint` together with `llm.baseURL`. Gateway appends the path from `/proxy/model/...` to the configured endpoint. A duplicated `/v1` path often causes upstream `404` errors. + +### MCP Tools Are Unavailable {#mcp-tools-unavailable} + +Check that the runtime config contains `url: "/meta/mcp"` and that user requests can reach `/meta/mcp` through Gateway. Gateway registers `/meta/mcp` only when MCP is enabled; it is enabled by default, but custom Gateway config can disable it. + +### Documentation Search Is Unavailable {#docs-search-unavailable} + +Check the `ydb_em_docs_search_*` settings, the embeddings endpoint path, and the tokenator entry used for embeddings requests. Also make sure documentation vectors are available in the YDB EM metabase. If docs search is not configured, the `search_docs` tool is not exposed. + +### Embeddings Endpoint Rejects Vector Size {#embeddings-vector-size} + +If `search_docs` fails with `variable embedding size not supported`, set `ydb_em_docs_search_vector_size: 0` and verify the embeddings endpoint and model. + +### Documentation Search Returns 504 {#docs-search-504} + +If `search_docs` returns `504 Gateway Timeout` and Gateway logs for `/meta/docs` contain `Member not found: doc_id`, check the YDB EM metabase table `ydb/Docs.db`. The current Gateway docs search query uses the `doc_id`, `embedding`, and `payload` columns. For useful results, the table should also contain rows and be built for an embeddings model compatible with `ydb_em_docs_search_embeddings_model`. diff --git a/ydb/docs/en/core/devops/enterprise-manager/index.md b/ydb/docs/en/core/devops/enterprise-manager/index.md index 3c17a8910bb..539c14ba7ee 100644 --- a/ydb/docs/en/core/devops/enterprise-manager/index.md +++ b/ydb/docs/en/core/devops/enterprise-manager/index.md @@ -81,3 +81,4 @@ The user interacts with Gateway through a browser or API. Gateway forwards reque - [{#T}](initial-deployment.md) - [{#T}](s3-backups.md) +- [{#T}](ai-assistant.md) diff --git a/ydb/docs/en/core/devops/enterprise-manager/toc_p.yaml b/ydb/docs/en/core/devops/enterprise-manager/toc_p.yaml index 62bbdf45692..9d215745076 100644 --- a/ydb/docs/en/core/devops/enterprise-manager/toc_p.yaml +++ b/ydb/docs/en/core/devops/enterprise-manager/toc_p.yaml @@ -3,3 +3,5 @@ items: href: initial-deployment.md - name: Configuring S3 backups href: s3-backups.md +- name: Configuring AI assistant + href: ai-assistant.md diff --git a/ydb/docs/en/core/downloads/ydb-ansible.md b/ydb/docs/en/core/downloads/ydb-ansible.md index 1bbaaec590b..bf92da7df0c 100644 --- a/ydb/docs/en/core/downloads/ydb-ansible.md +++ b/ydb/docs/en/core/downloads/ydb-ansible.md @@ -4,6 +4,7 @@ A set of automated playbooks for installing and maintaining the server side of [ | Version | Release date | Download | Changelog | | ------ | ------------ | ------- | ----------------- | +| v2.1.0 | 29.06.2026 | [ydb-ansible-2.1.0.tar.gz](https://github.com/ydb-platform/ydb-ansible/releases/download/v2.1.0/ydb_platform-ydb-2.1.0.tar.gz) | | | v2.0.0 | 23.12.2025 | [ydb-ansible-2.0.0.tar.gz](https://github.com/ydb-platform/ydb-ansible/releases/download/v2.0.0/ydb_platform-ydb-2.0.0.tar.gz) | | | v1.3.2 | 02.12.2025 | [ydb-ansible-1.3.2.tar.gz](https://github.com/ydb-platform/ydb-ansible/releases/download/v1.3.2/ydb_platform-ydb-1.3.2.tar.gz) | | | v1.3.1 | 01.12.2025 | [ydb-ansible-1.3.1.tar.gz](https://github.com/ydb-platform/ydb-ansible/releases/download/v1.3.1/ydb_platform-ydb-1.3.1.tar.gz) | | diff --git a/ydb/docs/en/core/recipes/json-search/index.md b/ydb/docs/en/core/recipes/json-search/index.md new file mode 100644 index 00000000000..b6568a26d8f --- /dev/null +++ b/ydb/docs/en/core/recipes/json-search/index.md @@ -0,0 +1,12 @@ +# Recipes for searching JSON documents + +This section contains ready-made recipes for using [JSON indexes](../../dev/json-indexes.md) to efficiently select data based on the contents of columns of type `Json` and `JsonDocument`. + +The recipes are built around the [JSON_EXISTS](../../yql/reference/builtins/json.md) and [JSON_VALUE](../../yql/reference/builtins/json.md) functions with [JsonPath](../../yql/reference/builtins/json.md#jsonpath) expressions. To guarantee index usage, all examples use explicit access via `VIEW IndexName`. + +* [{#T}](json-index-quickstart.md) — minimal scenario: creating a table, an index, inserting data, and basic queries. +* [{#T}](json-index-catalog.md) — example of a product catalog with nested attributes, range conditions, and searching nested arrays. +* [{#T}](json-index-parameters.md) — parameterized queries and passing variables to JsonPath via the `PASSING` section. +* [{#T}](json-index-typecheck.md) — checking the field type and path existence using JsonPath methods. + +Before running the examples, make sure that [JSON index support is enabled](../../reference/configuration/feature_flags.md) on the cluster. diff --git a/ydb/docs/en/core/recipes/json-search/json-index-catalog.md b/ydb/docs/en/core/recipes/json-search/json-index-catalog.md new file mode 100644 index 00000000000..f9d6fcda059 --- /dev/null +++ b/ydb/docs/en/core/recipes/json-search/json-index-catalog.md @@ -0,0 +1,133 @@ +# Catalog with nested attributes + +This recipe shows how to use a [JSON index](../../dev/json-indexes.md) to speed up access to a product catalog where product characteristics are stored as a JSON document with an arbitrary set of fields. This schema is convenient when: + +* The set of product attributes is unknown in advance or differs for different categories. +* Adding new attributes via `ALTER TABLE ADD COLUMN` is undesirable. +* You need to efficiently filter products by various combinations of attributes. + +A JSON index enables efficient retrieval by any path and value inside a JSON document without a full table scan. + +## Create a table and index + + +```yql +CREATE TABLE products ( + sku_id Uint64, + attrs JsonDocument, + PRIMARY KEY (sku_id), + INDEX attrs_json_idx GLOBAL USING json ON (attrs) +); +``` + + +In this schema: + +* `sku_id` — numeric product identifier. +* `attrs` — product attributes in `JsonDocument` format. Storing as `JsonDocument` saves space and speeds up deserialization compared to `Json`. +* `attrs_json_idx` — JSON index on column `attrs`. It is updated synchronously together with the main table. + +## Load test data + + +```yql +UPSERT INTO products (sku_id, attrs) VALUES + (10, JsonDocument(@@{ + "brand": "ACME", + "price": 49.90, + "category": "tools", + "warehouses": [{"id": 1, "stock": 12}, {"id": 2, "stock": 0}] + }@@)), + (11, JsonDocument(@@{ + "brand": "ACME", + "price": 199.00, + "category": "electronics", + "warehouses": [{"id": 1, "stock": 3}] + }@@)), + (12, JsonDocument(@@{ + "brand": "Globex", + "price": 25.00, + "category": "tools", + "warehouses": [{"id": 2, "stock": 0}] + }@@)); +``` + + +## Filter by brand and price range + + +```yql +SELECT sku_id, attrs +FROM products VIEW attrs_json_idx +WHERE JSON_VALUE(attrs, '$.brand' RETURNING Utf8) = "ACME"u + AND JSON_VALUE(attrs, '$.price' RETURNING Double) BETWEEN 10.0 AND 100.0; +``` + + +What happens: + +* For the `$.brand = "ACME"` condition, the token «path + value» goes into the index — this gives an exact match and maximum selectivity. +* For the `BETWEEN 10.0 AND 100.0` condition, only the path token `$.price` goes into the index. This narrows the set of rows to those where the `price` field is present, after which the query execution engine checks the range with an exact comparison (post-filter). +* The conditions are combined via `AND` — this allows the index to be used to check both fragments at once, resulting in minimal index reads. + +Result: + + +```text +sku_id attrs +10 {"brand":"ACME","price":49.9,...} +``` + + +## Search by nested array + +JsonPath supports accessing array elements and filters within the path. For example, you can find products that have stock in at least one warehouse: + + +```yql +SELECT sku_id +FROM products VIEW attrs_json_idx +WHERE JSON_EXISTS(attrs, '$.warehouses ? (@.stock > 0)'); +``` + + +For index search, the path token `$.warehouses.stock` is used, and the `@.stock > 0` condition is checked by a post-filter on each found record. This is efficient when the corresponding field is present in a relatively small portion of documents. + +Result: + + +```text +sku_id +10 +11 +``` + + +## Search by category and attribute presence + +Conditions can be combined with any `AND` and `OR` operators. Search for products in category `tools` that have the `price` field specified: + + +```yql +SELECT sku_id +FROM products VIEW attrs_json_idx +WHERE JSON_VALUE(attrs, '$.category' RETURNING Utf8) = "tools"u + AND JSON_EXISTS(attrs, '$.price'); +``` + + +Result: + + +```text +sku_id +10 +12 +``` + + +## See also + +* [Supported JSON index predicates](../../dev/json-indexes.md#predicates) — full rules for which expressions are indexed. +* [AND and OR handling](../../dev/json-indexes.md#predicates) — nuances of combining conditions, including with non-indexable predicates. +* [{#T}](json-index-parameters.md) — parameterized versions of the same queries. diff --git a/ydb/docs/en/core/recipes/json-search/json-index-parameters.md b/ydb/docs/en/core/recipes/json-search/json-index-parameters.md new file mode 100644 index 00000000000..6759f2215b3 --- /dev/null +++ b/ydb/docs/en/core/recipes/json-search/json-index-parameters.md @@ -0,0 +1,139 @@ +# Parameterized queries and JsonPath variables + +In most applications, query input data is not substituted into SQL text but is passed via [parameters](../../yql/reference/syntax/declare.md). Typical use cases for parameters when working with [JSON indexes](../../dev/json-indexes.md): + +1. direct comparison of the result of `JSON_VALUE` with a parameter. +2. checking whether the result is in a list of values using the `IN` expression. +3. passing a parameter to a JsonPath expression via the `PASSING` clause. + +## Prerequisites + +Below is an example of a table, a JSON index, and data population. + + +```yql +CREATE TABLE documents ( + id Uint64, + payload JsonDocument, + PRIMARY KEY (id), + INDEX json_idx GLOBAL USING json ON (payload) +); + +UPSERT INTO documents (id, payload) VALUES + (1, JsonDocument(@@{ + "owner_id": 100, + "tag": "active", + "archived": false, + "content": {"x": 1, "y": 1} + }@@)), + (2, JsonDocument(@@{ + "owner_id": 100, + "tag": "draft", + "archived": false, + "content": {"x": 1, "y": 2} + }@@)), + (3, JsonDocument(@@{ + "owner_id": 101, + "tag": "active", + "archived": true, + "content": {"x": 2, "y": 1} + }@@)), + (4, JsonDocument(@@{ + "owner_id": 102, + "tag": "pending", + "archived": false, + "content": {"x": 2, "y": 2} + }@@)); +``` + + +## Direct comparison with a parameter + +The most common case is that the parameter is substituted as the right operand of the comparison, and the left operand is a call to `JSON_VALUE` with an explicit `RETURNING`: + + +```yql +DECLARE $owner_id AS Int64; + +SELECT id, payload +FROM documents VIEW json_idx +WHERE JSON_VALUE(payload, '$.owner_id' RETURNING Int64) = $owner_id + AND JSON_EXISTS(payload, '$.archived ? (@ == false)'); +``` + + +The index receives a token «path + value» (`$.owner_id = $owner_id`) — the parameter is treated as a regular value. This allows performing a query with the same selectivity as when comparing with a literal. + +Running with `$owner_id = 100` returns rows `1` and `2`. + +## Search by a list of values + +To search by multiple values of a single field, you can use `IN`: + + +```yql +DECLARE $tags AS List<Utf8>; + +SELECT id +FROM documents VIEW json_idx +WHERE JSON_VALUE(payload, '$.tag' RETURNING Utf8) IN $tags; +``` + + +The behavior of the index differs depending on whether it is a literal or a parameter: + +* `IN ("active"u, "pending"u)` — a list of literals is converted into `OR` of several «path + value» tokens, one for each value in the list. +* `IN $tags` — for each parameter value during query execution, a «path + value» token is formed (`$.tag` + value from the list). At the compilation stage, the parameter values are not yet known, so in the query plan (`EXPLAIN`) such a condition is displayed as a «path + parameter» pair (`{"path": "$.tag", "param": "$tags"}`). + +Any collection of scalar values can serve as a parameter for `IN`: `List<T>`, `Tuple<T, ...>`, `Dict<K, V>`, or `Set<T>`. + +Running with `$tags = ["active"u, "pending"u]` returns rows `1`, `3`, and `4`. + +## Parameters inside JsonPath (PASSING) + +If a parameter must be used inside a JsonPath filter (`? (...)`), it is passed in the `PASSING` clause: + + +```yql +DECLARE $min_stock AS Int64; + +SELECT id +FROM documents VIEW json_idx +WHERE JSON_EXISTS( + payload, + '$.content ? (@.y > $threshold)' + PASSING $min_stock AS threshold +); +``` + + +For index search, the path token `$.content.y` is used, and the condition `@.y > $threshold` is checked by a post-filter. + +Similarly, `PASSING` works in `JSON_VALUE`: + + +```yql +DECLARE $v AS Int64; + +SELECT id +FROM documents VIEW json_idx +WHERE JSON_VALUE( + payload, + '$.content ? (@.y == $val)' + PASSING $v AS val + RETURNING Int64 +) = 10; +``` + + +## Supported parameter types + +For all three methods, parameters with the following types are supported: `Int8` … `Int64`, `Uint8` … `Uint64`, `Float`, `Double`, `Bytes` (`String`), `Text` (`Utf8`), `Bool`. Optional types (`Optional<T>`) are not supported in parameters. + +For more details about types, see [{#T}](../../dev/json-indexes.md#json-value). + +## See also + +* [{#T}](json-index-quickstart.md) — basic use case for a JSON index. +* [Passing parameters to JSON index predicates](../../dev/json-indexes.md#json-value) — detailed description of all options. +* [JsonPath](../../yql/reference/builtins/json.md#jsonpath) — JsonPath language syntax. diff --git a/ydb/docs/en/core/recipes/json-search/json-index-quickstart.md b/ydb/docs/en/core/recipes/json-search/json-index-quickstart.md new file mode 100644 index 00000000000..4a15dc1d7db --- /dev/null +++ b/ydb/docs/en/core/recipes/json-search/json-index-quickstart.md @@ -0,0 +1,110 @@ +# JSON index – quick start + +This guide shows how to create a [JSON index](../../dev/json-indexes.md) and run queries using the [JSON_EXISTS](../../yql/reference/builtins/json.md#json_exists) and [JSON_VALUE](../../yql/reference/builtins/json.md#json_value) functions in {{ ydb-short-name }}. + +## Create a table and a JSON index + + +```yql +CREATE TABLE documents ( + id Uint64, + payload JsonDocument, + PRIMARY KEY (id), + INDEX json_idx GLOBAL USING json ON (payload) +); +``` + + +The column type `JsonDocument` stores JSON in a compact binary format and is preferred for an indexed column. Alternatively, the type `Json` (text representation) can be used. + +The primary key of the table must consist of a single column of an integer type (`Uint64`, `Uint32`, `Int64`, or `Int32`) — this is a [current limitation](../../dev/json-indexes.md#limitations) of the JSON index implementation. + +## Add test data + + +```yql +UPSERT INTO documents (id, payload) VALUES + (1, JsonDocument(@@{"user": {"id": 100, "name": "Alice"}, "active": true}@@)), + (2, JsonDocument(@@{"user": {"id": 101, "name": "Bob"}, "active": false}@@)), + (3, JsonDocument(@@{"user": {"id": 102, "name": "Charlie"}, "archived": true}@@)); +``` + + +Here, the `@@...@@` construct is a [multiline string literal](../../yql/reference/syntax/expressions.md), convenient for writing JSON without escaping quotes. The `JsonDocument(...)` function converts text into a value of type `JsonDocument`. + +## Filter by the presence of a path in the document + +The [JSON_EXISTS](../../yql/reference/builtins/json.md#json_exists) function checks whether a path specified by a JsonPath expression exists in the document. + + +```yql +SELECT id +FROM documents VIEW json_idx +WHERE JSON_EXISTS(payload, '$.user.id'); +``` + + +Result: + + +```text +id +1 +2 +3 +``` + + +The path token `$.user.id` is used for index search. The index returns the result without scanning the main table. + +## Selecting rows with a specific document field value + +The [JSON_VALUE](../../yql/reference/builtins/json.md#json_value) function extracts a scalar value by JsonPath; to use the index, you must specify the return type in the `RETURNING` clause: + + +```yql +SELECT id +FROM documents VIEW json_idx +WHERE JSON_VALUE(payload, '$.user.name' RETURNING Utf8) = "Alice"u; +``` + + +Result: + + +```text +id +1 +``` + + +When checking equality, the token «path + value» (`$.user.name = "Alice"`) is placed into the index, which ensures the highest selectivity. + +## Combination of conditions + +Multiple calls `JSON_EXISTS` / `JSON_VALUE` on a single indexed JSON column can be combined using the `AND` and `OR` operators: + + +```yql +SELECT id +FROM documents VIEW json_idx +WHERE JSON_EXISTS(payload, '$.user.id') + AND JSON_VALUE(payload, '$.active' RETURNING Bool); +``` + + +Result: + + +```text +id +1 +``` + + +## See also + +* [JSON indexes](../../dev/json-indexes.md) — a complete overview of capabilities and limitations. +* [VIEW (JSON index)](../../yql/reference/syntax/select/json_index.md) — query syntax through `VIEW`. +* [INDEX (CREATE TABLE)](../../yql/reference/syntax/create_table/json_index.md) — syntax for creating a JSON index. +* [{#T}](json-index-catalog.md) — example of a product catalog with nested attributes. diff --git a/ydb/docs/en/core/recipes/json-search/json-index-typecheck.md b/ydb/docs/en/core/recipes/json-search/json-index-typecheck.md new file mode 100644 index 00000000000..1ec8fc63af4 --- /dev/null +++ b/ydb/docs/en/core/recipes/json-search/json-index-typecheck.md @@ -0,0 +1,94 @@ +# Checking field type and path existence + +This recipe shows how a [JSON index](../../dev/json-indexes.md) is used to check document structure: the presence of a specific path, the type of value at the path, etc. These tasks are common when working with heterogeneous JSON documents where some fields are optional or may have different types. + +## Prerequisites + + +```yql +CREATE TABLE documents ( + id Uint64, + payload JsonDocument, + PRIMARY KEY (id), + INDEX json_idx GLOBAL USING json ON (payload) +); + +UPSERT INTO documents (id, payload) VALUES + (1, JsonDocument(@@{"archived": false, "value": 1, "data": [1, 2, 3]}@@)), + (2, JsonDocument(@@{"archived": false, "value": 2, "data": "plain text"}@@)), + (3, JsonDocument(@@{"archived": true, "value": 3, "data": {"nested": true}}@@)), + (4, JsonDocument(@@{"archived": false, "value": null, "meta": "no data field"}@@)); +``` + + +## Find documents where a field contains an array + +The JsonPath method `.type()` returns a string name of the value type at the specified path. This allows filtering documents by content type: + + +```yql +SELECT id +FROM documents VIEW json_idx +WHERE JSON_VALUE(payload, '$.data.type()' RETURNING Utf8) = "array"u; +``` + + +The path token `$.data` is added to the index — the method `.type()` completes the path token construction. The exact check of the string value `"array"` is performed by a post-filter. + +Result: + + +```text +id +1 +``` + + +The full list of values returned by the method `.type()` is given in the [JsonPath](../../yql/reference/builtins/json.md#jsonpath) syntax description. + +## Find documents where a field is a non-empty array + +The method `.size()` returns the number of array elements (or 1 for a scalar, 0 for a missing path). In combination with a type filter: + + +```yql +SELECT id +FROM documents VIEW json_idx +WHERE JSON_VALUE(payload, '$.data.type()' RETURNING Utf8) = "array"u + AND JSON_VALUE(payload, '$.data.size()' RETURNING Int64) > 0; +``` + + +Both fragments are indexed by the corresponding paths, and the exact values are checked by a post-filter. Result: `id = 1`. + +## Find documents with value `false` or `null` + +To check for "value equals `false`" or "value equals `null`", use JsonPath inside `JSON_EXISTS`, not `JSON_VALUE(...) IS NULL`: + + +```yql +-- The value of the 'archived' field is false +SELECT id +FROM documents VIEW json_idx +WHERE JSON_EXISTS(payload, '$.archived ? (@ == false)'); + +-- The value of the 'value' field is null +SELECT id +FROM documents VIEW json_idx +WHERE JSON_EXISTS(payload, '$.value ? (@ == null)'); +``` + + +The index search operation includes the path `$.archived` (or `$.value`) and the values `false` or `null`, respectively. + +{% note info %} + +The above conditions select documents where the specified attribute is explicitly set to false or null. Checking for the absence of an attribute cannot be done using a JSON index. + +{% endnote %} + +## See also + +* [JSON_EXISTS](../../dev/json-indexes.md#json-exists) — what is allowed in JsonPath expressions. +* [JSON_VALUE](../../dev/json-indexes.md#json-value) — what is allowed when extracting values. +* [JsonPath: methods](../../yql/reference/builtins/json.md#jsonpath) — list of methods (`type`, `size`, `keyvalue`, ...) and JsonPath predicates. diff --git a/ydb/docs/en/core/recipes/json-search/toc_p.yaml b/ydb/docs/en/core/recipes/json-search/toc_p.yaml new file mode 100644 index 00000000000..3b1d05ecbe1 --- /dev/null +++ b/ydb/docs/en/core/recipes/json-search/toc_p.yaml @@ -0,0 +1,9 @@ +items: +- name: JSON index — quick start + href: json-index-quickstart.md +- name: Directory with nested attributes + href: json-index-catalog.md +- name: Parameterized queries and JsonPath variables + href: json-index-parameters.md +- name: Checking field type and path existence + href: json-index-typecheck.md diff --git a/ydb/docs/en/core/recipes/toc_p.yaml b/ydb/docs/en/core/recipes/toc_p.yaml index e6abaf532e0..b4552b3e512 100644 --- a/ydb/docs/en/core/recipes/toc_p.yaml +++ b/ydb/docs/en/core/recipes/toc_p.yaml @@ -32,6 +32,11 @@ items: include: mode: link path: fulltext-search/toc_p.yaml +- name: JSON search + href: json-search/index.md + include: + mode: link + path: json-search/toc_p.yaml - name: Local indexes href: local-indexes/index.md include: diff --git a/ydb/docs/en/core/recipes/ydb-sdk/debug-jaeger.md b/ydb/docs/en/core/recipes/ydb-sdk/debug-jaeger.md deleted file mode 100644 index bdb57ffd9f7..00000000000 --- a/ydb/docs/en/core/recipes/ydb-sdk/debug-jaeger.md +++ /dev/null @@ -1,170 +0,0 @@ -# Enabling tracing in Jaeger - -Below are code examples for enabling tracing in Jaeger in different {{ ydb-short-name }} SDK. - -{% list tabs %} - -- C++ - - This functionality is not currently supported. - -- Go - - {% list tabs %} - - - Native SDK - - ```go - package main - - import ( - "context" - "time" - - "github.com/opentracing/opentracing-go" - jaegerConfig "github.com/uber/jaeger-client-go/config" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - - tracing "github.com/ydb-platform/ydb-go-sdk-opentracing" - ) - - const ( - tracerURL = "localhost:5775" - serviceName = "ydb-go-sdk" - ) - - func main() { - tracer, closer, err := jaegerConfig.Configuration{ - ServiceName: serviceName, - Sampler: &jaegerConfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jaegerConfig.ReporterConfig{ - LogSpans: true, - BufferFlushInterval: 1 * time.Second, - LocalAgentHostPort: tracerURL, - }, - }.NewTracer() - if err != nil { - panic(err) - } - - defer closer.Close() - - // set global tracer of this application - opentracing.SetGlobalTracer(tracer) - - span, ctx := opentracing.StartSpanFromContext(context.Background(), "client") - defer span.Finish() - - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - tracing.WithTraces(tracing.WithDetails(trace.DetailsAll)), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - ... - } - ``` - - - database/sql - - ```go - package main - - import ( - "context" - "database/sql" - "time" - - "github.com/opentracing/opentracing-go" - jaegerConfig "github.com/uber/jaeger-client-go/config" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - - tracing "github.com/ydb-platform/ydb-go-sdk-opentracing" - ) - - const ( - tracerURL = "localhost:5775" - serviceName = "ydb-go-sdk" - ) - - func main() { - tracer, closer, err := jaegerConfig.Configuration{ - ServiceName: serviceName, - Sampler: &jaegerConfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jaegerConfig.ReporterConfig{ - LogSpans: true, - BufferFlushInterval: 1 * time.Second, - LocalAgentHostPort: tracerURL, - }, - }.NewTracer() - if err != nil { - panic(err) - } - - defer closer.Close() - - // set global tracer of this application - opentracing.SetGlobalTracer(tracer) - - span, ctx := opentracing.StartSpanFromContext(context.Background(), "client") - defer span.Finish() - - nativeDriver, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - tracing.WithTraces(tracing.WithDetails(trace.DetailsAll)), - ) - if err != nil { - panic(err) - } - defer nativeDriver.Close(ctx) - - connector, err := ydb.Connector(nativeDriver) - if err != nil { - panic(err) - } - - db := sql.OpenDB(connector) - defer db.Close() - ... - } - ``` - - {% endlist %} - -- Java - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- Python - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- C# - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- JavaScript - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- Rust - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- PHP - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -{% endlist %} diff --git a/ydb/docs/en/core/recipes/ydb-sdk/debug-logs-otel.md b/ydb/docs/en/core/recipes/ydb-sdk/debug-logs-otel.md deleted file mode 100644 index c90d1578c90..00000000000 --- a/ydb/docs/en/core/recipes/ydb-sdk/debug-logs-otel.md +++ /dev/null @@ -1,352 +0,0 @@ -# Export logs to OpenTelemetry - -Each {{ ydb-short-name }} SDK writes its internal logs (driver initialization, session pool, query execution, retries, etc.) through the standard logging facility of its language. Instead of outputting logs only to the console (see [Enable logging](debug-logs.md)), they can be redirected to the [OpenTelemetry](https://opentelemetry.io/) Logs SDK and exported via the standard OTLP protocol to a collector. The collector then forwards the records to the chosen backend for log storage and viewing. - -The principle is the same in all SDKs: - -1. Create an OpenTelemetry `LoggerProvider` with an OTLP log exporter and resource attribute `service.name`. -2. Redirect the SDK logger to this provider via an adapter (log appender / bridge) that converts each SDK log record into an OTel log record. For the list of adapters and log support status in different languages, see the [OpenTelemetry logs documentation](https://opentelemetry.io/docs/concepts/signals/logs/). -3. Run the workload — all internal SDK logs are now sent to the collector as OTLP log records. - -{% note info %} - -## Principles {#principles} - -The log collection method depends on the language and infrastructure — there is no single format: - -* **Programmatic bridges (log appender / bridge).** The application itself sends logs to the collector via OTLP directly from the process ("direct-to-Collector" workflow). This is the approach shown in the examples below. -* **Collector agents.** The application writes logs to a file or `stdout`, and a separate agent (e.g., [filelog receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/filelogreceiver) in [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/)) reads, parses, and forwards them to the backend — without changing application code. - -{% endnote %} - -## Connecting to the SDK {#integration} - -{% list tabs %} - -- Go - - For the {{ ydb-short-name }} Go SDK, there is a ready-made adapter [ydb-go-sdk-otel](https://github.com/ydb-platform/ydb-go-sdk-otel) that converts SDK events into OpenTelemetry signals: traces (`WithTracer`), metrics (`WithMetrics`), and logs (`WithLogger`). The adapter does not configure exporters itself — you create `LoggerProvider` with an OTLP log exporter, get a logger from it, and pass it to the `ydbOtel.WithLogger` option when calling `ydb.Open`. Each internal SDK log record (driver initialization, session pool, query execution, retries, etc.) is then sent to the collector as an OTLP log record. - - - ```bash - go get github.com/ydb-platform/ydb-go-sdk-otel - go get go.opentelemetry.io/otel/sdk/log - go get go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc - ``` - - - ```go - package main - - import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" - sdklog "go.opentelemetry.io/otel/sdk/log" - "go.opentelemetry.io/otel/sdk/resource" - - ydbOtel "github.com/ydb-platform/ydb-go-sdk-otel" - ) - - func main() { - ctx := context.Background() - - // 1. Configuring the OTel log provider with an OTLP exporter. - exporter, err := otlploggrpc.New(ctx, - otlploggrpc.WithEndpoint("localhost:4317"), - otlploggrpc.WithInsecure(), - ) - if err != nil { - panic(err) - } - res, _ := resource.Merge(resource.Default(), resource.NewSchemaless( - attribute.String("service.name", "ydb-go-sdk-otel-logs-sample"), - )) - lp := sdklog.NewLoggerProvider( - sdklog.WithResource(res), - sdklog.WithProcessor(sdklog.NewBatchProcessor(exporter)), - ) - defer lp.Shutdown(ctx) - - // 2. Opening the YDB driver with the ydb-go-sdk-otel adapter. - // WithLogger forwards SDK log events to OTel log records. - logger := lp.Logger("ydb-go-sdk") - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydbOtel.WithLogger(logger, ydbOtel.WithDetailer(trace.DetailsAll)), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - // ... use db ... - } - ``` - -- Python - - {{ ydb-short-name }} Python SDK writes logs through the standard `logging` module (loggers named `ydb.*`). Use the OpenTelemetry built-in `LoggingHandler` to redirect these records to `LoggerProvider` and export via OTLP: - - - ```bash - pip install opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc - ``` - - - ```python - import logging - - import ydb - from opentelemetry._logs import set_logger_provider - from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter - from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler - from opentelemetry.sdk._logs.export import BatchLogRecordProcessor - from opentelemetry.sdk.resources import Resource - - resource = Resource(attributes={"service.name": "ydb-otel-logs-example"}) - logger_provider = LoggerProvider(resource=resource) - logger_provider.add_log_record_processor( - BatchLogRecordProcessor(OTLPLogExporter(endpoint="http://localhost:4317")) - ) - set_logger_provider(logger_provider) - - # Bridge stdlib logging -> OpenTelemetry. Binding a handler to the root logger - # intercepts everything the SDK writes via logging.getLogger("ydb..."). - otel_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) - logging.basicConfig(level=logging.INFO, handlers=[otel_handler]) - logging.getLogger("ydb").setLevel(logging.INFO) - - with ydb.Driver(endpoint="grpc://localhost:2136", database="/local") as driver: - driver.wait(timeout=5) - with ydb.QuerySessionPool(driver) as pool: - pool.execute_with_retries("SELECT 1") - - logger_provider.shutdown() - ``` - -- C# - - {{ ydb-short-name }} C# SDK writes logs through the `ILoggerFactory` passed to it. Create a factory based on the OpenTelemetry logging provider with an OTLP exporter and pass it to the data source: - - - ```bash - dotnet add package OpenTelemetry - dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol - ``` - - - ```csharp - using Microsoft.Extensions.Logging; - using OpenTelemetry.Logs; - using OpenTelemetry.Resources; - using Ydb.Sdk.Ado; - - var resourceBuilder = ResourceBuilder.CreateDefault() - .AddService("ydb-sdk-otel-logs-sample"); - - using var loggerFactory = LoggerFactory.Create(builder => - { - builder.AddOpenTelemetry(options => - { - options.SetResourceBuilder(resourceBuilder); - options.AddOtlpExporter(o => o.Endpoint = new Uri("http://localhost:4317")); - }); - }); - - // Pass the factory to the SDK: each internal log (driver initialization, session pool, - // query execution, retries, ...) is exported to the collector as an OTLP log record. - await using var dataSource = new YdbDataSource( - new YdbConnectionStringBuilder("Host=localhost;Port=2136;Database=/local") - { - LoggerFactory = loggerFactory - }); - - await using var connection = await dataSource.OpenConnectionAsync(); - await new YdbCommand("SELECT 1", connection).ExecuteNonQueryAsync(); - ``` - -- Java - - {{ ydb-short-name }} Java SDK writes logs through `slf4j`. If logback is used as the `slf4j` implementation, connect the ready-made `opentelemetry-logback-appender-1.0` appender, which forwards each event to the OpenTelemetry Logs SDK: - - - ```xml - <dependency> - <groupId>io.opentelemetry</groupId> - <artifactId>opentelemetry-sdk</artifactId> - <version>${otel.version}</version> - </dependency> - <dependency> - <groupId>io.opentelemetry</groupId> - <artifactId>opentelemetry-exporter-otlp</artifactId> - <version>${otel.version}</version> - </dependency> - <dependency> - <groupId>io.opentelemetry.instrumentation</groupId> - <artifactId>opentelemetry-logback-appender-1.0</artifactId> - <version>${otel.instrumentation.version}</version> - </dependency> - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - <version>${logback.version}</version> - </dependency> - ``` - - - Connect the appender in `logback.xml`: - - - ```xml - <configuration> - <appender name="OTEL" class="io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender"> - </appender> - - <root level="INFO"> - <appender-ref ref="OTEL"/> - </root> - </configuration> - ``` - - - Build an instance of the OpenTelemetry SDK with an OTLP log exporter and set it in the appender by calling `OpenTelemetryAppender.install(...)` before starting the workload: - - - ```java - import java.util.concurrent.TimeUnit; - - import io.opentelemetry.api.OpenTelemetry; - import io.opentelemetry.api.common.Attributes; - import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; - import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender; - import io.opentelemetry.sdk.OpenTelemetrySdk; - import io.opentelemetry.sdk.logs.SdkLoggerProvider; - import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor; - import io.opentelemetry.sdk.resources.Resource; - import tech.ydb.core.grpc.GrpcTransport; - import tech.ydb.query.QueryClient; - - String serviceName = "ydb-java-sdk-otel-logs-sample"; - Resource resource = Resource.getDefault().merge( - Resource.create(Attributes.builder().put("service.name", serviceName).build())); - - SdkLoggerProvider loggerProvider = SdkLoggerProvider.builder() - .setResource(resource) - .addLogRecordProcessor(BatchLogRecordProcessor.builder( - OtlpGrpcLogRecordExporter.builder().setEndpoint("http://localhost:4317").build() - ).build()) - .build(); - - OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() - .setLoggerProvider(loggerProvider) - .build(); - - // Pass the OpenTelemetry SDK to the appender declared in logback.xml. - OpenTelemetryAppender.install(openTelemetry); - - try (GrpcTransport transport = GrpcTransport.forConnectionString("grpc://localhost:2136/local").build(); - QueryClient queryClient = QueryClient.newClient(transport).build()) { - // ... use queryClient ... - } finally { - loggerProvider.forceFlush().join(10, TimeUnit.SECONDS); - loggerProvider.shutdown().join(10, TimeUnit.SECONDS); - } - ``` - -- C++ - - {{ ydb-short-name }} C++ SDK writes logs through `TLogBackend` from `util`. Implement a backend that turns each record into an OTel log record, and pass it to the driver via `TDriverConfig::SetLog`. See [Getting Started](https://opentelemetry.io/docs/languages/cpp/getting-started/) for building the OpenTelemetry C++ SDK. - - - ```cpp - #include <ydb-cpp-sdk/client/driver/driver.h> - #include <opentelemetry/exporters/otlp/otlp_grpc_log_record_exporter_factory.h> - #include <opentelemetry/exporters/otlp/otlp_grpc_log_record_exporter_options.h> - #include <opentelemetry/logs/provider.h> - #include <opentelemetry/sdk/logs/batch_log_record_processor_factory.h> - #include <opentelemetry/sdk/logs/logger_provider_factory.h> - #include <opentelemetry/sdk/resource/resource.h> - #include <library/cpp/logger/backend.h> - #include <library/cpp/logger/record.h> - #include <memory> - - namespace otlp = opentelemetry::exporter::otlp; - namespace logs_sdk = opentelemetry::sdk::logs; - namespace logs_api = opentelemetry::logs; - namespace resource = opentelemetry::sdk::resource; - - namespace { - - class TOtelLogBackend final : public TLogBackend { - public: - explicit TOtelLogBackend(opentelemetry::nostd::shared_ptr<logs_api::Logger> logger) - : Logger_(std::move(logger)) {} - - void WriteData(const TLogRecord& rec) override { - Logger_->EmitLogRecord(MapSeverity(rec.Priority), - opentelemetry::nostd::string_view(rec.Data, rec.Len)); - } - - void ReopenLog() override {} - - private: - static logs_api::Severity MapSeverity(ELogPriority p) { - switch (p) { - case TLOG_EMERG: - case TLOG_ALERT: - case TLOG_CRIT: return logs_api::Severity::kFatal; - case TLOG_ERR: return logs_api::Severity::kError; - case TLOG_WARNING: return logs_api::Severity::kWarn; - case TLOG_NOTICE: - case TLOG_INFO: return logs_api::Severity::kInfo; - case TLOG_DEBUG: return logs_api::Severity::kDebug; - default: return logs_api::Severity::kTrace; - } - } - - opentelemetry::nostd::shared_ptr<logs_api::Logger> Logger_; - }; - } // namespace - - int main() { - // 1. Configuring the OTel log provider with an OTLP exporter - otlp::OtlpGrpcLogRecordExporterOptions exporterOpts; - exporterOpts.endpoint = "localhost:4317"; - auto exporter = otlp::OtlpGrpcLogRecordExporterFactory::Create(exporterOpts); - auto processor = logs_sdk::BatchLogRecordProcessorFactory::Create(std::move(exporter)); - auto res = resource::Resource::Create({{"service.name", "ydb-cpp-sdk-otel-logs-sample"}}); - - std::shared_ptr<logs_api::LoggerProvider> provider( - logs_sdk::LoggerProviderFactory::Create(std::move(processor), res)); - logs_api::Provider::SetLoggerProvider(provider); - - auto logger = provider->GetLogger("ydb-cpp-sdk"); - - // 2. Pass the bridge to the YDB driver - auto config = NYdb::TDriverConfig() - .SetEndpoint("localhost:2136") - .SetDatabase("/local") - .SetLog(std::make_unique<TOtelLogBackend>(logger)); - - NYdb::TDriver driver(config); - // ... use driver ... - driver.Stop(true); - } - ``` - -- JavaScript - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- Rust - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - - Track progress or vote for Rust SDK support: [ydb-rs-sdk#268](https://github.com/ydb-platform/ydb-rs-sdk/issues/268) - -{% endlist %} diff --git a/ydb/docs/en/core/recipes/ydb-sdk/debug-logs.md b/ydb/docs/en/core/recipes/ydb-sdk/debug-logs.md deleted file mode 100644 index b164a73bb7f..00000000000 --- a/ydb/docs/en/core/recipes/ydb-sdk/debug-logs.md +++ /dev/null @@ -1,411 +0,0 @@ -# Enabling logging - -Below are examples of code that enables logging in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% list tabs %} - - - Native SDK - - There are several ways to enable logs in an application that uses `ydb-go-sdk`: - - {% cut "Using the `YDB_LOG_SEVERITY_LEVEL` environment variable" %} - - This environment variable enables the built-in `ydb-go-sdk` logger (synchronous, non-block) and prints to the standard output stream. - You can set the environment variable as follows: - - ```shell - export YDB_LOG_SEVERITY_LEVEL=info - ``` - - (possible values: `trace`, `debug`, `info`, `warn`, `error`, `fatal`, and `quiet`, defaults to `quiet`). - - {% endcut %} - - {% cut "Enable a third-party logger `go.uber.org/zap`" %} - - ```go - package main - - import ( - "context" - "os" - - "go.uber.org/zap" - - ydbZap "github.com/ydb-platform/ydb-go-sdk-zap" - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var log *zap.Logger // zap-logger with init out of this scope - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydbZap.WithTraces( - log, - trace.DetailsAll, - ), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - ... - } - ``` - - {% endcut %} - - {% cut "Enable a third-party logger `github.com/rs/zerolog`" %} - - ```go - package main - - import ( - "context" - "os" - - "github.com/rs/zerolog" - - ydbZerolog "github.com/ydb-platform/ydb-go-sdk-zerolog" - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var log zerolog.Logger // zap-logger with init out of this scope - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydbZerolog.WithTraces( - &log, - trace.DetailsAll, - ), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - ... - } - ``` - - {% endcut %} - - {% include [overlay](_includes/debug-logs-go-appendix.md) %} - - {% cut "Enable a custom logger implementation `github.com/ydb-platform/ydb-go-sdk/v3/log.Logger`" %} - - ```go - package main - - import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/log" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var logger log.Logger // logger implementation with init out of this scope - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithLogger( - logger, - trace.DetailsAll, - ), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - ... - } - ``` - - {% endcut %} - - {% cut "Implement your own logging package" %} - - You can implement your own logging package based on the driver events in the `github.com/ydb-platform/ydb-go-sdk/v3/trace` tracing package. The `github.com/ydb-platform/ydb-go-sdk/v3/trace` tracing package describes all logged driver events. - - {% endcut %} - - {% cut "Iterate over server errors with `IterateByIssues`" %} - - When using {{ ydb-short-name }} through the Go SDK, you can enable logging of requests and responses and also obtain detailed information about server errors (issues) — extra messages YDB returns in the response when operations fail. To iterate over issues in the server response, use [IterateByIssues](https://pkg.go.dev/github.com/ydb-platform/ydb-go-sdk/v3#IterateByIssues). - - {% endcut %} - - - database/sql - - There are several ways to enable logs in an application that uses `ydb-go-sdk`: - - {% cut "Using the `YDB_LOG_SEVERITY_LEVEL` environment variable" %} - - This environment variable enables the built-in `ydb-go-sdk` logger (synchronous, non-block) and prints to the standard output stream. - You can set the environment variable as follows: - - ```shell - export YDB_LOG_SEVERITY_LEVEL=info - ``` - - (possible values: `trace`, `debug`, `info`, `warn`, `error`, `fatal`, and `quiet`, defaults to `quiet`). - - {% endcut %} - - {% cut "Enable a third-party logger `go.uber.org/zap`" %} - - ```go - package main - - import ( - "context" - "database/sql" - "os" - - "go.uber.org/zap" - - ydbZap "github.com/ydb-platform/ydb-go-sdk-zap" - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var log *zap.Logger // zap-logger with init out of this scope - nativeDriver, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydbZap.WithTraces( - log, - trace.DetailsAll, - ), - ) - if err != nil { - panic(err) - } - defer nativeDriver.Close(ctx) - - connector, err := ydb.Connector(nativeDriver) - if err != nil { - panic(err) - } - defer connector.Close() - - db := sql.OpenDB(connector) - defer db.Close() - ... - } - ``` - - {% endcut %} - - {% cut "Enable a third-party logger `github.com/rs/zerolog`" %} - - ```go - package main - - import ( - "context" - "database/sql" - "os" - - "github.com/rs/zerolog" - - ydbZerolog "github.com/ydb-platform/ydb-go-sdk-zerolog" - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var log zerolog.Logger // zap-logger with init out of this scope - nativeDriver, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydbZerolog.WithTraces( - &log, - trace.DetailsAll, - ), - ) - if err != nil { - panic(err) - } - defer nativeDriver.Close(ctx) - - connector, err := ydb.Connector(nativeDriver) - if err != nil { - panic(err) - } - defer connector.Close() - - db := sql.OpenDB(connector) - defer db.Close() - ... - } - ``` - - {% endcut %} - - {% include [overlay](_includes/debug-logs-go-sql-appendix.md) %} - - {% cut "Enable a custom logger implementation `github.com/ydb-platform/ydb-go-sdk/v3/log.Logger`" %} - - ```go - package main - - import ( - "context" - "database/sql" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/log" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - var logger log.Logger // logger implementation with init out of this scope - nativeDriver, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydb.WithLogger( - logger, - trace.DetailsAll, - ), - ) - if err != nil { - panic(err) - } - defer nativeDriver.Close(ctx) - - connector, err := ydb.Connector(nativeDriver) - if err != nil { - panic(err) - } - defer connector.Close() - - db := sql.OpenDB(connector) - defer db.Close() - ... - } - ``` - - {% endcut %} - - {% cut "Implement your own logging package" %} - - You can implement your own logging package based on the driver events in the `github.com/ydb-platform/ydb-go-sdk/v3/trace` tracing package. The `github.com/ydb-platform/ydb-go-sdk/v3/trace` tracing package describes all logged driver events. - - {% endcut %} - - {% endlist %} - - -- Java - - {% list tabs %} - - - Native SDK - - For logging purposes, the {{ ydb-short-name }} Java SDK uses the slf4j library, which supports multiple logging levels (`error`, `warn`, `info`, `debug`, `trace`) for one or many loggers. The current implementation supports the following loggers: - - * The `tech.ydb.core.grpc` logger provides information about the internal gRPC implementation - * The `debug` level logs all gRPC operations; use it only for debugging - * The `info` level is recommended by default - * On the `debug` level, the `tech.ydb.table.impl` logger lets you track the internal state of the {{ ydb-short-name }} driver, including the session pool - * On the `debug` level, the `tech.ydb.table.SessionRetryContext` logger reports the number of retries, query results, per-retry duration, and total operation time - * On the `debug` level, the `tech.ydb.table.Session` logger provides the query text, response status, and execution time for session operations - - Enabling and configuring the Java SDK loggers depends on the `slf4j-api` implementation you use. - Here is an example `log4j2` configuration for the `log4j-slf4j-impl` library: - - ```xml - <Configuration status="WARN"> - <Appenders> - <Console name="Console" target="SYSTEM_OUT"> - <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/> - </Console> - </Appenders> - - <Loggers> - <Logger name="io.netty" level="warn" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - <Logger name="io.grpc.netty" level="warn" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - <Logger name="tech.ydb.core.grpc" level="info" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - <Logger name="tech.ydb.table.impl" level="info" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - <Logger name="tech.ydb.table.SessionRetryContext" level="debug" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - <Logger name="tech.ydb.table.Session" level="debug" additivity="false"> - <AppenderRef ref="Console"/> - </Logger> - - <Root level="debug" > - <AppenderRef ref="Console"/> - </Root> - </Loggers> - </Configuration> - ``` - - - JDBC - - The JDBC driver uses the same logging stack via `slf4j`; configure the `tech.ydb.*` loggers the same way as in the native SDK. - - The same debug logs are available in other stacks built on JDBC (Spring Boot, ORMs, connection pools, and so on): they reach {{ ydb-short-name }} through this driver, so it is enough to attach the same `slf4j` / log4j2 / logback configuration in the application. - - {% endlist %} - -- PHP - - For logging purposes, you need to use a class, that implements `\Psr\Log\LoggerInterface`. - `ydb-php-sdk` has build-in loggers in `YdbPlatform\Ydb\Logger` namespace: - - * `NullLogger` - default logger, which writes nothing - * `SimpleStdLogger($level)` - logger, which writes to logs in stderr. - - Usage example: - - ```php - $config = [ - 'logger' => new \YdbPlatform\Ydb\Logger\SimpleStdLogger(\YdbPlatform\Ydb\Logger\SimpleStdLogger::INFO) - ] - $ydb = new \YdbPlatform\Ydb\Ydb($config); - ``` - -- Python - - The Python SDK uses the standard `logging` library. To enable a specific logging level: - - ```python - import logging - - logging.getLogger('ydb').setLevel(logging.DEBUG) - ``` - -- JavaScript - - The SDK uses the [debug](https://www.npmjs.com/package/debug) library for logging. - To enable logs, set the `DEBUG` environment variable to filter SDK events, for example `DEBUG=ydbjs:*`. - -{% endlist %} diff --git a/ydb/docs/en/core/recipes/ydb-sdk/debug-otel-metrics.md b/ydb/docs/en/core/recipes/ydb-sdk/debug-otel-metrics.md deleted file mode 100644 index 791ac9a4feb..00000000000 --- a/ydb/docs/en/core/recipes/ydb-sdk/debug-otel-metrics.md +++ /dev/null @@ -1,321 +0,0 @@ -# Metrics with OpenTelemetry - -{{ ydb-short-name }} SDK instruments Query Service operations with [OpenTelemetry](https://opentelemetry.io/) metrics, allowing you to observe the client state — the duration and number of operations, the state of the session pool — from the application code to gRPC calls to YDB. The metrics are exported via the standard OTLP protocol and are compatible with Prometheus, Grafana, VictoriaMetrics, and any other backend that supports OpenTelemetry. - -## List of metrics {#metrics-list} - -### Operation metrics {#operation-metrics} - -| Name | Type | Unit | Description | -| --- | --- | --- | --- | -| `ydb.client.operation.duration` | Histogram | `s` | Duration of a single attempt of a client operation (`ExecuteQuery`, `Commit`, `Rollback`, `CreateSession`). | -| `ydb.client.operation.failed` | Counter | `{operation}` | Number of unsuccessful client operations. | - -### Session pool metrics {#session-pool-metrics} - -| Name | Type | Unit | Description | -| --- | --- | --- | --- | -| `ydb.query.session.create_time` | Histogram | `s` | Duration of creating a new session. | -| `ydb.query.session.pending_requests` | Counter | `{request}` | Monotonic counter of session acquisition queries that have entered the wait queue since the pool was created. | -| `ydb.query.session.timeouts` | Counter | `{timeout}` | Monotonic counter of timeouts when waiting for a free session, since the pool creation. | -| `ydb.query.session.count` | Gauge | `{session}` | Current number of sessions in the pool, broken down by state. | -| `ydb.query.session.min` | Gauge | `{session}` | Configured minimum size of the session pool. | -| `ydb.query.session.max` | Gauge | `{session}` | Configured maximum size of the session pool. | - -### Retry metrics {#retry-metrics} - -| Name | Type | Unit | Description | -| --- | --- | --- | --- | -| `ydb.client.retry.duration` | Histogram | `s` | Total client-visible duration of a logical operation executed through a retry policy, including all attempts and backoff delays. | -| `ydb.client.retry.attempts` | Histogram | `{attempt}` | Distribution of the number of attempts per logical operation. The value `1` means success on the first attempt. | - -### Additional metrics (JavaScript) {#js-metrics} - -In the [JavaScript SDK](https://github.com/ydb-platform/ydb-js-sdk), the `@ydbjs/telemetry` package uses its own names and semantics for retry metrics — different from `ydb.client.retry.*` above: - -| Name | Type | Unit | Description | -| --- | --- | --- | --- | -| `ydb.retry.attempts` | Counter | `{attempt}` | Monotonic counter of retry attempts with outcome tag `ydb.retry.outcome`. Unlike histogram `ydb.client.retry.attempts`, it does not show the distribution of the number of attempts per query. | -| `ydb.retry.duration` | Histogram | `s` | Total retry cycle duration, including backoff (equivalent of `ydb.client.retry.duration`). | - -## Attributes {#attributes} - -| Name | Applies to | Value | -| --- | --- | --- | -| `database` | `ydb.client.operation.duration`, `ydb.client.operation.failed` | Database name {{ ydb-short-name }}. | -| `endpoint` | `ydb.client.operation.duration`, `ydb.client.operation.failed` | Discovery-endpoint in the `host:port` format. | -| `operation.name` | `ydb.client.operation.duration`, `ydb.client.operation.failed`, `ydb.client.retry.duration`, `ydb.client.retry.attempts` | Client operation name: `ExecuteQuery`, `Commit`, `Rollback`, `CreateSession`. | -| `status_code` | `ydb.client.operation.failed` | Status code {{ ydb-short-name }} (for example, `BAD_REQUEST`, `SCHEME_ERROR`). | -| `ydb.query.session.pool.name` | All `ydb.query.session.*` metrics | Session pool name. By default, it is formed as `<endpoint>/<database>`; configured through the API of a specific SDK. | -| `ydb.query.session.state` | `ydb.query.session.count` | Session state: `idle` or `used`. | -| `ydb.retry.outcome` | `ydb.retry.attempts` (JavaScript) | Outcome of a retry attempt: for example, `success`, `retried`. | - -## Connecting to the SDK {#integration} - -{% list tabs %} - -- Go - - Install the OpenTelemetry adapter for the {{ ydb-short-name }} Go SDK: - - - ```bash - go get github.com/ydb-platform/ydb-go-sdk-otel - ``` - - - Configure `MeterProvider`, obtain `Meter` from it, and pass it to the `ydbOtel.WithMetrics` adapter, which connects to the driver via the `ydb.Open` option. The granularity of collected metrics can be configured via `ydbOtel.WithDetailer`: - - - ```go - package main - - import ( - "context" - "os" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" - "go.opentelemetry.io/otel/sdk/metric" - "go.opentelemetry.io/otel/sdk/resource" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ydbOtel "github.com/ydb-platform/ydb-go-sdk-otel" - ) - - func main() { - ctx := context.Background() - - exporter, err := otlpmetricgrpc.New(ctx, - otlpmetricgrpc.WithEndpoint("localhost:4317"), - otlpmetricgrpc.WithInsecure(), - ) - if err != nil { - panic(err) - } - res, _ := resource.Merge(resource.Default(), resource.NewSchemaless( - attribute.String("service.name", "my-service"), - )) - mp := metric.NewMeterProvider( - metric.WithReader(metric.NewPeriodicReader(exporter)), - metric.WithResource(res), - ) - defer mp.Shutdown(ctx) - otel.SetMeterProvider(mp) - - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydbOtel.WithMetrics( - mp.Meter("ydb-go-sdk"), - ydbOtel.WithDetailer(trace.DetailsAll), - ), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - } - ``` - -- Python - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- C# - - Add the NuGet package: - - - ```bash - dotnet add package Ydb.Sdk.OpenTelemetry - ``` - - - Register the {{ ydb-short-name }} instrumentation in the OpenTelemetry metrics pipeline: - - - ```csharp - services.AddOpenTelemetry() - .WithMetrics(builder => builder - .AddYdb() - .AddOtlpExporter()); - ``` - - - Or using standalone `MeterProvider`: - - - ```csharp - using var meterProvider = Sdk.CreateMeterProviderBuilder() - .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("my-service")) - .AddYdb() - .AddOtlpExporter() - .Build(); - ``` - - - The session pool name is set by the `PoolName=` parameter in the `YdbDataSource` connection string. A full example with load and Grafana is in the SDK repository ( [Ydb.Sdk.AdoNet.OpenTelemetry/Metrics](https://github.com/ydb-platform/ydb-dotnet-sdk/blob/main/examples/Ydb.Sdk.AdoNet.OpenTelemetry/Metrics/Program.cs)). - -- Java - - {% note info %} - - Currently, only [session pool metrics](#session-pool-metrics) are supported in the Java SDK. - - {% endnote %} - - Pass your `OpenTelemetry` to the SDK via the `OpenTelemetryMeter` adapter and the `QueryClient.Builder#withMeter` method: - - - ```java - import io.opentelemetry.api.OpenTelemetry; - import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; - import io.opentelemetry.sdk.OpenTelemetrySdk; - import io.opentelemetry.sdk.metrics.SdkMeterProvider; - import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; - - import tech.ydb.core.grpc.GrpcTransport; - import tech.ydb.core.metrics.OpenTelemetryMeter; - import tech.ydb.query.QueryClient; - - SdkMeterProvider meterProvider = SdkMeterProvider.builder() - .registerMetricReader(PeriodicMetricReader - .builder(OtlpGrpcMetricExporter.getDefault()) - .build()) - .build(); - - OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() - .setMeterProvider(meterProvider) - .build(); - - try (GrpcTransport transport = GrpcTransport - .forConnectionString(System.getenv("YDB_CONNECTION_STRING")) - .build(); - QueryClient queryClient = QueryClient.newClient(transport) - .withMeter(OpenTelemetryMeter.fromOpenTelemetry(openTelemetry)) - .sessionPoolName("my-app") - .build()) { - // Use queryClient - } - meterProvider.close(); - ``` - - - If a global `GlobalOpenTelemetry` is already configured, you can use `OpenTelemetryMeter.createGlobal()`. `TableClient` also supports `withMeter(...)`. - -- C++ - - Include the OpenTelemetry metrics header from {{ ydb-short-name }} C++ SDK and register `MetricRegistry` in `TDriverConfig`: - - - ```cpp - #include <ydb-cpp-sdk/client/driver/driver.h> - #include <ydb-cpp-sdk/open_telemetry/metrics.h> - - #include <opentelemetry/exporters/otlp/otlp_http_metric_exporter_factory.h> - #include <opentelemetry/exporters/otlp/otlp_http_metric_exporter_options.h> - #include <opentelemetry/sdk/metrics/meter_provider.h> - #include <opentelemetry/sdk/metrics/view/view_registry.h> - #include <opentelemetry/sdk/metrics/export/periodic_exporting_metric_reader_factory.h> - #include <opentelemetry/sdk/resource/resource.h> - #include <opentelemetry/metrics/provider.h> - - namespace sdkmetrics = opentelemetry::sdk::metrics; - namespace otlp = opentelemetry::exporter::otlp; - namespace resource = opentelemetry::sdk::resource; - using namespace NYdb; - - // 1. Initialize the OTel metrics provider - otlp::OtlpHttpMetricExporterOptions opts; - opts.url = "http://localhost:4318/v1/metrics"; - - auto exporter = otlp::OtlpHttpMetricExporterFactory::Create(opts); - - sdkmetrics::PeriodicExportingMetricReaderOptions readerOpts; - readerOpts.export_interval_millis = std::chrono::milliseconds(5000); - auto reader = sdkmetrics::PeriodicExportingMetricReaderFactory::Create( - std::move(exporter), readerOpts); - - auto res = resource::Resource::Create({{"service.name", "my-service"}}); - auto rawProvider = std::make_shared<sdkmetrics::MeterProvider>( - std::unique_ptr<sdkmetrics::ViewRegistry>(new sdkmetrics::ViewRegistry()), res); - rawProvider->AddMetricReader(std::move(reader)); - - std::shared_ptr<opentelemetry::metrics::MeterProvider> meterProvider = rawProvider; - opentelemetry::metrics::Provider::SetMeterProvider(meterProvider); - - // 2. Wrap in the YDB metrics registrar - auto ydbMetricRegistry = NMetrics::CreateOtelMetricRegistry(meterProvider); - - // 3. Create a YDB driver with metrics enabled - auto driverConfig = TDriverConfig() - .SetEndpoint("localhost:2136") - .SetDatabase("/local") - .SetMetricRegistry(ydbMetricRegistry); - - TDriver driver(driverConfig); - ``` - - - Metrics and tracing can be connected together by registering both `MetricRegistry` and `TraceProvider` in `TDriverConfig` simultaneously. - -- JavaScript - - Install the {{ ydb-short-name }} JavaScript SDK telemetry package and the OpenTelemetry SDK: - - - ```bash - npm install @ydbjs/telemetry @opentelemetry/sdk-node @opentelemetry/exporter-metrics-otlp-http - ``` - - - Initialize `NodeSDK` before creating `Driver` and register the `@ydbjs/telemetry` instrumentation: - - - ```ts - import { NodeSDK } from '@opentelemetry/sdk-node' - import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http' - import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics' - import { Driver } from '@ydbjs/core' - import { query } from '@ydbjs/query' - import { register } from '@ydbjs/telemetry' - - const sdk = new NodeSDK({ - serviceName: 'my-service', - metricReader: new PeriodicExportingMetricReader({ - exporter: new OTLPMetricExporter({ - url: 'http://localhost:4318/v1/metrics', - }), - }), - }) - sdk.start() - - // Must be called BEFORE creating the Driver: the package subscribes to - // diagnostics_channel SDK events and converts them into OTel metrics. - const instrumentation = register() - - const driver = new Driver(process.env.YDB_CONNECTION_STRING) - await driver.ready() - - const sql = query(driver) - await sql`SELECT 1` - - instrumentation.disable() - await driver.close() - await sdk.shutdown() - ``` - - - The `@ydbjs/telemetry` package subscribes to `node:diagnostics_channel` events from `@ydbjs/core`, `@ydbjs/query`, `@ydbjs/auth`, and `@ydbjs/retry` and publishes metrics from the [list above](#metrics-list) and [additional JavaScript SDK metrics](#js-metrics). Operation duration is exported as `db.client.operation.duration` (OpenTelemetry semantic conventions), not `ydb.client.operation.duration`. The current number of requests waiting for a session is gauge `ydb.query.session.acquire.pending`, not counter `ydb.query.session.pending_requests`. For more information, see the [JavaScript SDK repository](https://github.com/ydb-platform/ydb-js-sdk). - -- Rust - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- PHP - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -{% endlist %} diff --git a/ydb/docs/en/core/recipes/ydb-sdk/debug-otel-tracing.md b/ydb/docs/en/core/recipes/ydb-sdk/debug-otel-tracing.md deleted file mode 100644 index 4bce69ac974..00000000000 --- a/ydb/docs/en/core/recipes/ydb-sdk/debug-otel-tracing.md +++ /dev/null @@ -1,371 +0,0 @@ -# Tracing with OpenTelemetry - -{{ ydb-short-name }} SDK instruments Query Service operations with [OpenTelemetry](https://opentelemetry.io/) spans, providing distributed tracing from application code to each gRPC call to YDB. The spans are exported via the standard OTLP protocol and are compatible with Jaeger, Grafana Tempo, Zipkin, and any other backend that supports OpenTelemetry. - -## Created spans {#spans} - -Currently, spans are supported for Query Service operations. Support for topics and other services is planned for future SDK versions. - -The following spans are created: - -| Span | Type | Description | -| --- | --- | --- | -| `ydb.RunWithRetry` | `Internal` | Covers the entire retry cycle for a single operation | -| `ydb.Try` | `Internal` | One span per attempt, including the first; child RPC spans are attached to it | -| `ydb.CreateSession` | `Client` | Session creation via gRPC `CreateSession` and `AttachStream` | -| `ydb.ExecuteQuery` | `Client` | Execution of a single YQL query | -| `ydb.BeginTransaction` | `Client` | Explicit call to start a transaction | -| `ydb.Commit` | `Client` | Transaction commit | -| `ydb.Rollback` | `Client` | Transaction rollback | -| `ydb.Driver.Initialize` | `Internal` | Initial driver initialization: cluster discovery and authentication | - -A typical span tree for a transactional operation with retry looks as follows: - - -```text -ydb.RunWithRetry (Internal) -├─ ydb.Try (Internal) ← 1st attempt: ERROR -│ ├─ ydb.ExecuteQuery (Client) -│ ├─ ydb.ExecuteQuery (Client) -│ └─ ydb.Commit (Client) ← ERROR: Transaction Lock Invalidated -└─ ydb.Try (Internal) ← 2nd attempt: SUCCESS, ydb.retry.backoff_ms=50 - ├─ ydb.ExecuteQuery (Client) - ├─ ydb.ExecuteQuery (Client) - └─ ydb.Commit (Client) -``` - - -`ydb.RunWithRetry` is the parent span for the entire retry operation. For each execution attempt, a separate child span `ydb.Try` is created: the first `ydb.Try` corresponds to the first attempt, the second to the first **retry** attempt, and so on. RPC spans of a specific attempt, for example `ydb.ExecuteQuery` and `ydb.Commit`, are created inside the corresponding `ydb.Try`. - -{% note info %} - -If an attempt fails, its span `ydb.Try` ends with an error status. On a retry, a new `ydb.Try` is created; starting from the second attempt, the attribute `ydb.retry.backoff_ms` is set on it — the wait time before this attempt in milliseconds. This wait is included in the duration of the next span `ydb.Try`: the span starts before the backoff pause, then after the pause the RPC calls of this attempt are executed. - -{% endnote %} - -## Span attributes {#attributes} - -The SDK uses both standard OpenTelemetry semantic convention attributes and YDB-specific extensions. - -### Standard OpenTelemetry attributes - -The following attributes belong to the stable [OpenTelemetry semantic conventions](https://opentelemetry.io/docs/specs/semconv/) and can be processed by tracing backends as standard: - -| Attribute | Where set | Description | -| --- | --- | --- | -| `db.system.name` | RPC spans | Always `"ydb"` | -| `db.namespace` | RPC spans | YDB database path | -| `server.address` | RPC spans | Primary host from the connection string | -| `server.port` | RPC spans | Primary port from the connection string | -| `network.peer.address` | RPC spans | Actual gRPC endpoint used for the call | -| `network.peer.port` | RPC spans | Actual port of the gRPC endpoint used for the call | -| `error.type` | Spans that ended with an error | Error type. For example: `"transport_error"`, `"ydb_error"`, or the full exception class name | -| `db.response.status_code` | RPC spans when `YdbException` | Text name of the YDB status from the error, for example `ABORTED`, `UNAVAILABLE`, `OVERLOADED` | - -### YDB-specific attributes - -The following attributes are YDB extensions on top of the standard semantic conventions: - -| Attribute | Where set | Description | -| --- | --- | --- | -| `ydb.node.id` | RPC spans | ID of the YDB node that processed the request | -| `ydb.node.dc` | RPC spans | Datacenter of the YDB node that processed the request | -| `ydb.retry.backoff_ms` | Spans `ydb.Try`, starting from the second attempt | Wait time before a retry attempt in milliseconds | - -## W3C trace context {#w3c} - -The SDK automatically propagates the W3C `traceparent` header in every outgoing gRPC call. This allows the YDB server to trace internal operations within the same trace — without additional configuration. For more details on server-side tracing, see the section [Passing an external trace-id to {{ ydb-short-name }}](../../reference/observability/tracing/external-traces.md). - -## Connecting to the SDK {#integration} - -{% list tabs %} - -- Go - - Install the OpenTelemetry adapter for the {{ ydb-short-name }} Go SDK: - - - ```bash - go get github.com/ydb-platform/ydb-go-sdk-otel - ``` - - - Configure `TracerProvider` and pass the adapter to `ydb.Open`: - - - ```go - package main - - import ( - "context" - "os" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - - "github.com/ydb-platform/ydb-go-sdk/v3" - ydbOtel "github.com/ydb-platform/ydb-go-sdk-otel" - ) - - func main() { - ctx := context.Background() - - exporter, err := otlptracegrpc.New(ctx, - otlptracegrpc.WithEndpoint("localhost:4317"), - otlptracegrpc.WithInsecure(), - ) - if err != nil { - panic(err) - } - res, _ := resource.Merge(resource.Default(), resource.NewSchemaless( - attribute.String("service.name", "my-service"), - )) - tp := sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exporter), - sdktrace.WithResource(res), - ) - defer tp.Shutdown(ctx) - otel.SetTracerProvider(tp) - - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydbOtel.WithTraces( - ydbOtel.WithTracer(tp.Tracer("ydb-go-sdk")), - ), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - } - ``` - -- Python - - Install the additional dependencies `opentelemetry` and the OTLP exporter: - - - ```bash - pip install ydb[opentelemetry] - pip install opentelemetry-exporter-otlp-proto-grpc - ``` - - - Call `enable_tracing()` after configuring the global `TracerProvider`: - - - ```python - from opentelemetry import trace - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter - from opentelemetry.sdk.resources import Resource - - import ydb - from ydb.opentelemetry import enable_tracing - - resource = Resource(attributes={"service.name": "my-service"}) - provider = TracerProvider(resource=resource) - provider.add_span_processor( - BatchSpanProcessor(OTLPSpanExporter(endpoint="http://localhost:4317")) - ) - trace.set_tracer_provider(provider) - - enable_tracing() - - with ydb.Driver(endpoint="grpc://localhost:2136", database="/local") as driver: - driver.wait(timeout=5) - with ydb.QuerySessionPool(driver) as pool: - pool.execute_with_retries("SELECT 1") - - provider.shutdown() - ``` - -- C# - - Add the NuGet package: - - - ```bash - dotnet add package Ydb.Sdk.OpenTelemetry - ``` - - - Register the {{ ydb-short-name }} instrumentation when configuring OpenTelemetry in your service: - - - ```csharp - services.AddOpenTelemetry() - .WithTracing(builder => builder - .AddYdb() - .AddOtlpExporter()); - ``` - -- Java - - Add the YDB SDK and OpenTelemetry dependencies (example for Maven): - - - ```xml - <dependency> - <groupId>tech.ydb</groupId> - <artifactId>ydb-sdk-core</artifactId> - <version>${ydb.sdk.version}</version> - </dependency> - <dependency> - <groupId>io.opentelemetry</groupId> - <artifactId>opentelemetry-sdk</artifactId> - <version>${otel.version}</version> - </dependency> - <dependency> - <groupId>io.opentelemetry</groupId> - <artifactId>opentelemetry-exporter-otlp</artifactId> - <version>${otel.version}</version> - </dependency> - ``` - - - Create an instance of the OpenTelemetry SDK and pass it to the transport via `OpenTelemetryTracer`: - - - ```java - import io.opentelemetry.api.OpenTelemetry; - import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; - import io.opentelemetry.sdk.OpenTelemetrySdk; - import io.opentelemetry.sdk.resources.Resource; - import io.opentelemetry.sdk.trace.SdkTracerProvider; - import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; - import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; - import tech.ydb.core.auth.CloudAuthHelper; - import tech.ydb.core.grpc.GrpcTransport; - import tech.ydb.core.opentelemetry.OpenTelemetryTracer; - import tech.ydb.query.QueryClient; - - Resource resource = Resource.getDefault().toBuilder() - .put(ResourceAttributes.SERVICE_NAME, "my-service") - .build(); - - SdkTracerProvider tracerProvider = SdkTracerProvider.builder() - .setResource(resource) - .addSpanProcessor(BatchSpanProcessor.builder( - OtlpGrpcSpanExporter.builder() - .setEndpoint("http://localhost:4317") - .build() - ).build()) - .build(); - - OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() - .setTracerProvider(tracerProvider) - .build(); - - try (GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) - .withAuthProvider(CloudAuthHelper.getAuthProviderFromEnviron()) - .withTracer(OpenTelemetryTracer.fromOpenTelemetry(openTelemetry)) - .build(); - QueryClient queryClient = QueryClient.newClient(transport).build()) { - // Use queryClient here - } - ``` - - - When using the JDBC driver, simply add the `enableOpenTelemetryTracer=true` parameter to the connection string — the driver will pick up the global OTel provider automatically: - - - ```text - jdbc:ydb://<host>:<port>/<database>?enableOpenTelemetryTracer=true - ``` - -- C++ - - Include the OpenTelemetry tracing header from the {{ ydb-short-name }} C++ SDK and add a dependency on the OTel C++ SDK: - - - ```cpp - #include <ydb-cpp-sdk/client/driver/driver.h> - #include <ydb-cpp-sdk/open_telemetry/trace.h> - - #include <opentelemetry/exporters/otlp/otlp_http_exporter_factory.h> - #include <opentelemetry/exporters/otlp/otlp_http_exporter_options.h> - #include <opentelemetry/sdk/trace/tracer_provider.h> - #include <opentelemetry/sdk/trace/simple_processor_factory.h> - #include <opentelemetry/sdk/resource/resource.h> - #include <opentelemetry/trace/provider.h> - - namespace sdktrace = opentelemetry::sdk::trace; - namespace otlp = opentelemetry::exporter::otlp; - namespace resource = opentelemetry::sdk::resource; - using namespace NYdb; - - // 1. Initialize the OTel tracing provider - otlp::OtlpHttpExporterOptions opts; - opts.url = "http://localhost:4318/v1/traces"; - auto exporter = otlp::OtlpHttpExporterFactory::Create(opts); - auto processor = sdktrace::SimpleSpanProcessorFactory::Create(std::move(exporter)); - auto res = resource::Resource::Create({{"service.name", "my-service"}}); - auto otelProvider = std::make_shared<sdktrace::TracerProvider>( - std::move(processor), res); - opentelemetry::trace::Provider::SetTracerProvider(otelProvider); - - // 2. Wrap in the YDB tracing provider - auto ydbTraceProvider = NTrace::CreateOtelTraceProvider(otelProvider); - - // 3. Create the YDB driver with tracing enabled - auto driverConfig = TDriverConfig() - .SetEndpoint("localhost:2136") - .SetDatabase("/local") - .SetTraceProvider(ydbTraceProvider); - - TDriver driver(driverConfig); - ``` - -- JavaScript - - Install `@ydbjs/telemetry` together with the OpenTelemetry Node SDK and the OTLP exporter: - - - ```bash - npm install @ydbjs/telemetry @opentelemetry/sdk-node @opentelemetry/exporter-trace-otlp-http - ``` - - - Initialize `NodeSDK` before creating the driver and call `register()` from `@ydbjs/telemetry`: - - - ```js - import { NodeSDK } from '@opentelemetry/sdk-node' - import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http' - import { Driver } from '@ydbjs/core' - import { query } from '@ydbjs/query' - import { register } from '@ydbjs/telemetry' - - const sdk = new NodeSDK({ - serviceName: 'my-service', - traceExporter: new OTLPTraceExporter({ url: 'http://localhost:4318/v1/traces' }), - }) - sdk.start() - - // Must be called BEFORE creating the Driver — W3C propagation middleware - // trace context is set once during driver construction. - const instrumentation = register() - - using driver = new Driver(process.env.YDB_CONNECTION_STRING) - await driver.ready() - await using sql = query(driver) - // ... - - instrumentation.disable() - await sdk.shutdown() - ``` - - - Alternatively, via `--import` for auto-loading before the application starts: - - - ```bash - node --import @opentelemetry/sdk-node/register --import @ydbjs/telemetry/register your-app.js - ``` - -{% endlist %} diff --git a/ydb/docs/en/core/recipes/ydb-sdk/debug-otel.md b/ydb/docs/en/core/recipes/ydb-sdk/debug-otel.md deleted file mode 100644 index e8b21dec519..00000000000 --- a/ydb/docs/en/core/recipes/ydb-sdk/debug-otel.md +++ /dev/null @@ -1,371 +0,0 @@ -# Tracing with OpenTelemetry - -{{ ydb-short-name }} SDK instruments Query Service operations with [OpenTelemetry](https://opentelemetry.io/) spans, providing distributed tracing from application code to each gRPC call to YDB. The spans are exported via the standard OTLP protocol and are compatible with Jaeger, Grafana Tempo, Zipkin, and any other backend that supports OpenTelemetry. - -## Created spans {#spans} - -Currently, spans are supported for Query Service operations. Support for topics and other services is planned for future SDK versions. - -The following spans are created: - -| Span | Type | Description | -| --- | --- | --- | -| `ydb.RunWithRetry` | `Internal` | Covers the entire retry cycle for a single operation | -| `ydb.Try` | `Internal` | One span per attempt, including the first; child RPC spans are attached to it | -| `ydb.CreateSession` | `Client` | Creating a session via gRPC `CreateSession` and `AttachStream` | -| `ydb.ExecuteQuery` | `Client` | Execution of one YQL query | -| `ydb.BeginTransaction` | `Client` | Explicit call to start a transaction | -| `ydb.Commit` | `Client` | Transaction commit | -| `ydb.Rollback` | `Client` | Transaction rollback | -| `ydb.Driver.Initialize` | `Internal` | Initial driver initialization: cluster discovery and authentication | - -A typical span tree for a transactional operation with retry looks as follows: - - -```text -ydb.RunWithRetry (Internal) -├─ ydb.Try (Internal) ← 1st attempt: ERROR -│ ├─ ydb.ExecuteQuery (Client) -│ ├─ ydb.ExecuteQuery (Client) -│ └─ ydb.Commit (Client) ← ERROR: Transaction Lock Invalidated -└─ ydb.Try (Internal) ← 2nd attempt: SUCCESS, ydb.retry.backoff_ms=50 - ├─ ydb.ExecuteQuery (Client) - ├─ ydb.ExecuteQuery (Client) - └─ ydb.Commit (Client) -``` - - -`ydb.RunWithRetry` is the parent span for the entire retry operation. For each execution attempt, a separate child span `ydb.Try` is created: the first `ydb.Try` corresponds to the first attempt, the second to the first **retry** attempt, and so on. RPC spans of a specific attempt, for example `ydb.ExecuteQuery` and `ydb.Commit`, are created inside the corresponding `ydb.Try`. - -{% note info %} - -If an attempt ends with an error, its span `ydb.Try` ends with an error status. On a retry, a new `ydb.Try` is created; starting from the second attempt, the attribute `ydb.retry.backoff_ms` — the wait time before this attempt in milliseconds — is set on it. This wait time is included in the duration of the next span `ydb.Try`: the span starts before the backoff pause, then after the pause, the RPC calls of this attempt are executed. - -{% endnote %} - -## Span attributes {#attributes} - -The SDK uses both standard OpenTelemetry semantic conventions attributes and YDB-specific extensions. - -### Standard OpenTelemetry attributes - -The following attributes belong to the stable [OpenTelemetry semantic conventions](https://opentelemetry.io/docs/specs/semconv/) and can be processed by tracing backends as standard: - -| Attribute | Where installed | Description | -| --- | --- | --- | -| `db.system.name` | RPC spans | Always `"ydb"` | -| `db.namespace` | RPC spans | Path to YDB database | -| `server.address` | RPC spans | Main host from the connection string | -| `server.port` | RPC spans | Main port from the connection string | -| `network.peer.address` | RPC spans | Actual gRPC endpoint used for the call | -| `network.peer.port` | RPC spans | Actual gRPC endpoint port used for the call | -| `error.type` | Spans that failed | Error type. For example: `"transport_error"`, `"ydb_error"` or the full exception class name | -| `db.response.status_code` | RPC spans during `YdbException` | Textual name of the YDB status from the error, for example `ABORTED`, `UNAVAILABLE`, `OVERLOADED` | - -### YDB-specific attributes - -The following attributes are YDB extensions on top of the standard semantic conventions: - -| Attribute | Where it is installed | Description | -| --- | --- | --- | -| `ydb.node.id` | RPC spans | ID of the YDB node that processed the query | -| `ydb.node.dc` | RPC spans | Data center of the YDB node that processed the query | -| `ydb.retry.backoff_ms` | Spans `ydb.Try`, starting from the second attempt | Wait time before retry in milliseconds | - -## W3C trace context {#w3c} - -The SDK automatically propagates the W3C `traceparent` header in every outgoing gRPC call. This allows the YDB server to trace internal operations within the same trace, without additional configuration. For more details about server-side tracing, see the section [Passing an external trace-id to {{ ydb-short-name }}](../../reference/observability/tracing/external-traces.md). - -## Connecting to the SDK {#integration} - -{% list tabs %} - -- Go - - Install the OpenTelemetry adapter for the {{ ydb-short-name }} Go SDK: - - - ```bash - go get github.com/ydb-platform/ydb-go-sdk-otel - ``` - - - Configure `TracerProvider` and pass the adapter to `ydb.Open`: - - - ```go - package main - - import ( - "context" - "os" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - - "github.com/ydb-platform/ydb-go-sdk/v3" - ydbOtel "github.com/ydb-platform/ydb-go-sdk-otel" - ) - - func main() { - ctx := context.Background() - - exporter, err := otlptracegrpc.New(ctx, - otlptracegrpc.WithEndpoint("localhost:4317"), - otlptracegrpc.WithInsecure(), - ) - if err != nil { - panic(err) - } - res, _ := resource.Merge(resource.Default(), resource.NewSchemaless( - attribute.String("service.name", "my-service"), - )) - tp := sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exporter), - sdktrace.WithResource(res), - ) - defer tp.Shutdown(ctx) - otel.SetTracerProvider(tp) - - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydbOtel.WithTraces( - ydbOtel.WithTracer(tp.Tracer("ydb-go-sdk")), - ), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - } - ``` - -- Python - - Install additional dependencies `opentelemetry` and the OTLP exporter: - - - ```bash - pip install ydb[opentelemetry] - pip install opentelemetry-exporter-otlp-proto-grpc - ``` - - - Call `enable_tracing()` after configuring the global `TracerProvider`: - - - ```python - from opentelemetry import trace - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter - from opentelemetry.sdk.resources import Resource - - import ydb - from ydb.opentelemetry import enable_tracing - - resource = Resource(attributes={"service.name": "my-service"}) - provider = TracerProvider(resource=resource) - provider.add_span_processor( - BatchSpanProcessor(OTLPSpanExporter(endpoint="http://localhost:4317")) - ) - trace.set_tracer_provider(provider) - - enable_tracing() - - with ydb.Driver(endpoint="grpc://localhost:2136", database="/local") as driver: - driver.wait(timeout=5) - with ydb.QuerySessionPool(driver) as pool: - pool.execute_with_retries("SELECT 1") - - provider.shutdown() - ``` - -- C# - - Add the NuGet package: - - - ```bash - dotnet add package Ydb.Sdk.OpenTelemetry - ``` - - - Register the {{ ydb-short-name }} instrumentation when configuring OpenTelemetry in your service: - - - ```csharp - services.AddOpenTelemetry() - .WithTracing(builder => builder - .AddYdb() - .AddOtlpExporter()); - ``` - -- Java - - Add YDB SDK and OpenTelemetry dependencies (example for Maven): - - - ```xml - <dependency> - <groupId>tech.ydb</groupId> - <artifactId>ydb-sdk-core</artifactId> - <version>${ydb.sdk.version}</version> - </dependency> - <dependency> - <groupId>io.opentelemetry</groupId> - <artifactId>opentelemetry-sdk</artifactId> - <version>${otel.version}</version> - </dependency> - <dependency> - <groupId>io.opentelemetry</groupId> - <artifactId>opentelemetry-exporter-otlp</artifactId> - <version>${otel.version}</version> - </dependency> - ``` - - - Create an instance of the OpenTelemetry SDK and pass it to the transport via `OpenTelemetryTracer`: - - - ```java - import io.opentelemetry.api.OpenTelemetry; - import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; - import io.opentelemetry.sdk.OpenTelemetrySdk; - import io.opentelemetry.sdk.resources.Resource; - import io.opentelemetry.sdk.trace.SdkTracerProvider; - import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; - import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; - import tech.ydb.core.auth.CloudAuthHelper; - import tech.ydb.core.grpc.GrpcTransport; - import tech.ydb.core.opentelemetry.OpenTelemetryTracer; - import tech.ydb.query.QueryClient; - - Resource resource = Resource.getDefault().toBuilder() - .put(ResourceAttributes.SERVICE_NAME, "my-service") - .build(); - - SdkTracerProvider tracerProvider = SdkTracerProvider.builder() - .setResource(resource) - .addSpanProcessor(BatchSpanProcessor.builder( - OtlpGrpcSpanExporter.builder() - .setEndpoint("http://localhost:4317") - .build() - ).build()) - .build(); - - OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() - .setTracerProvider(tracerProvider) - .build(); - - try (GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) - .withAuthProvider(CloudAuthHelper.getAuthProviderFromEnviron()) - .withTracer(OpenTelemetryTracer.fromOpenTelemetry(openTelemetry)) - .build(); - QueryClient queryClient = QueryClient.newClient(transport).build()) { - // Use queryClient here - } - ``` - - - When using the JDBC driver, simply add the `enableOpenTelemetryTracer=true` parameter to the connection string — the driver will pick up the global OTel provider automatically: - - - ```text - jdbc:ydb://<host>:<port>/<database>?enableOpenTelemetryTracer=true - ``` - -- C++ - - Include the OpenTelemetry tracing header from the {{ ydb-short-name }} C++ SDK and add a dependency on the OTel C++ SDK: - - - ```cpp - #include <ydb-cpp-sdk/client/driver/driver.h> - #include <ydb-cpp-sdk/open_telemetry/trace.h> - - #include <opentelemetry/exporters/otlp/otlp_http_exporter_factory.h> - #include <opentelemetry/exporters/otlp/otlp_http_exporter_options.h> - #include <opentelemetry/sdk/trace/tracer_provider.h> - #include <opentelemetry/sdk/trace/simple_processor_factory.h> - #include <opentelemetry/sdk/resource/resource.h> - #include <opentelemetry/trace/provider.h> - - namespace sdktrace = opentelemetry::sdk::trace; - namespace otlp = opentelemetry::exporter::otlp; - namespace resource = opentelemetry::sdk::resource; - using namespace NYdb; - - // 1. Initialize the OTel tracing provider - otlp::OtlpHttpExporterOptions opts; - opts.url = "http://localhost:4318/v1/traces"; - auto exporter = otlp::OtlpHttpExporterFactory::Create(opts); - auto processor = sdktrace::SimpleSpanProcessorFactory::Create(std::move(exporter)); - auto res = resource::Resource::Create({{"service.name", "my-service"}}); - auto otelProvider = std::make_shared<sdktrace::TracerProvider>( - std::move(processor), res); - opentelemetry::trace::Provider::SetTracerProvider(otelProvider); - - // 2. Wrap in the YDB tracing provider - auto ydbTraceProvider = NTrace::CreateOtelTraceProvider(otelProvider); - - // 3. Create a YDB driver with tracing enabled - auto driverConfig = TDriverConfig() - .SetEndpoint("localhost:2136") - .SetDatabase("/local") - .SetTraceProvider(ydbTraceProvider); - - TDriver driver(driverConfig); - ``` - -- JavaScript - - Install `@ydbjs/telemetry` together with the OpenTelemetry Node SDK and OTLP exporter: - - - ```bash - npm install @ydbjs/telemetry @opentelemetry/sdk-node @opentelemetry/exporter-trace-otlp-http - ``` - - - Initialize `NodeSDK` before creating the driver and call `register()` from `@ydbjs/telemetry`: - - - ```js - import { NodeSDK } from '@opentelemetry/sdk-node' - import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http' - import { Driver } from '@ydbjs/core' - import { query } from '@ydbjs/query' - import { register } from '@ydbjs/telemetry' - - const sdk = new NodeSDK({ - serviceName: 'my-service', - traceExporter: new OTLPTraceExporter({ url: 'http://localhost:4318/v1/traces' }), - }) - sdk.start() - - // Must be called BEFORE creating the Driver — W3C propagation middleware - // trace context is set once during driver construction. - const instrumentation = register() - - using driver = new Driver(process.env.YDB_CONNECTION_STRING) - await driver.ready() - await using sql = query(driver) - // ... - - instrumentation.disable() - await sdk.shutdown() - ``` - - - Alternatively, use `--import` for auto-loading before starting the application: - - - ```bash - node --import @opentelemetry/sdk-node/register --import @ydbjs/telemetry/register your-app.js - ``` - -{% endlist %} diff --git a/ydb/docs/en/core/recipes/ydb-sdk/debug-prometheus.md b/ydb/docs/en/core/recipes/ydb-sdk/debug-prometheus.md deleted file mode 100644 index 855ae9ff88d..00000000000 --- a/ydb/docs/en/core/recipes/ydb-sdk/debug-prometheus.md +++ /dev/null @@ -1,106 +0,0 @@ -# Enabling metrics in Prometheus - -Below are examples of the code for enabling metrics in Prometheus in different {{ ydb-short-name }} SDKs. - -{% list tabs %} - -- Go - - {% list tabs %} - - - Native SDK - - ```go - package main - - import ( - "context" - - "github.com/prometheus/client_golang/prometheus" - metrics "github.com/ydb-platform/ydb-go-sdk-prometheus/v2" - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx := context.Background() - registry := prometheus.NewRegistry() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - metrics.WithTraces( - registry, - metrics.WithDetails(trace.DetailsAll), - metrics.WithSeparator("_"), - ), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - ... - } - ``` - - - database/sql - - ```go - package main - - import ( - "context" - "database/sql" - - "github.com/prometheus/client_golang/prometheus" - metrics "github.com/ydb-platform/ydb-go-sdk-prometheus/v2" - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx := context.Background() - registry := prometheus.NewRegistry() - nativeDriver, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - metrics.WithTraces( - registry, - metrics.WithDetails(trace.DetailsAll), - metrics.WithSeparator("_"), - ), - ) - if err != nil { - panic(err) - } - defer nativeDriver.Close(ctx) - - connector, err := ydb.Connector(nativeDriver) - if err != nil { - panic(err) - } - - db := sql.OpenDB(connector) - defer db.Close() - ... - } - ``` - - {% endlist %} - -- Java - - This functionality is not currently supported. - -- Python - - This functionality is not currently supported. - -- JavaScript - - {% include [work-in-progress](../../_includes/work-in-progress.md) %} - -- Rust - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - - Track progress or vote for Rust SDK support: [ydb-rs-sdk#267](https://github.com/ydb-platform/ydb-rs-sdk/issues/267) - -{% endlist %} diff --git a/ydb/docs/en/core/recipes/ydb-sdk/debug.md b/ydb/docs/en/core/recipes/ydb-sdk/debug.md deleted file mode 100644 index 6b54b61c897..00000000000 --- a/ydb/docs/en/core/recipes/ydb-sdk/debug.md +++ /dev/null @@ -1,12 +0,0 @@ -# Diagnosing problems - -When diagnosing problems related to {{ ydb-short-name }}, diagnostic tools help: logging, metrics, and distributed tracing. It is recommended to enable them in advance, before problems occur, so that when investigating an incident, you can see the full picture of the system state before, during, and after the failure. - -This section contains code recipes for enabling diagnostic tools in different {{ ydb-short-name }} SDKs. - -Contents: - -- [Enable logging](debug-logs.md) -- [Connect metrics to Prometheus](debug-prometheus.md) -- [Tracing with OpenTelemetry](debug-otel.md) -- [Connect tracing to Jaeger](debug-jaeger.md) diff --git a/ydb/docs/en/core/recipes/ydb-sdk/index.md b/ydb/docs/en/core/recipes/ydb-sdk/index.md index 650af008091..c1217c4f34f 100644 --- a/ydb/docs/en/core/recipes/ydb-sdk/index.md +++ b/ydb/docs/en/core/recipes/ydb-sdk/index.md @@ -31,11 +31,8 @@ Contents: - [Service discovery](service-discovery.md) - [Configuration publishing](config-publication.md) - [Leader election](leader-election.md) -- [Problem diagnostics](debug.md) - - [Enable logging](debug-logs.md) - - [Connect metrics to Prometheus](debug-prometheus.md) - - [Tracing with OpenTelemetry](debug-otel.md) +Connecting {{ ydb-short-name }} SDK diagnostics — logging, metrics, and distributed tracing — is described in [{#T}](../../reference/ydb-sdk/observability/index.md). See also: diff --git a/ydb/docs/en/core/recipes/ydb-sdk/toc_i.yaml b/ydb/docs/en/core/recipes/ydb-sdk/toc_i.yaml index cb7d29a231c..de23ade42c8 100644 --- a/ydb/docs/en/core/recipes/ydb-sdk/toc_i.yaml +++ b/ydb/docs/en/core/recipes/ydb-sdk/toc_i.yaml @@ -55,15 +55,3 @@ items: href: service-discovery.md - name: Configuration publication href: config-publication.md -- name: Troubleshooting - items: - - name: Overview - href: debug.md - - name: Enable logging - href: debug-logs.md - - name: Enable metrics in Prometheus - href: debug-prometheus.md - - name: Tracing with OpenTelemetry - href: debug-otel.md - - name: Connect tracing in Jaeger - href: debug-jaeger.md diff --git a/ydb/docs/en/core/reference/configuration/feature_flags.md b/ydb/docs/en/core/reference/configuration/feature_flags.md index 72af624eca5..92867eb88c7 100644 --- a/ydb/docs/en/core/reference/configuration/feature_flags.md +++ b/ydb/docs/en/core/reference/configuration/feature_flags.md @@ -1,29 +1,35 @@ # feature_flags -The `feature_flags` section enables or disables specific {{ ydb-short-name }} features using boolean flags. To enable a feature, set the corresponding feature flag to `true` in the cluster configuration. For example, to enable support for auto-partitioning of topics in the CDC, you need to add the following lines to the configuration: +The `feature_flags` section enables or disables certain {{ ydb-short-name }} features using boolean flags. To enable a feature, set the corresponding feature flag to `true` in the cluster configuration. For example, to enable support for auto-partitioning of topics in CDC, add the following lines to the configuration: + ```yaml feature_flags: enable_topic_autopartitioning_for_cdc: true ``` -## Feature Flags -| Flag | Feature | -|---------------------------| ----------------------------------------------------| -| `enable_fulltext_index` | [Fulltext index](../../dev/fulltext-indexes.md) for fulltext search | -| `enable_local_bloom_filter_index` | [Local Bloom skip index](../../dev/bloom-skip-indexes.md#types) of type `bloom_filter` | -| `enable_local_bloom_ngram_filter_index` | [Local Bloom skip index](../../dev/bloom-skip-indexes.md#types) of type `bloom_ngram_filter` | -| `enable_topic_autopartitioning_for_cdc` | [Auto-partitioning topics](../../concepts/cdc.md#topic-partitions) for row-oriented tables in CDC | -| `enable_access_to_index_impl_tables` | Support for [followers (read replicas)](../../yql/reference/syntax/alter_table/indexes.md) for covered secondary indexes | -| `enable_changefeeds_export`, `enable_changefeeds_import` | Support for changefeeds in backup and restore operations | -| `enable_view_export` | Support for views in backup and restore operations | -| `enable_export_auto_dropping` | Automatic cleanup of temporary tables and directories during export to S3 | -| `enable_followers_stats` | System views with information about [history of overloaded partitions](../../dev/system-views.md#top-overload-partitions) | -| `enable_strict_acl_check` | Strict ACL checks — do not allow granting rights to non-existent users and delete users with permissions | -| `enable_strict_user_management` | Strict checks for local users — only the cluster or database administrator can administer local users | -| `enable_database_admin` | The role of a database administrator | -| `enable_kafka_native_balancing` | Client balancing of partitions when reading using the [Kafka protocol](https://kafka.apache.org/documentation/#consumerconfigs_partition.assignment.strategy) | -| `enable_topic_compactification_by_key` | Enabling topic compactification in the [YDB Topics Kafka API](../../reference/kafka-api/index.md) | -| `enable_kafka_transactions` | Enabling transactions in the [YDB Topics Kafka API](../../reference/kafka-api/index.md) | -| `enable_grpc_audit` | Enabling [audit](../../security/audit-log.md#grpc-connection) of gRPC connection state changes | +## Feature flags + +| Flag | Function | +| --- | --- | +| `enable_json_index` | [JSON indexes](../../dev/json-indexes.md) to speed up search in JSON fields | +| `enable_json_index_auto_select` | Automatic selection of [JSON indexes](../../dev/json-indexes.md) when executing queries | +| `enable_fulltext_index` | [Full-text index](../../dev/fulltext-indexes.md) for full-text search | +| `enable_local_bloom_filter_index` | [Local Bloom index](../../dev/bloom-skip-indexes.md#types) of type `bloom_filter` | +| `enable_local_bloom_ngram_filter_index` | [Local Bloom index](../../dev/bloom-skip-indexes.md#types) of type `bloom_ngram_filter` | +| `enable_topic_autopartitioning_for_cdc` | [Auto-partitioning of topics](../../concepts/cdc.md#topic-partitions) in CDC for row tables | +| `enable_access_to_index_impl_tables` | Ability to [specify the number of replicas](../../yql/reference/syntax/alter_table/indexes.md) for a secondary index | +| `enable_changefeeds_export`, `enable_changefeeds_import` | Support for change feeds (changefeed) in backup and restore operations | +| `enable_view_export` | Support for views (`VIEW`) in backup and restore operations | +| `enable_export_auto_dropping` | Auto-deletion of temporary directories and tables when exporting to S3 | +| `enable_followers_stats` | System views with information about [history of overloaded partitions](../../dev/system-views#top-overload-partitions) | +| `enable_strict_acl_check` | Prohibition on granting permissions to non-existent users and on deleting users who have been granted permissions | +| `enable_strict_user_management` | Strict rules for administering local users (i.e., only the cluster or database administrator can administer local users) | +| `enable_database_admin` | Adding the database administrator role | +| `enable_kafka_native_balancing` | Client-side partition balancing when reading via [Kafka protocol](https://kafka.apache.org/documentation/#consumerconfigs_partition.assignment.strategy) | +| `enable_topic_compactification_by_key` | Enabling topic compaction in [YDB Topics Kafka API](../../reference/kafka-api/index.md) | +| `enable_kafka_transactions` | Enabling transactions in [YDB Topics Kafka API](../../reference/kafka-api/index.md) | +| `enable_external_data_sources` | Enabling [external data sources](../../concepts/datamodel/external_data_source.md) | +| `enable_grpc_audit` | Enabling [audit](../../security/audit-log.md#grpc-connection) of gRPC connection state changes | +| `enable_fs_backups` | Enabling [backup and restore operations to a network file system](../../concepts/backup.md#nfs) | diff --git a/ydb/docs/en/core/reference/configuration/hive.md b/ydb/docs/en/core/reference/configuration/hive_config.md index 8f65c4e39ab..8f65c4e39ab 100644 --- a/ydb/docs/en/core/reference/configuration/hive.md +++ b/ydb/docs/en/core/reference/configuration/hive_config.md diff --git a/ydb/docs/en/core/reference/configuration/host_configs.md b/ydb/docs/en/core/reference/configuration/host_configs.md index b26b51119b8..43e236ece91 100644 --- a/ydb/docs/en/core/reference/configuration/host_configs.md +++ b/ydb/docs/en/core/reference/configuration/host_configs.md @@ -10,16 +10,19 @@ host_configs: drive: - path: <path_to_device> type: <type> + disk_scope: <disk_scope> # optional attribute - path: ... - host_config_id: 2 ... ``` -The `host_config_id` attribute specifies a numeric configuration ID. The `drive` attribute contains a collection of descriptions of connected drives. Each description consists of two attributes: +The `host_config_id` attribute specifies a numeric configuration ID. The `drive` attribute contains a collection of descriptions of connected drives. Each description consists of two mandatory attributes: - `path`: Path to the mounted block device, for example, `/dev/disk/by-partlabel/ydb_disk_ssd_01` - `type`: Type of the device's physical media: `ssd`, `nvme`, or `rot` (rotational - HDD) +Additionally, an optional `disk_scope` attribute can be specified — a label for calculating fail domains for some reduced configurations, see [Configuring disk_scope](#disk-scope). + ## Examples One configuration with ID 1 and one SSD disk accessible via `/dev/disk/by-partlabel/ydb_disk_ssd_01`: @@ -52,6 +55,54 @@ host_configs: type: SSD ``` +## Configuring disk_scope {#disk-scope} + +`disk_scope` is an optional string attribute of a disk that defines a finer-grained failure zone within a single node. It is taken into account when calculating [fail domains](../../concepts/glossary.md#fail-domain) during the selection of disks for placing [VDisks](../../concepts/glossary.md#vdisk) of [storage groups](../../concepts/glossary.md#storage-group). + +A fail domain is determined by the physical location of a disk (data center, rack, server, physical device), so either no two VDisks of the same group can be placed on the same node (if the fail domain corresponds to a server or a server rack), or several VDisks of the same group can be placed on the same node on different disks (if the fail domain corresponds to an individual physical device). In the first case, it is impossible to create a configuration in `block-4-2` mode with fewer than 8 storage nodes, and in the second case the configuration might not stand the failure of a single node. To limit the number of VDisks of the same group placed on a single node, physical devices can be labeled with the `disk_scope` attribute, and fail domain calculation can be enabled at the `disk_scope` level. This makes it possible to build some fault-tolerant configurations in installations that have fewer servers than the number of fail domains required by the chosen [fault tolerance mode](../../concepts/topology.md). + +### Example {#disk-scope-example} + +A cluster of 4 servers with 4 disks on each server, in the `block-4-2` fault tolerance mode: + +``` yaml +host_configs: +- host_config_id: 1 + drive: + - path: /dev/disk/by-partlabel/ydb_disk_ssd_01 + type: SSD + disk_scope: fail-domain-1 + - path: /dev/disk/by-partlabel/ydb_disk_ssd_02 + type: SSD + disk_scope: fail-domain-1 + - path: /dev/disk/by-partlabel/ydb_disk_ssd_03 + type: SSD + disk_scope: fail-domain-2 + - path: /dev/disk/by-partlabel/ydb_disk_ssd_04 + type: SSD + disk_scope: fail-domain-2 +``` + +To calculate fail domains at the `disk_scope` level for the static group and other storage groups, specify the corresponding fail domain type in the yaml config (top-level key): + +``` yaml +fail_domain_type: disk_scope +``` + +And in the domain configuration: + +```yaml +domains: +- domain_name: <domain name> + ... + storage_pool_kinds: + - kind: <type of physical devices used> + fail_domain_type: disk_scope + ... +``` + +With this configuration, two VDisks of each storage group will be placed on each server, and the maximum tolerated failure in such a configuration is the failure of a single server. + ## Kubernetes Features {#host-configs-k8s} The {{ ydb-short-name }} Kubernetes operator mounts NBS disks for Storage nodes at the path `/dev/kikimr_ssd_00`. To use them, the following `host_configs` configuration must be specified: diff --git a/ydb/docs/en/core/reference/configuration/index.md b/ydb/docs/en/core/reference/configuration/index.md index 76ab6831c2f..af930764d40 100644 --- a/ydb/docs/en/core/reference/configuration/index.md +++ b/ydb/docs/en/core/reference/configuration/index.md @@ -21,10 +21,10 @@ The following top-level configuration sections are available, listed in alphabet || [{#T}](domains_config.md) | No | Cluster domain configuration including Blob Storage and State Storage || || [{#T}](feature_flags.md) | No | Feature flags to enable or disable specific {{ ydb-short-name }} features || || [{#T}](healthcheck_config.md) | No | Health check service thresholds and timeout settings || -|| [{#T}](hive.md) | No | Hive component configuration for tablet management || +|| [{#T}](hive_config.md) | No | Hive component configuration for tablet management || || [{#T}](host_configs.md) | No | Typical host configurations for cluster nodes || || [{#T}](hosts.md) | Yes | Static cluster nodes configuration || -|| [{#T}](kafka.md) | No | [Kafka Proxy](../../reference/kafka-api/index.md) configuration || +|| [{#T}](kafka_proxy_config.md) | No | [Kafka Proxy](../../reference/kafka-api/index.md) configuration || || [{#T}](log_config.md) | No | Logging configuration and parameters || || [{#T}](memory_controller_config.md) | No | Memory allocation and limits for database components || || [{#T}](node_broker_config.md) | No | Stable node names configuration || diff --git a/ydb/docs/en/core/reference/configuration/kafka.md b/ydb/docs/en/core/reference/configuration/kafka_proxy_config.md index a4593cd1d6e..065a248530b 100644 --- a/ydb/docs/en/core/reference/configuration/kafka.md +++ b/ydb/docs/en/core/reference/configuration/kafka_proxy_config.md @@ -8,6 +8,7 @@ The `kafka_proxy_config` section of the {{ ydb-short-name }} configuration file | --- | --- | --- | --- | | `enable_kafka_proxy` | bool | `false` | Enables or disables Kafka Proxy. | | `listening_port` | int32 | `9092` | The port on which the Kafka API will be available. | +| `listening_address` | string | `[::]` | The network address on which Kafka Proxy listens for incoming connections. Use `[::]` to listen on all interfaces (dual-stack, requires IPv6 to be enabled), `127.0.0.1` or `[::1]` to restrict access to localhost. | | `transaction_timeout_ms` | uint32 | `300000` (5 minutes) | The maximum timeout for Kafka transactions, after which the transaction will be cancelled. | | `auto_create_topics_enable` | bool | `false` | Enables automatic creation of topics when they are accessed. Analogous to [the same option](https://kafka.apache.org/documentation/#brokerconfigs_auto.create.topics.enable) in Apache Kafka. | | `auto_create_consumers_enable` | bool | `true` | Enables automatic registration of consumers when they are accessed. | @@ -22,6 +23,7 @@ The `kafka_proxy_config` section of the {{ ydb-short-name }} configuration file kafka_proxy_config: enable_kafka_proxy: true listening_port: 9092 + listening_address: "[::]" transaction_timeout_ms: 300000 # 5 minutes auto_create_topics_enable: true auto_create_consumers_enable: true diff --git a/ydb/docs/en/core/reference/configuration/monitoring_config.md b/ydb/docs/en/core/reference/configuration/monitoring_config.md new file mode 100644 index 00000000000..ea734d88044 --- /dev/null +++ b/ydb/docs/en/core/reference/configuration/monitoring_config.md @@ -0,0 +1,47 @@ +# monitoring_config + +The `monitoring_config` section of the {{ ydb-short-name }} configuration file configures [YDB Monitoring](../embedded-ui/ydb-monitoring.md). The parameters below control [authentication](../../security/authentication.md) on individual embedded monitoring pages. + +```yaml +monitoring_config: + # authentication on the /counters and /healthcheck pages + require_counters_authentication: false + require_healthcheck_authentication: false +``` + +## Authentication on Monitoring Pages {#authentication} + +#| +|| Parameter | Description || +|| `require_counters_authentication` | Selects mandatory [authentication](../../security/authentication.md) mode for the `/counters` and `/counters/hosts` pages. + +Valid values: + +- `true`: Access to `/counters` and `/counters/hosts` requires an [auth token](../../concepts/glossary.md#auth-token). Requests undergo authentication and authorization. + + The `true` value is allowed only when mandatory [authentication](../../security/authentication.md) is enabled in the [security_config](./security_config.md) section of the {{ ydb-short-name }} configuration file. + +- `false`: Requests to `/counters` and `/counters/hosts` can be made without an [auth token](../../concepts/glossary.md#auth-token). + +Default value: `false`. + || +|| `require_healthcheck_authentication` | Adds an [authentication](../../security/authentication.md) requirement for the `/healthcheck` endpoint on top of the cluster-wide rules. + +Valid values: + +- `true`: Any `/healthcheck` response, including [Prometheus format](https://prometheus.io/docs/instrumenting/exposition_formats/) output (the `format=prometheus` parameter), is returned only for requests with an [auth token](../../concepts/glossary.md#auth-token). Requests undergo authentication and authorization. + + The `true` value is allowed only when mandatory [authentication](../../security/authentication.md) is enabled in the [security_config](./security_config.md) section of the {{ ydb-short-name }} configuration file. + +- `false`: When mandatory authentication is enabled in the cluster, requests to `/healthcheck` without a token are still allowed if [Prometheus format](https://prometheus.io/docs/instrumenting/exposition_formats/) output is requested (`format=prometheus`). Cluster-wide rules apply to all other `/healthcheck` response formats (see the note below). + +Default value: `false`. + +{% note info %} + +If mandatory [authentication](../../security/authentication.md) is enabled in [security_config](./security_config.md), an auth token is required for `/healthcheck` responses in any format other than Prometheus, regardless of the `require_healthcheck_authentication` value. + +{% endnote %} + +|| +|# diff --git a/ydb/docs/en/core/reference/configuration/toc_p.yaml b/ydb/docs/en/core/reference/configuration/toc_p.yaml index a5b0c7aca1c..530013aa1b7 100644 --- a/ydb/docs/en/core/reference/configuration/toc_p.yaml +++ b/ydb/docs/en/core/reference/configuration/toc_p.yaml @@ -21,14 +21,20 @@ items: href: query_service_config.md - name: healthcheck_config href: healthcheck_config.md +- name: hive_config + href: hive_config.md - name: host_configs href: host_configs.md - name: hosts href: hosts.md +- name: kafka_proxy_config + href: kafka_proxy_config.md - name: log_config href: log_config.md - name: memory_controller_config href: memory_controller_config.md +- name: monitoring_config + href: monitoring_config.md - name: node_broker_config href: node_broker_config.md - name: resource_broker_config @@ -43,7 +49,3 @@ items: href: table_service_config.md - name: tli_config href: tli_config.md -- name: hive_config - href: hive.md -- name: kafka_proxy_config - href: kafka.md diff --git a/ydb/docs/en/core/reference/kafka-api/auth.md b/ydb/docs/en/core/reference/kafka-api/auth.md index 6645cf358f3..e493f1cdfdc 100644 --- a/ydb/docs/en/core/reference/kafka-api/auth.md +++ b/ydb/docs/en/core/reference/kafka-api/auth.md @@ -154,7 +154,7 @@ ssl.endpoint.identification.algorithm= #### YDB configuration -It is necessary to specify the required fields in the [kafka_proxy_config](../configuration/kafka.md). +It is necessary to specify the required fields in the [kafka_proxy_config](../configuration/kafka_proxy_config.md). ```yaml kafka_proxy_config: diff --git a/ydb/docs/en/core/reference/observability/tracing/external-traces.md b/ydb/docs/en/core/reference/observability/tracing/external-traces.md index a293348daa5..5dec01a970b 100644 --- a/ydb/docs/en/core/reference/observability/tracing/external-traces.md +++ b/ydb/docs/en/core/reference/observability/tracing/external-traces.md @@ -19,4 +19,4 @@ If the [`external_throttling`](./setup.md#external-throttling) section is presen ## SDK support -Some SDKs support trace-id passing; you can find their list and usage examples on the [{#T}](../../../recipes/ydb-sdk/debug-otel.md) page. +Some SDKs support trace-id passing; you can find their list and usage examples on the [{#T}](../../ydb-sdk/observability/tracing/opentelemetry.md) page. diff --git a/ydb/docs/en/core/yql/reference/syntax/_includes/index_grammar_explanation.md b/ydb/docs/en/core/yql/reference/syntax/_includes/index_grammar_explanation.md index 5a526a3acaf..d1b89d58256 100644 --- a/ydb/docs/en/core/yql/reference/syntax/_includes/index_grammar_explanation.md +++ b/ydb/docs/en/core/yql/reference/syntax/_includes/index_grammar_explanation.md @@ -1,24 +1,22 @@ -* `GLOBAL/LOCAL` — global or local index; depending on the index type (`<index_type>`), only one of them may be available: - - * `GLOBAL` — an index implemented as a separate table or set of tables. Synchronous updates to such an index require distributed transactions. - * `LOCAL` — a local index within a shard of a row-oriented or column-oriented table. Does not require distributed transactions for updates, but does not provide pruning during search. +* `GLOBAL/LOCAL` — global or local index, depending on the index type (`<index_type>`), only one of them may be available: -* `<index_name>` — a unique name of the index that will be used to access data. -* `UNIQUE` — required to create a [unique index](../../../../concepts/query_execution/secondary_indexes.md#unique). A unique index must always be created as global and synchronous (`GLOBAL UNIQUE SYNC`) and must not specify a `USING <index_type>` clause. -* `SYNC/ASYNC` — the index synchronization mode. + * `GLOBAL` — an index implemented as a separate table or a set of tables. Synchronous update of such an index requires distributed transactions. + * `LOCAL` — a local index within a shard of a columnar or row-based table, does not require distributed transactions during update, but does not provide pruning during search. +* `<index_name>` — unique index name by which data can be accessed. +* `SYNC/ASYNC` — indicator of index synchrony. - * `SYNC` — a [synchronous](../../../../concepts/query_execution/secondary_indexes.md#sync) index. This is the default value. - * `ASYNC` — an [asynchronous](../../../../concepts/query_execution/secondary_indexes.md#async) index. + * `SYNC` - [synchronous](../../../../concepts/query_execution/secondary_indexes.md#sync) index. Default value. + * `ASYNC` - [asynchronous](../../../../concepts/query_execution/secondary_indexes.md#async) index. +* `UNIQUE` — indicator of a [unique secondary index](../../../../concepts/query_execution/secondary_indexes.md#unique). A unique index must be global synchronous (`GLOBAL UNIQUE SYNC`) and must not contain the `USING <index_type>` construct. +* `<index_type>` - index type, currently supported: -* `<index_type>` — index type, currently supported: - - * `secondary` — secondary index. Only `GLOBAL` mode is available for secondary indexes. This is the default index type. - * `vector_kmeans_tree` — vector index. Described in detail in [{#T}](../create_table/vector_index.md). - * `fulltext_plain` — basic fulltext index. Described in detail in [{#T}](../create_table/fulltext_index.md). - * `fulltext_relevance` — fulltext index with [BM25](https://en.wikipedia.org/wiki/Okapi_BM25) statistics for relevance scoring. Described in detail in [{#T}](../create_table/fulltext_index.md). - * `bloom_filter` — local Bloom skip index. Only `LOCAL` is available. See [ALTER TABLE ADD INDEX](../alter_table/indexes.md#local-bloom). - * `bloom_ngram_filter` — local N-gram Bloom skip index. Only `LOCAL` is available. See [ALTER TABLE ADD INDEX](../alter_table/indexes.md#local-bloom). - -* `<index_columns>` — comma-separated list of column names for the table being created. This list defines the composition and order of columns included in the index key. Must be specified. The index key will include both the columns listed and the columns from the table's primary key. -* `<cover_columns>` — comma-separated list of column names from the created table that will be saved in the index in addition to index key columns, providing the ability to get additional data without accessing the table. Empty by default. -* `<parameter_name>` and `<parameter_value>` — index parameters specific to a particular `<index_type>`. + * `secondary` — secondary index. Only the `GLOBAL` mode is available for secondary indexes. This is the default index type. + * `vector_kmeans_tree` — vector index. More details are described in the [{#T}](../create_table/vector_index.md) section. + * `fulltext_plain` — basic full-text index. More details are described in [{#T}](../create_table/fulltext_index.md). + * `fulltext_relevance` — full-text index with [BM25](https://en.wikipedia.org/wiki/Okapi_BM25) statistics for relevance calculation. More details are described in [{#T}](../create_table/fulltext_index.md). + * `json` — JSON index to speed up predicates `JSON_EXISTS` and `JSON_VALUE` on a column of type `Json` or `JsonDocument`. More details are described in [{#T}](../create_table/json_index.md). + * `bloom_filter` — local Bloom index. Only `LOCAL` is available. See [ALTER TABLE ADD INDEX](../alter_table/indexes.md#local-bloom). + * `bloom_ngram_filter` — local N-gram Bloom index. Only `LOCAL` is available. See [ALTER TABLE ADD INDEX](../alter_table/indexes.md#local-bloom). +* `<index_columns>` — comma-separated list of column names of the table being created, which determines the composition and order of columns included in the index key. Must be specified. The index key will consist of these columns with the addition of the table's primary key columns. +* `<cover_columns>` — comma-separated list of column names of the table being created that will be stored in the index in addition to the index key columns, allowing you to get additional data without accessing the table. Empty by default. +* `<parameter_name>` and `<parameter_value>` are index parameters specific to a particular `<index_type>`. diff --git a/ydb/docs/en/core/yql/reference/syntax/alter-resource-pool-classifier.md b/ydb/docs/en/core/yql/reference/syntax/alter-resource-pool-classifier.md index 4414c69b316..6c3ebd41593 100644 --- a/ydb/docs/en/core/yql/reference/syntax/alter-resource-pool-classifier.md +++ b/ydb/docs/en/core/yql/reference/syntax/alter-resource-pool-classifier.md @@ -1,6 +1,6 @@ # ALTER RESOURCE POOL CLASSIFIER -`ALTER RESOURCE POOL CLASSIFIER` changes the definition of a [resource pool classifier](../../../concepts/glossary.md#resource-pool-classifier). +`ALTER RESOURCE POOL CLASSIFIER` changes the definition of a [resource pool classifier](../../../concepts/glossary.md#resource-pool-classifier.md). ## Syntax @@ -8,47 +8,57 @@ The syntax for changing any resource pool classifier parameter is as follows: + ```yql ALTER RESOURCE POOL CLASSIFIER <name> SET (<key> = <value>); ``` + `<key>` is the parameter name, `<value>` is its new value. For example, the following command changes the user to which the rule applies: + ```yql ALTER RESOURCE POOL CLASSIFIER olap_classifier SET (MEMBER_NAME = "user2@domain"); ``` + ### Resetting parameters The command to reset a resource pool classifier parameter is as follows: + ```yql ALTER RESOURCE POOL CLASSIFIER <name> RESET (<key>); ``` + `<key>` is the parameter name. For example, the following command resets the `MEMBER_NAME` setting: + ```yql ALTER RESOURCE POOL CLASSIFIER olap_classifier RESET (MEMBER_NAME); ``` + ## Permissions The `ALL` [permission](grant.md#permissions-list) on the database is required. Example of granting it: + ```yql GRANT 'ALL' ON `/my_db` TO `user1@domain`; ``` + ## Parameters * `RANK` (Int64) — Optional field that defines the order in which resource pool classifiers are chosen. If omitted, the maximum existing `RANK` is taken and 1000 is added. Allowed values: a unique number in the range $[0, 2^{63}-1]$. * `RESOURCE_POOL` (String) — Required field: name of the resource pool to which queries matching the classifier criteria are sent. -* `MEMBER_NAME` (String) — Optional field specifying which user or group is routed to the given resource pool. If omitted, the classifier ignores `MEMBER_NAME` and classification uses other criteria. +* `MEMBER_NAME` (String) — Optional field specifying which user or group is routed to the given resource pool. If omitted, the classifier ignores ⟦C2⟧ and classification uses other criteria. ## See also diff --git a/ydb/docs/en/core/yql/reference/syntax/alter_table/indexes.md b/ydb/docs/en/core/yql/reference/syntax/alter_table/indexes.md index 83eee7b18e5..fc382b2728c 100644 --- a/ydb/docs/en/core/yql/reference/syntax/alter_table/indexes.md +++ b/ydb/docs/en/core/yql/reference/syntax/alter_table/indexes.md @@ -1,8 +1,9 @@ -# Adding, removing, and renaming a index +# Adding, deleting, and renaming an index ## Adding an index {#add-index} -`ADD INDEX` — adds an index with the specified name and type for a given set of columns. Grammar: +`ADD INDEX` — adds an index with the specified name and type for the given set of columns in {% if backend_name == "YDB" and oss == true %}row tables.{% else %}tables.{% endif %} Grammar: + ```yql ALTER TABLE `<table_name>` @@ -17,14 +18,16 @@ ALTER TABLE `<table_name>` [, ...] ``` + {% include [index_grammar_explanation.md](../_includes/index_grammar_explanation.md) %} Parameters for all index types: -* `parallel` - maximum number of parallel [partition](../../../../concepts/glossary.md#partition)-based workers used during index build (an integer between `1` and `MaxBuildIndexShardsInFlight` from `SchemeShardConfig`). - - If not specified, currently defaults to `32` or `MaxBuildIndexShardsInFlight` if it's lower. Default `MaxBuildIndexShardsInFlight` is `1000`. Default parallelism selection logic may be changed in future versions. - - You may set a smaller limit to reduce the impact of index build on the DB performance. - - You may also set a larger limit to speed up the index build if you have enough hardware resources. +* - maximum number of `parallel` handlers based on [partitions](../../../../concepts/glossary.md#partition) involved in index building (an integer between `1` and `MaxBuildIndexShardsInFlight` from `SchemeShardConfig`). + +- If the parameter is not specified, the default value `32` or `MaxBuildIndexShardsInFlight` is currently used, whichever is smaller. `MaxBuildIndexShardsInFlight` defaults to `1000`. In future versions, the default parallelism selection logic may change. +- You can set a lower limit to reduce the impact of index building on database performance. +- You can also set a higher limit to speed up index building if you have sufficient hardware resources. Parameters specific to vector indexes: @@ -32,29 +35,29 @@ Parameters specific to vector indexes: {% note info %} -For vector indexes, the vector_type and vector_dimension parameters can be omitted if the table is not empty — they are determined automatically based on the row contents. The levels and clusters parameters are also determined automatically, and the table may be empty for them, but this is highly unrecommended because the default values in that case are levels=1, clusters=2. It is far better to create the index on a table that already has data loaded, so that the values can be determined correctly. +For vector indexes, the `vector_type` and `vector_dimension` parameters can be omitted if the table is not empty — they are determined automatically from the row contents. The `levels` and `clusters` parameters are also determined automatically, and for them the table can be empty, but doing so is strongly not recommended because the default values in this case are `levels`=1, `clusters`=2; it is much better to create the index on a table that already has data loaded, so that the values can be correctly determined. {% endnote %} -Parameters specific to fulltext indexes: +Parameters specific to full-text indexes: {% include [fulltext_index_parameters.md](../_includes/fulltext_index_parameters.md) %} -### Local Bloom skip index parameters {#local-bloom} +### Parameters of local bloom indexes {#local-bloom} {% include [bloom_skip_index_parameters.md](../_includes/bloom_skip_index_parameters.md) %} -{% if backend_name == "YDB" %} +{% if backend_name == "YDB" and oss == true %} -You can also add a secondary index using the {{ ydb-short-name }} CLI [table index](../../../../reference/ydb-cli/commands/secondary_index.md#add) command. +You can also add a secondary index using the [table index](../../../../reference/ydb-cli/commands/secondary_index.md#add) {{ ydb-short-name }} CLI command. {% endif %} ### Limitations -The `ADD INDEX` operation for creating global secondary (`GLOBAL`, `UNIQUE`, and so on) and vector indexes is supported only for row-oriented tables. For [column-oriented tables](../../../../concepts/datamodel/table.md#column-oriented-tables), `ADD INDEX` [supports only local Bloom skip indexes](#local-bloom). +The `ADD INDEX` operation for creating global secondary (`GLOBAL`, `UNIQUE`, etc.) and vector indexes is supported only for row tables. For [columnar tables](../../../../concepts/datamodel/table.md#column-oriented-tables), via `ADD INDEX`, [only local bloom indexes are supported](#local-bloom). -Local Bloom skip index behavior: +Features of local bloom indexes: {% include [bloom_skip_index_features.md](../_includes/bloom_skip_index_features.md) %} @@ -66,7 +69,8 @@ Local Bloom skip index behavior: ### Examples -A regular secondary index: +Secondary index: + ```yql ALTER TABLE `series` @@ -74,8 +78,10 @@ ALTER TABLE `series` GLOBAL ON (`title`); ``` + [Vector index](../../../../dev/vector-indexes.md): + ```yql ALTER TABLE `series` ADD INDEX emb_cosine_idx GLOBAL SYNC USING vector_kmeans_tree @@ -85,7 +91,9 @@ ALTER TABLE `series` ); ``` -A fulltext index: + +Full-text index: + ```yql ALTER TABLE `series` @@ -94,7 +102,19 @@ ALTER TABLE `series` WITH (tokenizer=standard, use_filter_lowercase=true); ``` -A bloom index: + +[JSON index](../../../../dev/json-indexes.md): + + +```yql +ALTER TABLE `series` + ADD INDEX json_idx GLOBAL USING json + ON (metadata); +``` + + +[Bloom index](../../../../dev/bloom-skip-indexes.md): + ```yql ALTER TABLE `/Root/Table` @@ -103,7 +123,9 @@ ALTER TABLE `/Root/Table` WITH (false_positive_probability = 0.01); ``` -A bloom ngram index: + +Bloom n-gram index: + ```yql ALTER TABLE `/Root/Table` @@ -114,55 +136,58 @@ ALTER TABLE `/Root/Table` false_positive_probability = 0.01, case_sensitive = true ); -``` -## Altering an index {#alter-index} +# # Changing index parameters {#alter-index} -Indexes have type-specific parameters that can be tuned. Global indexes, whether [synchronous]({{ concept_secondary_index }}#sync) or [asynchronous]({{ concept_secondary_index }}#async), are implemented as hidden tables, and their automatic partitioning and followers settings can be adjusted just like those of regular tables. +Индексы имеют параметры, зависящие от типа, которые можно настраивать. Глобальные индексы, [синхронные](yfmvar-0-yfmvarend#sync) или [асинхронные](yfmvar-1-yfmvarend#async), реализованы в виде скрытых таблиц, и их параметры автоматического партиционирования и реплик можно регулировать так же, как и настройки обычных таблиц. {% note info %} -Currently, specifying secondary index partitioning settings during index creation is not supported in either the [`ALTER TABLE ADD INDEX`](#add-index) or the [`CREATE TABLE INDEX`](../create_table/secondary_index.md) statements. +В настоящее время задание настроек партиционирования вторичных индексов при создании индекса не поддерживается ни в операторе [`ALTER TABLE ADD INDEX`](#add-index), ни в операторе [`CREATE TABLE INDEX`](../create_table/secondary_index.md). {% endnote %} -```sql +```yql ALTER TABLE <table_name> ALTER INDEX <index_name> SET <setting_name> <value>; ALTER TABLE <table_name> ALTER INDEX <index_name> SET (<setting_name_1> = <value_1>, ...); ``` -* `<table_name>`: The name of the table whose index is to be modified. -* `<index_name>`: The name of the index to be modified. -* `<setting_name>`: The name of the setting to be modified. Allowed settings depend on the index type: - * for global secondary indexes: - * [AUTO_PARTITIONING_BY_SIZE]({{ concept_table }}#auto_partitioning_by_size) - * [AUTO_PARTITIONING_BY_LOAD]({{ concept_table }}#auto_partitioning_by_load) - * [AUTO_PARTITIONING_PARTITION_SIZE_MB]({{ concept_table }}#auto_partitioning_partition_size_mb) - * [AUTO_PARTITIONING_MIN_PARTITIONS_COUNT]({{ concept_table }}#auto_partitioning_min_partitions_count) - * [AUTO_PARTITIONING_MAX_PARTITIONS_COUNT]({{ concept_table }}#auto_partitioning_max_partitions_count) - * [READ_REPLICAS_SETTINGS]({{ concept_table }}#read_only_replicas) - * for local Bloom skip indexes (see [Local Bloom skip index parameters](#local-bloom)): - * `FALSE_POSITIVE_PROBABILITY` - * `NGRAM_SIZE` and `CASE_SENSITIVE` (for `bloom_ngram_filter` only) +* `<table_name>` - name of the table whose index needs to be changed. +* `<index_name>` - name of the index to change. +* `<setting_name>` - name of the parameter to change. The set of allowed parameters depends on the index type: + + * for global secondary indexes: + + * [AUTO_PARTITIONING_BY_SIZE]({{ concept_table }}#auto_partitioning_by_size) + * [AUTO_PARTITIONING_BY_LOAD]({{ concept_table }}#auto_partitioning_by_load) + * [AUTO_PARTITIONING_PARTITION_SIZE_MB]({{ concept_table }}#auto_partitioning_partition_size_mb) + * [AUTO_PARTITIONING_MIN_PARTITIONS_COUNT]({{ concept_table }}#auto_partitioning_min_partitions_count) + * [AUTO_PARTITIONING_MAX_PARTITIONS_COUNT]({{ concept_table }}#auto_partitioning_max_partitions_count) + * [READ_REPLICAS_SETTINGS]({{ concept_table }}#read_only_replicas) + * for local bloom indexes (see [Parameters of local bloom indexes](#local-bloom)): + + * `FALSE_POSITIVE_PROBABILITY` + * `NGRAM_SIZE` and `CASE_SENSITIVE` (only for `bloom_ngram_filter`) {% note info %} -`RESET` is not supported for `ALTER INDEX`. +The `RESET` operation for `ALTER INDEX` is not supported. {% endnote %} -* `<value>`: The new value for the setting. Possible values include: - * `ENABLED` or `DISABLED` for the `AUTO_PARTITIONING_BY_SIZE` and `AUTO_PARTITIONING_BY_LOAD` settings - * `"PER_AZ:<count>"` or `"ANY_AZ:<count>"` where `<count>` is the number of replicas for the `READ_REPLICAS_SETTINGS` - * An integer of `Uint64` type for the other settings - * A floating-point value in `(0, 1)` for `FALSE_POSITIVE_PROBABILITY`; smaller values usually reduce false positives but increase index size - * An integer value from `3` to `8` for `NGRAM_SIZE` (a typical starting point is `3`) - * `true` or `false` for `CASE_SENSITIVE` +* `<value>` - new parameter value. Possible values include: + + * `ENABLED` or `DISABLED` for the `AUTO_PARTITIONING_BY_SIZE` and `AUTO_PARTITIONING_BY_LOAD` parameters + * `"PER_AZ:<count>"` or `"ANY_AZ:<count>"` where `<count>` is the number of replicas for `READ_REPLICAS_SETTINGS` + * for other parameters — an integer of type `Uint64` + * for `FALSE_POSITIVE_PROBABILITY` — a floating-point number in the range `(0, 1)`; a smaller value usually reduces the number of false positives but increases the index size + * for `NGRAM_SIZE` — an integer in the range from `3` to `8` (usually recommended to start with `3`) + * for `CASE_SENSITIVE` — `true` or `false` ### Example -The query in the following example enables automatic partitioning by load for the index named `title_index` of the table `series`, sets its minimum partition count to 5, and enables one follower per AZ for every partition: +The code in the following example enables automatic partitioning by load for the index named `title_index` in the table `series`, sets the minimum number of partitions to 5, and starts one replica in each availability zone (AZ) for each partition: ```yql @@ -173,7 +198,9 @@ ALTER TABLE `series` ALTER INDEX `title_index` SET ( ); ``` -For local Bloom skip indexes, you can also alter index-specific parameters, for example: + +For local bloom indexes, you can also change their specific parameters, for example: + ```yql ALTER TABLE `/Root/Table` ALTER INDEX idx_ngram SET ( @@ -183,35 +210,37 @@ ALTER TABLE `/Root/Table` ALTER INDEX idx_ngram SET ( ); ``` + ## Deleting an index {#drop-index} -`DROP INDEX`: Deletes the index with the specified name. The code below deletes the index named `title_index`. +`DROP INDEX` — deletes the index with the specified name. The code below will delete the index named `title_index`. + ```yql ALTER TABLE `series` DROP INDEX `title_index`; ``` -{% if backend_name == "YDB" %} -You can also remove a index using the {{ ydb-short-name }} CLI [table index](../../../../reference/ydb-cli/commands/secondary_index.md#drop) command. +{% if backend_name == "YDB" and oss == true %} -{% endif %} +You can also delete an index using the [table index](../../../../reference/ydb-cli/commands/secondary_index.md#drop) {{ ydb-short-name }} CLI command. -## Renaming an index {#rename-index} +{% endif %} -`RENAME INDEX`: Renames the index with the specified name. +## Renaming a secondary index {#rename-secondary-index} -If an index with the new name exists, an error is returned. +`RENAME INDEX` — renames the index with the specified name. If an index with the new name already exists, an error will be returned. -{% if backend_name == "YDB" %} +{% if backend_name == "YDB" and oss == true %} -Atomically replacing an index under load is supported by the [{{ ydb-cli }} table index rename](../../../../reference/ydb-cli/commands/secondary_index.md#rename) command in the {{ ydb-short-name }} CLI and by {{ ydb-short-name }} SDK methods. +The ability to atomically replace an index under load is supported by the [{{ ydb-cli }} table index rename](../../../../reference/ydb-cli/commands/secondary_index.md#rename) {{ ydb-short-name }} CLI command and specialized {{ ydb-short-name }} SDK methods. -This applies to global secondary indexes (the hidden index table and the `--replace` mode). Local Bloom skip indexes are not covered by this atomic under-load replacement flow. +This applies to global secondary indexes (hidden index table and `--replace` mode). Local bloom indexes are not applicable to such atomic replacement under load. {% endif %} -Example of index renaming: +Example of renaming an index: + ```yql ALTER TABLE `series` RENAME INDEX `title_index` TO `title_index_new`; diff --git a/ydb/docs/en/core/yql/reference/syntax/create-resource-pool-classifier.md b/ydb/docs/en/core/yql/reference/syntax/create-resource-pool-classifier.md index 25d76d45304..eb4a7c06ebc 100644 --- a/ydb/docs/en/core/yql/reference/syntax/create-resource-pool-classifier.md +++ b/ydb/docs/en/core/yql/reference/syntax/create-resource-pool-classifier.md @@ -1,43 +1,59 @@ # CREATE RESOURCE POOL CLASSIFIER -`CREATE RESOURCE POOL CLASSIFIER` creates a [resource pool classifier](../../../concepts/glossary.md#resource-pool-classifier). +`CREATE RESOURCE POOL CLASSIFIER` creates a [resource pool classifier](../../../concepts/glossary.md#resource-pool-classifier.md). ## Syntax + ```yql CREATE RESOURCE POOL CLASSIFIER <name> WITH ( <parameter_name> [= <parameter_value>] [, ... ] ) ``` -- `name` — name of the resource pool classifier to create. Must be unique and must not contain characters forbidden for schema objects. -- `WITH ( <parameter_name> [= <parameter_value>] [, ... ] )` — parameters that define classifier behavior. + +- `name` — the name of the resource pool classifier being created. It must be unique. The name must not contain characters prohibited for schema objects. +- `WITH ( <parameter_name> [= <parameter_value>] [, ... ] )` — allows you to set parameter values that define the behavior of the resource pool classifier. ### Parameters -* `RANK` (Int64) — Optional: order in which classifiers are evaluated. If omitted, the maximum existing `RANK` plus 1000 is used. Allowed values: a unique number in $[0, 2^{63}-1]$. -* `RESOURCE_POOL` (String) — Required: name of the resource pool for queries that match the classifier. -* `MEMBER_NAME` (String) — Optional: user or group routed to that pool. If omitted, the classifier ignores `MEMBER_NAME` and uses other criteria. +* `RANK` (Int64) — an optional field that specifies the selection order of the resource pool classifier. If the value is not specified, the maximum existing `RANK` is taken and 1000 is added to it. Valid values: a unique number in the range $[0, 2^{63}-1]$. +* `RESOURCE_POOL` (String) — a required field that specifies the name of the resource pool to which queries that meet the classifier criteria will be sent. +* `MEMBER_NAME` (String) — an optional field that determines which user or group of users will be sent to the specified resource pool. The value is compared with the user's SID or any group SID from their authentication token; see [below](#member-name-format) for details on the format. If the field is not specified, the classifier ignores `MEMBER_NAME`, and classification is performed based on other criteria. + +### MEMBER_NAME format {#member-name-format} + +`MEMBER_NAME` is compared character by character with the user's [SID](../../../concepts/glossary.md#access-sid) or any group SID from their authentication token. The SID format depends on how the user logged into the system. + +- **Built-in users {{ ydb-short-name }} (login/password)** — the SID matches the username, without a suffix. For example, `user1`. For more information, see [{#T}](../../../security/authentication.md#static-credentials). +- **Cloud users (Access Service)** — the SID has the form `<subject_id>@as`, where `<subject_id>` is the user ID in IAM. The suffix is set by the [`access_service_domain`](../../../reference/configuration/auth_config.md#iam-auth-config) parameter (default `as`). For example, `ajeb89hv69nujke769fa@as`. For more information, see [{#T}](../../../security/authentication.md#iam). +- **LDAP** — the SID has the form `<login>@<domain>`, where the domain is set by the [`ldap_authentication_domain`](../../../reference/configuration/auth_config.md#ldap-auth-config) parameter (default `ldap`). For example, `user1@ldap`. For more information, see [{#T}](../../../security/authentication.md#ldap). +- **External identity providers (OIDC)** — the SID has the form `<login>@<domain>`, where the domain is set by the `external_idp_authentication_domain` parameter in the [authentication configuration](../../../reference/configuration/auth_config.md) (default `sso`). For example, `user1@sso`. + +You can specify either the SID of a specific user or the SID of a group. The group `all-users@well-known` is automatically added to all authenticated users — it is convenient to use if you need to direct queries from all authenticated clients to the pool. ## Notes {#remarks} -If `RANK` is omitted in the DDL, the default is $RANK = MAX(existing\_ranks) + 1000$. All `RANK` values must be unique so pool choice is deterministic when rules conflict. This allows inserting new classifiers between existing ones. +If `RANK` is not specified in the DDL for creating a resource pool classifier, it will be assigned the default value $RANK = MAX(existing_ranks) + 1000$. All `RANK` values must be unique to ensure a strictly deterministic order of resource pool selection in case of conflicting conditions. This behavior is chosen to allow adding new resource pool classifiers between existing ones. -A classifier may reference a non-existent pool or a pool the user cannot access; such classifiers are skipped. +It is also possible to have a classifier that references a non-existent resource pool or one to which the user does not have access. In such a case, such classifiers will be skipped. -Classifier count limits are described on the [limits](../../../concepts/limits-ydb.md#resource_pool) page. +For limitations on the number of classifiers, see the [limitations](../../../../concepts/limits-ydb#resource_pool) page. ## Permissions -The `ALL` [permission](grant.md#permissions-list) on the database is required. +The [permission](./grant.md#permissions-list) `ALL` on the database is required. + +Example of granting such a permission: -Example: ```yql GRANT 'ALL' ON `/my_db` TO `user1@domain`; ``` + ## Examples {#examples} + ```yql CREATE RESOURCE POOL CLASSIFIER olap_classifier WITH ( RANK=1000, @@ -46,7 +62,8 @@ CREATE RESOURCE POOL CLASSIFIER olap_classifier WITH ( ) ``` -The example above creates a resource pool classifier named `olap_classifier` that routes queries from user `user1@domain` to the resource pool named `olap`. Queries from all other users go to the `default` resource pool, assuming no other resource pool classifiers exist. + +In the example above, a resource pool classifier named `olap_classifier` is created, which directs queries from user `user1@domain` to a resource pool named `olap`. Queries from all other users will be sent to the resource pool `default`, provided that no other resource pool classifiers exist. ## See also diff --git a/ydb/docs/en/core/yql/reference/syntax/create_table/index.md b/ydb/docs/en/core/yql/reference/syntax/create_table/index.md index 6f77c0b80a9..cd5a9ee304d 100644 --- a/ydb/docs/en/core/yql/reference/syntax/create_table/index.md +++ b/ydb/docs/en/core/yql/reference/syntax/create_table/index.md @@ -2,16 +2,17 @@ {% if feature_bulk_tables %} -The table is automatically created upon the first [INSERT INTO](../insert_into.md){% if feature_mapreduce %} in the database specified by the [USE](../use.md) operator{% endif %}. The schema is defined automatically in this process. +The table is created automatically on the first [INSERT INTO](../insert_into.md){% if feature_mapreduce %}, in the database specified by the [USE](../use.md) statement{% endif %}. The schema is determined automatically. {% else %} -The invocation of `CREATE TABLE` creates {% if concept_table %}a [table]({{ concept_table }}){% else %}a table{% endif %} with the specified data schema{% if feature_map_tables %} and primary key columns (`PRIMARY KEY`){% endif %}.{% if feature_secondary_index == true %} It also allows defining secondary indexes on the created table. +The `CREATE TABLE` call creates {% if concept_table %} [a table]({{ concept_table }}){% else %}a table{% endif %} with the specified data schema{% if feature_map_tables %} and key columns (`PRIMARY KEY`){% endif %}.{% if feature_secondary_index == true %} It allows defining secondary indexes on the created table. {% endif %} {% endif %} + ```yql CREATE TABLE [IF NOT EXISTS] <table_name> ( [<column_name> <column_data_type>] [FAMILY <family_name>] [NULL | NOT NULL] [DEFAULT <default_value>] @@ -36,90 +37,100 @@ CREATE TABLE [IF NOT EXISTS] <table_name> ( [AS SELECT ...] ``` + {% if oss == true and backend_name == "YDB" %} -## Request parameters {#request-parameters} +## Query parameters ### table_name -The path of the table to be created. +Path of the table being created. -When choosing a name for the table, consider the common [schema object naming rules](../../../../concepts/datamodel/cluster-namespace.md#object-naming-rules). +When choosing a table name, follow the general [naming rules for schema objects](../../../../concepts/datamodel/cluster-namespace.md#object-naming-rules). ### IF NOT EXISTS -If the table with the specified name already exists, the execution of the operator is completely skipped — no checks or schema matching is performed, and no error occurs. Note that the existing table may differ in structure from the one you would like to create with this query — no comparison or equivalence check is performed. +If a table with the specified name already exists, the statement execution is completely skipped — no checks or schema comparison are performed, and no error occurs. Note that the existing table may differ in structure from the one you intended to create with this query — no comparison or equivalence check is performed. ### column_name -The name of the column to be created in the new table. +Name of the column being created in the new table. -When choosing a name for the column, consider the common [column naming rules](../../../../concepts/datamodel/table.md#column-naming-rules). +When choosing a column name, follow the general [column naming rules](../../../../concepts/datamodel/table.md#column-naming-rules). ### column_data_type -The data type of the column. The complete list of data types supported by {{ ydb-short-name }} is available in the [{#T}](../../types/index.md) section. +Data type of the column. The full list of data types supported by {{ ydb-short-name }} is available in the [{#T}](../../types/index.md) section. {% include [column_option_list.md](../_includes/column_option_list.md) %} ### INDEX -Definition of an index on the table. [Secondary indexes](secondary_index.md), [vector indexes](vector_index.md), [fulltext indexes](fulltext_index.md), and [Bloom skip indexes](bloom_skip_index.md) are supported. +Defining an index on the table. Supported types: + +* [secondary indexes](secondary_index.md), +* [vector indexes](vector_index.md), +* [full-text indexes](fulltext_index.md), +* [Bloom indexes](bloom_skip_index.md), +* [JSON indexes](json_index.md). ### PRIMARY KEY -Definition of the primary key of the table. Specifies the columns that make up the primary key in the order of enumeration. For more information on selecting a primary key, see the [{#T}](../../../../dev/primary-key/index.md) article. +Defining the table's primary key. Specifies the columns that make up the primary key in the order listed. For more details on choosing a primary key, see the [{#T}](../../../../dev/primary-key/index.md) section. ### PARTITION BY HASH -Definition of the columns on which partitioning will occur for **column-oriented** tables. Specifies the columns on which [partitioning](../../../../concepts/glossary.md#partition) will occur using the hash function. The columns must be part of the primary key. The columns do not necessarily have to be a prefix or suffix — the requirement is to be part of the primary key. +Defining partitioning keys for **column-oriented** tables. Specifies the columns by whose hash the data [partitioning](../../../../concepts/glossary.md#partition) is performed. The columns must be part of the primary key. However, the columns do not necessarily have to be a prefix or suffix — the requirement is to be part of the primary key. -If the parameter is not specified, the table will be partitioned on the same columns as those included in the primary key. For more information on selecting and working with partition keys in column-oriented tables, see the [{#T}](../../../../dev/primary-key/column-oriented.md) article. +If the parameter is not specified, the table will be split into partitions by the same columns that are part of the primary key. For guidance on how to choose partitioning keys for column-oriented tables, see the article [{#T}](../../../../dev/primary-key/column-oriented.md). -For more information on partitioning column-oriented tables, see the [{#T}](../../../../concepts/datamodel/table.md#olap-tables-partitioning) section. +For more details on partitioning column-oriented tables, see the [{#T}](../../../../concepts/datamodel/table.md#olap-tables-partitioning) section. -### FAMILY <column_family> (column group setting) +### FAMILY <column_family> (column group settings) -Definition of a column group with specified parameters. For more information, see the [{#T}](family.md) section. +Defining a column group with specified parameters. For more details, see the [{#T}](family.md) section. ### WITH -Additional parameters for creating a table. For more information, see the [{#T}](with.md) section. +Additional table creation parameters. For more details, see the [{#T}](with.md) section. {% note info %} {{ ydb-short-name }} supports two types of tables: -* [Row-oriented](../../../../concepts/datamodel/table.md#row-oriented-tables) tables. -* [Column-oriented](../../../../concepts/datamodel/table.md#column-oriented-tables) tables. +* [Row-oriented](../../../../concepts/datamodel/table.md#row-oriented-tables). +* [Column-oriented](../../../../concepts/datamodel/table.md#column-oriented-tables). + +The table type when created is specified by the `STORE` parameter in the `WITH` block, where `ROW` means [row-oriented table](../../../../concepts/datamodel/table.md#row-oriented-tables) and `COLUMN` means [column-oriented table](../../../../concepts/datamodel/table.md#column-oriented-tables): -The table type is specified by the `STORE` parameter in the `WITH` clause, where `ROW` indicates a [row-oriented](../../../../concepts/datamodel/table.md#row-oriented-tables) table and `COLUMN` indicates a [column-oriented](../../../../concepts/datamodel/table.md#column-oriented-tables) table: ```yql CREATE <table_name> ( columns ... ) + WITH ( STORE = COLUMN -- Default value ROW ) ``` + By default, if the `STORE` parameter is not specified, a row-oriented table is created. {% endnote %} {% note info %} -When choosing a name for the table, consider the common [schema object naming rules](../../../../concepts/datamodel/cluster-namespace.md#object-naming-rules). +When choosing a table name, follow the general [naming rules for schema objects](../../../../concepts/datamodel/cluster-namespace.md#object-naming-rules). {% endnote %} ### AS SELECT -Creating and filling a table with data from a `SELECT` query. For more information, see the [{#T}](as_select.md) section. +Creating and populating a table based on the results of the `SELECT` query. For more details, see the [{#T}](as_select.md) section. -## Examples of table creation +## Table creation examples {% list tabs %} @@ -135,7 +146,7 @@ Creating and filling a table with data from a `SELECT` query. For more informati d "List<List<Int32>>" PRIMARY KEY (a, b) ); - ``` + ``` {% else %} @@ -146,11 +157,12 @@ Creating and filling a table with data from a `SELECT` query. For more informati c Float, PRIMARY KEY (a, b) ); - ``` + ``` {% endif %} - Example of creating a table with a DEFAULT value: + Example of creating a table using a default value (DEFAULT): + ```yql CREATE TABLE table_with_default ( @@ -161,15 +173,16 @@ Creating and filling a table with data from a `SELECT` query. For more informati ); ``` + {% if feature_column_container_type == true %} - For non-key columns, any data types are allowed{% if feature_serial %}, except [serial](../../types/serial.md) types{% endif %}, whereas for key columns only [primitive](../../types/primitive.md) types{% if feature_serial %} and [serial](../../types/serial.md) types{% endif %} are permitted. When specifying complex types (for example, `List<String>`), the type should be enclosed in double quotes. + For non-key columns, any data types are allowed{% if feature_serial %}, except [serial](../../types/serial.md) {% endif %}; for key columns, only [primitive](../../types/primitive.md){% if feature_serial %} and [serial](../../types/serial.md){% endif %} types are allowed. When specifying complex types (e.g., `List<String>`), the type is enclosed in double quotes. {% else %} {% if feature_serial %} - For key columns, only [primitive](../../types/primitive.md) and [serial](../../types/serial.md) data types are allowed; for non-key columns, only [primitive](../../types/primitive.md) types are allowed. + For key columns, only [primitive](../../types/primitive.md) and [serial](../../types/serial.md) data types are allowed; for non-key columns, only [primitive](../../types/primitive.md) data types are allowed. {% else %} @@ -181,17 +194,17 @@ Creating and filling a table with data from a `SELECT` query. For more informati {% if feature_not_null == true %} - Without additional modifiers, a column acquires an [optional](../../types/optional.md) type and allows `NULL` values. To designate a non-optional type, use the `NOT NULL` constraint. + Without additional modifiers, the column acquires an [optional type](../../types/optional.md) and allows `NULL` to be written as values. To obtain a non-optional type, use `NOT NULL`. {% else %} {% if feature_not_null_for_pk %} - By default, all columns are [optional](../../types/optional.md) and can have `NULL` values. The `NOT NULL` constraint can only be specified for columns that are part of the primary key. + By default, all columns are [optional](../../types/optional.md) and can have a NULL value. The `NOT NULL` constraint can only be specified for columns that are part of the primary key. {% else %} - All columns allow NULL values, meaning they are [optional](../../types/optional.md). + All columns allow `NULL` as a value, meaning they are [optional](../../types/optional.md). {% endif %} @@ -199,11 +212,12 @@ Creating and filling a table with data from a `SELECT` query. For more informati {% if feature_map_tables %} - Specifying a `PRIMARY KEY` with a non-empty list of columns is mandatory. These columns become part of the key in the order they are listed. + It is mandatory to specify `PRIMARY KEY` with a non-empty list of columns. These columns become part of the key in the order they are listed. {% endif %} - Example of creating a row-oriented table using partitioning options: + Example of creating a row table with partitioning options: + ```yql CREATE TABLE <table_name> ( @@ -218,9 +232,10 @@ Creating and filling a table with data from a `SELECT` query. For more informati ); ``` - Such code will create a row-oriented table with automatic partitioning by partition size (`AUTO_PARTITIONING_BY_SIZE`) enabled, and with the preferred size of each partition (`AUTO_PARTITIONING_PARTITION_SIZE_MB`) set to 512 megabytes. The full list of row-oriented table partitioning options can be found in the [Partitioning Row-Oriented Tables](../../../../concepts/datamodel/table.md#partitioning_row_table) section of the [{#T}](../../../../concepts/datamodel/table.md) article. -- Creating a column-oriented table + This code will create a row table with automatic partitioning enabled by partition size (`AUTO_PARTITIONING_BY_SIZE`) and a preferred partition size (`AUTO_PARTITIONING_PARTITION_SIZE_MB`) of 512 megabytes. The full list of row table partitioning options is in the [Row table partitioning](../../../../concepts/datamodel/table.md#partitioning_row_table) section of the [{#T}](../../../../concepts/datamodel/table.md) article. + +- Creating a column table ```yql CREATE TABLE table_name ( @@ -235,9 +250,11 @@ Creating and filling a table with data from a `SELECT` query. For more informati ); ``` - For column-oriented tables, you can explicitly specify the columns on which partitioning will occur using the `PARTITION BY HASH` construct. Usually, these are columns of the primary key with a large number of unique values, such as `Timestamp`. If `PARTITION BY HASH` is not specified, partitioning will occur automatically on all columns included in the primary key. For more information on selecting and working with partition keys in column-oriented tables, see the [{#T}](../../../../dev/primary-key/column-oriented.md) article. - Column-oriented tables do not currently support automatic repartitioning, so it is important to specify the correct number of partitions when creating a table with the `AUTO_PARTITIONING_MIN_PARTITIONS_COUNT` parameter: + For column tables, you can explicitly specify which columns will be used for partitioning using the `PARTITION BY HASH` construct. Typically, primary key columns with a large number of unique values are chosen for this, for example, `Timestamp`. If `PARTITION BY HASH` is not specified, partitioning will occur automatically across all columns that are part of the primary key. For more details on selecting and working with partitioning keys in column tables, see the [{#T}](../../../../dev/primary-key/column-oriented.md) article. + + Currently, column tables do not support automatic repartitioning, so it is important to specify the correct number of partitions when creating a table using the `AUTO_PARTITIONING_MIN_PARTITIONS_COUNT` parameter: + ```yql CREATE TABLE table_name ( @@ -253,8 +270,8 @@ Creating and filling a table with data from a `SELECT` query. For more informati ); ``` - This code will create a column-oriented table with 10 partitions. The full list of column-oriented table partitioning options can be found in the [{#T}](../../../../concepts/datamodel/table.md#olap-tables-partitioning) section of the [{#T}](../../../../concepts/datamodel/table.md) article. + This code will create a column table with 10 partitions. For a full list of column table partitioning options, see the [{#T}](../../../../concepts/datamodel/table.md#olap-tables-partitioning) section of the [{#T}](../../../../concepts/datamodel/table.md) article. {% endlist %} @@ -262,27 +279,27 @@ Creating and filling a table with data from a `SELECT` query. For more informati {% if feature_column_container_type == true %} -For non-key columns, any data types are allowed, whereas for key columns only [primitive](../../types/primitive.md) types are permitted. When specifying complex types (for example, `List<String>`), the type should be enclosed in double quotes. +For non-key columns, any data types are allowed; for key columns, only [primitive](../../types/primitive.md) types are allowed. When specifying complex types (e.g., `List<String>`), the type is enclosed in double quotes. {% else %} -For both key and non-key columns, only [primitive](../../types/primitive.md) data types are allowed. +Only [primitive](../../types/primitive.md) data types are allowed for both key and non-key columns. {% endif %} {% if feature_not_null == true %} -Without additional modifiers, a column acquires an [optional](../../types/optional.md) type and allows `NULL` values. To designate a non-optional type, use the `NOT NULL` constraint. +Without additional modifiers, a column acquires an [optional type](../../types/optional.md) and allows `NULL` as a value. To get a non-optional type, use `NOT NULL`. {% else %} {% if feature_not_null_for_pk %} -By default, all columns are [optional](../../types/optional.md) and can have `NULL` values. The `NOT NULL` constraint can only be specified for columns that are part of the primary key. +By default, all columns are [optional](../../types/optional.md) and can have a NULL value. The `NOT NULL` constraint can only be specified for columns that are part of the primary key. {% else %} -All columns allow NULL values, meaning they are [optional](../../types/optional.md). +All columns allow `NULL` as a value, meaning they are [optional](../../types/optional.md). {% endif %} @@ -290,12 +307,13 @@ All columns allow NULL values, meaning they are [optional](../../types/optional. {% if feature_map_tables %} -Specifying a `PRIMARY KEY` with a non-empty list of columns is mandatory. These columns become part of the key in the order they are listed. +It is mandatory to specify `PRIMARY KEY` with a non-empty list of columns. These columns become part of the key in the order they are listed. {% endif %} Example: + ```yql CREATE TABLE <table_name> ( a Uint64, @@ -309,21 +327,22 @@ CREATE TABLE <table_name> ( {% if backend_name == "YDB" and oss == true %} -When creating row-oriented tables, it is possible to specify: +When creating row tables, you can specify: -* [A secondary index](secondary_index.md). -* [A vector index](vector_index.md). -* [A fulltext index](fulltext_index.md). -* [A Bloom skip index](bloom_skip_index.md). +* [Secondary index](secondary_index.md). +* [Vector index](vector_index.md). +* [Full-text index](fulltext_index.md). +* [JSON index](json_index.md). +* [Bloom index](bloom_skip_index.md). * [Column groups](family.md). * [Additional parameters](with.md). -* [Creating a table filled with query results](as_select.md). +* [Creating and populating a table based on query results](as_select.md). -When creating column-oriented tables, it is possible to specify: +For column tables, when creating them, you can specify: -* [A Bloom skip index](bloom_skip_index.md). +* [Bloom index](bloom_skip_index.md). * [Column groups](family.md). * [Additional parameters](with.md). -* [Creating a table filled with query results](as_select.md). +* [Creating and populating a table based on query results](as_select.md). {% endif %} diff --git a/ydb/docs/en/core/yql/reference/syntax/create_table/json_index.md b/ydb/docs/en/core/yql/reference/syntax/create_table/json_index.md new file mode 100644 index 00000000000..d53b623719a --- /dev/null +++ b/ydb/docs/en/core/yql/reference/syntax/create_table/json_index.md @@ -0,0 +1,42 @@ +# JSON-index + +{% if backend_name == 'YDB' %} [JSON indexes](../../../../dev/json-indexes.md){% else %}JSON indexes{% endif %} in {% if backend_name == 'YDB' %}[row](../../../../concepts/datamodel/table.md#row-oriented-tables){% else %}row{% endif %} tables are created using the same syntax as [secondary indexes](secondary_index.md), by specifying `json` as the index type. A subset of the syntax available for JSON indexes: + + +```yql +CREATE TABLE `<table_name>` ( + ... + INDEX `<index_name>` + GLOBAL + [SYNC] + USING json + ON ( <json_column> ) + [, ...] +) +``` + + +Where: + +* `<index_name>` — unique index name for data access. +* `SYNC` — specifies synchronous index update. This is the only mode available for JSON indexes; explicit specification is not required. +* `<json_column>` — table column of type `Json` or `JsonDocument`. A JSON index is built on a single column only. + +A JSON index does not support the `COVER` expression — attempting to specify it will result in an error. + +{% include [not_allow_for_olap](../../../../_includes/not_allow_for_olap_note.md) %} + +## Example + + +```yql +CREATE TABLE documents ( + id Uint64 NOT NULL, + payload JsonDocument NOT NULL, + INDEX json_idx GLOBAL USING json ON (payload), + PRIMARY KEY (id) +) +``` + + +In this example, a table `documents` is created with a JSON index `json_idx` on column `payload`. The index will be used by queries whose `WHERE` predicate contains calls to `JSON_EXISTS` or `JSON_VALUE` on column `payload`. diff --git a/ydb/docs/en/core/yql/reference/syntax/create_table/toc_i.yaml b/ydb/docs/en/core/yql/reference/syntax/create_table/toc_i.yaml index d51714fb9d8..6644cea4776 100644 --- a/ydb/docs/en/core/yql/reference/syntax/create_table/toc_i.yaml +++ b/ydb/docs/en/core/yql/reference/syntax/create_table/toc_i.yaml @@ -1,8 +1,9 @@ items: -- { name: Overview, href: index.md } +- { name: Overview, href: index.md } - { name: SECONDARY INDEX, href: secondary_index.md } - { name: VECTOR INDEX, href: vector_index.md } - { name: FULLTEXT INDEX, href: fulltext_index.md } +- { name: JSON INDEX, href: json_index.md } - { name: BLOOM INDEX, href: bloom_skip_index.md } - { name: FAMILY, href: family.md } - { name: WITH, href: with.md } diff --git a/ydb/docs/en/core/yql/reference/syntax/discard.md b/ydb/docs/en/core/yql/reference/syntax/discard.md index 480fe5dedaa..f19f75e553f 100644 --- a/ydb/docs/en/core/yql/reference/syntax/discard.md +++ b/ydb/docs/en/core/yql/reference/syntax/discard.md @@ -4,6 +4,34 @@ Calculates {% if select_command == "SELECT STREAM" %}[`SELECT STREAM`](select_st It's good to combine it with [`Ensure`](../builtins/basic.md#ensure) to check the final calculation result against the user's criteria. +{% if backend_name == "YDB" %} + +A query with `DISCARD` is executed in full — with all its filters, aggregations, and `Ensure` checks — but the result set is not returned to the client. The data is not sent over the network either: the client receives only the query execution status. For large outputs, this saves a noticeable amount of traffic and memory. + +In a query consisting of multiple statements, `DISCARD` applies to an individual statement: + +```yql +SELECT 1; -- returned +DISCARD SELECT 2; -- executed, but not included in the response +SELECT 3; -- returned +``` + +The client will receive two result sets — from the first and third statements. + +`DISCARD` applies only to a statement as a whole and cannot be used inside expressions. In particular, it is not allowed: + +* in subqueries — `SELECT * FROM (DISCARD SELECT 1)`; +* in `WHERE ... IN (...)` — `SELECT * FROM my_table WHERE Key IN (DISCARD SELECT 1)`; +* in `UNION ALL` operands — `SELECT 1 UNION ALL DISCARD SELECT 2`. + +{% note info %} + +`DISCARD` is supported for queries executed via the Query Service starting from {{ ydb-short-name }} version 26.2. When a query is executed via the legacy interfaces (Table Service, scan queries), `DISCARD` is ignored and the result is returned to the client — this behavior is preserved for backward compatibility. + +{% endnote %} + +{% endif %} + {% if select_command != true or select_command == "SELECT" %} ## Examples @@ -12,6 +40,20 @@ It's good to combine it with [`Ensure`](../builtins/basic.md#ensure) to check th DISCARD SELECT 1; ``` +{% if backend_name == "YDB" %} + +```yql +DISCARD SELECT Ensure( + Data, + Data < 1000000, + "Value too big" +) FROM `result_table`; +``` + +If the `Ensure` condition is violated, the query fails with an error. Otherwise, the query completes successfully, and the client does not receive the result set that would otherwise have to be downloaded in full. + +{% else %} + ```yql INSERT INTO result_table WITH TRUNCATE SELECT * FROM @@ -29,3 +71,4 @@ DISCARD SELECT Ensure( {% endif %} +{% endif %} diff --git a/ydb/docs/en/core/yql/reference/syntax/index.md b/ydb/docs/en/core/yql/reference/syntax/index.md index ca8bb8acc76..c251f9d48b8 100644 --- a/ydb/docs/en/core/yql/reference/syntax/index.md +++ b/ydb/docs/en/core/yql/reference/syntax/index.md @@ -79,12 +79,8 @@ {% endif %} -{% if backend_name != "YDB" %} - * [DISCARD](discard.md) -{% endif %} - * [INTO RESULT](into_result.md) {% if feature_mapreduce %} diff --git a/ydb/docs/en/core/yql/reference/syntax/select/index.md b/ydb/docs/en/core/yql/reference/syntax/select/index.md index db34322bed5..a416ce4b424 100644 --- a/ydb/docs/en/core/yql/reference/syntax/select/index.md +++ b/ydb/docs/en/core/yql/reference/syntax/select/index.md @@ -1,112 +1,234 @@ -<!-- markdownlint-disable blanks-around-fences --> - -# SELECT +## SELECT Returns the result of evaluating the expressions specified after `SELECT`. -It can be used in combination with other operations to obtain other effect. +Can be used in combination with other operations to achieve a different effect. + +### Examples -## Examples ```yql SELECT "Hello, world!"; ``` + ```yql SELECT 2 + 2; ``` -## SELECT execution procedure {#selectexec} -The `SELECT` query result is calculated as follows: +## Procedure for executing SELECT {#selectexec} + +The result of the `SELECT` query is computed as follows: + +* the set of input tables is determined: expressions after [FROM](../select/from.md) are evaluated -* Determine the set of input tables by evaluating the [FROM](from.md) clauses. {% if feature_match_recogznize==true %} -* Apply [MATCH_RECOGNIZE](match_recognize.md) to input tables. -{% endif %} -{% if feature_tablesample==true %} -* Evaluate [SAMPLE](sample.md)/[TABLESAMPLE](sample.md). + +* [MATCH_RECOGNIZE](match_recognize.md) is applied to the input tables + {% endif %} -* Execute [FLATTEN COLUMNS](flatten.md#flatten-columns) or [FLATTEN BY](flatten.md); aliases set in `FLATTEN BY` become visible after this point. + +* is computed [SAMPLE](sample.md) / [TABLESAMPLE](sample.md) +* [FLATTEN COLUMNS](flatten.md#flatten-columns) or [FLATTEN BY](flatten.md) is performed; aliases specified in `FLATTEN BY` become visible after this point. + {% if feature_join %} -* Execute every [JOIN](join.md). +* All [JOIN](join.md) are executed. {% endif %} -* Add to (or replace in) the data the columns listed in [GROUP BY ... AS ...](group-by.md). -* Execute [WHERE](where.md) — Discard all the data mismatching the predicate. -* Execute [GROUP BY](group-by.md), evaluate aggregate functions. -* Apply the filter [HAVING](group-by.md#having). + +* columns specified in [GROUP BY ... AS ...](group-by.md) are added (or replaced) to the resulting data +* the [WHERE](where.md) clause is executed: all data that does not satisfy the predicate is filtered out. +* [GROUP BY](group-by.md) is performed, aggregate function values are computed. +* Filtering is performed using [HAVING](group-by.md#having) + {% if feature_window_functions %} -* Evaluate [window functions](window.md); +* Values of [window functions](window.md) are computed {% endif %} -* Evaluate expressions in `SELECT`. -* Assign names set by aliases to expressions in `SELECT`. -* Apply top-level [DISTINCT](distinct.md) to the resulting columns. -* Execute similarly every subquery inside [UNION ALL](union.md#union-all), combine them (see [PRAGMA AnsiOrderByLimitInUnionAll](../pragma.md#pragmas)). -* Perform sorting with [ORDER BY](order_by.md). -* Apply [OFFSET and LIMIT](limit_offset.md) to the result. + +* Expressions in `SELECT` are evaluated. +* expressions in `SELECT` are assigned names defined by aliases. +* a top-level [DISTINCT](distinct.md) is applied to the columns obtained in this way +* All subqueries in [UNION ALL](union.md#union-all) are computed in the same way and combined (see [PRAGMA AnsiOrderByLimitInUnionAll](../pragma.md#pragmas)). +* Sorting is performed according to [ORDER BY](order_by.md) +* [OFFSET and LIMIT](limit_offset.md) are applied to the result. ## Column order in YQL {#orderedcolumns} -The standard SQL is sensitive to the order of columns in projections (that is, in `SELECT`). While the order of columns must be preserved in the query results or when writing data to a new table, some SQL constructs use this order. -This applies, for example, to [UNION ALL](union.md#union-all) and positional [ORDER BY](order_by.md) (ORDER BY ordinal). +In standard SQL, the order of columns specified in the projection (in `SELECT`) matters. Besides the fact that the column order must be preserved when displaying query results or when writing to a new table, some SQL constructs use this order. This applies in particular to [UNION ALL](union.md#union-all) and positional [ORDER BY](order_by.md) (ORDER BY ordinal). -The column order is ignored in YQL by default: +By default, the order of columns is ignored in YQL: -* The order of columns in the output tables and query results is undefined -* The data scheme of the `UNION ALL` result is output by column names rather than positions +* the order of columns in output tables and in query results is undefined +* The data schema of the `UNION ALL` result is output by column names, not by positions. -If you enable `PRAGMA OrderedColumns;`, the order of columns is preserved in the query results and is derived from the order of columns in the input tables using the following rules: +When `PRAGMA OrderedColumns;` is enabled, the order of columns is preserved in the query results and is derived from the order of columns in the input tables according to the following rules: -* `SELECT`: an explicit column enumeration dictates the result order. +* `SELECT` with explicit column enumeration sets the corresponding order. * `SELECT` with an asterisk (`SELECT * FROM ...`) inherits the order from its input. + {% if feature_join %} -* The order of columns after [JOIN](join.md): First output the left-hand columns, then the right-hand ones. If the column order in any of the sides in the `JOIN` output is undefined, the column order in the result is also undefined. +* the order of columns after [JOIN](join.md): first the columns from the left side, then from the right. If the order of either side present in the output `JOIN` is not defined, the order of the result columns is also not defined; {% endif %} -* The order in `UNION ALL` depends on the [UNION ALL](union.md#union-all) execution mode. -* The column order for [AS_TABLE](from_as_table.md) is undefined. - -{% note warning %} - -In the YT table schema, key columns always precede non-key columns. The order of key columns is determined by the order of the composite key. -When `PRAGMA OrderedColumns;` is enabled, non-key columns preserve their output order. +* the order of depends on the execution mode of [`UNION ALL`](union.md#union-all). +* Column order for [AS_TABLE](from_as_table.md) is not defined. -{% endnote %} +## Combination of queries {#combining-queries} -## Combining queries {#combining-queries} +Results of multiple SELECT (or subqueries) can be combined using the keywords `UNION` and `UNION ALL`. -Results of several SELECT statements (or subqueries) can be combined using `UNION` and `UNION ALL` keywords. ```yql query1 UNION [ALL] query2 (UNION [ALL] query3 ...) ``` -Union of more than two queries is interpreted as a left-associative operation, that is + +Union of more than two queries is interpreted as a left-associative operation, i.e. + ```yql query1 UNION query2 UNION ALL query3 ``` + is interpreted as + ```yql (query1 UNION query2) UNION ALL query3 ``` -If the underlying queries have one of the `ORDER BY/LIMIT/DISCARD/INTO RESULT` operators, the following rules apply: -* `ORDER BY/LIMIT/INTO RESULT` is only allowed after the last query -* `DISCARD` is only allowed before the first query -* the operators apply to the `UNION [ALL]` as a whole, instead of referring to one of the queries -* to apply the operator to one of the queries, enclose the query in parentheses +If `ORDER BY/LIMIT/DISCARD/INTO RESULT` is present in the combined subqueries, the following rules apply: + +* `ORDER BY/LIMIT/INTO RESULT` is allowed only after the last subquery +* `DISCARD` is allowed only before the first subquery. +* the specified operators act on the result `UNION [ALL]`, not on the subquery +* to apply an operator to a subquery, the subquery must be enclosed in parentheses. + +## Accessing multiple tables in a single query + +In standard SQL, [UNION ALL](union.md#union-all) is used to query multiple tables, which combines the results of two or more `SELECT`. This is not very convenient for a use case where you need to run the same query across multiple tables (for example, containing data for different dates). In YQL, for convenience, in `SELECT` after `FROM` you can specify not only a single table or subquery, but also call built-in functions that allow combining data from multiple tables. + +The following functions are defined for these purposes: + +``` CONCAT(`table1`, `table2`, `table3` VIEW view_name, ...) ``` — combines all tables listed in the arguments. + +`EACH($list_of_strings)` or `EACH($list_of_strings VIEW view_name)` — combines all tables whose names are listed in the string list. Optionally, you can pass multiple lists in separate arguments, similar to `CONCAT`. + +``` RANGE(`prefix`, `min`, `max`, `suffix`, `view`) ```: combines a range of tables. Arguments: + +* prefix — directory for searching tables, specified without a trailing slash. The only required argument; if only it is specified, all tables in this directory are used. +* min, max — the next two arguments specify a range of names for including tables. The range is inclusive on both ends. If the range is not specified, all tables in the prefix directory are used. Names of tables or directories located in the directory specified in prefix are compared with the range `[min, max]` lexicographically, not concatenated, so it is important to specify the range without leading slashes. +* suffix — table name. Expected without a leading slash. If suffix is not specified, the arguments `[min, max]` specify a range of table names. If suffix is specified, the arguments `[min, max]` specify a range of folders in which a table with the name specified in the suffix argument exists. + +``` LIKE(`prefix`, `pattern`, `suffix`, `view`)` и `REGEXP(`prefix`, `pattern`, `suffix`, `view`) ``` — the pattern argument is specified in a format similar to the binary operators of the same name: [LIKE](../expressions.md#like) and [REGEXP](../expressions.md#regexp). + +``` FILTER(`prefix`, `callable`, `suffix`, `view`) ``` — the callable argument must be a callable expression with signature `(String)->Bool`, which will be called for each table/subdirectory in the prefix directory. Only those tables for which the callable value returned `true` will participate in the query. As a callable value, it is most convenient to use [lambda functions](../expressions.md#lambda){% if yql == true %}, or UDFs in Python or JavaScript{% endif %}. + +{% note warning %} + +The order in which tables are merged by all the above functions is not guaranteed. + +The list of tables is computed **before** the query itself is executed. Therefore, tables created during the query will not be included in the function results. + +{% endnote %} + +By default, schemas of all participating tables are merged according to the rules of [UNION ALL](union.md#union-all). If schema merging is not desired, you can use functions with the suffix `_STRICT`, for example `CONCAT_STRICT` or `RANGE_STRICT`, which work exactly like the original ones but treat any discrepancy in table schemas as an error. + +To specify the cluster of the merged tables, you need to specify it before the function name. + +All arguments of the functions described above can be declared separately using [named expressions](../expressions.md#named-nodes). In this case, simple expressions are also allowed in them by implicitly calling [EvaluateExpr](../../builtins/basic.md#evaluate_expr_atom). + +The name of the source table from which each row was originally obtained can be obtained using the [TablePath()](../../builtins/basic.md#tablepath) function. + +### Examples + + +```yql +SELECT * FROM CONCAT( + `table1`, + `table2`, + `table3`); +``` -## Clauses supported in SELECT + +```yql +$indices = ListFromRange(1, 4); +$tables = ListMap($indices, ($index) -> { + RETURN "table" || CAST($index AS String); +}); +SELECT * FROM EACH($tables); -- identical to the previous example +``` + + +```yql +SELECT * FROM RANGE(`my_folder`); +``` + + +```yql +SELECT * FROM some_cluster.RANGE( -- The cluster can be specified before the function name + `my_folder`, + `from_table`, + `to_table`); +``` + + +```yql +SELECT * FROM RANGE( + `my_folder`, + `from_folder`, + `to_folder`, + `my_table`); +``` + + +```yql +SELECT * FROM RANGE( + `my_folder`, + `from_table`, + `to_table`, + ``, + `my_view`); +``` + + +```yql +SELECT * FROM LIKE( + `my_folder`, + "2017-03-%" +); +``` + + +```yql +SELECT * FROM REGEXP( + `my_folder`, + "2017-03-1[2-4]?" +); +``` + + +```yql +$callable = ($table_name) -> { + return $table_name > "2017-03-13"; +}; + +SELECT * FROM FILTER( + `my_folder`, + $callable +); +``` + + +## Supported constructs in SELECT * [FROM](from.md) * [FROM AS_TABLE](from_as_table.md) @@ -114,26 +236,34 @@ If the underlying queries have one of the `ORDER BY/LIMIT/DISCARD/INTO RESULT` o * [DISTINCT](distinct.md) * [UNIQUE DISTINCT](unique_distinct_hints.md) * [UNION](union.md) -* [WITH](with.md) +* WITH * [WITHOUT](without.md) * [WHERE](where.md) * [ORDER BY](order_by.md) * [ASSUME ORDER BY](assume_order_by.md) * [LIMIT OFFSET](limit_offset.md) -{% if feature_tablesample==true %} * [SAMPLE](sample.md) * [TABLESAMPLE](sample.md) -{% endif %} + {% if feature_match_recogznize==true %} + * [MATCH_RECOGNIZE](match_recognize.md) + {% endif %} + {% if feature_join %} + * [JOIN](join.md) + {% endif %} + * [GROUP BY](group-by.md) * [FLATTEN](flatten.md) + {% if feature_window_functions %} + * [WINDOW](window.md) + {% endif %} {% if yt %} @@ -164,8 +294,8 @@ If the underlying queries have one of the `ORDER BY/LIMIT/DISCARD/INTO RESULT` o {% if feature_secondary_index %} * [VIEW secondary_index](secondary_index.md) - * [VIEW vector_index](vector_index.md) * [VIEW fulltext_index](fulltext_index.md) +* [VIEW json_index](json_index.md) {% endif %} diff --git a/ydb/docs/en/core/yql/reference/syntax/select/json_index.md b/ydb/docs/en/core/yql/reference/syntax/select/json_index.md new file mode 100644 index 00000000000..e9985894235 --- /dev/null +++ b/ydb/docs/en/core/yql/reference/syntax/select/json_index.md @@ -0,0 +1,125 @@ +# VIEW (JSON index) + +To execute a `SELECT` query on a row table with explicit use of a [JSON index](../../../../dev/json-indexes.md), use the `VIEW` expression: + + +```yql +SELECT ... +FROM documents VIEW json_idx +WHERE <предикат на основе JSON_EXISTS / JSON_VALUE> +ORDER BY ... +``` + + +In the query example above, `documents` is the name of the table containing a column of type `Json` or `JsonDocument`, and `json_idx` is the name of the JSON index created on that column. + +If the predicate is not supported for execution via a JSON index, a query with an explicit `VIEW` expression fails with a compile-time error. Without the `VIEW` expression, the optimizer cannot select this index for such a predicate, and the query will be executed by another method (for example, selecting a different index or performing a full scan of the base table). + +{% note info %} + +A JSON index can be selected automatically by the [optimizer](../../../../concepts/glossary.md#optimizer) if the predicate meets the requirements for index usage. For debugging and guaranteed index usage, specify it explicitly using `VIEW IndexName`. + +For a description of supported predicates, see the [{#T}](../../../../dev/json-indexes.md#predicates) section. + +{% endnote %} + +## Automatic index selection {#auto} + +If the `WHERE` predicate contains `JSON_EXISTS` / `JSON_VALUE` calls on an indexed JSON column, the [optimizer](../../../../concepts/glossary.md#optimizer) may use the JSON index without an explicit `VIEW`. Automatic selection follows these rules: + +* The JSON index is considered by the optimizer with the lowest priority — it is a fallback option that is selected only after other access methods have been ruled out. +* The JSON index is not selected if the query can already be served by a [primary key](../../../../concepts/glossary.md#primary-index) or another, more specific [secondary index](../../../../concepts/glossary.md#secondary-index). +* Explicit specification of `VIEW` overrides the optimizer's decision and forces index usage. +* All indexed subexpressions in a single query must refer to the same indexed JSON column. If predicates on different indexed columns are combined via `AND` / `OR`, automatic selection is not applied. + +{% note info %} + +Automatic JSON index selection is rule-based: it uses the query structure and schema metadata, without data statistics. The selection logic may change in future versions of {{ ydb-short-name }}, so for guaranteed index usage, specify it explicitly via `VIEW`. + +{% endnote %} + +## JSON_EXISTS + +[JSON_EXISTS(doc, jsonpath)](../../builtins/json.md#json_exists) checks for the existence of a path or value inside a JSON document: + + +```yql +SELECT id, payload +FROM documents VIEW json_idx +WHERE JSON_EXISTS(payload, '$.user.id'); +``` + + +The JSON index supports almost all JsonPath syntax, except for nested predicates and boolean expressions at the context object level (`$`) in `JSON_EXISTS`. A more complex example of a supported expression: + + +```yql +SELECT id, payload +FROM documents VIEW json_idx +WHERE JSON_EXISTS(payload, '$.user ? (@.id > 100)'); +``` + + +## JSON_VALUE + +[JSON_VALUE(doc, jsonpath RETURNING <type>)](../../builtins/json.md#json_value) extracts a scalar value by a JsonPath and returns it in the specified type. To use the JSON index, the `RETURNING <type>` clause is required: + + +```yql +SELECT id, payload +FROM documents VIEW json_idx +WHERE JSON_VALUE(payload, '$.user.name' RETURNING Utf8) = "Charlie"u; +``` + + +When checking equality conditions or whether a value is in a given list (`IN`), the index search is performed by the 'path + value' token, which provides the highest selectivity. For other comparisons (`!=`, `<`, `>=`, `BETWEEN`, etc.), only the path token is used, and the final comparison accuracy is ensured by a post-filter. + +### Parameters + +Three ways to pass query parameters are supported: + +1. Direct comparison of the `JSON_VALUE` result with a parameter: + + + ```yql + DECLARE $id AS Int64; + SELECT * FROM documents VIEW json_idx + WHERE JSON_VALUE(payload, '$.owner_id' RETURNING Int64) = $id; + ``` + +2. Checking whether the result is in a list of values: + + + ```yql + DECLARE $tags AS List<Utf8>; + SELECT * FROM documents VIEW json_idx + WHERE JSON_VALUE(payload, '$.tag' RETURNING Utf8) IN $tags; + ``` + +3. Passing a parameter to JsonPath via the `PASSING` clause: + + + ```yql + DECLARE $v AS Int64; + SELECT * FROM documents VIEW json_idx + WHERE JSON_EXISTS(payload, '$.k ? (@ == $v)' PASSING $v AS v); + ``` + +## AND and OR combinations + +`JSON_EXISTS` and `JSON_VALUE` on the same JSON column can be combined in a single `WHERE` using the `AND` and `OR` operators: + + +```yql +SELECT id, payload +FROM documents VIEW json_idx +WHERE JSON_VALUE(payload, '$.user.name' RETURNING Utf8) = "Charlie"u + AND JSON_VALUE(payload, '$.user.id' RETURNING Int64) BETWEEN 100 AND 200; +``` + + +{% note info %} + +Query execution via a JSON index can only use a subset of expressions based on `JSON_EXISTS` and `JSON_VALUE` (see [{#T}](../../../../dev/json-indexes.md#predicates)). Conditions that do not fall under these rules (for example, negations, comparisons with another column, comparison of two `JSON_VALUE` from different columns) are not indexed; in `AND` they go into a post-filter, in `OR` they lead to rejection of index usage for the entire group of conditions combined via `OR`. + +{% endnote %} diff --git a/ydb/docs/en/core/yql/reference/syntax/select/toc_i.yaml b/ydb/docs/en/core/yql/reference/syntax/select/toc_i.yaml index abb7e9b1c22..8a6bee82b9b 100644 --- a/ydb/docs/en/core/yql/reference/syntax/select/toc_i.yaml +++ b/ydb/docs/en/core/yql/reference/syntax/select/toc_i.yaml @@ -1,5 +1,5 @@ items: -- { name: Overview, href: index.md } +- { name: Overview, href: index.md } - { name: TEMPORARY TABLE, href: temporary_table.md, when: feature_temp_table } - { name: FROM, href: from.md } - { name: FROM AS_TABLE, href: from_as_table.md } @@ -18,16 +18,13 @@ items: - { name: VIEW SECONDARY INDEX, href: secondary_index.md, when: feature_secondary_index } - { name: VIEW VECTOR INDEX, href: vector_index.md, when: feature_secondary_index } - { name: VIEW FULLTEXT INDEX, href: fulltext_index.md, when: feature_secondary_index } +- { name: VIEW JSON INDEX, href: json_index.md, when: feature_secondary_index } - { name: ORDER BY HybridRank, href: hybrid_search.md, when: feature_secondary_index } -- name: WITH - href: with.md - include: { mode: link, path: with/toc_p.yaml } - { name: WITHOUT, href: without.md } - { name: WHERE, href: where.md } - { name: ORDER BY, href: order_by.md } - { name: ASSUME ORDER BY, href: assume_order_by.md } - { name: LIMIT OFFSET, href: limit_offset.md } -- { name: SAMPLE, href: sample.md, when: feature_tablesample } -- { name: TABLESAMPLE, href: sample.md, when: feature_tablesample } +- { name: SAMPLE, href: sample.md } - { name: MATCH_RECOGNIZE, href: match_recognize.md } - { name: STREAMING, href: streaming.md } diff --git a/ydb/docs/en/core/yql/reference/syntax/toc_i.yaml b/ydb/docs/en/core/yql/reference/syntax/toc_i.yaml index d0c57876b81..b9fd62bf1f2 100644 --- a/ydb/docs/en/core/yql/reference/syntax/toc_i.yaml +++ b/ydb/docs/en/core/yql/reference/syntax/toc_i.yaml @@ -44,7 +44,7 @@ items: - { name: COMMIT, href: commit.md } - { name: DECLARE, href: declare.md } - { name: DELETE, href: delete.md, when: feature_map_tables } -- { name: DISCARD, href: discard.md, when: backend_name != "YDB" } +- { name: DISCARD, href: discard.md } - { name: DROP ASYNC REPLICATION, href: drop-async-replication.md, when: feature_async_replication } - { name: DROP BACKUP COLLECTION, href: drop-backup-collection.md, when: feature_backup_collections } - { name: DROP GROUP, href: drop-group.md, when: feature_user_and_group } diff --git a/ydb/docs/en/core/yql/reference/types/primitive.md b/ydb/docs/en/core/yql/reference/types/primitive.md index e68db6350b7..d41ee113642 100644 --- a/ydb/docs/en/core/yql/reference/types/primitive.md +++ b/ydb/docs/en/core/yql/reference/types/primitive.md @@ -60,7 +60,7 @@ Floating-point number with fixed precision, 16 bytes in size. Precision is the m || `DyNumber` | Binary representation of a floating-point number with up to 38 digits of precision. Valid values: positive from 1×10<sup>-130</sup> to 1×10<sup>126</sup>–1, negative from -1×10<sup>126</sup>–1 to -1×10<sup>-130</sup> and 0. -Compatible with the `Number` type of AWS DynamoDB. Not recommended for use in {{ backend_name_lower }}-native applications. | Not supported in columnar tables +Compatible with the `Number` type of AWS DynamoDB. Not recommended for use in {{ backend_name_lower }}-native applications. | || {% endif %} |# @@ -95,7 +95,7 @@ Does not support comparison{% if feature_map_tables %}, cannot be used in the pr Does not support comparison{% if feature_map_tables %}, cannot be used in the primary key and in the columns that form the secondary index key{% endif %} || || `Uuid` | -Universal identifier [UUID](https://tools.ietf.org/html/rfc4122) | Not supported in columnar tables +Universal identifier [UUID](https://tools.ietf.org/html/rfc4122) | || |# {% note info "Size limitations" %} @@ -199,7 +199,6 @@ from -136 years to +136 years | 8 | -Not supported in columnar tables || || @@ -211,7 +210,6 @@ from -292277 years to +292277 years | 8 | -Not supported in columnar tables || || diff --git a/ydb/docs/en/core/yql/toc_i.yaml b/ydb/docs/en/core/yql/toc_i.yaml index f5b02d179a6..28a10223cb7 100644 --- a/ydb/docs/en/core/yql/toc_i.yaml +++ b/ydb/docs/en/core/yql/toc_i.yaml @@ -1,6 +1,3 @@ items: -- name: Overview - href: reference/index.md -- include: { mode: link, path: reference/toc_i.yaml } - name: Query plans href: query_plans.md diff --git a/ydb/docs/en/core/yql/toc_p.yaml b/ydb/docs/en/core/yql/toc_p.yaml index 50818e81233..0be250eb322 100644 --- a/ydb/docs/en/core/yql/toc_p.yaml +++ b/ydb/docs/en/core/yql/toc_p.yaml @@ -1,3 +1,5 @@ items: -- include: { mode: link, path: toc_i.yaml } +- name: Overview + href: reference/index.md - include: { mode: link, path: reference/toc_p.yaml } +- include: { mode: link, path: toc_i.yaml } diff --git a/ydb/docs/redirects.yaml b/ydb/docs/redirects.yaml index c0f191371f2..e738be43d12 100644 --- a/ydb/docs/redirects.yaml +++ b/ydb/docs/redirects.yaml @@ -544,20 +544,38 @@ ru: to: /reference/ydb-sdk/observability/metrics/opentelemetry.md - from: /recipes/ydb-sdk/debug-jaeger.md to: /reference/ydb-sdk/observability/tracing/jaeger.md + - from: /recipes/ydb-sdk/debug-otel.md + to: /reference/ydb-sdk/observability/tracing/opentelemetry.md - from: /recipes/ydb-sdk/debug-otel-tracing.md to: /reference/ydb-sdk/observability/tracing/opentelemetry.md - from: /integrations/sql-dialect-converter.md to: /integrations/sql-translation/sql-dialect-converter.md en: - - from: /reference/ydb-sdk/recipes/debug-jaeger.md - to: /recipes/ydb-sdk/debug-jaeger.md + - from: /reference/ydb-sdk/recipes/debug.md + to: /reference/ydb-sdk/observability/index.md - from: /reference/ydb-sdk/recipes/debug-logs.md - to: /recipes/ydb-sdk/debug-logs.md + to: /reference/ydb-sdk/observability/logging/logging.md - from: /reference/ydb-sdk/recipes/debug-prometheus.md - to: /recipes/ydb-sdk/debug-prometheus.md - - from: /reference/ydb-sdk/recipes/debug.md - to: /recipes/ydb-sdk/debug.md + to: /reference/ydb-sdk/observability/metrics/prometheus.md + - from: /reference/ydb-sdk/recipes/debug-jaeger.md + to: /reference/ydb-sdk/observability/tracing/jaeger.md + - from: /recipes/ydb-sdk/debug.md + to: /reference/ydb-sdk/observability/index.md + - from: /recipes/ydb-sdk/debug-logs.md + to: /reference/ydb-sdk/observability/logging/logging.md + - from: /recipes/ydb-sdk/debug-logs-otel.md + to: /reference/ydb-sdk/observability/logging/opentelemetry.md + - from: /recipes/ydb-sdk/debug-prometheus.md + to: /reference/ydb-sdk/observability/metrics/prometheus.md + - from: /recipes/ydb-sdk/debug-otel-metrics.md + to: /reference/ydb-sdk/observability/metrics/opentelemetry.md + - from: /recipes/ydb-sdk/debug-jaeger.md + to: /reference/ydb-sdk/observability/tracing/jaeger.md + - from: /recipes/ydb-sdk/debug-otel.md + to: /reference/ydb-sdk/observability/tracing/opentelemetry.md + - from: /recipes/ydb-sdk/debug-otel-tracing.md + to: /reference/ydb-sdk/observability/tracing/opentelemetry.md - from: /devops/deployment-options/manual/initial-deployment.md to: /devops/deployment-options/manual/initial-deployment/index.md - from: /devops/manual/initial-deployment.md diff --git a/ydb/docs/ru/core/_assets/resources_weight.drawio b/ydb/docs/ru/core/_assets/resources_weight.drawio index 9fe02ceb6cf..ef2a1a4bdd1 100644 --- a/ydb/docs/ru/core/_assets/resources_weight.drawio +++ b/ydb/docs/ru/core/_assets/resources_weight.drawio @@ -40,7 +40,7 @@ <mxPoint x="490" y="230" as="targetPoint" /> </mxGeometry> </mxCell> - <mxCell id="cv_4T4Yt_JHGhEiulPHa-16" value="RESOURCES_WEIGHT<div>=100</div>" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="cv_4T4Yt_JHGhEiulPHa-15"> + <mxCell id="cv_4T4Yt_JHGhEiulPHa-16" value="RESOURCE_WEIGHT<div>=100</div>" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="cv_4T4Yt_JHGhEiulPHa-15"> <mxGeometry x="-0.1738" y="6" relative="1" as="geometry"> <mxPoint x="65" y="-12" as="offset" /> </mxGeometry> @@ -146,10 +146,10 @@ <mxPoint x="-97" y="7" as="offset" /> </mxGeometry> </mxCell> - <mxCell id="cv_4T4Yt_JHGhEiulPHa-42" value="<div style="text-align: center;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; float: none; display: inline !important;">Cтарый лимит (RESOURCES_WEIGHT=100):</span></div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="forced-color-adjust: none; color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5</span><div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;"><br></span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">Новый лимит (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCES_WEIGHT=100)</span><span style="text-wrap: nowrap; background-color: initial;">:</span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 70/3 ~= 2.3 vCPU</span><br style="text-wrap: nowrap;"><span style="text-wrap: nowrap;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 ~= 1.15&nbsp; vCPU</span><span style="text-wrap: nowrap;"><br></span></div>" style="text;whiteSpace=wrap;html=1;" vertex="1" parent="1"> + <mxCell id="cv_4T4Yt_JHGhEiulPHa-42" value="<div style="text-align: center;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; float: none; display: inline !important;">Cтарый лимит (RESOURCE_WEIGHT=100):</span></div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="forced-color-adjust: none; color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5</span><div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;"><br></span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">Новый лимит (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCE_WEIGHT=100)</span><span style="text-wrap: nowrap; background-color: initial;">:</span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 70/3 ~= 2.3 vCPU</span><br style="text-wrap: nowrap;"><span style="text-wrap: nowrap;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 ~= 1.15&nbsp; vCPU</span><span style="text-wrap: nowrap;"><br></span></div>" style="text;whiteSpace=wrap;html=1;" vertex="1" parent="1"> <mxGeometry x="170.00000000000009" y="820" width="360" height="50" as="geometry" /> </mxCell> - <mxCell id="cv_4T4Yt_JHGhEiulPHa-43" value="<div style="text-align: center;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; float: none; display: inline !important;">Cтарый лимит (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCES_WEIGHT=200)</span><span style="text-wrap: nowrap;">:</span></div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="forced-color-adjust: none; color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5</span><div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;"><br></span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">Новый лимит (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCES_WEIGHT=200)</span><span style="text-wrap: nowrap; background-color: initial;">:</span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="text-wrap: nowrap;"><span style="text-wrap: nowrap;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5 vCPU</span><span style="text-wrap: nowrap;"><br></span></div>" style="text;whiteSpace=wrap;html=1;" vertex="1" parent="1"> + <mxCell id="cv_4T4Yt_JHGhEiulPHa-43" value="<div style="text-align: center;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; float: none; display: inline !important;">Cтарый лимит (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCE_WEIGHT=200)</span><span style="text-wrap: nowrap;">:</span></div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="forced-color-adjust: none; color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial;"><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5</span><div><span style="color: rgb(0, 0, 0); font-family: Helvetica; font-size: 12px; font-style: normal; font-variant-ligatures: normal; font-variant-caps: normal; font-weight: 400; letter-spacing: normal; orphans: 2; text-align: center; text-indent: 0px; text-transform: none; widows: 2; word-spacing: 0px; -webkit-text-stroke-width: 0px; white-space: nowrap; background-color: rgb(251, 251, 251); text-decoration-thickness: initial; text-decoration-style: initial; text-decoration-color: initial; display: inline !important; float: none;"><br></span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">Новый лимит (</span><span style="text-wrap: nowrap; background-color: initial;">(RESOURCE_WEIGHT=200)</span><span style="text-wrap: nowrap; background-color: initial;">:</span></div><div style="text-align: center;"><span style="text-wrap: nowrap;">TOTAL_CPU_LIMIT_PERCENT_PER_NODE = 30 = 3 vCPU</span><br style="text-wrap: nowrap;"><span style="text-wrap: nowrap;">QUERY_CPU_LIMIT_PERCENT_PER_NODE = 50 = 1.5 vCPU</span><span style="text-wrap: nowrap;"><br></span></div>" style="text;whiteSpace=wrap;html=1;" vertex="1" parent="1"> <mxGeometry x="555" y="820" width="360" height="50" as="geometry" /> </mxCell> </root> diff --git a/ydb/docs/ru/core/_assets/resources_weight.png b/ydb/docs/ru/core/_assets/resources_weight.png Binary files differindex 6e84265c170..25e7b38ba52 100644 --- a/ydb/docs/ru/core/_assets/resources_weight.png +++ b/ydb/docs/ru/core/_assets/resources_weight.png diff --git a/ydb/docs/ru/core/_includes/olap-data-types.md b/ydb/docs/ru/core/_includes/olap-data-types.md deleted file mode 100644 index ca4b37293e0..00000000000 --- a/ydb/docs/ru/core/_includes/olap-data-types.md +++ /dev/null @@ -1,24 +0,0 @@ -Тип данных | Можно использовать<br/>в колоночных таблицах | Можно использовать<br/>в качестве первичного ключа ----|---|--- -`Bool` | ✓ | ✓ -`Int8` | ✓ | ☓ -`Int16` | ✓ | ☓ -`Int32` | ✓ | ✓ -`Int64` | ✓ | ✓ -`Uint8` | ✓ | ✓ -`Uint16` | ✓ | ✓ -`Uint32` | ✓ | ✓ -`Uint64` | ✓ | ✓ -`Float` | ✓ | ☓ -`Double` | ✓ | ☓ -`Decimal` | ✓ | ☓ -`String` | ✓ | ✓ -`Utf8` | ✓ | ✓ -`Json` | ✓ | ☓ -`JsonDocument` | ✓ | ☓ -`Yson` | ✓ | ☓ -`Uuid` | ☓ | ☓ -`Date` | ✓ | ✓ -`Datetime` | ✓ | ✓ -`Timestamp` | ✓ | ✓ -`Interval` | ☓ | ☓ diff --git a/ydb/docs/ru/core/concepts/query_execution/index.md b/ydb/docs/ru/core/concepts/query_execution/index.md index d9613c4466c..da166f2bb57 100644 --- a/ydb/docs/ru/core/concepts/query_execution/index.md +++ b/ydb/docs/ru/core/concepts/query_execution/index.md @@ -22,7 +22,7 @@ ## Запросы к топикам {#streaming} -Запросы к [топикам](../datamodel/topic.md) позволяют обрабатывать неограниченные потоки данных. Поскольку поток данных бесконечен, такой запрос не завершается после получения результата, а работает до явной отмены. Есть два варианта таких запросов. +Запросы к [топикам](../datamodel/topic.md) позволяют обрабатывать данные из очередей сообщений. Есть два варианта таких запросов. ### Потоковые запросы @@ -32,6 +32,6 @@ ### Запросы к топикам в табличном режиме -Прочитать данные из топика можно и обычным `SELECT`, указав `STREAMING = "TRUE"` в блоке `WITH`. Без ограничения `LIMIT` такой запрос работает бесконечно, возвращая результат клиенту по мере поступления сообщений. В отличие от потоковых запросов, он не создаёт постоянного объекта схемы и не восстанавливается после сбоев, поэтому предназначен в первую очередь для отладки и проверки данных в топике. +Прочитать данные из топика можно и обычным [`SELECT`](../../yql/reference/syntax/select/topics.md). В отличие от потоковых запросов, он не создаёт постоянного объекта схемы и не восстанавливается после сбоев, поэтому предназначен в первую очередь для отладки и проверки данных в топике. -Подробнее см. в разделе [{#T}](../../yql/reference/syntax/select/streaming.md). +Подробнее см. в разделе [{#T}](topics.md). diff --git a/ydb/docs/ru/core/concepts/query_execution/topics.md b/ydb/docs/ru/core/concepts/query_execution/topics.md index 26a77ea0a8e..f1a1be7f90d 100644 --- a/ydb/docs/ru/core/concepts/query_execution/topics.md +++ b/ydb/docs/ru/core/concepts/query_execution/topics.md @@ -37,11 +37,11 @@ SELECT * FROM ext_source.input_topic WITH (FORMAT = json_each_row, SCHEMA = (... ## Чтение из топика {#topic-read} -Чтение из топика можно выполнять в [табличном](#table-read) и [потоковом](#streaming-read) режимах (не путать с [потоковыми запросами](../streaming-query/streaming-query.md)). +Чтение из топика может ограничиваться только текущими данными топика или работать с ожиданием новых записанных сообщений. -### Табличное чтение {#table-read} +### Чтение текущих данных {#table-read} -В табличном режиме чтение выполняется от первого до последнего смещения, хранящегося в топике на момент запуска запроса. Если в топик продолжается запись данных, то запрос остановится после достижения последнего смещения, известного на момент запуска. Указание фильтров по [Служебным полям](#system-metadata) ускоряет чтение, так как чтение происходит только по указанным диапазонам. +В данном режиме чтение выполняется от первого до последнего смещения, хранящегося в топике на момент запуска запроса. Если в топик продолжается запись данных, то запрос остановится после достижения последнего смещения, известного на момент запуска. Указание фильтров по [Служебным полям](#system-metadata) ускоряет чтение, так как чтение происходит только по указанным диапазонам. ```yql SELECT @@ -51,9 +51,9 @@ FROM LIMIT 10; ``` -### Потоковое чтение {#streaming-read} +### Чтение с ожиданием данных {#streaming-read} -Для чтения новых сообщений используйте опцию `WITH (STREAMING = "TRUE")` — подробнее в разделе [Потоковое чтение данных из топика](../../yql/reference/syntax/select/streaming.md). Чтение начинается с текущего момента и продолжается до тех пор, пока не будет прочитано заданное в выражении `LIMIT` количество сообщений. Параметр `LIMIT` обязателен — без него запрос не завершится, так как будет ожидать новые сообщения бесконечно. +Для ожидания новых сообщений используйте опцию `WITH (STREAMING = "TRUE")`. Чтение начинается с текущего момента и продолжается до тех пор, пока не будет прочитано заданное в выражении `LIMIT` количество сообщений. Параметр `LIMIT` обязателен — без него запрос не завершится, так как будет ожидать новые сообщения бесконечно. ```yql @@ -135,7 +135,19 @@ FROM ### Служебные поля {#system-metadata} -При чтении можно запрашивать служебные поля: +При чтении можно запрашивать служебные поля и [пользовательские атрибуты сообщений](../datamodel/topic.md#message): + +| Поле | [Тип](../../yql/reference/types/index.md) | Описание | +|------|-----|----------| +| `__ydb_create_time` | `Timestamp` | Время создания сообщения | +| `__ydb_write_time` | `Timestamp` | Время записи сообщения в топик | +| `__ydb_offset` | `Uint64` | Смещение сообщения в партиции | +| `__ydb_partition_id` | `Uint64` | Номер партиции | +| `__ydb_message_group_id` | `String` | Идентификатор группы сообщений | +| `__ydb_seq_no` | `Uint64` | Порядковый номер сообщения внутри группы | +| `__ydb_user_attributes` | `Dict<String,String>` | [Пользовательские атрибуты сообщения](../datamodel/topic.md#message) | + +Пример использования служебных полей: ```yql SELECT @@ -165,6 +177,18 @@ WHERE AND __ydb_write_time > CurrentUtcTimestamp() - Interval("PT2H"); ``` +Пример использования пользовательских атрибутов: + +```yql +SELECT + COUNT(*) AS ErrorCount +FROM + input_topic -- локальный топик; для внешнего: ext_source.input_topic +WHERE + __ydb_user_attributes["type"] = "log" + AND __ydb_user_attributes["level"] = "error"; +``` + ## Запись в топик {#topic-write} ### Запись одного сообщения {#simple-write} @@ -193,7 +217,7 @@ FROM {% note warning %} -Чтение и запись [пользовательских атрибутов](../datamodel/topic.md#message) не поддерживаются. +Запись [пользовательских атрибутов](../datamodel/topic.md#message) через YQL не поддерживается. {% endnote %} diff --git a/ydb/docs/ru/core/concepts/streaming-query/watermarks.md b/ydb/docs/ru/core/concepts/streaming-query/watermarks.md index 9da73888d6e..aead307dbc2 100644 --- a/ydb/docs/ru/core/concepts/streaming-query/watermarks.md +++ b/ydb/docs/ru/core/concepts/streaming-query/watermarks.md @@ -2,9 +2,13 @@ Watermark в потоковой обработке данных ([stream processing](https://en.wikipedia.org/wiki/Stream_processing)) — это монотонно возрастающая нижняя оценка времён событий, которые ещё могут поступить в поток. Когда watermark достигает значения X, система объявляет, что все события с временем меньше X с высокой вероятностью уже получены. +{{ ydb-short-name }} реализует механизм watermarks в [потоковых запросах](streaming-query.md): они используются для корректного закрытия временных окон агрегации ([HoppingWindow](../../yql/reference/syntax/select/group-by.md#group-by-hopping_window)) и гарантируют, что результат окна выдаётся только тогда, когда система убедилась в полноте входных данных за этот период. + В потоковой обработке каждое событие имеет две временны́е метки: **время события** (event time) — момент, когда событие произошло в реальном мире, и **время обработки** (processing time) — момент, когда система получила событие. Из-за сетевых задержек, сбоев и неравномерной нагрузки эти два значения могут существенно расходиться. Именно поэтому системе нужен механизм watermarks: без него она не знает, когда можно считать прошедший временной диапазон достаточно полным, чтобы выдать результат. -## Компромисс точности и задержки {#tradeoff} +## Компромисс Точности и Задержки {#tradeoff} + +В {{ ydb-short-name }} этот компромисс регулируется явно: параметр [`WATERMARK`](../../dev/streaming-query/watermarks.md#configuration) в предложении `GROUP BY` задаёт выражение для вычисления watermark, в том числе величину отставания от времени последнего события. Это позволяет подобрать баланс между актуальностью результатов и полнотой учёта запоздавших событий. Watermark не может одновременно учитывать сколь угодно большие задержки событий и продвигаться быстро: чем дольше система ждёт опоздавших событий, тем позже она выдаёт результаты. Это фундаментальный компромисс потоковой обработки. diff --git a/ydb/docs/ru/core/concepts/topology.md b/ydb/docs/ru/core/concepts/topology.md index 4339c5ced2d..2b42b5e97c8 100644 --- a/ydb/docs/ru/core/concepts/topology.md +++ b/ydb/docs/ru/core/concepts/topology.md @@ -31,6 +31,7 @@ | `mirror-3-dc` *(3 узла)*, переживает отказ 1 узла или 1 дата-центра | 3 | 3 | Сервер | Дата-центр | 3 | Не важно | | `block-4-2`, переживает отказ 2 стоек | 1.5 | 8 ([10 рекомендовано](*recommended-node-count)) | Стойка | Дата-центр | 1 | 8 | | `block-4-2` *(упрощённый)*, переживает отказ 1 стойки | 1.5 | 10 | ½ стойки | Дата-центр | 1 | 5 | +| `block-4-2` *(упрощённый отказоустойчивый)*, переживает отказ 1 узла | 1.5 | 4 | Сервер | Дата-центр | 1 | Не важно | | `none`, избыточность отсутствует | 1 | 1 | Узел | Узел | 1 | 1 | {% note info %} @@ -61,7 +62,11 @@ В случаях, когда невозможно использовать [рекомендованное количество](#cluster-config) оборудования, можно разделить серверы одной стойки на 2 фиктивных домена отказа. В такой конфигурации отказ одной стойки будет означать отказ не одного, а сразу двух доменов. При использовании таких упрощённых конфигураций {{ ydb-short-name }} сохраняет работоспособность при отказе сразу двух доменов. Минимальное количество стоек в кластере для режима `block-4-2` составляет 5, а для `mirror-3-dc` — по 2 в каждом дата-центре (т.е. суммарно 6 стоек). -Минимальная отказоустойчивая конфигурация кластера {{ ydb-short-name }} представляет собой вариант режима работы `mirror-3-dc` с 3 узлами, который требует всего три сервера с тремя дисками на каждом. В этой конфигурации каждый сервер выступает как домен отказа, так и как область отказа. Таким образом, кластер может выдержать отказ только одного сервера. Каждый сервер должен находиться в своём независимом дата-центре для обеспечения должного уровня отказоустойчивости. +Есть 2 варианта минимальной отказоустойчивой конфигурации кластера {{ ydb-short-name }}: + +- Вариант режима работы `mirror-3-dc` с 3 узлами, который требует всего три сервера с тремя дисками на каждом. В этой конфигурации каждый сервер выступает как домен отказа, так и как область отказа. Таким образом, кластер может выдержать отказ только одного сервера. Каждый сервер должен находиться в своём независимом дата-центре для обеспечения должного уровня отказоустойчивости. + +- Вариант режима работы `block-4-2` с 4 узлами, который требует 4 сервера с 2 или более дисками на каждом. В этой конфигурации диски каждого сервера делятся на 2 домена отказа с помощью атрибута [`disk_scope`](../reference/configuration/host_configs.md#disk-scope), что в сумме даёт 8 доменов отказа, необходимых для работы режима `block-4-2`. Такой кластер сохраняет работоспособность при отказе одного сервера. Кластеры {{ ydb-short-name }}, настроенные с использованием одного из этих подходов, могут использоваться в production окружениях, если в них не требуются повышенные гарантии отказоустойчивости. diff --git a/ydb/docs/ru/core/dev/resource-consumption-management.md b/ydb/docs/ru/core/dev/resource-consumption-management.md index 01f41e1638c..c33c75eb4f6 100644 --- a/ydb/docs/ru/core/dev/resource-consumption-management.md +++ b/ydb/docs/ru/core/dev/resource-consumption-management.md @@ -22,13 +22,13 @@ CREATE RESOURCE POOL olap WITH ( CONCURRENT_QUERY_LIMIT=10, QUEUE_SIZE=1000, DATABASE_LOAD_CPU_THRESHOLD=80, - RESOURCES_WEIGHT=100, + RESOURCE_WEIGHT=100, QUERY_CPU_LIMIT_PERCENT_PER_NODE=50, TOTAL_CPU_LIMIT_PERCENT_PER_NODE=70 ) ``` -Ознакомиться с полным перечнем параметров пулов ресурсов можно в справке по [{#T}](../yql/reference/syntax/create-resource-pool.md#parameters). Часть параметров является глобальными для всей базы данных (например, `CONCURRENT_QUERY_LIMIT`, `QUEUE_SIZE`, `DATABASE_LOAD_CPU_THRESHOLD`), а другие — применяются только к одному вычислительному узлу (например, `QUERY_CPU_LIMIT_PERCENT_PER_NODE`, `TOTAL_CPU_LIMIT_PERCENT_PER_NODE`, `TOTAL_MEMORY_LIMIT_PERCENT_PER_NODE`). Между всеми пулами может разделяться CPU в случае переподписки на одном вычислительном узле с помощью `RESOURCES_WEIGHT`. +Ознакомиться с полным перечнем параметров пулов ресурсов можно в справке по [{#T}](../yql/reference/syntax/create-resource-pool.md#parameters). Часть параметров является глобальными для всей базы данных (например, `CONCURRENT_QUERY_LIMIT`, `QUEUE_SIZE`, `DATABASE_LOAD_CPU_THRESHOLD`), а другие — применяются только к одному вычислительному узлу (например, `QUERY_CPU_LIMIT_PERCENT_PER_NODE`, `TOTAL_CPU_LIMIT_PERCENT_PER_NODE`, `TOTAL_MEMORY_LIMIT_PERCENT_PER_NODE`). Между всеми пулами может разделяться CPU в случае переподписки на одном вычислительном узле с помощью `RESOURCE_WEIGHT`.  @@ -64,18 +64,18 @@ Issues: Как и в случае `CONCURRENT_QUERY_LIMIT`, при превышении указанного порога нагрузки запросы отправляются в очередь ожидания. -### Распределение ресурсов в соответствии с RESOURCES_WEIGHT {#resources_weight} +### Распределение ресурсов в соответствии с RESOURCE_WEIGHT {#resources_weight}  -Параметр `RESOURCES_WEIGHT` начинает работать только в случае переподписки и при наличии более одного пула ресурсов в системе. В текущей реализации `RESOURCES_WEIGHT` влияет только на распределение ресурсов `vCPU`. Когда в пуле ресурсов появляются запросы, он начинает участвовать в распределении ресурсов. Для этого в пулах происходит перерасчёт лимитов согласно алгоритму [Max-min fairness](https://en.wikipedia.org/wiki/Max-min_fairness). Само перераспределение ресурсов выполняется на каждом вычислительном узле индивидуально, как показано на рисунке выше. +Параметр `RESOURCE_WEIGHT` начинает работать только в случае переподписки и при наличии более одного пула ресурсов в системе. В текущей реализации `RESOURCE_WEIGHT` влияет только на распределение ресурсов `vCPU`. Когда в пуле ресурсов появляются запросы, он начинает участвовать в распределении ресурсов. Для этого в пулах происходит перерасчёт лимитов согласно алгоритму [Max-min fairness](https://en.wikipedia.org/wiki/Max-min_fairness). Само перераспределение ресурсов выполняется на каждом вычислительном узле индивидуально, как показано на рисунке выше. Допустим, у нас есть узел в системе с доступными $10 vCPU$. Установлены ограничения: - $TOTAL\_CPU\_LIMIT\_PERCENT\_PER\_NODE = 30$, - $QUERY\_CPU\_LIMIT\_PERCENT\_PER\_NODE = 50$. -В этом случае у пула ресурсов будет ограничение $3 vCPU$ на узел и $1.5 vCPU$ на один запрос в этом пуле (рисунок *a*). Если в системе существует 4 таких пула, и все они пытаются использовать максимальные ресурсы, это составит $12 vCPU$, что превышает предел доступных ресурсов на узле ($10 vCPU$). В этом случае начинают действовать `RESOURCES_WEIGHT`, и каждому пулу будет выделено по $2.5 vCPU$ (рисунок *b*). +В этом случае у пула ресурсов будет ограничение $3 vCPU$ на узел и $1.5 vCPU$ на один запрос в этом пуле (рисунок *a*). Если в системе существует 4 таких пула, и все они пытаются использовать максимальные ресурсы, это составит $12 vCPU$, что превышает предел доступных ресурсов на узле ($10 vCPU$). В этом случае начинают действовать `RESOURCE_WEIGHT`, и каждому пулу будет выделено по $2.5 vCPU$ (рисунок *b*). Если необходимо увеличить выделяемые ресурсы для конкретного пула, можно изменить его вес, например, на 200. Тогда этот пул получит $3 vCPU$, а остальные пулы поделят поровну оставшиеся $7 vCPU$, что составит $\frac{7}{3} vCPU$ на каждый пул (рисунок *c*). @@ -94,7 +94,7 @@ CREATE RESOURCE POOL default WITH ( CONCURRENT_QUERY_LIMIT=-1, QUEUE_SIZE=-1, DATABASE_LOAD_CPU_THRESHOLD=-1, - RESOURCES_WEIGHT=-1, + RESOURCE_WEIGHT=-1, TOTAL_MEMORY_LIMIT_PERCENT_PER_NODE=-1, QUERY_CPU_LIMIT_PERCENT_PER_NODE=-1, TOTAL_CPU_LIMIT_PERCENT_PER_NODE=-1 @@ -136,7 +136,7 @@ WITH ( ``` - `RESOURCE_POOL` — имя пула ресурсов, в который будет отправлен запрос, удовлетворяющий требованиям, заданным в классификаторе пула ресурсов. -- `MEMBER_NAME` — группа пользователей или пользователь, запросы которых будут отправлены в указанный пул ресурсов. +- `MEMBER_NAME` — группа пользователей или пользователь, запросы которых будут отправлены в указанный пул ресурсов. Подробнее о формате значения см. [CREATE RESOURCE POOL CLASSIFIER](../yql/reference/syntax/create-resource-pool-classifier.md#member-name-format). ## Управление ACL классификатора пула ресурсов @@ -183,7 +183,7 @@ CREATE RESOURCE POOL olap WITH ( CONCURRENT_QUERY_LIMIT=20, QUEUE_SIZE=100, DATABASE_LOAD_CPU_THRESHOLD=80, - RESOURCES_WEIGHT=20, + RESOURCE_WEIGHT=20, QUERY_CPU_LIMIT_PERCENT_PER_NODE=80, TOTAL_CPU_LIMIT_PERCENT_PER_NODE=100 ); @@ -191,7 +191,7 @@ CREATE RESOURCE POOL olap WITH ( CREATE RESOURCE POOL the_ceo WITH ( CONCURRENT_QUERY_LIMIT=20, QUEUE_SIZE=100, - RESOURCES_WEIGHT=100, + RESOURCE_WEIGHT=100, QUERY_CPU_LIMIT_PERCENT_PER_NODE=100, TOTAL_CPU_LIMIT_PERCENT_PER_NODE=100 ); diff --git a/ydb/docs/ru/core/dev/streaming-query/watermarks.md b/ydb/docs/ru/core/dev/streaming-query/watermarks.md index 21ed92747aa..d8ed9c1bab6 100644 --- a/ydb/docs/ru/core/dev/streaming-query/watermarks.md +++ b/ydb/docs/ru/core/dev/streaming-query/watermarks.md @@ -4,7 +4,7 @@ Watermark — монотонно возрастающая нижняя оцен� ## Время события {#event-time} -В потоковой обработке каждое событие имеет временную метку, по которой система отслеживает прогресс времени в потоке. В текущей реализации источником времени события может быть только время записи события в [топик](../../concepts/datamodel/topic.md), доступное через системную колонку `__ydb_write_time`. +В потоковой обработке каждое событие имеет временную метку, по которой система отслеживает прогресс времени в потоке. В текущей реализации источником времени события может быть только время записи события в [топик](../../concepts/datamodel/topic.md), доступное через системную колонку [`__ydb_write_time`](../../concepts/query_execution/topics.md#system-metadata). {% note info %} @@ -35,7 +35,7 @@ sequenceDiagram ## Вычисление watermark {#watermark-computation} -Когда система получает событие, она обновляет watermark — **продвигает** его вперёд по временной оси. Watermark вычисляется как `максимальное наблюдаемое время события − delay`, где `delay` — величина отставания, заданная в выражении [`WATERMARK`](#configuration) (например, `Interval("PT5S")` в `WATERMARK = __ydb_write_time - Interval("PT5S")`). +Когда система получает событие, она обновляет watermark — **продвигает** его вперёд по временной оси. Watermark вычисляется как `максимальное наблюдаемое время события − delay`, где `delay` — величина отставания, заданная в выражении [`WATERMARK`](#configuration) (например, `Interval("PT5S")` в `WATERMARK = __ydb_write_time - Interval("PT5S")`; [`__ydb_write_time`](../../concepts/query_execution/topics.md#system-metadata) — служебная колонка топика). События в потоке могут приходить не в хронологическом порядке: событие с временем 10:00:03 может быть обработано после события с временем 10:00:05. Причины: расхождение часов в распределённой системе, сетевые задержки, неравномерная нагрузка на [партиции](../../concepts/datamodel/topic.md#partitioning) топика. @@ -61,7 +61,7 @@ Watermarks включаются и настраиваются в секции [W {% note warning %} -При использовании [HoppingWindow](../../yql/reference/syntax/select/group-by.md#group-by-hopping_window) первый параметр (time extractor) и источник времени в выражении WATERMARK должны совпадать. В текущей реализации оба должны использовать `__ydb_write_time`. +При использовании [HoppingWindow](../../yql/reference/syntax/select/group-by.md#group-by-hopping_window) первый параметр (time extractor) и источник времени в выражении WATERMARK должны совпадать. В текущей реализации оба должны использовать [`__ydb_write_time`](../../concepts/query_execution/topics.md#system-metadata). {% endnote %} @@ -120,7 +120,7 @@ END DO; Где: - [`CREATE STREAMING QUERY`](../../yql/reference/syntax/create-streaming-query.md) — создаёт именованный потоковый запрос. -- `__ydb_write_time` — системная колонка, содержащая время записи события в [топик](../../concepts/datamodel/topic.md). +- [`__ydb_write_time`](../../concepts/query_execution/topics.md#system-metadata) — системная колонка, содержащая время записи события в [топик](../../concepts/datamodel/topic.md). - `FORMAT = json_each_row` — [формат данных](streaming-query-formats.md) в топике, каждая строка содержит отдельный JSON-объект. - `WATERMARK = __ydb_write_time - Interval("PT5S")` — watermark с отставанием 5 секунд. `Interval("PT5S")` задаёт интервал в формате [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601#Durations). - [`AGGREGATE_LIST`](../../yql/reference/builtins/aggregation.md#agg-list) — агрегатная функция, собирающая значения в список. diff --git a/ydb/docs/ru/core/devops/deployment-options/manual/initial-deployment/deployment-configuration-v1.md b/ydb/docs/ru/core/devops/deployment-options/manual/initial-deployment/deployment-configuration-v1.md index 8efa15c447b..433145ac6e4 100644 --- a/ydb/docs/ru/core/devops/deployment-options/manual/initial-deployment/deployment-configuration-v1.md +++ b/ydb/docs/ru/core/devops/deployment-options/manual/initial-deployment/deployment-configuration-v1.md @@ -403,53 +403,51 @@ - drive: - path: /dev/disk/by-partlabel/ydb_disk_ssd_01 type: SSD - - path: /dev/disk/by-partlabel/ydb_disk_ssd_02 - type: SSD host_config_id: 1 hosts: - - host: ydb-node-zone-a-1.local + - host: static-node-1.ydb-cluster.com host_config_id: 1 walle_location: body: 1 data_center: 'zone-a' rack: '1' - - host: ydb-node-zone-a-2.local + - host: static-node-2.ydb-cluster.com host_config_id: 1 walle_location: body: 2 data_center: 'zone-a' rack: '2' - - host: ydb-node-zone-a-3.local + - host: static-node-3.ydb-cluster.com host_config_id: 1 walle_location: body: 3 data_center: 'zone-a' rack: '3' - - host: ydb-node-zone-a-4.local + - host: static-node-4.ydb-cluster.com host_config_id: 1 walle_location: body: 4 data_center: 'zone-a' rack: '4' - - host: ydb-node-zone-a-5.local + - host: static-node-5.ydb-cluster.com host_config_id: 1 walle_location: body: 5 data_center: 'zone-a' rack: '5' - - host: ydb-node-zone-a-6.local + - host: static-node-6.ydb-cluster.com host_config_id: 1 walle_location: body: 6 data_center: 'zone-a' rack: '6' - - host: ydb-node-zone-a-7.local + - host: static-node-7.ydb-cluster.com host_config_id: 1 walle_location: body: 7 data_center: 'zone-a' rack: '7' - - host: ydb-node-zone-a-8.local + - host: static-node-8.ydb-cluster.com host_config_id: 1 walle_location: body: 8 @@ -458,11 +456,21 @@ domains_config: security_config: enforce_user_token_requirement: true - default_users: - - name: "root" - password: "" - default_access: - - "+(F):root" + monitoring_allowed_sids: + - "root" + - "ADMINS" + - "DATABASE-ADMINS" + administration_allowed_sids: + - "root" + - "ADMINS" + - "DATABASE-ADMINS" + viewer_allowed_sids: + - "root" + - "ADMINS" + - "DATABASE-ADMINS" + register_dynamic_node_allowed_sids: + - databaseNodes@cert + - root@builtin domain: - name: Root storage_pool_types: @@ -513,55 +521,55 @@ rings: - fail_domains: - vdisk_locations: - - node_id: "ydb-node-zone-a-1.local" + - node_id: "static-node-1.ydb-cluster.com" pdisk_category: SSD path: /dev/disk/by-partlabel/ydb_disk_ssd_01 - fail_domains: - vdisk_locations: - - node_id: "ydb-node-zone-a-2.local" + - node_id: "static-node-2.ydb-cluster.com" pdisk_category: SSD path: /dev/disk/by-partlabel/ydb_disk_ssd_01 - fail_domains: - vdisk_locations: - - node_id: "ydb-node-zone-a-3.local" + - node_id: "static-node-3.ydb-cluster.com" pdisk_category: SSD path: /dev/disk/by-partlabel/ydb_disk_ssd_01 - fail_domains: - vdisk_locations: - - node_id: "ydb-node-zone-a-4.local" + - node_id: "static-node-4.ydb-cluster.com" pdisk_category: SSD path: /dev/disk/by-partlabel/ydb_disk_ssd_01 - fail_domains: - vdisk_locations: - - node_id: "ydb-node-zone-a-5.local" + - node_id: "static-node-5.ydb-cluster.com" pdisk_category: SSD path: /dev/disk/by-partlabel/ydb_disk_ssd_01 - fail_domains: - vdisk_locations: - - node_id: "ydb-node-zone-a-6.local" + - node_id: "static-node-6.ydb-cluster.com" pdisk_category: SSD path: /dev/disk/by-partlabel/ydb_disk_ssd_01 - fail_domains: - vdisk_locations: - - node_id: "ydb-node-zone-a-7.local" + - node_id: "static-node-7.ydb-cluster.com" pdisk_category: SSD path: /dev/disk/by-partlabel/ydb_disk_ssd_01 - fail_domains: - vdisk_locations: - - node_id: "ydb-node-zone-a-8.local" + - node_id: "static-node-8.ydb-cluster.com" pdisk_category: SSD path: /dev/disk/by-partlabel/ydb_disk_ssd_01 channel_profile_config: profile: - channel: - erasure_species: block-4-2 - pdisk_category: 1 + pdisk_category: 0 storage_pool_kind: ssd - erasure_species: block-4-2 - pdisk_category: 1 + pdisk_category: 0 storage_pool_kind: ssd - erasure_species: block-4-2 - pdisk_category: 1 + pdisk_category: 0 storage_pool_kind: ssd profile_id: 0 interconnect_config: @@ -579,7 +587,7 @@ client_certificate_authorization: request_client_certificate: true client_certificate_definitions: - - member_groups: ["registerNode@cert"] + - member_groups: ["databaseNodes@cert"] subject_terms: - short_name: "O" values: ["YDB"] diff --git a/ydb/docs/ru/core/devops/enterprise-manager/ai-assistant.md b/ydb/docs/ru/core/devops/enterprise-manager/ai-assistant.md new file mode 100644 index 00000000000..e61f084664d --- /dev/null +++ b/ydb/docs/ru/core/devops/enterprise-manager/ai-assistant.md @@ -0,0 +1,205 @@ +# Настройка AI-ассистента в YDB EM + +В этой инструкции описано, как включить AI-ассистента в {{ ydb-short-name }} Enterprise Manager (YDB EM). После настройки пользователи увидят ассистента в веб-интерфейсе YDB EM. Ассистент отправляет запросы к модели через [Gateway](index.md#architecture) и может использовать инструменты Model Context Protocol (MCP), предоставляемые Gateway. + +## Перед началом {#before-start} + +Эту инструкцию можно использовать до первого развёртывания YDB EM или для уже работающей установки. При первом развёртывании добавьте переменные в inventory до запуска initial setup playbook. Инструкцию по развёртыванию см. в разделе [{#T}](initial-deployment.md). + +Убедитесь, что у вас есть: + +1. Доступ к Ansible inventory, который используется для развёртывания YDB EM. +1. Endpoint OpenAI-compatible модели, который будет доступен с хоста Gateway. +1. Права на изменение [token-файла Gateway](#configure-model-access). В рамках настройки будет необходимо прописать секрет для доступа к модели в token-файл. Gateway читает этот файл и использует значение поля `Token` записи, у которой поле `Name` равно `ydb_em_ai_model_token_name`, например `model-token`, как upstream-заголовок `Authorization`. +1. Если вы обновляете существующую установку, доступ к развёрнутому хосту Gateway. + +Для уже работающей установки сначала убедитесь, что хост Gateway управляется тем же Ansible inventory. Перед запуском playbook проверьте активный service manager unit, владельца процесса, путь к конфигурации, token-файл и порт, который слушает Gateway. Если на хосте используется custom или ручной layout Gateway, применяйте те же настройки через операционную процедуру этой установки, а не запускайте playbook вслепую. + +{% note warning %} + +Не помещайте API-ключи модели, OAuth-токены и другие секреты в `ydb_em_ai_assistant_client_runtime_config`. Gateway возвращает это значение в браузер через `GET /meta/ai_assistant_client_config`. Секреты держите в token-файле. + +{% endnote %} + +## Как это работает {#how-it-works} + +Браузер работает с ассистентом через Gateway: + +1. UI проверяет `GET /capabilities`. Кнопка ассистента отображается, если `Settings.Proxy.Model` равно `true` и включена пользовательская настройка AI-ассистента. Когда backend capability появляется впервые, YDB EM инициализирует эту пользовательскую настройку включённой. +1. UI получает runtime-настройки из `GET /meta/ai_assistant_client_config`. +1. UI отправляет запросы к модели в `/proxy/model/...`. +1. Gateway перенаправляет запросы к модели в `ydb_em_ai_model_endpoint` и добавляет заголовок Authorization из tokenator. +1. UI отправляет MCP-запросы в `/meta/mcp`. Gateway публикует MCP-инструменты через этот endpoint и выполняет их через API YDB EM и proxy endpoints кластеров. Если поиск по документации настроен, `search_docs` публикуется через тот же MCP endpoint и доступен ассистенту. + +## Настройте доступ к модели {#configure-model-access} + +`ydb_em_ai_model_token_name` — это имя записи [tokenator](../../concepts/glossary.md#tokenator), которую Gateway использует для запросов к модели. + +Пример записи tokenator: + +```textproto +StaticTokenInfo { + Name: "model-token" + Token: "Bearer <model-api-token>" +} +``` + +Способ доставки токена зависит от вашего процесса развёртывания. Главное требование: tokenator-файл Gateway содержит запись, у которой `Name` совпадает с `ydb_em_ai_model_token_name`. + +{% note info %} + +Стандартный Ansible-шаблон токенов YDB EM генерирует только запись `meta` для доступа к metabase. Если в вашей установке используется этот шаблон без изменений, расширьте или переопределите tokenator-файл через ваш процесс развёртывания, чтобы в нём также были `model-token` и, если он отличается, токен для embeddings. + +{% endnote %} + +## Настройте Ansible-переменные {#configure-ansible-variables} + +Добавьте переменные AI-ассистента в inventory YDB EM, например в `examples/inventory/50-inventory.yaml` или другой inventory-файл вашего развёртывания. + +```yaml +ydb_em_ai_assistant_enabled: true +ydb_em_ai_model_endpoint: "https://llm.example.com" +ydb_em_ai_model_token_name: "model-token" + +ydb_em_ai_assistant_client_runtime_config: + llm: + baseURL: "/proxy/model/v1" + model: "<model-name>" + temperature: 0.2 + mcp: + - id: "ydb-meta" + url: "/meta/mcp" + transport: "streamable" + toolCallTimeoutMs: 600000 +``` + +Параметр | Описание +--- | --- +`ydb_em_ai_assistant_enabled` | Включает настройки AI-ассистента в Gateway. +`ydb_em_ai_model_endpoint` | Upstream model API endpoint, который использует Gateway. +`ydb_em_ai_model_token_name` | Имя записи tokenator для запросов к модели. +`ydb_em_ai_assistant_client_runtime_config.llm.baseURL` | URL модели, который видит браузер. Используйте Gateway proxy, обычно `/proxy/model/v1`. +`ydb_em_ai_assistant_client_runtime_config.llm.model` | Имя модели, которое отправляется в OpenAI-compatible API. +`ydb_em_ai_assistant_client_runtime_config.mcp` | MCP-серверы, доступные ассистенту. Для YDB EM используйте `/meta/mcp`. + +Gateway добавляет путь из `/proxy/model/...` к `ydb_em_ai_model_endpoint`. При `baseURL: "/proxy/model/v1"` запрос chat completions будет перенаправлен в `/v1/chat/completions` на upstream endpoint. Настройте endpoint так, чтобы этот путь был корректен для вашего provider. + +## Настройте поиск по документации {#configure-docs-search} + +Рекомендуется включить поиск по документации, чтобы ассистент получил MCP-инструмент `search_docs`. Для этого инструмента Gateway вызывает OpenAI-compatible embeddings endpoint и добавляет `/embeddings` к настроенному base URL, если suffix отсутствует. Когда поиск по документации включён, ассистент получает `search_docs` через настроенный MCP-сервер `/meta/mcp`. + +```yaml +ydb_em_docs_search_enabled: true +ydb_em_docs_search_embeddings_upstream_base_url: "https://llm.example.com/v1" +ydb_em_docs_search_embeddings_token_name: "model-token" +ydb_em_docs_search_embeddings_model: "<embeddings-model-name>" +ydb_em_docs_search_vector_size: 0 +ydb_em_docs_search_limit: 6 +ydb_em_docs_search_score: 0.6 +``` + +Параметр | Описание +--- | --- +`ydb_em_docs_search_enabled` | Настраивает backend семантического поиска по документации и публикует `search_docs` через MCP, если все обязательные настройки поиска валидны. +`ydb_em_docs_search_embeddings_upstream_base_url` | Базовый URL OpenAI-совместимого endpoint для embeddings. +`ydb_em_docs_search_embeddings_token_name` | Имя записи tokenator для embeddings-запросов. +`ydb_em_docs_search_embeddings_model` | Имя embeddings-модели. +`ydb_em_docs_search_vector_size` | Необязательное переопределение размерности вектора эмбеддинга. Не задавайте переменную, чтобы использовать Ansible default `0`. При `0` Gateway не отправляет OpenAI-compatible поле `dimensions` и отключает проверку размера ответа. Положительное значение задавайте только если embeddings provider и модель поддерживают explicit dimensions и ожидаемый размер известен. +`ydb_em_docs_search_limit` | Максимальное количество возвращаемых документов. +`ydb_em_docs_search_score` | Минимальная оценка релевантности от `0` до `1`. Документы с меньшей оценкой не возвращаются. + +## Примените конфигурацию {#apply-configuration} + +После обновления inventory запустите playbook YDB EM из каталога с вашим inventory. Один и тот же playbook используется для первого развёртывания и для применения этого изменения к существующей установке: + +```bash +ansible-playbook ydb_platform.ydb_em.initial_setup +``` + +Если vault-файл зашифрован, добавьте vault-параметр, который используется в вашем развёртывании: + +```bash +ansible-playbook ydb_platform.ydb_em.initial_setup --ask-vault-pass +``` + +Роль сгенерирует конфигурацию и стандартный token-файл Gateway и убедится, что сервис Gateway запущен. Если дополнительные учётные данные модели доставляются через переопределённый token-файл или отдельный шаг развёртывания, убедитесь, что развёрнутый token-файл содержит настроенное имя записи. Если вы применяете эти настройки к уже работающему развёртыванию, перезапустите Gateway по вашей операционной процедуре, если ваш Ansible-запуск не перезапускает сервис после изменения конфигурации или token-файла. + +## Проверьте настройку {#verify} + +Откройте веб-интерфейс YDB EM: + +```text +https://<gateway-host>:8789/ui/clusters +``` + +Проверьте, что model proxy включён: + +```bash +curl -k https://<gateway-host>:8789/capabilities +``` + +Ответ должен содержать: + +```json +{ + "Settings": { + "Proxy": { + "Model": true + } + } +} +``` + +Проверьте runtime-конфигурацию, которую получает UI: + +```bash +curl -k https://<gateway-host>:8789/meta/ai_assistant_client_config +``` + +Ответ должен содержать блок `llm` и не должен содержать секреты модели. + +Проверьте model proxy: + +```bash +curl -k https://<gateway-host>:8789/proxy/model/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"<model-name>","messages":[{"role":"user","content":"ping"}],"max_tokens":1}' +``` + +Если в вашем развёртывании Gateway требует аутентификацию пользователя, добавьте те же authentication headers, которые используются для UI YDB EM. + +Если включён поиск по документации, также проверьте, что `/meta/mcp` публикует `search_docs`, и выполните безопасный поисковый запрос через ваш MCP-клиент или tooling ассистента. Это проверяет и регистрацию MCP-инструмента, и путь embeddings/metabase. + +## Устранение неполадок {#troubleshooting} + +### Кнопка ассистента не отображается {#button-not-shown} + +Проверьте, что `ydb_em_ai_assistant_enabled` равно `true`, а `/capabilities` содержит `Settings.Proxy.Model: true`. Также проверьте пользовательскую настройку AI-ассистента в UI YDB EM: YDB EM инициализирует её включённой, когда backend capability присутствует, но пользователь может её выключить. + +### Runtime-конфигурация невалидна {#runtime-config-invalid} + +Проверьте `GET /meta/ai_assistant_client_config`. Для включённого ассистента ответ должен быть JSON-объектом со строковыми полями `llm.baseURL` и `llm.model`. Ответ `null` означает, что Gateway не загрузил включённую клиентскую конфигурацию AI-ассистента. + +### Model proxy возвращает ошибку авторизации {#model-authorization-error} + +Проверьте, что `ydb_em_ai_model_token_name` совпадает с именем записи tokenator, а запись возвращает заголовок Authorization, который ожидает endpoint модели. Если tokenator не находит запись, Gateway отправляет upstream-запрос без ожидаемого заголовка Authorization. + +### Model proxy вызывает неправильный upstream path {#wrong-upstream-path} + +Проверьте `ydb_em_ai_model_endpoint` вместе с `llm.baseURL`. Gateway добавляет путь из `/proxy/model/...` к настроенному endpoint. Дублирование `/v1` часто приводит к upstream `404`. + +### MCP-инструменты недоступны {#mcp-tools-unavailable} + +Проверьте, что runtime-конфигурация содержит `url: "/meta/mcp"` и что пользовательские запросы могут обращаться к `/meta/mcp` через Gateway. Gateway регистрирует `/meta/mcp` только если MCP включён; по умолчанию он включён, но custom Gateway config может его отключить. + +### Поиск по документации недоступен {#docs-search-unavailable} + +Проверьте настройки `ydb_em_docs_search_*`, путь embeddings endpoint и запись tokenator для embeddings-запросов. Также убедитесь, что documentation vectors доступны в metabase YDB EM. Если поиск по документации не настроен, инструмент `search_docs` не публикуется. + +### Embeddings endpoint отклоняет размер вектора {#embeddings-vector-size} + +Если `search_docs` падает с `variable embedding size not supported`, установите `ydb_em_docs_search_vector_size: 0` и проверьте embeddings endpoint и модель. + +### Поиск по документации возвращает 504 {#docs-search-504} + +Если `search_docs` возвращает `504 Gateway Timeout`, а в логах Gateway для `/meta/docs` есть `Member not found: doc_id`, проверьте таблицу metabase `ydb/Docs.db`. Текущий запрос Gateway docs search использует колонки `doc_id`, `embedding` и `payload`. Для полезных результатов в таблице также должны быть строки, а сама таблица должна быть построена для embeddings model, совместимой с `ydb_em_docs_search_embeddings_model`. diff --git a/ydb/docs/ru/core/devops/enterprise-manager/index.md b/ydb/docs/ru/core/devops/enterprise-manager/index.md index 3fda6e30032..bbc397de54a 100644 --- a/ydb/docs/ru/core/devops/enterprise-manager/index.md +++ b/ydb/docs/ru/core/devops/enterprise-manager/index.md @@ -81,3 +81,4 @@ flowchart LR - [{#T}](initial-deployment.md) - [{#T}](s3-backups.md) +- [{#T}](ai-assistant.md) diff --git a/ydb/docs/ru/core/devops/enterprise-manager/toc_p.yaml b/ydb/docs/ru/core/devops/enterprise-manager/toc_p.yaml index d5d01a59696..58cc63c869c 100644 --- a/ydb/docs/ru/core/devops/enterprise-manager/toc_p.yaml +++ b/ydb/docs/ru/core/devops/enterprise-manager/toc_p.yaml @@ -3,3 +3,5 @@ items: href: initial-deployment.md - name: Настройка резервного копирования в S3 href: s3-backups.md +- name: Настройка AI-ассистента + href: ai-assistant.md diff --git a/ydb/docs/ru/core/downloads/ydb-ansible.md b/ydb/docs/ru/core/downloads/ydb-ansible.md index d424f9e3a1c..5b9f36e95a7 100644 --- a/ydb/docs/ru/core/downloads/ydb-ansible.md +++ b/ydb/docs/ru/core/downloads/ydb-ansible.md @@ -4,6 +4,7 @@ | Версия | Дата выпуска | Скачать | Список изменений | | ------ | ------------ | ------- | ----------------- | +| v2.1.0 | 29.06.2026 | [ydb-ansible-2.1.0.tar.gz](https://github.com/ydb-platform/ydb-ansible/releases/download/v2.1.0/ydb_platform-ydb-2.1.0.tar.gz) | | | v2.0.0 | 23.12.2025 | [ydb-ansible-2.0.0.tar.gz](https://github.com/ydb-platform/ydb-ansible/releases/download/v2.0.0/ydb_platform-ydb-2.0.0.tar.gz) | | | v1.3.2 | 02.12.2025 | [ydb-ansible-1.3.2.tar.gz](https://github.com/ydb-platform/ydb-ansible/releases/download/v1.3.2/ydb_platform-ydb-1.3.2.tar.gz) | | | v1.3.1 | 01.12.2025 | [ydb-ansible-1.3.1.tar.gz](https://github.com/ydb-platform/ydb-ansible/releases/download/v1.3.1/ydb_platform-ydb-1.3.1.tar.gz) | | diff --git a/ydb/docs/ru/core/recipes/streaming_queries/debug-read.md b/ydb/docs/ru/core/recipes/streaming_queries/debug-read.md index 46c9f856d15..622853532d4 100644 --- a/ydb/docs/ru/core/recipes/streaming_queries/debug-read.md +++ b/ydb/docs/ru/core/recipes/streaming_queries/debug-read.md @@ -13,5 +13,5 @@ ## См. также * [{#T}](../../concepts/query_execution/topics.md) -* [{#T}](../../yql/reference/syntax/select/streaming.md) +* [{#T}](../../yql/reference/syntax/select/topics.md) * [{#T}](../../dev/streaming-query/streaming-query-formats.md) diff --git a/ydb/docs/ru/core/recipes/ydb-sdk/debug-jaeger.md b/ydb/docs/ru/core/recipes/ydb-sdk/debug-jaeger.md deleted file mode 100644 index 591aa6cb092..00000000000 --- a/ydb/docs/ru/core/recipes/ydb-sdk/debug-jaeger.md +++ /dev/null @@ -1,172 +0,0 @@ -# Включение трассировки в Jaeger - -Ниже приведены примеры кода для включения трассировки в Jaeger в различных {{ ydb-short-name }} SDK. - -{% list tabs %} - -- C++ - - Функциональность в настоящее время не поддерживается. - -- Go - - {% list tabs %} - - - Нативный SDK - - ```go - package main - - import ( - "context" - "time" - - "github.com/opentracing/opentracing-go" - jaegerConfig "github.com/uber/jaeger-client-go/config" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - - tracing "github.com/ydb-platform/ydb-go-sdk-opentracing" - ) - - const ( - tracerURL = "localhost:5775" - serviceName = "ydb-go-sdk" - ) - - func main() { - tracer, closer, err := jaegerConfig.Configuration{ - ServiceName: serviceName, - Sampler: &jaegerConfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jaegerConfig.ReporterConfig{ - LogSpans: true, - BufferFlushInterval: 1 * time.Second, - LocalAgentHostPort: tracerURL, - }, - }.NewTracer() - if err != nil { - panic(err) - } - - defer closer.Close() - - // set global tracer of this application - opentracing.SetGlobalTracer(tracer) - - span, ctx := opentracing.StartSpanFromContext(context.Background(), "client") - defer span.Finish() - - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - tracing.WithTraces(tracing.WithDetails(trace.DetailsAll)), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - ... - } - ``` - - - database/sql - - ```go - package main - - import ( - "context" - "database/sql" - "time" - - "github.com/opentracing/opentracing-go" - jaegerConfig "github.com/uber/jaeger-client-go/config" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - - tracing "github.com/ydb-platform/ydb-go-sdk-opentracing" - ) - - const ( - tracerURL = "localhost:5775" - serviceName = "ydb-go-sdk" - ) - - func main() { - tracer, closer, err := jaegerConfig.Configuration{ - ServiceName: serviceName, - Sampler: &jaegerConfig.SamplerConfig{ - Type: "const", - Param: 1, - }, - Reporter: &jaegerConfig.ReporterConfig{ - LogSpans: true, - BufferFlushInterval: 1 * time.Second, - LocalAgentHostPort: tracerURL, - }, - }.NewTracer() - if err != nil { - panic(err) - } - - defer closer.Close() - - // set global tracer of this application - opentracing.SetGlobalTracer(tracer) - - span, ctx := opentracing.StartSpanFromContext(context.Background(), "client") - defer span.Finish() - - nativeDriver, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - tracing.WithTraces(tracing.WithDetails(trace.DetailsAll)), - ) - if err != nil { - panic(err) - } - defer nativeDriver.Close(ctx) - - connector, err := ydb.Connector(nativeDriver) - if err != nil { - panic(err) - } - - db := sql.OpenDB(connector) - defer db.Close() - ... - } - ``` - - {% endlist %} - -- Java - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- Python - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- C# - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- JavaScript - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- Rust - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - - Используйте экосистему [`tracing`](https://docs.rs/tracing) и экспорт OpenTelemetry ([#268](https://github.com/ydb-platform/ydb-rs-sdk/issues/268)). - -- PHP - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -{% endlist %} diff --git a/ydb/docs/ru/core/recipes/ydb-sdk/debug-logs-otel.md b/ydb/docs/ru/core/recipes/ydb-sdk/debug-logs-otel.md deleted file mode 100644 index 8b294c69ac2..00000000000 --- a/ydb/docs/ru/core/recipes/ydb-sdk/debug-logs-otel.md +++ /dev/null @@ -1,352 +0,0 @@ -# Экспорт логов в OpenTelemetry - -Каждый {{ ydb-short-name }} SDK записывает свои внутренние логи (инициализация драйвера, пул сессий, выполнение запросов, повторные попытки и т.д.) через стандартный механизм логирования своего языка. Вместо вывода логов только в консоль (см. [Включение логирования](debug-logs.md)), их можно перенаправить в [OpenTelemetry](https://opentelemetry.io/) Logs SDK и экспортировать по стандартному протоколу OTLP в коллектор. Затем коллектор пересылает записи в выбранный бэкенд для хранения и просмотра логов. - -Принцип одинаков для всех SDK: - -1. Создайте OpenTelemetry `LoggerProvider` с экспортером логов OTLP и атрибутом ресурса `service.name`. -2. Перенаправьте логгер SDK в этот провайдер через адаптер (log appender / bridge), который преобразует каждую запись лога SDK в запись лога OTel. Список адаптеров и статус поддержки логов в разных языках см. в [документации по логам OpenTelemetry](https://opentelemetry.io/docs/concepts/signals/logs/). -3. Запустите рабочую нагрузку — теперь все внутренние логи SDK отправляются в коллектор как записи логов OTLP. - -{% note info %} - -## Принципы {#principles} - -Метод сбора логов зависит от языка и инфраструктуры — единого формата нет: - -* **Программные мосты (log appender / bridge).** Приложение само отправляет логи в коллектор по OTLP непосредственно из процесса (рабочий процесс "direct-to-Collector"). Именно этот подход показан в примерах ниже. -* **Агенты коллектора.** Приложение записывает логи в файл или `stdout`, а отдельный агент (например, [filelog receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/filelogreceiver) в [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/)) читает, разбирает и пересылает их в бэкенд — без изменения кода приложения. - -{% endnote %} - -## Подключение к SDK {#integration} - -{% list tabs %} - -- Go - - Для {{ ydb-short-name }} Go SDK существует готовый адаптер [ydb-go-sdk-otel](https://github.com/ydb-platform/ydb-go-sdk-otel), который преобразует события SDK в сигналы OpenTelemetry: трассы (`WithTracer`), метрики (`WithMetrics`) и логи (`WithLogger`). Адаптер сам не настраивает экспортеры — вы создаете `LoggerProvider` с OTLP-экспортером логов, получаете из него логгер и передаете его в опцию `ydbOtel.WithLogger` при вызове `ydb.Open`. Каждая внутренняя запись лога SDK (инициализация драйвера, пул сессий, выполнение запросов, повторные попытки и т.д.) затем отправляется в коллектор как запись лога OTLP. - - - ```bash - go get github.com/ydb-platform/ydb-go-sdk-otel - go get go.opentelemetry.io/otel/sdk/log - go get go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc - ``` - - - ```go - package main - - import ( - "context" - "os" - - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc" - sdklog "go.opentelemetry.io/otel/sdk/log" - "go.opentelemetry.io/otel/sdk/resource" - - ydbOtel "github.com/ydb-platform/ydb-go-sdk-otel" - ) - - func main() { - ctx := context.Background() - - // 1. Configuring the OTel log provider with an OTLP exporter. - exporter, err := otlploggrpc.New(ctx, - otlploggrpc.WithEndpoint("localhost:4317"), - otlploggrpc.WithInsecure(), - ) - if err != nil { - panic(err) - } - res, _ := resource.Merge(resource.Default(), resource.NewSchemaless( - attribute.String("service.name", "ydb-go-sdk-otel-logs-sample"), - )) - lp := sdklog.NewLoggerProvider( - sdklog.WithResource(res), - sdklog.WithProcessor(sdklog.NewBatchProcessor(exporter)), - ) - defer lp.Shutdown(ctx) - - // 2. Opening the YDB driver with the ydb-go-sdk-otel adapter. - // WithLogger forwards SDK log events to OTel log records. - logger := lp.Logger("ydb-go-sdk") - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - ydbOtel.WithLogger(logger, ydbOtel.WithDetailer(trace.DetailsAll)), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - // ... use db ... - } - ``` - -- Python - - {{ ydb-short-name }} Python SDK записывает логи через стандартный модуль `logging` (логгеры с именем `ydb.*`). Используйте встроенный `LoggingHandler` OpenTelemetry для перенаправления этих записей в `LoggerProvider` и экспорта через OTLP: - - - ```bash - pip install opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc - ``` - - - ```python - import logging - - import ydb - from opentelemetry._logs import set_logger_provider - from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter - from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler - from opentelemetry.sdk._logs.export import BatchLogRecordProcessor - from opentelemetry.sdk.resources import Resource - - resource = Resource(attributes={"service.name": "ydb-otel-logs-example"}) - logger_provider = LoggerProvider(resource=resource) - logger_provider.add_log_record_processor( - BatchLogRecordProcessor(OTLPLogExporter(endpoint="http://localhost:4317")) - ) - set_logger_provider(logger_provider) - - # Bridge stdlib logging -> OpenTelemetry. Binding a handler to the root logger - # intercepts everything the SDK writes via logging.getLogger("ydb..."). - otel_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) - logging.basicConfig(level=logging.INFO, handlers=[otel_handler]) - logging.getLogger("ydb").setLevel(logging.INFO) - - with ydb.Driver(endpoint="grpc://localhost:2136", database="/local") as driver: - driver.wait(timeout=5) - with ydb.QuerySessionPool(driver) as pool: - pool.execute_with_retries("SELECT 1") - - logger_provider.shutdown() - ``` - -- C# - - {{ ydb-short-name }} C# SDK записывает логи через переданный ему `ILoggerFactory`. Создайте фабрику на основе провайдера логирования OpenTelemetry с экспортером OTLP и передайте ее источнику данных: - - - ```bash - dotnet add package OpenTelemetry - dotnet add package OpenTelemetry.Exporter.OpenTelemetryProtocol - ``` - - - ```csharp - using Microsoft.Extensions.Logging; - using OpenTelemetry.Logs; - using OpenTelemetry.Resources; - using Ydb.Sdk.Ado; - - var resourceBuilder = ResourceBuilder.CreateDefault() - .AddService("ydb-sdk-otel-logs-sample"); - - using var loggerFactory = LoggerFactory.Create(builder => - { - builder.AddOpenTelemetry(options => - { - options.SetResourceBuilder(resourceBuilder); - options.AddOtlpExporter(o => o.Endpoint = new Uri("http://localhost:4317")); - }); - }); - - // Pass the factory to the SDK: each internal log (driver initialization, session pool, - // query execution, retries, ...) is exported to the collector as an OTLP log record. - await using var dataSource = new YdbDataSource( - new YdbConnectionStringBuilder("Host=localhost;Port=2136;Database=/local") - { - LoggerFactory = loggerFactory - }); - - await using var connection = await dataSource.OpenConnectionAsync(); - await new YdbCommand("SELECT 1", connection).ExecuteNonQueryAsync(); - ``` - -- Java - - {{ ydb-short-name }} Java SDK записывает логи через `slf4j`. Если logback используется в качестве реализации `slf4j`, подключите готовый аппендер `opentelemetry-logback-appender-1.0`, который пересылает каждое событие в OpenTelemetry Logs SDK: - - - ```xml - <dependency> - <groupId>io.opentelemetry</groupId> - <artifactId>opentelemetry-sdk</artifactId> - <version>${otel.version}</version> - </dependency> - <dependency> - <groupId>io.opentelemetry</groupId> - <artifactId>opentelemetry-exporter-otlp</artifactId> - <version>${otel.version}</version> - </dependency> - <dependency> - <groupId>io.opentelemetry.instrumentation</groupId> - <artifactId>opentelemetry-logback-appender-1.0</artifactId> - <version>${otel.instrumentation.version}</version> - </dependency> - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - <version>${logback.version}</version> - </dependency> - ``` - - - Подключите аппендер в `logback.xml`: - - - ```xml - <configuration> - <appender name="OTEL" class="io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender"> - </appender> - - <root level="INFO"> - <appender-ref ref="OTEL"/> - </root> - </configuration> - ``` - - - Создайте экземпляр OpenTelemetry SDK с экспортером логов OTLP и установите его в аппендере, вызвав `OpenTelemetryAppender.install(...)` перед началом рабочей нагрузки: - - - ```java - import java.util.concurrent.TimeUnit; - - import io.opentelemetry.api.OpenTelemetry; - import io.opentelemetry.api.common.Attributes; - import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; - import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender; - import io.opentelemetry.sdk.OpenTelemetrySdk; - import io.opentelemetry.sdk.logs.SdkLoggerProvider; - import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor; - import io.opentelemetry.sdk.resources.Resource; - import tech.ydb.core.grpc.GrpcTransport; - import tech.ydb.query.QueryClient; - - String serviceName = "ydb-java-sdk-otel-logs-sample"; - Resource resource = Resource.getDefault().merge( - Resource.create(Attributes.builder().put("service.name", serviceName).build())); - - SdkLoggerProvider loggerProvider = SdkLoggerProvider.builder() - .setResource(resource) - .addLogRecordProcessor(BatchLogRecordProcessor.builder( - OtlpGrpcLogRecordExporter.builder().setEndpoint("http://localhost:4317").build() - ).build()) - .build(); - - OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() - .setLoggerProvider(loggerProvider) - .build(); - - // Pass the OpenTelemetry SDK to the appender declared in logback.xml. - OpenTelemetryAppender.install(openTelemetry); - - try (GrpcTransport transport = GrpcTransport.forConnectionString("grpc://localhost:2136/local").build(); - QueryClient queryClient = QueryClient.newClient(transport).build()) { - // ... use queryClient ... - } finally { - loggerProvider.forceFlush().join(10, TimeUnit.SECONDS); - loggerProvider.shutdown().join(10, TimeUnit.SECONDS); - } - ``` - -- C++ - - {{ ydb-short-name }}C++ SDK записывает логи через `TLogBackend` из `util`. Реализуйте бэкенд, который преобразует каждую запись в лог-запись OTel, и передайте его драйверу через `TDriverConfig::SetLog`. См. [Начало работы](https://opentelemetry.io/docs/languages/cpp/getting-started/) для сборки OpenTelemetry C++ SDK. - - - ```cpp - #include <ydb-cpp-sdk/client/driver/driver.h> - #include <opentelemetry/exporters/otlp/otlp_grpc_log_record_exporter_factory.h> - #include <opentelemetry/exporters/otlp/otlp_grpc_log_record_exporter_options.h> - #include <opentelemetry/logs/provider.h> - #include <opentelemetry/sdk/logs/batch_log_record_processor_factory.h> - #include <opentelemetry/sdk/logs/logger_provider_factory.h> - #include <opentelemetry/sdk/resource/resource.h> - #include <library/cpp/logger/backend.h> - #include <library/cpp/logger/record.h> - #include <memory> - - namespace otlp = opentelemetry::exporter::otlp; - namespace logs_sdk = opentelemetry::sdk::logs; - namespace logs_api = opentelemetry::logs; - namespace resource = opentelemetry::sdk::resource; - - namespace { - - class TOtelLogBackend final : public TLogBackend { - public: - explicit TOtelLogBackend(opentelemetry::nostd::shared_ptr<logs_api::Logger> logger) - : Logger_(std::move(logger)) {} - - void WriteData(const TLogRecord& rec) override { - Logger_->EmitLogRecord(MapSeverity(rec.Priority), - opentelemetry::nostd::string_view(rec.Data, rec.Len)); - } - - void ReopenLog() override {} - - private: - static logs_api::Severity MapSeverity(ELogPriority p) { - switch (p) { - case TLOG_EMERG: - case TLOG_ALERT: - case TLOG_CRIT: return logs_api::Severity::kFatal; - case TLOG_ERR: return logs_api::Severity::kError; - case TLOG_WARNING: return logs_api::Severity::kWarn; - case TLOG_NOTICE: - case TLOG_INFO: return logs_api::Severity::kInfo; - case TLOG_DEBUG: return logs_api::Severity::kDebug; - default: return logs_api::Severity::kTrace; - } - } - - opentelemetry::nostd::shared_ptr<logs_api::Logger> Logger_; - }; - } // namespace - - int main() { - // 1. Configuring the OTel log provider with an OTLP exporter - otlp::OtlpGrpcLogRecordExporterOptions exporterOpts; - exporterOpts.endpoint = "localhost:4317"; - auto exporter = otlp::OtlpGrpcLogRecordExporterFactory::Create(exporterOpts); - auto processor = logs_sdk::BatchLogRecordProcessorFactory::Create(std::move(exporter)); - auto res = resource::Resource::Create({{"service.name", "ydb-cpp-sdk-otel-logs-sample"}}); - - std::shared_ptr<logs_api::LoggerProvider> provider( - logs_sdk::LoggerProviderFactory::Create(std::move(processor), res)); - logs_api::Provider::SetLoggerProvider(provider); - - auto logger = provider->GetLogger("ydb-cpp-sdk"); - - // 2. Pass the bridge to the YDB driver - auto config = NYdb::TDriverConfig() - .SetEndpoint("localhost:2136") - .SetDatabase("/local") - .SetLog(std::make_unique<TOtelLogBackend>(logger)); - - NYdb::TDriver driver(config); - // ... use driver ... - driver.Stop(true); - } - ``` - -- JavaScript - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - -- Rust - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - - Отслеживайте прогресс или голосуйте за поддержку Rust SDK: [ydb-rs-sdk#268](https://github.com/ydb-platform/ydb-rs-sdk/issues/268) - -{% endlist %} diff --git a/ydb/docs/ru/core/recipes/ydb-sdk/debug-prometheus.md b/ydb/docs/ru/core/recipes/ydb-sdk/debug-prometheus.md deleted file mode 100644 index b466129199e..00000000000 --- a/ydb/docs/ru/core/recipes/ydb-sdk/debug-prometheus.md +++ /dev/null @@ -1,106 +0,0 @@ -# Включение метрик в Prometheus - -Ниже приведены примеры кода для включения метрик в Prometheus в различных {{ ydb-short-name }} SDK. - -{% list tabs %} - -- Go - - {% list tabs %} - - - Нативный SDK - - ```go - package main - - import ( - "context" - - "github.com/prometheus/client_golang/prometheus" - metrics "github.com/ydb-platform/ydb-go-sdk-prometheus/v2" - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx := context.Background() - registry := prometheus.NewRegistry() - db, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - metrics.WithTraces( - registry, - metrics.WithDetails(trace.DetailsAll), - metrics.WithSeparator("_"), - ), - ) - if err != nil { - panic(err) - } - defer db.Close(ctx) - ... - } - ``` - - - database/sql - - ```go - package main - - import ( - "context" - "database/sql" - - "github.com/prometheus/client_golang/prometheus" - metrics "github.com/ydb-platform/ydb-go-sdk-prometheus/v2" - "github.com/ydb-platform/ydb-go-sdk/v3" - "github.com/ydb-platform/ydb-go-sdk/v3/trace" - ) - - func main() { - ctx := context.Background() - registry := prometheus.NewRegistry() - nativeDriver, err := ydb.Open(ctx, - os.Getenv("YDB_CONNECTION_STRING"), - metrics.WithTraces( - registry, - metrics.WithDetails(trace.DetailsAll), - metrics.WithSeparator("_"), - ), - ) - if err != nil { - panic(err) - } - defer nativeDriver.Close(ctx) - - connector, err := ydb.Connector(nativeDriver) - if err != nil { - panic(err) - } - - db := sql.OpenDB(connector) - defer db.Close() - ... - } - ``` - - {% endlist %} - -- Java - - Данная функциональность в настоящее время не поддерживается. - -- Python - - Данная функциональность в настоящее время не поддерживается. - -- JavaScript - - {% include [work-in-progress](../../_includes/work-in-progress.md) %} - -- Rust - - {% include [feature-not-supported](../../_includes/feature-not-supported.md) %} - - Отслеживайте прогресс или голосуйте за поддержку Rust SDK: [ydb-rs-sdk#267](https://github.com/ydb-platform/ydb-rs-sdk/issues/267) - -{% endlist %} diff --git a/ydb/docs/ru/core/reference/configuration/host_configs.md b/ydb/docs/ru/core/reference/configuration/host_configs.md index f99b5214c65..eac783db2dc 100644 --- a/ydb/docs/ru/core/reference/configuration/host_configs.md +++ b/ydb/docs/ru/core/reference/configuration/host_configs.md @@ -10,16 +10,19 @@ host_configs: drive: - path: <path_to_device> type: <type> + disk_scope: <disk_scope> # необязательный атрибут - path: ... - host_config_id: 2 ... ``` -Атрибут `host_config_id` задает числовой идентификатор конфигурации. В атрибуте `drive` содержится коллекция описаний подключенных дисков. Каждое описание состоит из двух атрибутов: +Атрибут `host_config_id` задает числовой идентификатор конфигурации. В атрибуте `drive` содержится коллекция описаний подключенных дисков. Каждое описание состоит из двух обязательных атрибутов: - `path` : Путь к смонтированному блочному устройству, например `/dev/disk/by-partlabel/ydb_disk_ssd_01` - `type` : Тип физического носителя устройства: `ssd`, `nvme` или `rot` (rotational - HDD) +Дополнительно может быть указан необязательный атрибут `disk_scope` — метка для вычисления домена отказа для некоторых упрощенных конфигураций, см. [Конфигурирование disk_scope](#disk-scope). + ## Примеры Одна конфигурация с идентификатором 1, с одним диском типа SSD, доступным по пути `/dev/disk/by-partlabel/ydb_disk_ssd_01`: @@ -52,6 +55,54 @@ host_configs: type: SSD ``` +## Конфигурирование disk_scope {#disk-scope} + +`disk_scope` — необязательный строковый атрибут диска, задающий более детальную зону отказа внутри одного узла. Он учитывается при вычислении [доменов отказа](../../concepts/glossary.md#fail-domain) во время выбора дисков для размещения [VDisk'ов](../../concepts/glossary.md#vdisk) [групп хранения](../../concepts/glossary.md#storage-group). + +Домен отказа определяется физическим расположением диска (дата-центр, стойка, сервер, физическое устройство), поэтому либо никакие два VDisk'а одной группы не могут быть размещены на одном узле (если домен отказа соответствует серверу или серверной стойке), либо несколько VDisk'ов одной группы может быть размещено на одном узле на разных дисках (если домен отказа соответствует отдельному физическому устройству). В первом случае невозможно создать конфигурацию в режиме `block-4-2` менее чем с 8 узлами хранения, а во втором случае конфигурация может не переживать отказ 1 узла. Чтобы ограничить количество VDisk'ов одной группы, размещаемых на одном узле, физические устройства можно пометить атрибутом `disk_scope` и включить расчёт доменов отказа по уровню `disk_scope`. Это позволит собирать некоторые отказоустойчивые конфигурации в инсталляциях, в которых серверов меньше, чем требуется доменов отказа для выбранного [режима отказоустойчивости](../../concepts/topology.md). + +### Пример {#disk-scope-example} + +Кластер из 4 серверов, на каждом сервере 4 диска в режиме отказоустойчивости `block-4-2`: + +``` yaml +host_configs: +- host_config_id: 1 + drive: + - path: /dev/disk/by-partlabel/ydb_disk_ssd_01 + type: SSD + disk_scope: fail-domain-1 + - path: /dev/disk/by-partlabel/ydb_disk_ssd_02 + type: SSD + disk_scope: fail-domain-1 + - path: /dev/disk/by-partlabel/ydb_disk_ssd_03 + type: SSD + disk_scope: fail-domain-2 + - path: /dev/disk/by-partlabel/ydb_disk_ssd_04 + type: SSD + disk_scope: fail-domain-2 +``` + +Чтобы домены отказа рассчитывались по уровню `disk_scope` для статической группы и других групп хранения, укажите соответствующий тип домена отказа в yaml-конфиге (верхнеуровневый ключ): + +``` yaml +fail_domain_type: disk_scope +``` + +И в конфигурации домена: + +```yaml +domains: +- domain_name: <имя домена> + ... + storage_pool_kinds: + - kind: <тип используемых физических устройств> + fail_domain_type: disk_scope + ... +``` + +При такой конфигурации на каждом сервере будут размещены по два VDisk'а каждой группы хранения, и максимальным отказом в такой конфигурации будет отказ одного сервера. + ## Особенности Kubernetes {#host-configs-k8s} {{ ydb-short-name }} Kubernetes operator монтирует NBS диски для Storage узлов на путь `/dev/kikimr_ssd_00`. Для их использования должна быть указана следующая конфигурация `host_configs`: diff --git a/ydb/docs/ru/core/reference/configuration/kafka_proxy_config.md b/ydb/docs/ru/core/reference/configuration/kafka_proxy_config.md index adf90e80d3a..1cc5a76a55c 100644 --- a/ydb/docs/ru/core/reference/configuration/kafka_proxy_config.md +++ b/ydb/docs/ru/core/reference/configuration/kafka_proxy_config.md @@ -8,6 +8,7 @@ || Параметр | Тип | Значение по умолчанию | Описание || || `enable_kafka_proxy` | bool | `false` | Включает или отключает Kafka Proxy. || || `listening_port` | int32 | `9092` | Порт, на котором будет доступен Kafka API. || +|| `listening_address` | string | `[::]` | Сетевой адрес, на котором Kafka Proxy принимает входящие соединения. Значение `[::]` — все интерфейсы (dual-stack, должна быть включена поддержка IPv6), `127.0.0.1` или `[::1]` — только localhost. || || `transaction_timeout_ms` | uint32 | `300000` (5 минут) | Максимальный таймаут для Kafka транзакций, после которого транзакция будет отменена. || || `auto_create_topics_enable` | bool | `false` | Включает автоматическое создание топиков при обращении к ним. Аналог [такой же опции](https://kafka.apache.org/documentation/#brokerconfigs_auto.create.topics.enable) в Apache Kafka. || || `auto_create_consumers_enable` | bool | `true` | Включает автоматическое заведение консьюмеров при обращении к ним. || @@ -26,6 +27,7 @@ kafka_proxy_config: enable_kafka_proxy: true listening_port: 9092 + listening_address: "[::]" transaction_timeout_ms: 300000 # 5 минут auto_create_topics_enable: true auto_create_consumers_enable: true diff --git a/ydb/docs/ru/core/reference/ydb-cli/operation-list.md b/ydb/docs/ru/core/reference/ydb-cli/operation-list.md index 6fc668a8c38..4975d9e9c64 100644 --- a/ydb/docs/ru/core/reference/ydb-cli/operation-list.md +++ b/ydb/docs/ru/core/reference/ydb-cli/operation-list.md @@ -20,7 +20,8 @@ * `import/nfs` — операции импорта с NFS; * `scriptexec` — операции выполнения скриптов; * `incbackup` — операции инкрементального резервного копирования; - * `restore` — операции восстановления из резервной копии. + * `restore` — операции восстановления из резервной копии; + * `setnotnull` — операции выставления ограничения `NOT NULL`. Посмотрите описание команды получения списка фоновых операций: diff --git a/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list.md b/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list.md index d31065e389c..1764b77b9ed 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list.md +++ b/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list.md @@ -19,11 +19,11 @@ ### NULL -Данная колонка может содержать значения `NULL` (по умолчанию). +Опция колонки, которое разрешает запись `NULL` в качестве значений колонки. Это является поведением по умолчанию. ### NOT NULL -Данная колонка не принимает значения `NULL`. +Ограничение колонки, которое запрещает запись `NULL` в качестве значений колонки. По умолчанию это ограничение отсутствует. ### COMPRESSION([algorithm=<algorithm_name>[, level=<value>]]) diff --git a/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list_alter.md b/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list_alter.md index 71aff46669e..c04c18f81a8 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list_alter.md +++ b/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list_alter.md @@ -17,13 +17,12 @@ ### NOT NULL -{% note warning %} - -На текущий момент поддерживается только операция `DROP NOT NULL`. +Существуют две операции: -{% endnote %} +* `SET NOT NULL` — устанавливает ограничение `NOT NULL` для колонки. +* `DROP NOT NULL` — снимает ограничение `NOT NULL` с колонки, снова разрешая значения `NULL`. -Снимает ограничение `NOT NULL` с колонки, снова разрешая значения `NULL`. +Подробнее о работе этих операций можно прочитать [здесь](../alter_table/not_null.md). ### COMPRESSION([algorithm=<algorithm_name>[, level=<value>]]) diff --git a/ydb/docs/ru/core/yql/reference/syntax/alter-resource-pool-classifier.md b/ydb/docs/ru/core/yql/reference/syntax/alter-resource-pool-classifier.md index b5b42e2dc5b..97a9326e7dd 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/alter-resource-pool-classifier.md +++ b/ydb/docs/ru/core/yql/reference/syntax/alter-resource-pool-classifier.md @@ -48,7 +48,7 @@ GRANT 'ALL' ON `/my_db` TO `user1@domain`; * `RANK` (Int64) — опциональное поле, задающее порядок выбора классификатора пула ресурсов. Если значение не указано, берётся максимальный существующий `RANK` и к нему прибавляется 1000. Допустимые значения: уникальное число в диапазоне $[0, 2^{63}-1]$. * `RESOURCE_POOL` (String) — обязательное поле, задающее имя пула ресурсов, в который будут отправлены запросы, удовлетворяющие критериям классификатора. -* `MEMBER_NAME` (String) — опциональное поле, определяющее, какой пользователь или группа пользователей будут отправлены в указанный пул ресурсов. Если поле не указано, классификатор игнорирует `MEMBER_NAME`, и классификация осуществляется по другим признакам. +* `MEMBER_NAME` (String) — опциональное поле, определяющее, какой пользователь или группа пользователей будут отправлены в указанный пул ресурсов. Формат значения описан в [CREATE RESOURCE POOL CLASSIFIER](create-resource-pool-classifier.md#member-name-format). ## См. также diff --git a/ydb/docs/ru/core/yql/reference/syntax/alter_table/columns.md b/ydb/docs/ru/core/yql/reference/syntax/alter_table/columns.md index 7024279300b..8390ee072d7 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/alter_table/columns.md +++ b/ydb/docs/ru/core/yql/reference/syntax/alter_table/columns.md @@ -46,7 +46,7 @@ ALTER TABLE episodes ADD COLUMN rate Double (DEFAULT 5.0, NOT NULL); -- альт Изменяет свойства существующей колонки в указанной таблице. Изменение свойства происходит без пересоздания колонки. Некоторые свойства применяются только к свежим записанным данным или в процессе компакшена (детали можно найти в описании конкретного свойства) ```yql -ALTER TABLE table_name ALTER COLUMN column_name SET [FAMILY <family_name>] [DEFAULT <default_value>] [COMPRESSION([algorithm=<algorithm_name>[, level=<value>]])] [ENCODING([OFF|DICT])]; +ALTER TABLE table_name ALTER COLUMN column_name SET [FAMILY <family_name>] [NOT NULL] [DEFAULT <default_value>] [COMPRESSION([algorithm=<algorithm_name>[, level=<value>]])] [ENCODING([OFF|DICT])]; ALTER TABLE table_name ALTER COLUMN column_name DROP [FAMILY] [NOT NULL] [DEFAULT] [COMPRESSION] [ENCODING]; ``` diff --git a/ydb/docs/ru/core/yql/reference/syntax/alter_table/index.md b/ydb/docs/ru/core/yql/reference/syntax/alter_table/index.md index 217b12ebf98..f5fa7d1a512 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/alter_table/index.md +++ b/ydb/docs/ru/core/yql/reference/syntax/alter_table/index.md @@ -17,4 +17,5 @@ ALTER TABLE table_name action1, action2, ..., actionN; {% if backend_name == "YDB" and oss == true %} * Изменение [дополнительных параметров таблиц](set.md). {% endif %} -* [Запуск принудительного компакшена](compact.md).
\ No newline at end of file +* [Запуск принудительного компакшена](compact.md). +* [Выставление и сброс ограничения NOT NULL](not_null.md).
\ No newline at end of file diff --git a/ydb/docs/ru/core/yql/reference/syntax/alter_table/not_null.md b/ydb/docs/ru/core/yql/reference/syntax/alter_table/not_null.md new file mode 100644 index 00000000000..c9227e16933 --- /dev/null +++ b/ydb/docs/ru/core/yql/reference/syntax/alter_table/not_null.md @@ -0,0 +1,43 @@ +# Выставление и сброс ограничения `NOT NULL` + +Ограничение целостности данных на уровне колонки, которое запрещает запись `NULL` в качестве значений. Это ограничение гарантирует, что колонка всегда содержит действительное значение. + +В YDB операция `SET NOT NULL` выполняется как синхронная SQL-операция, ожидающая применения изменения схемы. При этом создаётся фоновая операция для проверки таблицы на наличие `NULL`-значений в существующих данных. + +## Выставление `NOT NULL` + +`SET NOT NULL` устанавливает ограничение `NOT NULL` для указанной колонки. + +Например, следующий запрос установит ограничение `NOT NULL` для колонки `column_name` в таблице `table_name`: + +```yql +ALTER TABLE table_name ALTER COLUMN column_name SET NOT NULL; +``` + +Примечания: + +* `SET NOT NULL` поддерживается только для [строковых таблиц](../../../../concepts/datamodel/table.md#row-oriented-tables). +* `SET NOT NULL` может занять длительное время: перед установкой ограничения YDB проверяет таблицу на наличие `NULL`-значений для существующих данных в указанной колонке. +* SQL-операция выполняется синхронно и ожидает завершения. При этом создаётся фоновая операция для наблюдаемости. +* За ходом выполнения операций можно следить с помощью [CLI-команды](../../../../reference/ydb-cli/operation-list.md) `ydb operation list setnotnull`. Также доступны команды, позволяющие [получить статус конкретной операции](../../../../reference/ydb-cli/operation-get.md), [отменить операцию](../../../../reference/ydb-cli/operation-cancel.md) или [удалить запись о завершённой операции](../../../../reference/ydb-cli/operation-forget.md). +* После запуска операции `SET NOT NULL` и до её завершения в указанную колонку нельзя записывать `NULL`-значения. При попытке записать такие значения, вы получите ошибку вида ``Can't set NULL or optional value to column: <column>. `SET NOT NULL` operation is currently in progress for this column``. +* Если валидация не пройдена, операция `SET NOT NULL` завершится с ошибкой `Validation failed for SET NOT NULL on table ...: one or more columns contain NULL values`. + +## Сброс `NOT NULL` + +`DROP NOT NULL` снимает ограничение `NOT NULL` с указанной колонки. + +Например, следующий запрос снимет ограничение `NOT NULL` с колонки `column_name` в таблице `table_name`: + +```yql +ALTER TABLE table_name ALTER COLUMN column_name DROP NOT NULL; +``` + +Примечание: + +* `DROP NOT NULL` поддерживается только для [строковых таблиц](../../../../concepts/datamodel/table.md#row-oriented-tables). + +## См. также + +* [ALTER COLUMN](columns.md) + diff --git a/ydb/docs/ru/core/yql/reference/syntax/alter_table/toc_i.yaml b/ydb/docs/ru/core/yql/reference/syntax/alter_table/toc_i.yaml index ddf6ca251af..a91b2276111 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/alter_table/toc_i.yaml +++ b/ydb/docs/ru/core/yql/reference/syntax/alter_table/toc_i.yaml @@ -1,9 +1,12 @@ items: - { name: Обзор, href: index.md } - { name: INDEX, href: indexes.md, when: feature_secondary_index } -- { name: COLUMN, href: columns.md } +- name: COLUMN + href: columns.md + items: + - { name: FAMILY, href: family.md, when: backend_name == "YDB" } + - { name: NOT NULL, href: not_null.md, when: backend_name == "YDB" } - { name: SET, href: set.md, when: backend_name == "YDB" } - { name: CHANGEFEED, href: changefeed.md, when: feature_changefeed and backend_name == "YDB" } - { name: RENAME, href: rename.md, when: feature_map_tables } -- { name: FAMILY, href: family.md, when: backend_name == "YDB" } -- { name: COMPACT, href: compact.md, when: backend_name == "YDB" }
\ No newline at end of file +- { name: COMPACT, href: compact.md, when: backend_name == "YDB" } diff --git a/ydb/docs/ru/core/yql/reference/syntax/create-resource-pool-classifier.md b/ydb/docs/ru/core/yql/reference/syntax/create-resource-pool-classifier.md index f14947dd664..aa1645c6f59 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/create-resource-pool-classifier.md +++ b/ydb/docs/ru/core/yql/reference/syntax/create-resource-pool-classifier.md @@ -16,7 +16,18 @@ WITH ( <parameter_name> [= <parameter_value>] [, ... ] ) * `RANK` (Int64) — опциональное поле, задающее порядок выбора классификатора пула ресурсов. Если значение не указано, берётся максимальный существующий `RANK` и к нему прибавляется 1000. Допустимые значения: уникальное число в диапазоне $[0, 2^{63}-1]$. * `RESOURCE_POOL` (String) — обязательное поле, задающее имя пула ресурсов, в который будут отправлены запросы, удовлетворяющие критериям классификатора. -* `MEMBER_NAME` (String) — опциональное поле, определяющее, какой пользователь или группа пользователей будут отправлены в указанный пул ресурсов. Если поле не указано, классификатор игнорирует `MEMBER_NAME`, и классификация осуществляется по другим признакам. +* `MEMBER_NAME` (String) — опциональное поле, определяющее, какой пользователь или группа пользователей будут отправлены в указанный пул ресурсов. Значение сравнивается с SID пользователя или любым SID группы из его токена аутентификации; подробнее о формате см. [ниже](#member-name-format). Если поле не указано, классификатор игнорирует `MEMBER_NAME`, и классификация осуществляется по другим признакам. + +### Формат MEMBER_NAME {#member-name-format} + +`MEMBER_NAME` сравнивается посимвольно с [SID](../../../concepts/glossary.md#access-sid) пользователя или любым SID группы из его токена аутентификации. Формат SID зависит от того, каким способом пользователь пришёл в систему. + +- **Встроенные пользователи {{ ydb-short-name }} (логин/пароль)** — SID совпадает с именем пользователя, без суффикса. Например, `user1`. Подробнее — [{#T}](../../../security/authentication.md#static-credentials). +- **Облачные пользователи (Access Service)** — SID имеет вид `<subject_id>@as`, где `<subject_id>` — идентификатор пользователя в IAM. Суффикс задаётся параметром [`access_service_domain`](../../../reference/configuration/auth_config.md#iam-auth-config) (по умолчанию `as`). Например, `ajeb89hv69nujke769fa@as`. Подробнее — [{#T}](../../../security/authentication.md#iam). +- **LDAP** — SID имеет вид `<логин>@<домен>`, где домен задаётся параметром [`ldap_authentication_domain`](../../../reference/configuration/auth_config.md#ldap-auth-config) (по умолчанию `ldap`). Например, `user1@ldap`. Подробнее — [{#T}](../../../security/authentication.md#ldap). +- **Внешние провайдеры идентификации (OIDC)** — SID имеет вид `<логин>@<домен>`, где домен задаётся параметром `external_idp_authentication_domain` в [конфигурации аутентификации](../../../reference/configuration/auth_config.md) (по умолчанию `sso`). Например, `user1@sso`. + +В качестве значения можно указать как SID конкретного пользователя, так и SID группы. Ко всем аутентифицированным пользователям автоматически добавляется группа `all-users@well-known` — её удобно использовать, если нужно направить в пул запросы от всех аутентифицированных клиентов. ## Замечания {#remarks} diff --git a/ydb/docs/ru/core/yql/reference/syntax/discard.md b/ydb/docs/ru/core/yql/reference/syntax/discard.md index 4ccf9cc0567..d6299aece2f 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/discard.md +++ b/ydb/docs/ru/core/yql/reference/syntax/discard.md @@ -1,18 +1,60 @@ # DISCARD -Вычисляет {% if select_command == "SELECT STREAM" %}[`SELECT STREAM`](select_stream.md){% else %}[`SELECT`](select/index.md){% endif %}{% if feature_mapreduce %}{% if reduce_command %}, [`{{ reduce_command }}`](reduce.md){% endif %} или [`{{ process_command }}`](process.md){% endif %}, но не возвращает результат ни в клиент, ни в таблицу. {% if feature_mapreduce %}Не может быть задано одновременно с [INTO RESULT](into_result.md).{% endif %} +Вычисляет {% if select_command == "SELECT STREAM" %}[`SELECT STREAM`](select_stream.md){% else %}[`SELECT`](select/index.md){% endif %}{% if feature_mapreduce %}{% if reduce_command %}, [`{{ reduce_command }}`](reduce.md){% endif %} или [`{{ process_command }}`](process.md){% endif %}, но не возвращает результат ни в клиент, ни в таблицу.{% if feature_mapreduce %} Не может быть задано одновременно с [INTO RESULT](into_result.md).{% endif %} Полезно использовать в сочетании с [`Ensure`](../builtins/basic.md#ensure) для проверки выполнения пользовательских условий на финальный результат вычислений. +{% if backend_name == "YDB" %} + +Запрос с `DISCARD` выполняется полностью — со всеми фильтрами, агрегациями и проверками `Ensure`, — но результирующий набор (result set) не возвращается клиенту. Данные при этом не пересылаются по сети: клиент получает только статус выполнения запроса. На больших выборках это заметно экономит трафик и память. + +В запросе из нескольких операторов `DISCARD` действует на отдельный оператор: + +```yql +SELECT 1; -- вернётся +DISCARD SELECT 2; -- выполнится, но в ответ не попадёт +SELECT 3; -- вернётся +``` + +Клиент получит два результирующих набора — от первого и третьего оператора. + +`DISCARD` применяется только к оператору целиком, использовать его внутри выражений нельзя. В частности, недопустимо: + +* в подзапросах — `SELECT * FROM (DISCARD SELECT 1)`; +* в `WHERE ... IN (...)` — `SELECT * FROM my_table WHERE Key IN (DISCARD SELECT 1)`; +* в операндах `UNION ALL` — `SELECT 1 UNION ALL DISCARD SELECT 2`. + +{% note info %} + +`DISCARD` поддерживается при выполнении запросов через сервис исполнения запросов (Query Service) начиная с версии {{ ydb-short-name }} 26.2. При выполнении через устаревшие интерфейсы (Table Service, скан-запросы) `DISCARD` игнорируется, и результат возвращается клиенту — поведение сохранено для обратной совместимости. + +{% endnote %} + +{% endif %} + {% if select_command != true or select_command == "SELECT" %} -### Примеры +## Примеры ```yql DISCARD SELECT 1; ``` +{% if backend_name == "YDB" %} + +```yql +DISCARD SELECT Ensure( + Data, + Data < 1000000, + "Value too big" +) FROM `result_table`; +``` + +Если условие `Ensure` нарушено, запрос завершится ошибкой. Если всё в порядке — запрос отработает успешно, а клиент не получит результирующий набор, который иначе пришлось бы выкачивать целиком. + +{% else %} + ```yql INSERT INTO result_table WITH TRUNCATE SELECT * FROM @@ -30,3 +72,5 @@ DISCARD SELECT Ensure( ``` {% endif %} + +{% endif %} diff --git a/ydb/docs/ru/core/yql/reference/syntax/index.md b/ydb/docs/ru/core/yql/reference/syntax/index.md index 243e56f2d92..c5d3ea27465 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/index.md +++ b/ydb/docs/ru/core/yql/reference/syntax/index.md @@ -75,12 +75,8 @@ {% endif %} -{% if backend_name != "YDB" %} - * [DISCARD](discard.md) -{% endif %} - * [INTO RESULT](into_result.md) {% if feature_mapreduce %} diff --git a/ydb/docs/ru/core/yql/reference/syntax/select/toc_i.yaml b/ydb/docs/ru/core/yql/reference/syntax/select/toc_i.yaml index d198e718af7..82cf0b75811 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/select/toc_i.yaml +++ b/ydb/docs/ru/core/yql/reference/syntax/select/toc_i.yaml @@ -4,6 +4,7 @@ items: - { name: FROM, href: from.md } - { name: FROM AS_TABLE, href: from_as_table.md } - { name: FROM SELECT, href: from_select.md } +- { name: FROM Topic, href: topics.md } - { name: FOLDER, href: folder.md, when: yt } - { name: FLATTEN, href: flatten.md } - { name: GROUP BY, href: group-by.md } @@ -31,4 +32,3 @@ items: - { name: SAMPLE, href: sample.md, when: feature_tablesample } - { name: TABLESAMPLE, href: sample.md, when: feature_tablesample } - { name: MATCH_RECOGNIZE, href: match_recognize.md, when: feature_match_recogznize } -- { name: STREAMING, href: streaming.md } diff --git a/ydb/docs/ru/core/yql/reference/syntax/select/streaming.md b/ydb/docs/ru/core/yql/reference/syntax/select/topics.md index 179939d1d12..cc87d2fa5e9 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/select/streaming.md +++ b/ydb/docs/ru/core/yql/reference/syntax/select/topics.md @@ -1,6 +1,6 @@ -# Потоковое чтение данных из топика +# Чтение данных из топика -Можно выполнять чтение данных из [топика](../../../../concepts/datamodel/topic.md) обычным `SELECT` без создания [потокового запроса](../../../../concepts/streaming-query/streaming-query.md). Для этого необходимо указать `STREAMING = "TRUE"` в блоке `WITH` и задать ограничение на количество выходных строк через `LIMIT`, иначе запрос не завершится. +Можно выполнять чтение данных из [топика](../../../../concepts/datamodel/topic.md) обычным `SELECT` без создания [потокового запроса](../../../../concepts/streaming-query/streaming-query.md). {% note warning %} @@ -19,7 +19,9 @@ {% endnote %} -## Пример +## Примеры + +### Чтение текущих данных ```yql SELECT @@ -30,14 +32,32 @@ WITH ( FORMAT = raw, SCHEMA = ( Data String + ) +) +``` + +### Чтение с ожиданием новых данных + +```yql +SELECT + Data +FROM + input_topic -- или внешний топик ext_source.input_topic +WITH ( + FORMAT = raw, + SCHEMA = ( + Data String ), STREAMING = "TRUE" ) LIMIT 1 ``` +Подробнее — [чтение из топика](../../../../concepts/query_execution/topics.md#topic-read). + + ## См. также -* [{#T}](../../../../concepts/query_execution/topics.md#streaming-read) — описание потокового чтения из топика +* [{#T}](../../../../concepts/query_execution/topics.md#topic-read) * [{#T}](../../../../concepts/streaming-query/streaming-query.md) * [{#T}](../create-streaming-query.md) diff --git a/ydb/docs/ru/core/yql/reference/syntax/toc_i.yaml b/ydb/docs/ru/core/yql/reference/syntax/toc_i.yaml index 72752ddc1de..384ca5771bd 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/toc_i.yaml +++ b/ydb/docs/ru/core/yql/reference/syntax/toc_i.yaml @@ -41,7 +41,7 @@ items: - { name: COMMIT, href: commit.md } - { name: DECLARE, href: declare.md } - { name: DELETE, href: delete.md, when: feature_map_tables } -- { name: DISCARD, href: discard.md, when: backend_name != "YDB" } +- { name: DISCARD, href: discard.md } - { name: DROP ASYNC REPLICATION, href: drop-async-replication.md } - { name: DROP BACKUP COLLECTION, href: drop-backup-collection.md, when: feature_backup_collections } - { name: DROP GROUP, href: drop-group.md, when: feature_user_and_group } diff --git a/ydb/docs/ru/core/yql/reference/types/primitive.md b/ydb/docs/ru/core/yql/reference/types/primitive.md index a48af7d1150..e01ed7ff615 100644 --- a/ydb/docs/ru/core/yql/reference/types/primitive.md +++ b/ydb/docs/ru/core/yql/reference/types/primitive.md @@ -61,7 +61,7 @@ || `DyNumber` | Бинарное представление вещественного числа точностью до 38 знаков. Допустимые значения: положительные от 1×10<sup>-130</sup> до 1×10<sup>126</sup>-1, отрицательные от -1×10<sup>126</sup>-1 до -1×10<sup>-130</sup> и 0. -Совместим с типом `Number` AWS DynamoDB. Не рекомендуется для использования в {{ backend_name_lower }}-native приложениях. | Не поддерживается в колоночных таблицах +Совместим с типом `Number` AWS DynamoDB. Не рекомендуется для использования в {{ backend_name_lower }}-native приложениях. | || {% endif %} |# @@ -97,7 +97,7 @@ Не поддерживает возможность сравнения{% if feature_map_tables %}, не может использоваться в первичном ключе и в колонках, формирующих ключ вторичного индекса{% endif %} || || `Uuid` | -Универсальный идентификатор [UUID](https://tools.ietf.org/html/rfc4122) | Не поддерживается в колоночных таблицах +Универсальный идентификатор [UUID](https://tools.ietf.org/html/rfc4122) | || |# @@ -205,7 +205,6 @@ | 8 | -Не поддерживается в колоночных таблицах || || @@ -217,7 +216,6 @@ | 8 | -Не поддерживается в колоночных таблицах || || @@ -265,7 +263,6 @@ | 8 и метка таймзоны | -— || || diff --git a/ydb/docs/ru/core/yql/toc_i.yaml b/ydb/docs/ru/core/yql/toc_i.yaml index 2d19742b88a..4fac8bfcd8f 100644 --- a/ydb/docs/ru/core/yql/toc_i.yaml +++ b/ydb/docs/ru/core/yql/toc_i.yaml @@ -1,8 +1,3 @@ items: -- name: Обзор - href: reference/index.md -- include: - mode: link - path: reference/toc_i.yaml - name: Планы запросов href: query_plans.md diff --git a/ydb/docs/ru/core/yql/toc_p.yaml b/ydb/docs/ru/core/yql/toc_p.yaml index 50818e81233..1ab6f7ee53c 100644 --- a/ydb/docs/ru/core/yql/toc_p.yaml +++ b/ydb/docs/ru/core/yql/toc_p.yaml @@ -1,3 +1,5 @@ items: -- include: { mode: link, path: toc_i.yaml } +- name: Обзор + href: reference/index.md - include: { mode: link, path: reference/toc_p.yaml } +- include: { mode: link, path: toc_i.yaml } diff --git a/ydb/library/actors/core/actor.cpp b/ydb/library/actors/core/actor.cpp index 804f1846cb8..6e7b0aae725 100644 --- a/ydb/library/actors/core/actor.cpp +++ b/ydb/library/actors/core/actor.cpp @@ -93,6 +93,16 @@ namespace NActors { return TActivationContext::Send(ev); } + bool IActor::SendActorLivenessCheck(const TActorId& target, ui64 cookie) const noexcept { + return Send(new IEventHandle( + TEvents::TSystem::CheckActorLiveness, + TEvents::TEvCheckActorLiveness::RequestFlags, + target, + SelfId(), + nullptr, + cookie)); + } + bool IActor::Send(const TActorId& recipient, IEventBase* ev, ui32 flags, ui64 cookie, NWilson::TTraceId traceId) const noexcept { return SelfActorId.Send(recipient, ev, flags, cookie, std::move(traceId)); } diff --git a/ydb/library/actors/core/actor.h b/ydb/library/actors/core/actor.h index 491535fe667..ede3d97ca7a 100644 --- a/ydb/library/actors/core/actor.h +++ b/ydb/library/actors/core/actor.h @@ -570,9 +570,16 @@ namespace NActors { public: using TReceiveFunc = void (IActor::*)(TAutoPtr<IEventHandle>& ev); + enum class ESystemFlag : ui64 { + // Notify this actor only after an activation that processed one of + // its events. Activations of other actors in the same mailbox are + // not reported. + MailboxProcessingFinished = 1ull << 0, + }; private: TReceiveFunc StateFunc_; + ui64 SystemFlags = 0; private: friend class NDetail::TActorAsyncHandlerPromise; @@ -690,6 +697,22 @@ namespace NActors { } // must not be called for registered actors, see Die method instead protected: + void SetSystemFlag(ESystemFlag flag) noexcept { + SystemFlags |= static_cast<ui64>(flag); + } + + void ClearSystemFlag(ESystemFlag flag) noexcept { + SystemFlags &= ~static_cast<ui64>(flag); + } + + ui64 GetSystemFlags() const noexcept { + return SystemFlags; + } + + bool HasSystemFlag(ESystemFlag flag) const noexcept { + return GetSystemFlags() & static_cast<ui64>(flag); + } + virtual void Die(const TActorContext& ctx); // would unregister actor so call exactly once and only from inside of message processing virtual void PassAway(); @@ -793,6 +816,7 @@ namespace NActors { void Describe(IOutputStream&) const override; bool Send(TAutoPtr<IEventHandle> ev) const noexcept; + bool SendActorLivenessCheck(const TActorId& target, ui64 cookie = 0) const noexcept; bool Send(const TActorId& recipient, IEventBase* ev, TEventFlags flags = 0, ui64 cookie = 0, NWilson::TTraceId traceId = {}) const noexcept final; bool Send(const TActorId& recipient, THolder<IEventBase> ev, TEventFlags flags = 0, ui64 cookie = 0, NWilson::TTraceId traceId = {}) const{ return Send(recipient, ev.Release(), flags, cookie, std::move(traceId)); diff --git a/ydb/library/actors/core/events.h b/ydb/library/actors/core/events.h index 3fb872be9b3..a782816fcbf 100644 --- a/ydb/library/actors/core/events.h +++ b/ydb/library/actors/core/events.h @@ -111,7 +111,7 @@ namespace NActors { CoroTimeout, InvokeQuery, Wilson, - Preemption, + MailboxProcessingFinished, ResumeRunnable, CheckActorLiveness, ActorAlive, @@ -139,12 +139,25 @@ namespace NActors { const ui64 Tag = 0; }; - struct TEvPreemption: public TEventLocal<TEvPreemption, TSystem::Preemption> { - bool ByEventCount = false; - bool ByCycles = false; - bool ByTailSend = false; - ui32 EventCount = 0; - ui64 Cycles = 0; + struct TEvMailboxProcessingFinished + : public TEventLocal<TEvMailboxProcessingFinished, TSystem::MailboxProcessingFinished> { + enum class EReason : ui8 { + QueueEmpty, + EventCountLimitReached, + TimeLimitReached, + SoftDeadlineReached, + TailSend, + }; + + const EReason Reason; + const ui32 ExecutedEvents; + const ui64 ElapsedCycles; + + TEvMailboxProcessingFinished(EReason reason, ui32 executedEvents, ui64 elapsedCycles) + : Reason(reason) + , ExecutedEvents(executedEvents) + , ElapsedCycles(elapsedCycles) + {} }; struct TEvSubscribe: public TEventLocal<TEvSubscribe, TSystem::Subscribe> { diff --git a/ydb/library/actors/core/executor_thread.cpp b/ydb/library/actors/core/executor_thread.cpp index f020bdb0217..ca8520d1e86 100644 --- a/ydb/library/actors/core/executor_thread.cpp +++ b/ydb/library/actors/core/executor_thread.cpp @@ -28,6 +28,8 @@ #include <util/system/type_name.h> #include <util/system/datetime.h> +#include <algorithm> + LWTRACE_USING(ACTORLIB_PROVIDER) @@ -184,10 +186,6 @@ namespace NActors { ThreadCtx.SetOverwrittenTimePerMailboxTs(Max(value, ThreadCtx.TimePerMailboxTs())); } - void TExecutorThread::SubscribeToPreemption(TActorId actorId) { - ThreadCtx.ExecutionContext.PreemptionSubscribed.push_back(actorId); - } - TExecutorThread::TProcessingResult TExecutorThread::Execute(TMailbox* mailbox, bool isTailExecution, NHPTimer::STime mailboxScheduledTimestampTs) { EXECUTOR_THREAD_DEBUG(EDebugLevel::Activation, "Execute mailbox"); @@ -206,16 +204,58 @@ namespace NActors { ui32 prevActivityType = std::numeric_limits<ui32>::max(); TActorId recipient; bool firstEvent = true; - bool preemptedByEventCount = false; - bool preemptedByCycles = false; - bool preemptedByTailSend = false; + using EFinishReason = TEvents::TEvMailboxProcessingFinished::EReason; + EFinishReason finishReason = EFinishReason::EventCountLimitReached; + ui32 finishedExecutedEvents = execCtx.ExecutedEvents; + bool isPreempted = false; bool wasWorking = false; + TStackVec<TActorId, 1> mailboxProcessingFinishedActors; NHPTimer::STime hpnow = execCtx.HPStart; NHPTimer::STime hpprev = TlsThreadContext->UpdateStartOfProcessingEventTS(hpnow); ExecutionStats.AddElapsedCycles(ActorSystemIndex, hpnow - hpprev); NHPTimer::STime eventStart = execCtx.HPStart; TlsThreadContext->ActivityContext.ActivationStartTS.store(execCtx.HPStart, std::memory_order_release); + auto finishActorEvent = [&](IActor*& currentActor, IEventHandle* ev, ui32 eventType, + const std::type_info* currentActorType, ui32 activityType) { + hpnow = GetCycleCountFast(); + hpprev = TlsThreadContext->UpdateStartOfProcessingEventTS(hpnow); + + const size_t dyingActorsCnt = DyingActors.size(); + EXECUTOR_THREAD_DEBUG(EDebugLevel::Event, "dyingActorsCnt ", dyingActorsCnt); + ExecutionStats.UpdateActorsStats(dyingActorsCnt, ThreadCtx.Pool()); + if (dyingActorsCnt) { + DropUnregistered(); + currentActor = nullptr; + } + + if (mailbox->IsEmpty()) { + // had actors and became empty, prepare to reclaim mailbox + mailbox->LockToFree(); + } + + ExecutionStats.AddElapsedCycles(activityType, hpnow - hpprev); + const NHPTimer::STime elapsed = ExecutionStats.AddEventProcessingStats( + eventStart, hpnow, activityType, CurrentActorScheduledEventsCounter); + mailbox->AddElapsedCycles(elapsed); + if (elapsed > 1000000) { + LwTraceSlowEvent( + ev, + eventType, + currentActorType, + ThreadCtx.PoolId(), + CurrentRecipient, + NHPTimer::GetSeconds(elapsed) * 1000.0); + } + + // The actor might have been destroyed + if (currentActor) { + currentActor->AddElapsedTicks(elapsed); + } + + CurrentRecipient = TActorId(); + }; + ThreadCtx.ResetOverwrittenEventsPerMailbox(); ThreadCtx.ResetOverwrittenTimePerMailboxTs(); Y_ABORT_UNLESS(mailboxScheduledTimestampTs, "mailbox scheduled timestamp must be set"); @@ -278,34 +318,21 @@ namespace NActors { actor->Receive(ev); - hpnow = GetCycleCountFast(); - hpprev = TlsThreadContext->UpdateStartOfProcessingEventTS(hpnow); - - size_t dyingActorsCnt = DyingActors.size(); - EXECUTOR_THREAD_DEBUG(EDebugLevel::Event, "dyingActorsCnt ", dyingActorsCnt); - ExecutionStats.UpdateActorsStats(dyingActorsCnt, ThreadCtx.Pool()); - if (dyingActorsCnt) { - DropUnregistered(); - actor = nullptr; - } - - if (mailbox->IsEmpty()) { - // had actors and became empty, prepare to reclaim mailbox - mailbox->LockToFree(); - } - - ExecutionStats.AddElapsedCycles(activityType, hpnow - hpprev); - NHPTimer::STime elapsed = ExecutionStats.AddEventProcessingStats(eventStart, hpnow, activityType, CurrentActorScheduledEventsCounter); - mailbox->AddElapsedCycles(elapsed); - if (elapsed > 1000000) { - LwTraceSlowEvent(ev.Get(), evTypeForTracing, actorType, ThreadCtx.PoolId(), CurrentRecipient, NHPTimer::GetSeconds(elapsed) * 1000.0); + const ui64 systemFlags = actor->GetSystemFlags(); + if (Y_UNLIKELY(systemFlags != 0)) { + if (systemFlags & static_cast<ui64>( + IActor::ESystemFlag::MailboxProcessingFinished)) { + const TActorId actorId = actor->SelfId(); + if (std::find( + mailboxProcessingFinishedActors.begin(), + mailboxProcessingFinishedActors.end(), + actorId) == mailboxProcessingFinishedActors.end()) { + mailboxProcessingFinishedActors.push_back(actorId); + } + } } - // The actor might have been destroyed - if (actor) - actor->AddElapsedTicks(elapsed); - - CurrentRecipient = TActorId(); + finishActorEvent(actor, ev.Get(), evTypeForTracing, actorType, activityType); } else { EXECUTOR_THREAD_DEBUG(EDebugLevel::Event, "actor is null"); actorType = nullptr; @@ -321,6 +348,7 @@ namespace NActors { ExecutionStats.AddElapsedCycles(ActorSystemIndex, hpnow - hpprev); } eventStart = hpnow; + finishedExecutedEvents = execCtx.ExecutedEvents + 1; if (TlsThreadContext->CheckCapturedSendingType(ESendingType::Tail)) { ExecutionStats.IncrementMailboxPushedOutByTailSending(); @@ -333,7 +361,7 @@ namespace NActors { ThreadCtx.WorkerId(), recipient.ToString(), SafeTypeName(actorType)); - preemptedByTailSend = true; + finishReason = EFinishReason::TailSend; break; } @@ -349,7 +377,8 @@ namespace NActors { ThreadCtx.WorkerId(), recipient.ToString(), SafeTypeName(actorType)); - preemptedByCycles = true; + finishReason = EFinishReason::SoftDeadlineReached; + isPreempted = true; break; } @@ -365,7 +394,8 @@ namespace NActors { ThreadCtx.WorkerId(), recipient.ToString(), SafeTypeName(actorType)); - preemptedByCycles = true; + finishReason = EFinishReason::TimeLimitReached; + isPreempted = true; break; } @@ -380,7 +410,8 @@ namespace NActors { ThreadCtx.WorkerId(), recipient.ToString(), SafeTypeName(actorType)); - preemptedByEventCount = true; + finishReason = EFinishReason::EventCountLimitReached; + isPreempted = true; break; } } else { @@ -395,24 +426,45 @@ namespace NActors { ThreadCtx.WorkerId(), recipient.ToString(), SafeTypeName(actor)); + finishReason = EFinishReason::QueueEmpty; break; // empty queue, leave } } - if (execCtx.PreemptionSubscribed.size()) { - std::unique_ptr<TEvents::TEvPreemption> event = std::make_unique<TEvents::TEvPreemption>(); - event->ByEventCount = preemptedByEventCount; - event->ByCycles = preemptedByCycles; - event->ByTailSend = preemptedByTailSend; - event->EventCount = execCtx.ExecutedEvents; - event->Cycles = hpnow - execCtx.HPStart; - TAutoPtr<IEventHandle> ev = new IEventHandle(TActorId(), TActorId(), event.release()); - for (const auto& actorId : execCtx.PreemptionSubscribed) { - IActor *actor = mailbox->FindActor(actorId.LocalId()); - if (actor) { - actor->Receive(ev); - } + const ui64 finishedElapsedCycles = hpnow - execCtx.HPStart; + // Deliver notifications directly so they do not enter the mailbox + // queue or consume its event processing budget. + for (const TActorId& actorId : mailboxProcessingFinishedActors) { + if (IActor* actor = mailbox->FindActor(actorId.LocalId()); + actor && (actor->GetSystemFlags() & static_cast<ui64>( + IActor::ESystemFlag::MailboxProcessingFinished))) { + const TActorId recipient = actor->SelfId(); + TActorContext ctx(*mailbox, *this, eventStart, recipient); + TlsActivationContext = &ctx; + TAutoPtr<IEventHandle> ev = new IEventHandle( + actor->SelfId(), + TActorId(), + new TEvents::TEvMailboxProcessingFinished( + finishReason, + finishedExecutedEvents, + finishedElapsedCycles)); + + const std::type_info* notificationActorType = &typeid(*actor); + const ui32 activityType = actor->GetActivityType().GetIndex(); + NProfiling::TMemoryTagScope::Reset(activityType); + TlsThreadContext->ActivityContext.ElapsingActorActivity.store(activityType, std::memory_order_release); + CurrentRecipient = recipient; + CurrentActorScheduledEventsCounter = 0; + + actor->Receive(ev); + + finishActorEvent( + actor, + ev.Get(), + TEvents::TSystem::MailboxProcessingFinished, + notificationActorType, + activityType); + eventStart = hpnow; } - execCtx.PreemptionSubscribed.clear(); } TlsThreadContext->ActivityContext.ActivationStartTS.store(hpnow, std::memory_order_release); TlsThreadContext->ActivityContext.ElapsingActorActivity.store(ActorSystemIndex, std::memory_order_release); @@ -425,7 +477,7 @@ namespace NActors { } else { mailbox->Unlock(ThreadCtx.Pool(), hpnow, RevolvingWriteCounter); } - return {preemptedByEventCount || preemptedByCycles, wasWorking}; + return {isPreempted, wasWorking}; } TThreadId TExecutorThread::GetThreadId() const { diff --git a/ydb/library/actors/core/executor_thread.h b/ydb/library/actors/core/executor_thread.h index 2ee87349560..ddc892c3d9d 100644 --- a/ydb/library/actors/core/executor_thread.h +++ b/ydb/library/actors/core/executor_thread.h @@ -78,7 +78,6 @@ namespace NActors { TThreadId GetThreadId() const; // blocks, must be called after Start() TWorkerId GetWorkerId() const; - void SubscribeToPreemption(TActorId actorId); ui32 GetOverwrittenEventsPerMailbox() const; void SetOverwrittenEventsPerMailbox(ui32 value); ui64 GetOverwrittenTimePerMailboxTs() const; diff --git a/ydb/library/actors/core/subsystems/cgroup/cgroup_events.h b/ydb/library/actors/core/subsystems/cgroup/cgroup_events.h index 1145dd011e0..7c2cd5c2115 100644 --- a/ydb/library/actors/core/subsystems/cgroup/cgroup_events.h +++ b/ydb/library/actors/core/subsystems/cgroup/cgroup_events.h @@ -15,6 +15,7 @@ namespace NActors { V2Stats, MemoryStats, OomAlert, + OomTrend, End, }; @@ -64,4 +65,38 @@ namespace NActors { } }; + struct TEvCGroupOomTrend + : TEventLocal<TEvCGroupOomTrend, static_cast<ui32>(ECGroupEvent::OomTrend)> + { + // ReadTrend responses are Active when a trend was found and Stopped + // otherwise. Subscriptions additionally receive Stopped when their + // forecast condition ceases to hold. + ECGroupOomTrendState State; + + // Empty when the calculation completed without finding a trend. A + // Stopped subscription notification may still contain the current + // trend when it exists but no longer meets the subscription threshold. + std::optional<TCGroupOomTrend> Trend; + + // Set only for subscription notifications so an actor can distinguish + // multiple subscriptions for the same window. + std::optional<TDuration> TimeToOomThreshold; + + explicit TEvCGroupOomTrend( + std::optional<TCGroupOomTrend> trend, + ECGroupOomTrendState state = ECGroupOomTrendState::Active, + std::optional<TDuration> timeToOomThreshold = std::nullopt) + : State(state) + , Trend(std::move(trend)) + , TimeToOomThreshold(timeToOomThreshold) + { + } + + explicit TEvCGroupOomTrend(TCGroupOomTrend trend) + : State(ECGroupOomTrendState::Active) + , Trend(std::move(trend)) + { + } + }; + } // namespace NActors diff --git a/ydb/library/actors/core/subsystems/cgroup/cgroup_oom.cpp b/ydb/library/actors/core/subsystems/cgroup/cgroup_oom.cpp index 6219141d671..60dcbcec214 100644 --- a/ydb/library/actors/core/subsystems/cgroup/cgroup_oom.cpp +++ b/ydb/library/actors/core/subsystems/cgroup/cgroup_oom.cpp @@ -1,6 +1,7 @@ #include "cgroup_oom.h" #include "cgroup_events.h" +#include "cgroup_oom_trend.h" #include "cgroup_v1.h" #include "cgroup_v2.h" @@ -11,9 +12,12 @@ #include <ydb/library/actors/core/events.h> #include <ydb/library/actors/core/hfunc.h> -#include <util/generic/hash.h> #include <util/generic/hash_set.h> #include <util/generic/vector.h> +#include <util/stream/output.h> + +#include <algorithm> +#include <array> namespace NActors { @@ -33,50 +37,74 @@ namespace NActors { enum : ui32 { EvPoll = EventSpaceBegin(TEvents::ES_PRIVATE), - EvSubscribeCallback, - EvUnsubscribeCallback, EvSubscribeActor, EvUnsubscribeActor, + EvReadTrend, + EvSubscribeTrendActor, + EvUnsubscribeTrendActor, }; struct TEvPoll : TEventLocal<TEvPoll, EvPoll> { }; - struct TEvSubscribeCallback : TEventLocal<TEvSubscribeCallback, EvSubscribeCallback> { - TCGroupOomSubSystem::TSubscriptionId SubscriptionId; - TCGroupOomSubSystem::TCallback Callback; + struct TEvSubscribeActor : TEventLocal<TEvSubscribeActor, EvSubscribeActor> { + TActorId ActorId; - TEvSubscribeCallback( - TCGroupOomSubSystem::TSubscriptionId subscriptionId, - TCGroupOomSubSystem::TCallback callback) - : SubscriptionId(subscriptionId) - , Callback(std::move(callback)) + explicit TEvSubscribeActor(TActorId actorId) + : ActorId(actorId) { } }; - struct TEvUnsubscribeCallback : TEventLocal<TEvUnsubscribeCallback, EvUnsubscribeCallback> { - TCGroupOomSubSystem::TSubscriptionId SubscriptionId; + struct TEvUnsubscribeActor : TEventLocal<TEvUnsubscribeActor, EvUnsubscribeActor> { + TActorId ActorId; - explicit TEvUnsubscribeCallback(TCGroupOomSubSystem::TSubscriptionId subscriptionId) - : SubscriptionId(subscriptionId) + explicit TEvUnsubscribeActor(TActorId actorId) + : ActorId(actorId) { } }; - struct TEvSubscribeActor : TEventLocal<TEvSubscribeActor, EvSubscribeActor> { + struct TEvReadTrend : TEventLocal<TEvReadTrend, EvReadTrend> { + TActorId Recipient; + ECGroupOomTrendWindow Window; + ui64 Cookie; + + TEvReadTrend( + TActorId recipient, + ECGroupOomTrendWindow window, + ui64 cookie) + : Recipient(recipient) + , Window(window) + , Cookie(cookie) + { + } + }; + + struct TEvSubscribeTrendActor + : TEventLocal<TEvSubscribeTrendActor, EvSubscribeTrendActor> + { TActorId ActorId; + ECGroupOomTrendWindow Window; + TDuration TimeToOomThreshold; - explicit TEvSubscribeActor(TActorId actorId) + TEvSubscribeTrendActor( + TActorId actorId, + ECGroupOomTrendWindow window, + TDuration timeToOomThreshold) : ActorId(actorId) + , Window(window) + , TimeToOomThreshold(timeToOomThreshold) { } }; - struct TEvUnsubscribeActor : TEventLocal<TEvUnsubscribeActor, EvUnsubscribeActor> { + struct TEvUnsubscribeTrendActor + : TEventLocal<TEvUnsubscribeTrendActor, EvUnsubscribeTrendActor> + { TActorId ActorId; - explicit TEvUnsubscribeActor(TActorId actorId) + explicit TEvUnsubscribeTrendActor(TActorId actorId) : ActorId(actorId) { } @@ -89,6 +117,7 @@ namespace NActors { TCGroupOomConfig config) : Providers(std::move(providers)) , Config(std::move(config)) + , TrendCalculator(Config.TrendWindows) { } @@ -100,14 +129,42 @@ namespace NActors { STRICT_STFUNC(StateWork, hFunc(TEvPoll, Handle) hFunc(TEvCGroupMemoryStats, Handle) - hFunc(TEvSubscribeCallback, Handle) - hFunc(TEvUnsubscribeCallback, Handle) hFunc(TEvSubscribeActor, Handle) hFunc(TEvUnsubscribeActor, Handle) + hFunc(TEvReadTrend, Handle) + hFunc(TEvSubscribeTrendActor, Handle) + hFunc(TEvUnsubscribeTrendActor, Handle) cFunc(TEvents::TSystem::Poison, PassAway) ) private: + struct TTrendSubscription { + TDuration TimeToOomThreshold; + bool Active = false; + + explicit TTrendSubscription(TDuration timeToOomThreshold) + : TimeToOomThreshold(timeToOomThreshold) + { + } + }; + + struct TPendingTrendRequest { + TActorId Recipient; + ui64 Cookie; + }; + + struct TTrendActorSubscription : TTrendSubscription { + TActorId ActorId; + + TTrendActorSubscription( + TActorId actorId, + TDuration timeToOomThreshold) + : TTrendSubscription(timeToOomThreshold) + , ActorId(actorId) + { + } + }; + void Handle(TEvPoll::TPtr&) { RequestProvider(0); } @@ -123,7 +180,9 @@ namespace NActors { const size_t providerIndex = ev->Cookie; if (ev->Get()->Stats) { - Evaluate(std::move(ev->Get()->Stats)); + if (!Evaluate(std::move(ev->Get()->Stats))) { + return; + } Schedule(Config.PollPeriod, new TEvPoll()); } else if (providerIndex + 1 < Providers.size()) { RequestProvider(providerIndex + 1); @@ -132,14 +191,6 @@ namespace NActors { } } - void Handle(TEvSubscribeCallback::TPtr& ev) { - Callbacks.emplace(ev->Get()->SubscriptionId, std::move(ev->Get()->Callback)); - } - - void Handle(TEvUnsubscribeCallback::TPtr& ev) { - Callbacks.erase(ev->Get()->SubscriptionId); - } - void Handle(TEvSubscribeActor::TPtr& ev) { ActorSubscribers.insert(ev->Get()->ActorId); } @@ -148,15 +199,88 @@ namespace NActors { ActorSubscribers.erase(ev->Get()->ActorId); } - void Evaluate(TCGroupMemoryStatsPtr stats) { - if (!PreviousVersion || *PreviousVersion != stats->Version || - PreviousCGroupPath != stats->CGroupPath) { - PreviousVersion = stats->Version; - PreviousCGroupPath = stats->CGroupPath; - HasPreviousMemory = false; - ActiveLevelReasons = 0; + void Handle(TEvReadTrend::TPtr& ev) { + if (TrendCalculator.HasResult(ev->Get()->Window)) { + const auto trend = GetTrend(ev->Get()->Window); + SendTrend( + ev->Get()->Recipient, + trend + ? ECGroupOomTrendState::Active + : ECGroupOomTrendState::Stopped, + trend, + std::nullopt, + ev->Get()->Cookie); + } else { + PendingTrendRequests[CGroupOomTrendWindowIndex( + ev->Get()->Window)].push_back({ + .Recipient = ev->Get()->Recipient, + .Cookie = ev->Get()->Cookie, + }); + } + } + + void Handle(TEvSubscribeTrendActor::TPtr& ev) { + const auto* event = ev->Get(); + auto& subscriptions = + TrendActors[CGroupOomTrendWindowIndex(event->Window)]; + const auto existing = std::find_if( + subscriptions.begin(), + subscriptions.end(), + [event](const TTrendActorSubscription& subscription) { + return subscription.ActorId == event->ActorId && + subscription.TimeToOomThreshold == event->TimeToOomThreshold; + }); + if (existing == subscriptions.end()) { + subscriptions.push_back({ + event->ActorId, + event->TimeToOomThreshold, + }); + if (TrendCalculator.HasResult(event->Window)) { + const auto trend = GetTrend(event->Window); + UpdateTrendActorSubscription( + trend, + subscriptions.back()); + } + } + } + + void Handle(TEvUnsubscribeTrendActor::TPtr& ev) { + const TActorId actorId = ev->Get()->ActorId; + for (auto& subscriptions : TrendActors) { + subscriptions.erase( + std::remove_if( + subscriptions.begin(), + subscriptions.end(), + [actorId](const TTrendActorSubscription& subscription) { + return subscription.ActorId == actorId; + }), + subscriptions.end()); + } + } + + bool Evaluate(TCGroupMemoryStatsPtr stats) { + if (!CGroupVersion) { + CGroupVersion = stats->Version; + CGroupPath = stats->CGroupPath; + } else if (*CGroupVersion != stats->Version || + CGroupPath != stats->CGroupPath) { + Cerr + << "ERROR: cgroup memory stats source changed from version " + << static_cast<ui32>(*CGroupVersion) + << " path " << CGroupPath + << " to version " << static_cast<ui32>(stats->Version) + << " path " << stats->CGroupPath << Endl; + Cerr + << "Stopping cgroup OOM monitoring; restart the process " + << "in the target cgroup" << Endl; + PassAway(); + return false; } + const auto recalculated = TrendCalculator.AddSample( + TActivationContext::Monotonic(), + stats); + TCGroupOomAlert alert; alert.Stats = std::move(stats); @@ -201,6 +325,13 @@ namespace NActors { if (alert.Reasons) { Notify(alert); } + for (ECGroupOomTrendWindow window : CGroupOomTrendWindows) { + if (recalculated.Contains(window)) { + ReplyToPendingTrendRequests(window); + UpdateTrendSubscriptions(window); + } + } + return true; } void Notify(const TCGroupOomAlert& alert) { @@ -208,26 +339,93 @@ namespace NActors { for (const TActorId& actorId : ActorSubscribers) { actorContext.Send(actorId, new TEvCGroupOomAlert(alert)); } + } + + void UpdateTrendSubscriptions(ECGroupOomTrendWindow window) { + const auto windowIndex = CGroupOomTrendWindowIndex(window); + const auto trend = GetTrend(window); + + for (auto& subscription : TrendActors[windowIndex]) { + UpdateTrendActorSubscription(trend, subscription); + } + } - for (const auto& item : Callbacks) { - try { - item.second(alert); - } catch (...) { - // One callback must not prevent other subscribers from - // releasing memory. - } + void UpdateTrendActorSubscription( + const std::optional<TCGroupOomTrend>& trend, + TTrendActorSubscription& subscription) { + const bool active = + trend && trend->TimeToOom <= subscription.TimeToOomThreshold; + if (!active && !subscription.Active) { + return; + } + + SendTrend( + subscription.ActorId, + active + ? ECGroupOomTrendState::Active + : ECGroupOomTrendState::Stopped, + trend, + subscription.TimeToOomThreshold); + subscription.Active = active; + } + + std::optional<TCGroupOomTrend> GetTrend( + ECGroupOomTrendWindow window) const { + if (const auto* trend = TrendCalculator.FindTrend(window)) { + return *trend; + } + return std::nullopt; + } + + void SendTrend( + const TActorId& recipient, + ECGroupOomTrendState state, + const std::optional<TCGroupOomTrend>& trend, + std::optional<TDuration> timeToOomThreshold = std::nullopt, + ui64 cookie = 0) { + Send( + recipient, + new TEvCGroupOomTrend(trend, state, timeToOomThreshold), + 0, + cookie); + } + + void ReplyToPendingTrendRequests(ECGroupOomTrendWindow window) { + auto& requests = + PendingTrendRequests[CGroupOomTrendWindowIndex(window)]; + if (!TrendCalculator.HasResult(window)) { + return; + } + const auto trend = GetTrend(window); + const auto state = trend + ? ECGroupOomTrendState::Active + : ECGroupOomTrendState::Stopped; + for (const auto& request : requests) { + SendTrend( + request.Recipient, + state, + trend, + std::nullopt, + request.Cookie); } + requests.clear(); } private: const TVector<const ICGroupMemoryStatsProvider*> Providers; const TCGroupOomConfig Config; + TCGroupOomTrendCalculator TrendCalculator; - THashMap<TCGroupOomSubSystem::TSubscriptionId, TCGroupOomSubSystem::TCallback> Callbacks; THashSet<TActorId> ActorSubscribers; + std::array< + TVector<TTrendActorSubscription>, + CGroupOomTrendWindows.size()> TrendActors; + std::array< + TVector<TPendingTrendRequest>, + CGroupOomTrendWindows.size()> PendingTrendRequests; - std::optional<ECGroupVersion> PreviousVersion; - TString PreviousCGroupPath; + std::optional<ECGroupVersion> CGroupVersion; + TString CGroupPath; TCGroupMemoryStats PreviousMemory; ui32 ActiveLevelReasons = 0; bool HasPreviousMemory = false; @@ -247,6 +445,30 @@ namespace NActors { (*Config.MemoryPressureFullAvg10Threshold >= 0 && *Config.MemoryPressureFullAvg10Threshold <= 100), "cgroup OOM PSI threshold must be in [0, 100]"); + + const auto validateTrendWindow = [this]( + const std::optional<TCGroupOomTrendWindowConfig>& window) { + if (!window) { + return; + } + Y_ABORT_UNLESS(window->Duration >= Config.PollPeriod, + "cgroup OOM trend window must be at least one poll period"); + Y_ABORT_UNLESS( + window->RecalculationPeriod >= Config.PollPeriod && + window->RecalculationPeriod <= window->Duration, + "cgroup OOM trend recalculation period must be between the poll period and window"); + Y_ABORT_UNLESS( + window->MinimumRSquared >= 0 && window->MinimumRSquared <= 1, + "cgroup OOM minimum trend R-squared must be in [0, 1]"); + }; + validateTrendWindow(Config.TrendWindows.ShortWindow); + validateTrendWindow(Config.TrendWindows.LongWindow); + Y_ABORT_UNLESS( + !Config.TrendWindows.ShortWindow || + !Config.TrendWindows.LongWindow || + Config.TrendWindows.ShortWindow->Duration < + Config.TrendWindows.LongWindow->Duration, + "cgroup OOM short trend window must be shorter than the long trend window"); } TSubSystemDependencies TCGroupOomSubSystem::GetDependencies() const { @@ -266,37 +488,72 @@ namespace NActors { } } - TCGroupOomSubSystem::TSubscriptionId TCGroupOomSubSystem::Subscribe(TCallback callback) { - Y_ABORT_UNLESS(callback, "cannot subscribe an empty cgroup OOM callback"); + void TCGroupOomSubSystem::Subscribe(const TActorId& actorId) { + Y_ABORT_UNLESS(actorId, "cannot subscribe an empty actor id to cgroup OOM alerts"); Y_ABORT_UNLESS(ActorSystem, "cgroup OOM subsystem is not running"); - const TSubscriptionId subscriptionId = - NextSubscriptionId.fetch_add(1, std::memory_order_relaxed); - ActorSystem->Send(MonitorActorId, - new NDetail::TEvSubscribeCallback(subscriptionId, std::move(callback))); - return subscriptionId; + ActorSystem->Send(MonitorActorId, new NDetail::TEvSubscribeActor(actorId)); } - void TCGroupOomSubSystem::Unsubscribe(TSubscriptionId subscriptionId) { + void TCGroupOomSubSystem::Unsubscribe(const TActorId& actorId) { Y_ABORT_UNLESS(ActorSystem, "cgroup OOM subsystem is not running"); - ActorSystem->Send(MonitorActorId, - new NDetail::TEvUnsubscribeCallback(subscriptionId)); + ActorSystem->Send(MonitorActorId, new NDetail::TEvUnsubscribeActor(actorId)); } - void TCGroupOomSubSystem::Subscribe(const TActorId& actorId) { - Y_ABORT_UNLESS(actorId, "cannot subscribe an empty actor id to cgroup OOM alerts"); + void TCGroupOomSubSystem::ReadTrend( + const TActorId& recipient, + ECGroupOomTrendWindow window, + ui64 cookie) const { + Y_ABORT_UNLESS(recipient, "cannot send a cgroup OOM trend to an empty actor id"); + Y_ABORT_UNLESS(ActorSystem, "cgroup OOM subsystem is not running"); + ValidateTrendWindow(window); + + ActorSystem->Send( + MonitorActorId, + new NDetail::TEvReadTrend(recipient, window, cookie)); + } + void TCGroupOomSubSystem::SubscribeToTrend( + const TActorId& actorId, + ECGroupOomTrendWindow window, + TDuration timeToOomThreshold) { + Y_ABORT_UNLESS( + actorId, + "cannot subscribe an empty actor id to cgroup OOM trends"); Y_ABORT_UNLESS(ActorSystem, "cgroup OOM subsystem is not running"); + ValidateTrendSubscription(window, timeToOomThreshold); - ActorSystem->Send(MonitorActorId, new NDetail::TEvSubscribeActor(actorId)); + ActorSystem->Send( + MonitorActorId, + new NDetail::TEvSubscribeTrendActor( + actorId, + window, + timeToOomThreshold)); } - void TCGroupOomSubSystem::Unsubscribe(const TActorId& actorId) { + void TCGroupOomSubSystem::UnsubscribeFromTrend(const TActorId& actorId) { Y_ABORT_UNLESS(ActorSystem, "cgroup OOM subsystem is not running"); - ActorSystem->Send(MonitorActorId, new NDetail::TEvUnsubscribeActor(actorId)); + ActorSystem->Send( + MonitorActorId, + new NDetail::TEvUnsubscribeTrendActor(actorId)); + } + + void TCGroupOomSubSystem::ValidateTrendWindow(ECGroupOomTrendWindow window) const { + Y_ABORT_UNLESS( + Config.TrendWindows.Find(window), + "cgroup OOM trend window is not configured"); + } + + void TCGroupOomSubSystem::ValidateTrendSubscription( + ECGroupOomTrendWindow window, + TDuration timeToOomThreshold) const { + ValidateTrendWindow(window); + Y_ABORT_UNLESS( + timeToOomThreshold > TDuration::Zero(), + "cgroup OOM trend time threshold must be positive"); } void TCGroupOomSubSystem::OnAfterStart(TActorSystem& actorSystem) { diff --git a/ydb/library/actors/core/subsystems/cgroup/cgroup_oom.h b/ydb/library/actors/core/subsystems/cgroup/cgroup_oom.h index e2004074462..5dda2bba635 100644 --- a/ydb/library/actors/core/subsystems/cgroup/cgroup_oom.h +++ b/ydb/library/actors/core/subsystems/cgroup/cgroup_oom.h @@ -9,8 +9,6 @@ #include <util/datetime/base.h> #include <util/generic/vector.h> -#include <atomic> -#include <functional> #include <memory> #include <optional> @@ -53,23 +51,76 @@ namespace NActors { } }; + enum class ECGroupOomTrendWindow : ui8 { + ShortWindow = 0, + LongWindow = 1, + }; + + struct TCGroupOomTrendWindowConfig { + // The fixed sample window used for linear regression. + TDuration Duration; + + // Samples are collected on every poll, while regression and the OOM + // forecast are updated no more often than this period. + TDuration RecalculationPeriod = TDuration::Seconds(1); + + // A trend is available only when the coefficient of determination + // reaches this value. + double MinimumRSquared = 0.8; + }; + + struct TCGroupOomTrendWindowsConfig { + std::optional<TCGroupOomTrendWindowConfig> ShortWindow; + std::optional<TCGroupOomTrendWindowConfig> LongWindow; + + const TCGroupOomTrendWindowConfig* Find(ECGroupOomTrendWindow window) const { + switch (window) { + case ECGroupOomTrendWindow::ShortWindow: + return ShortWindow ? &*ShortWindow : nullptr; + case ECGroupOomTrendWindow::LongWindow: + return LongWindow ? &*LongWindow : nullptr; + } + return nullptr; + } + }; + + struct TCGroupOomTrend { + TCGroupMemoryStatsPtr Stats; + ECGroupOomTrendWindow Window; + TDuration WindowDuration; + double GrowthBytesPerSecond = 0; + double RSquared = 0; + TDuration TimeToOom; + ui64 SampleCount = 0; + }; + + enum class ECGroupOomTrendState { + Active, + Stopped, + }; + struct TCGroupOomConfig { - // The monitor actor and callbacks run in this pool. Actual cgroup file - // reads use the pools configured by the selected stats providers. + // The monitor actor runs in this pool. Actual cgroup file reads use the + // pools configured by the selected stats providers. ui32 ExecutorPoolId = 0; TDuration PollPeriod = TDuration::Seconds(1); + // The cgroup version and path are fixed by the first memory snapshot. + // Moving a running process between cgroups stops OOM monitoring; restart + // the process in the target cgroup instead. + // An empty optional disables the corresponding threshold. std::optional<double> MemoryUsageThreshold = 0.9; std::optional<ui64> AvailableBytesThreshold; std::optional<double> MemoryPressureFullAvg10Threshold; + + // Trend windows are fixed for the lifetime of the subsystem. Each + // window is disabled when the corresponding optional is empty. + TCGroupOomTrendWindowsConfig TrendWindows; }; class TCGroupOomSubSystem : public ISubSystem { public: - using TSubscriptionId = ui64; - using TCallback = std::function<void(const TCGroupOomAlert&)>; - explicit TCGroupOomSubSystem(TCGroupOomConfig config); const TCGroupOomConfig& GetConfig() const { @@ -79,30 +130,46 @@ namespace NActors { TSubSystemDependencies GetDependencies() const override; void OnDependenciesResolved(const TResolvedSubSystemDependencies& dependencies) override; - // Subscription changes are applied asynchronously by the monitor - // actor. The actor system must be running when these methods are - // called. A notification already ahead of Unsubscribe in the actor's - // mailbox may still be delivered. Threshold notifications are - // edge-triggered; counter deltas trigger notifications whenever they - // increase. - TSubscriptionId Subscribe(TCallback callback); - void Unsubscribe(TSubscriptionId subscriptionId); - // Actor subscriptions are local and keyed by ActorId. Repeated - // Subscribe and Unsubscribe calls are idempotent. + // Subscribe and Unsubscribe calls are idempotent. Subscription changes + // are applied asynchronously by the monitor actor. void Subscribe(const TActorId& actorId); void Unsubscribe(const TActorId& actorId); + // Sends TEvCGroupOomTrend after the first calculation for the window. + // The request remains pending while the window is accumulating. Once + // calculated, the response is sent even when no trend was found. + void ReadTrend( + const TActorId& recipient, + ECGroupOomTrendWindow window, + ui64 cookie = 0) const; + + // Repeated actor subscriptions with the same window and threshold are + // idempotent. Active is delivered after every window recalculation + // while the threshold is met; Stopped is delivered once when it ceases + // to hold. UnsubscribeFromTrend removes all trend subscriptions for the + // actor. + void SubscribeToTrend( + const TActorId& actorId, + ECGroupOomTrendWindow window, + TDuration timeToOomThreshold); + void UnsubscribeFromTrend(const TActorId& actorId); + void OnAfterStart(TActorSystem& actorSystem) override; void OnBeforeStop(TActorSystem& actorSystem) override; private: + void ValidateTrendWindow(ECGroupOomTrendWindow window) const; + void ValidateTrendSubscription( + ECGroupOomTrendWindow window, + TDuration timeToOomThreshold) const; + + private: const TCGroupOomConfig Config; TVector<const ICGroupMemoryStatsProvider*> Providers; TActorSystem* ActorSystem = nullptr; TActorId MonitorActorId; - alignas(64) std::atomic<TSubscriptionId> NextSubscriptionId = 1; }; std::unique_ptr<TCGroupOomSubSystem> MakeCGroupOomSubSystem(TCGroupOomConfig config); diff --git a/ydb/library/actors/core/subsystems/cgroup/cgroup_oom_trend.cpp b/ydb/library/actors/core/subsystems/cgroup/cgroup_oom_trend.cpp new file mode 100644 index 00000000000..3d42f9da39f --- /dev/null +++ b/ydb/library/actors/core/subsystems/cgroup/cgroup_oom_trend.cpp @@ -0,0 +1,185 @@ +#include "cgroup_oom_trend.h" + +#include <algorithm> +#include <cmath> + +namespace NActors::NDetail { + + namespace { + + double RelativeBytes(ui64 value, ui64 base) { + return value >= base + ? static_cast<double>(value - base) + : -static_cast<double>(base - value); + } + + } // namespace + + TCGroupOomTrendCalculator::TCGroupOomTrendCalculator( + TCGroupOomTrendWindowsConfig windows) { + if (windows.ShortWindow) { + MaxWindow = std::max(MaxWindow, windows.ShortWindow->Duration); + Windows[CGroupOomTrendWindowIndex( + ECGroupOomTrendWindow::ShortWindow)].Config = + std::move(windows.ShortWindow); + } + if (windows.LongWindow) { + MaxWindow = std::max(MaxWindow, windows.LongWindow->Duration); + Windows[CGroupOomTrendWindowIndex( + ECGroupOomTrendWindow::LongWindow)].Config = + std::move(windows.LongWindow); + } + } + + TRecalculatedCGroupOomTrendWindows TCGroupOomTrendCalculator::AddSample( + TMonotonic timestamp, + TCGroupMemoryStatsPtr stats) { + TRecalculatedCGroupOomTrendWindows recalculated; + if (MaxWindow == TDuration::Zero()) { + return recalculated; + } + + if (!Samples.empty() && timestamp <= Samples.back().Timestamp) { + Reset(); + } + + if (!HistoryStartedAt) { + HistoryStartedAt = timestamp; + } + Samples.push_back({ + .Timestamp = timestamp, + .CurrentBytes = stats->CurrentBytes, + }); + Prune(timestamp); + + for (ECGroupOomTrendWindow window : CGroupOomTrendWindows) { + auto& state = Windows[CGroupOomTrendWindowIndex(window)]; + if (!state.Config) { + continue; + } + if (!state.LastCalculatedAt || + timestamp - *state.LastCalculatedAt >= + state.Config->RecalculationPeriod) { + state.HasResult = + timestamp - *HistoryStartedAt >= state.Config->Duration; + state.Trend = + CalculateTrend(window, state, timestamp, stats); + state.LastCalculatedAt = timestamp; + recalculated.Add(window); + } + } + return recalculated; + } + + bool TCGroupOomTrendCalculator::HasResult(ECGroupOomTrendWindow window) const { + const auto& state = Windows[CGroupOomTrendWindowIndex(window)]; + return state.Config && state.HasResult; + } + + const TCGroupOomTrend* TCGroupOomTrendCalculator::FindTrend( + ECGroupOomTrendWindow window) const { + const auto& state = Windows[CGroupOomTrendWindowIndex(window)]; + return state.Trend + ? &*state.Trend + : nullptr; + } + + void TCGroupOomTrendCalculator::Reset() { + Samples.clear(); + HistoryStartedAt.reset(); + for (auto& state : Windows) { + state.Trend.reset(); + state.LastCalculatedAt.reset(); + state.HasResult = false; + } + } + + void TCGroupOomTrendCalculator::Prune(TMonotonic timestamp) { + const TMonotonic cutoff = timestamp - MaxWindow; + while (Samples.size() > 1 && Samples[1].Timestamp <= cutoff) { + Samples.pop_front(); + } + } + + std::optional<TCGroupOomTrend> TCGroupOomTrendCalculator::CalculateTrend( + ECGroupOomTrendWindow window, + const TWindowState& state, + TMonotonic timestamp, + const TCGroupMemoryStatsPtr& stats) const { + const auto& config = *state.Config; + if (!HistoryStartedAt || timestamp - *HistoryStartedAt < config.Duration || + !stats->MaxBytes || stats->MaxBytes->Unlimited || + !stats->MaxBytes->Value) { + return std::nullopt; + } + + const TMonotonic cutoff = timestamp - config.Duration; + size_t first = 0; + while (first + 1 < Samples.size() && Samples[first + 1].Timestamp <= cutoff) { + ++first; + } + + const size_t sampleCount = Samples.size() - first; + // Two points always form a perfect line and do not provide a useful + // linearity check. + if (sampleCount < 3) { + return std::nullopt; + } + + const TMonotonic baseTimestamp = Samples[first].Timestamp; + const ui64 baseBytes = Samples[first].CurrentBytes; + double sumX = 0; + double sumY = 0; + for (size_t index = first; index < Samples.size(); ++index) { + sumX += (Samples[index].Timestamp - baseTimestamp).SecondsFloat(); + sumY += RelativeBytes(Samples[index].CurrentBytes, baseBytes); + } + + const double meanX = sumX / sampleCount; + const double meanY = sumY / sampleCount; + double sumXX = 0; + double sumXY = 0; + double sumYY = 0; + for (size_t index = first; index < Samples.size(); ++index) { + const double x = + (Samples[index].Timestamp - baseTimestamp).SecondsFloat() - meanX; + const double y = + RelativeBytes(Samples[index].CurrentBytes, baseBytes) - meanY; + sumXX += x * x; + sumXY += x * y; + sumYY += y * y; + } + + if (sumXX <= 0 || sumYY <= 0) { + return std::nullopt; + } + + const double growthBytesPerSecond = sumXY / sumXX; + const double rSquared = std::clamp( + sumXY * sumXY / (sumXX * sumYY), + 0.0, + 1.0); + if (!std::isfinite(growthBytesPerSecond) || + !std::isfinite(rSquared) || + growthBytesPerSecond <= 0 || + rSquared < config.MinimumRSquared) { + return std::nullopt; + } + + const ui64 maxBytes = stats->MaxBytes->Value; + const double remainingBytes = stats->CurrentBytes < maxBytes + ? static_cast<double>(maxBytes - stats->CurrentBytes) + : 0.0; + + return TCGroupOomTrend{ + .Stats = stats, + .Window = window, + .WindowDuration = config.Duration, + .GrowthBytesPerSecond = growthBytesPerSecond, + .RSquared = rSquared, + .TimeToOom = TDuration::Seconds(remainingBytes / growthBytesPerSecond), + .SampleCount = sampleCount, + }; + } + +} // namespace NActors::NDetail diff --git a/ydb/library/actors/core/subsystems/cgroup/cgroup_oom_trend.h b/ydb/library/actors/core/subsystems/cgroup/cgroup_oom_trend.h new file mode 100644 index 00000000000..9c957833d99 --- /dev/null +++ b/ydb/library/actors/core/subsystems/cgroup/cgroup_oom_trend.h @@ -0,0 +1,78 @@ +#pragma once + +#include "cgroup_oom.h" + +#include <ydb/library/actors/core/monotonic.h> + +#include <util/generic/deque.h> + +#include <array> +#include <optional> + +namespace NActors::NDetail { + + inline constexpr std::array<ECGroupOomTrendWindow, 2> CGroupOomTrendWindows = { + ECGroupOomTrendWindow::ShortWindow, + ECGroupOomTrendWindow::LongWindow, + }; + + constexpr size_t CGroupOomTrendWindowIndex(ECGroupOomTrendWindow window) { + return static_cast<size_t>(window); + } + + class TRecalculatedCGroupOomTrendWindows { + public: + void Add(ECGroupOomTrendWindow window) { + Recalculated[CGroupOomTrendWindowIndex(window)] = true; + } + + bool Contains(ECGroupOomTrendWindow window) const { + return Recalculated[CGroupOomTrendWindowIndex(window)]; + } + + private: + std::array<bool, CGroupOomTrendWindows.size()> Recalculated = {}; + }; + + class TCGroupOomTrendCalculator { + public: + explicit TCGroupOomTrendCalculator( + TCGroupOomTrendWindowsConfig windows); + + // Stores every sample and reports only windows whose regression was + // recalculated for this sample. + TRecalculatedCGroupOomTrendWindows AddSample( + TMonotonic timestamp, + TCGroupMemoryStatsPtr stats); + bool HasResult(ECGroupOomTrendWindow window) const; + const TCGroupOomTrend* FindTrend(ECGroupOomTrendWindow window) const; + + private: + struct TSample { + TMonotonic Timestamp; + ui64 CurrentBytes = 0; + }; + + struct TWindowState { + std::optional<TCGroupOomTrendWindowConfig> Config; + std::optional<TCGroupOomTrend> Trend; + std::optional<TMonotonic> LastCalculatedAt; + bool HasResult = false; + }; + + void Reset(); + void Prune(TMonotonic timestamp); + std::optional<TCGroupOomTrend> CalculateTrend( + ECGroupOomTrendWindow window, + const TWindowState& state, + TMonotonic timestamp, + const TCGroupMemoryStatsPtr& stats) const; + + private: + std::array<TWindowState, CGroupOomTrendWindows.size()> Windows; + TDeque<TSample> Samples; + TDuration MaxWindow; + std::optional<TMonotonic> HistoryStartedAt; + }; + +} // namespace NActors::NDetail diff --git a/ydb/library/actors/core/subsystems/ya.make b/ydb/library/actors/core/subsystems/ya.make index 41b3f865f29..746d3b5be89 100644 --- a/ydb/library/actors/core/subsystems/ya.make +++ b/ydb/library/actors/core/subsystems/ya.make @@ -15,6 +15,7 @@ ENDIF() SRCS( cgroup/cgroup_oom.cpp + cgroup/cgroup_oom_trend.cpp cgroup/cgroup_v1.cpp cgroup/cgroup_v2.cpp stats.cpp diff --git a/ydb/library/actors/core/thread_context.h b/ydb/library/actors/core/thread_context.h index 0d71288ebf6..de292ff9375 100644 --- a/ydb/library/actors/core/thread_context.h +++ b/ydb/library/actors/core/thread_context.h @@ -68,7 +68,6 @@ namespace NActors { ui32 ExecutedEvents = 0; ui32 OverwrittenEventsPerMailbox = 0; ui64 OverwrittenTimePerMailboxTs = 0; - TStackVec<TActorId, 1> PreemptionSubscribed; bool IsNeededToWaitNextActivation = true; ESendingType SendingType = ESendingType::Common; NHPTimer::STime HPStart = 0; diff --git a/ydb/library/actors/core/ut/actor_ut.cpp b/ydb/library/actors/core/ut/actor_ut.cpp index e76b960f2ef..fe7efe2ed7b 100644 --- a/ydb/library/actors/core/ut/actor_ut.cpp +++ b/ydb/library/actors/core/ut/actor_ut.cpp @@ -5,6 +5,7 @@ #include "scheduler_basic.h" #include "actor_bootstrapped.h" #include "actor_benchmark_helper.h" +#include "thread_context.h" #include "subsystems/stats.h" #include <ydb/library/actors/testlib/test_runtime.h> @@ -781,7 +782,7 @@ Y_UNIT_TEST_SUITE(TestActorLiveness) { void Bootstrap() { Become(&TThis::StateWork); - CheckActorLiveness(AliveCookie); + SendActorLivenessCheck(Target, AliveCookie); } STRICT_STFUNC(StateWork, @@ -795,7 +796,7 @@ Y_UNIT_TEST_SUITE(TestActorLiveness) { Y_ABORT_UNLESS(ev->Cookie == AliveCookie); Send(Target, new TEvents::TEvPoison()); - CheckActorLiveness(DeadCookie); + SendActorLivenessCheck(Target, DeadCookie); } void Handle(TEvents::TEvActorDead::TPtr& ev) { @@ -812,16 +813,6 @@ Y_UNIT_TEST_SUITE(TestActorLiveness) { PassAway(); } - void CheckActorLiveness(ui64 cookie) { - Send(new IEventHandle( - TEvents::TSystem::CheckActorLiveness, - TEvents::TEvCheckActorLiveness::RequestFlags, - Target, - SelfId(), - nullptr, - cookie)); - } - const TActorId Target; TThreadParkPad* const DonePad; std::atomic<bool>* const DoneFlag; @@ -838,13 +829,7 @@ Y_UNIT_TEST_SUITE(TestActorLiveness) { void Bootstrap() { Become(&TThis::StateWork); - Send(new IEventHandle( - TEvents::TSystem::CheckActorLiveness, - TEvents::TEvCheckActorLiveness::RequestFlags, - Target, - SelfId(), - nullptr, - Cookie)); + SendActorLivenessCheck(Target, Cookie); } STRICT_STFUNC(StateWork, @@ -924,6 +909,195 @@ Y_UNIT_TEST_SUITE(TestActorLiveness) { } } +Y_UNIT_TEST_SUITE(MailboxProcessingFinished) { + using EFinishReason = TEvents::TEvMailboxProcessingFinished::EReason; + using TActorBenchmark = NActors::NTests::TActorBenchmark<>; + + struct TResult { + EFinishReason Reason = EFinishReason::QueueEmpty; + ui32 ExecutedEvents = 0; + ui64 ElapsedCycles = 0; + }; + + class TObserverActor : public TActorBootstrapped<TObserverActor> { + public: + TObserverActor(EFinishReason expectedReason, TResult& result, TThreadParkPad& done) + : ExpectedReason(expectedReason) + , Result(result) + , Done(done) + {} + + void Bootstrap() { + SetSystemFlag(ESystemFlag::MailboxProcessingFinished); + ForceFinishReason(); + Become(&TThis::StateWork); + } + + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvents::TEvMailboxProcessingFinished, Handle); + } + } + + void Handle(TEvents::TEvMailboxProcessingFinished::TPtr& ev) { + RestoreExecutorState(); + Result.Reason = ev->Get()->Reason; + Result.ExecutedEvents = ev->Get()->ExecutedEvents; + Result.ElapsedCycles = ev->Get()->ElapsedCycles; + Done.Unpark(); + PassAway(); + } + + private: + void ForceFinishReason() { + OriginalEventsPerMailbox = TlsThreadContext->WorkerContext.EventsPerMailbox; + OriginalTimePerMailboxTs = TlsThreadContext->WorkerContext.TimePerMailboxTs; + OriginalSoftDeadlineTs = TlsThreadContext->WorkerContext.SoftDeadlineTs; + OriginalCapturedSendingType = + TlsThreadContext->ExecutionContext.CapturedActivation.SendingType; + + switch (ExpectedReason) { + case EFinishReason::QueueEmpty: + break; + case EFinishReason::EventCountLimitReached: + TlsThreadContext->WorkerContext.EventsPerMailbox = 1; + break; + case EFinishReason::TimeLimitReached: + TlsThreadContext->WorkerContext.TimePerMailboxTs = 0; + break; + case EFinishReason::SoftDeadlineReached: + TlsThreadContext->WorkerContext.SoftDeadlineTs = 0; + break; + case EFinishReason::TailSend: + TlsThreadContext->ExecutionContext.CapturedActivation.SendingType = + ESendingType::Tail; + break; + } + } + + void RestoreExecutorState() { + TlsThreadContext->WorkerContext.EventsPerMailbox = OriginalEventsPerMailbox; + TlsThreadContext->WorkerContext.TimePerMailboxTs = OriginalTimePerMailboxTs; + TlsThreadContext->WorkerContext.SoftDeadlineTs = OriginalSoftDeadlineTs; + TlsThreadContext->ExecutionContext.CapturedActivation.SendingType = + OriginalCapturedSendingType; + } + + private: + const EFinishReason ExpectedReason; + TResult& Result; + TThreadParkPad& Done; + ui32 OriginalEventsPerMailbox = 0; + ui64 OriginalTimePerMailboxTs = 0; + ui64 OriginalSoftDeadlineTs = 0; + ESendingType OriginalCapturedSendingType = ESendingType::Common; + }; + + class TPersistentObserverActor : public TActorBootstrapped<TPersistentObserverActor> { + public: + TPersistentObserverActor(ui32& notifications, TThreadParkPad& done) + : Notifications(notifications) + , Done(done) + {} + + void Bootstrap() { + SetSystemFlag(ESystemFlag::MailboxProcessingFinished); + Become(&TThis::StateWork); + } + + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvents::TEvMailboxProcessingFinished, Handle); + hFunc(TEvents::TEvWakeup, Handle); + } + } + + void Handle(TEvents::TEvMailboxProcessingFinished::TPtr&) { + ++Notifications; + if (Notifications == 1) { + Send(SelfId(), new TEvents::TEvWakeup(1)); + } else if (Notifications == 2) { + ClearSystemFlag(ESystemFlag::MailboxProcessingFinished); + Send(SelfId(), new TEvents::TEvWakeup(2)); + } else { + Done.Unpark(); + PassAway(); + } + } + + void Handle(TEvents::TEvWakeup::TPtr& ev) { + if (ev->Get()->Tag == 2) { + Send(SelfId(), new TEvents::TEvWakeup(3)); + } else if (ev->Get()->Tag == 3) { + Done.Unpark(); + PassAway(); + } + } + + private: + ui32& Notifications; + TThreadParkPad& Done; + }; + + TResult Run(EFinishReason expectedReason) { + auto setup = TActorBenchmark::GetActorSystemSetup(); + TActorBenchmark::AddBasicPool(setup, 1, false, false); + + TActorSystem actorSystem(setup); + actorSystem.Start(); + + TResult result; + TThreadParkPad done; + actorSystem.Register( + new TObserverActor(expectedReason, result, done), + TMailboxType::HTSwap, + 0); + + done.Park(); + actorSystem.Stop(); + return result; + } + + Y_UNIT_TEST(NotifiesForEveryFinishReason) { + const EFinishReason expectedReasons[] = { + EFinishReason::QueueEmpty, + EFinishReason::EventCountLimitReached, + EFinishReason::TimeLimitReached, + EFinishReason::SoftDeadlineReached, + EFinishReason::TailSend, + }; + + for (const EFinishReason expectedReason : expectedReasons) { + const TResult result = Run(expectedReason); + + UNIT_ASSERT_VALUES_EQUAL( + static_cast<ui8>(result.Reason), + static_cast<ui8>(expectedReason)); + UNIT_ASSERT_VALUES_EQUAL(result.ExecutedEvents, 1); + UNIT_ASSERT_C( + result.ElapsedCycles > 0, + "Expected non-zero mailbox processing time"); + } + } + + Y_UNIT_TEST(SystemFlagStaysSetUntilExplicitlyCleared) { + auto setup = TActorBenchmark::GetActorSystemSetup(); + TActorBenchmark::AddBasicPool(setup, 1, true, false); + + TActorSystem actorSystem(setup); + actorSystem.Start(); + + ui32 notifications = 0; + TThreadParkPad done; + actorSystem.Register(new TPersistentObserverActor(notifications, done), TMailboxType::HTSwap, 0); + + done.Park(); + actorSystem.Stop(); + + UNIT_ASSERT_VALUES_EQUAL(notifications, 2); + } +} + Y_UNIT_TEST_SUITE(TestThreadContextQueueTimestamps) { using namespace NThreading; diff --git a/ydb/library/actors/core/ut/cgroup_ut.cpp b/ydb/library/actors/core/ut/cgroup_ut.cpp index 8919466083b..0f194b57856 100644 --- a/ydb/library/actors/core/ut/cgroup_ut.cpp +++ b/ydb/library/actors/core/ut/cgroup_ut.cpp @@ -1,5 +1,6 @@ #include <subsystems/cgroup/cgroup_events.h> #include <subsystems/cgroup/cgroup_oom.h> +#include <subsystems/cgroup/cgroup_oom_trend.h> #include <subsystems/cgroup/cgroup_v1.h> #include <subsystems/cgroup/cgroup_v2.h> @@ -11,12 +12,14 @@ #include <scheduler_basic.h> #include <library/cpp/testing/unittest/registar.h> -#include <library/cpp/threading/future/core/future.h> #include <util/folder/path.h> #include <util/folder/tempdir.h> #include <util/stream/file.h> +#include <util/string/cast.h> +#include <util/system/event.h> +#include <atomic> #include <functional> using namespace NActors; @@ -27,6 +30,27 @@ namespace { constexpr ui32 IoPoolId = 1; constexpr ui64 RequestCookie = 42; + template<class TResult> + class TResultWaiter { + public: + void Set(TResult result) { + Result = std::move(result); + Ready.Signal(); + } + + bool Wait(TDuration timeout) { + return Ready.WaitT(timeout); + } + + const TResult& GetValue() const { + return Result; + } + + private: + TManualEvent Ready; + TResult Result; + }; + void WriteFile(const TFsPath& root, const TString& path, const TString& content) { const TFsPath fullPath = root / path; fullPath.Parent().MkDirs(); @@ -89,9 +113,9 @@ namespace { using TThis = TRequestActor<TEvent, TResult>; using TRequest = std::function<void(const TActorId&, ui64)>; - TRequestActor(TRequest request, NThreading::TPromise<TResult> promise) + TRequestActor(TRequest request, TResultWaiter<TResult>& result) : Request(std::move(request)) - , Promise(std::move(promise)) + , Result(result) { } @@ -109,29 +133,29 @@ namespace { private: void Handle(typename TResultEvent::TPtr& ev) { Y_ABORT_UNLESS(ev->Cookie == RequestCookie); - Promise.SetValue(std::move(ev->Get()->Stats)); + Result.Set(std::move(ev->Get()->Stats)); this->PassAway(); } private: const TRequest Request; - NThreading::TPromise<TResult> Promise; + TResultWaiter<TResult>& Result; }; template<class TEvent, class TResult> - NThreading::TFuture<TResult> RequestStats( + TResult RequestStats( TActorSystem& actorSystem, typename TRequestActor<TEvent, TResult>::TRequest request) { - auto promise = NThreading::NewPromise<TResult>(); - auto future = promise.GetFuture(); + TResultWaiter<TResult> result; actorSystem.Register( - new TRequestActor<TEvent, TResult>(std::move(request), std::move(promise)), + new TRequestActor<TEvent, TResult>(std::move(request), result), TMailboxType::Simple, SystemPoolId); - return future; + UNIT_ASSERT(result.Wait(TDuration::Seconds(5))); + return result.GetValue(); } - NThreading::TFuture<TCGroupV2StatsPtr> RequestV2Stats( + TCGroupV2StatsPtr RequestV2Stats( TActorSystem& actorSystem, const TCGroupV2StatsSubSystem& subsystem) { return RequestStats<TEvCGroupV2Stats, TCGroupV2StatsPtr>( @@ -141,7 +165,7 @@ namespace { }); } - NThreading::TFuture<TCGroupMemoryStatsPtr> RequestMemoryStats( + TCGroupMemoryStatsPtr RequestMemoryStats( TActorSystem& actorSystem, const ICGroupMemoryStatsProvider& provider) { return RequestStats<TEvCGroupMemoryStats, TCGroupMemoryStatsPtr>( @@ -151,6 +175,143 @@ namespace { }); } + class TTrendRequestActor : public TActorBootstrapped<TTrendRequestActor> { + public: + TTrendRequestActor( + const TCGroupOomSubSystem& subsystem, + ECGroupOomTrendWindow window, + TResultWaiter<std::optional<TCGroupOomTrend>>& result) + : Subsystem(subsystem) + , Window(window) + , Result(result) + { + } + + void Bootstrap() { + Become(&TThis::StateWork); + Subsystem.ReadTrend(SelfId(), Window, RequestCookie); + } + + STRICT_STFUNC(StateWork, + hFunc(TEvCGroupOomTrend, Handle) + ) + + private: + void Handle(TEvCGroupOomTrend::TPtr& ev) { + Y_ABORT_UNLESS(ev->Cookie == RequestCookie); + Y_ABORT_UNLESS( + ev->Get()->State == + (ev->Get()->Trend + ? ECGroupOomTrendState::Active + : ECGroupOomTrendState::Stopped)); + Y_ABORT_UNLESS(!ev->Get()->TimeToOomThreshold); + Result.Set(std::move(ev->Get()->Trend)); + PassAway(); + } + + private: + const TCGroupOomSubSystem& Subsystem; + const ECGroupOomTrendWindow Window; + TResultWaiter<std::optional<TCGroupOomTrend>>& Result; + }; + + void RequestOomTrend( + TActorSystem& actorSystem, + const TCGroupOomSubSystem& subsystem, + ECGroupOomTrendWindow window, + TResultWaiter<std::optional<TCGroupOomTrend>>& result) { + actorSystem.Register( + new TTrendRequestActor(subsystem, window, result), + TMailboxType::Simple, + SystemPoolId); + } + + class TOomAlertSubscriberActor : public TActorBootstrapped<TOomAlertSubscriberActor> { + public: + explicit TOomAlertSubscriberActor(TResultWaiter<TCGroupOomAlert>& result) + : Result(result) + { + } + + void Bootstrap() { + Become(&TThis::StateWork); + } + + STRICT_STFUNC(StateWork, + hFunc(TEvCGroupOomAlert, Handle) + ) + + private: + void Handle(TEvCGroupOomAlert::TPtr& ev) { + Result.Set(std::move(ev->Get()->Alert)); + PassAway(); + } + + private: + TResultWaiter<TCGroupOomAlert>& Result; + }; + + class TTrendSubscriberActor : public TActorBootstrapped<TTrendSubscriberActor> { + public: + TTrendSubscriberActor( + TResultWaiter<std::optional<TCGroupOomTrend>>& activeResult, + TResultWaiter<std::optional<TCGroupOomTrend>>& stoppedResult, + TDuration timeToOomThreshold, + std::atomic<ui32>& activeNotifications, + std::atomic<ui32>& stoppedNotifications) + : ActiveResult(activeResult) + , StoppedResult(stoppedResult) + , TimeToOomThreshold(timeToOomThreshold) + , ActiveNotifications(activeNotifications) + , StoppedNotifications(stoppedNotifications) + { + } + + void Bootstrap() { + Become(&TThis::StateWork); + } + + STRICT_STFUNC(StateWork, + hFunc(TEvCGroupOomTrend, Handle) + ) + + private: + void Handle(TEvCGroupOomTrend::TPtr& ev) { + Y_ABORT_UNLESS( + ev->Get()->TimeToOomThreshold == TimeToOomThreshold); + if (ev->Get()->State == ECGroupOomTrendState::Active) { + ActiveNotifications.fetch_add(1, std::memory_order_relaxed); + if (!HasActiveResult) { + ActiveResult.Set(std::move(ev->Get()->Trend)); + HasActiveResult = true; + } + } else { + StoppedNotifications.fetch_add(1, std::memory_order_relaxed); + StoppedResult.Set(std::move(ev->Get()->Trend)); + PassAway(); + } + } + + private: + TResultWaiter<std::optional<TCGroupOomTrend>>& ActiveResult; + TResultWaiter<std::optional<TCGroupOomTrend>>& StoppedResult; + const TDuration TimeToOomThreshold; + std::atomic<ui32>& ActiveNotifications; + std::atomic<ui32>& StoppedNotifications; + bool HasActiveResult = false; + }; + + TCGroupMemoryStatsPtr MakeMemoryStats(ui64 currentBytes, ui64 maxBytes) { + auto stats = std::make_shared<TCGroupMemoryStats>(); + stats->Version = ECGroupVersion::V2; + stats->CGroupPath = "/trend"; + stats->CurrentBytes = currentBytes; + stats->MaxBytes = TCGroupMemoryLimit{ + .Value = maxBytes, + }; + return stats; + } + } // namespace Y_UNIT_TEST_SUITE(TCGroupStatsSubSystemTest) { @@ -166,7 +327,7 @@ Y_UNIT_TEST_SUITE(TCGroupStatsSubSystemTest) { actorSystem->Start(); auto stats = RequestV2Stats( *actorSystem, - GetCGroupV2StatsSubSystem(*actorSystem)).GetValueSync(); + GetCGroupV2StatsSubSystem(*actorSystem)); UNIT_ASSERT(stats); UNIT_ASSERT_VALUES_EQUAL(stats->CGroupPath, "/test"); @@ -185,9 +346,9 @@ Y_UNIT_TEST_SUITE(TCGroupStatsSubSystemTest) { MakeV2Config(root, TDuration::Hours(1)))); actorSystem->Start(); const auto& subsystem = GetCGroupV2StatsSubSystem(*actorSystem); - auto first = RequestV2Stats(*actorSystem, subsystem).GetValueSync(); + auto first = RequestV2Stats(*actorSystem, subsystem); WriteFile(root, "sys/fs/cgroup/memory.current", "200\n"); - auto second = RequestV2Stats(*actorSystem, subsystem).GetValueSync(); + auto second = RequestV2Stats(*actorSystem, subsystem); UNIT_ASSERT(first); UNIT_ASSERT(first == second); @@ -208,7 +369,7 @@ Y_UNIT_TEST_SUITE(TCGroupStatsSubSystemTest) { actorSystem->Start(); auto stats = RequestMemoryStats( *actorSystem, - GetCGroupV1StatsSubSystem(*actorSystem)).GetValueSync(); + GetCGroupV1StatsSubSystem(*actorSystem)); UNIT_ASSERT(stats); UNIT_ASSERT(stats->Version == ECGroupVersion::V1); @@ -218,7 +379,7 @@ Y_UNIT_TEST_SUITE(TCGroupStatsSubSystemTest) { actorSystem->Stop(); } - Y_UNIT_TEST(OomControllerNotifiesCallback) { + Y_UNIT_TEST(OomControllerNotifiesActor) { TTempDir tempDir; const TFsPath root = tempDir.Path(); WriteFile(root, "proc/self/cgroup", "0::/\n"); @@ -236,21 +397,223 @@ Y_UNIT_TEST_SUITE(TCGroupStatsSubSystemTest) { MakeCGroupOomSubSystem(oomConfig)); actorSystem->Start(); - auto promise = NThreading::NewPromise<TCGroupOomAlert>(); - auto future = promise.GetFuture(); + TResultWaiter<TCGroupOomAlert> result; + const TActorId subscriber = actorSystem->Register( + new TOomAlertSubscriberActor(result), + TMailboxType::Simple, + SystemPoolId); auto& oom = GetCGroupOomSubSystem(*actorSystem); - const auto subscriptionId = oom.Subscribe( - [promise](const TCGroupOomAlert& alert) mutable { - promise.TrySetValue(alert); - }); + oom.Subscribe(subscriber); WriteFile(root, "sys/fs/cgroup/memory.current", "950\n"); - UNIT_ASSERT(future.Wait(TDuration::Seconds(5))); - const auto& alert = future.GetValue(); + UNIT_ASSERT(result.Wait(TDuration::Seconds(5))); + const auto& alert = result.GetValue(); UNIT_ASSERT(alert.HasReason(ECGroupOomReason::MemoryUsageThreshold)); UNIT_ASSERT_DOUBLES_EQUAL(*alert.MemoryUsage, 0.95, 1e-12); - oom.Unsubscribe(subscriptionId); + oom.Unsubscribe(subscriber); + actorSystem->Stop(); + } + + Y_UNIT_TEST(CalculatesOomTrendForMultipleWindows) { + constexpr ui64 MaxBytes = 10'000; + constexpr ui64 InitialBytes = 1'000; + constexpr ui64 GrowthBytesPerSecond = 2; + + NActors::NDetail::TCGroupOomTrendCalculator calculator({ + .ShortWindow = TCGroupOomTrendWindowConfig{ + .Duration = TDuration::Minutes(1), + .RecalculationPeriod = TDuration::Seconds(1), + .MinimumRSquared = 0.99, + }, + .LongWindow = TCGroupOomTrendWindowConfig{ + .Duration = TDuration::Hours(1), + .RecalculationPeriod = TDuration::Minutes(1), + .MinimumRSquared = 0.99, + }, + }); + + for (ui64 seconds = 0; seconds <= 3'600; seconds += 30) { + const auto recalculated = calculator.AddSample( + TMonotonic::Seconds(seconds), + MakeMemoryStats( + InitialBytes + GrowthBytesPerSecond * seconds, + MaxBytes)); + UNIT_ASSERT( + recalculated.Contains(ECGroupOomTrendWindow::ShortWindow)); + UNIT_ASSERT_VALUES_EQUAL( + recalculated.Contains(ECGroupOomTrendWindow::LongWindow), + seconds % 60 == 0); + + if (seconds == 60) { + const auto* shortTrend = + calculator.FindTrend(ECGroupOomTrendWindow::ShortWindow); + UNIT_ASSERT(shortTrend); + UNIT_ASSERT_DOUBLES_EQUAL( + shortTrend->GrowthBytesPerSecond, + GrowthBytesPerSecond, + 1e-12); + UNIT_ASSERT_DOUBLES_EQUAL(shortTrend->RSquared, 1.0, 1e-12); + UNIT_ASSERT_VALUES_EQUAL(shortTrend->TimeToOom.Seconds(), 4'440); + UNIT_ASSERT_VALUES_EQUAL(shortTrend->SampleCount, 3); + UNIT_ASSERT( + !calculator.FindTrend(ECGroupOomTrendWindow::LongWindow)); + } + } + + const auto* shortTrend = + calculator.FindTrend(ECGroupOomTrendWindow::ShortWindow); + const auto* longTrend = + calculator.FindTrend(ECGroupOomTrendWindow::LongWindow); + UNIT_ASSERT(shortTrend); + UNIT_ASSERT(longTrend); + UNIT_ASSERT_VALUES_EQUAL(shortTrend->TimeToOom.Seconds(), 900); + UNIT_ASSERT_VALUES_EQUAL(longTrend->TimeToOom.Seconds(), 900); + UNIT_ASSERT_VALUES_EQUAL(shortTrend->SampleCount, 3); + UNIT_ASSERT_VALUES_EQUAL(longTrend->SampleCount, 121); + + auto recalculated = calculator.AddSample( + TMonotonic::Seconds(3'630), + MakeMemoryStats( + InitialBytes + GrowthBytesPerSecond * 3'630, + MaxBytes)); + UNIT_ASSERT( + recalculated.Contains(ECGroupOomTrendWindow::ShortWindow)); + UNIT_ASSERT( + !recalculated.Contains(ECGroupOomTrendWindow::LongWindow)); + shortTrend = calculator.FindTrend(ECGroupOomTrendWindow::ShortWindow); + longTrend = calculator.FindTrend(ECGroupOomTrendWindow::LongWindow); + UNIT_ASSERT_VALUES_EQUAL(shortTrend->Stats->CurrentBytes, 8'260); + UNIT_ASSERT_VALUES_EQUAL(longTrend->Stats->CurrentBytes, 8'200); + + recalculated = calculator.AddSample( + TMonotonic::Seconds(3'660), + MakeMemoryStats( + InitialBytes + GrowthBytesPerSecond * 3'660, + MaxBytes)); + UNIT_ASSERT( + recalculated.Contains(ECGroupOomTrendWindow::ShortWindow)); + UNIT_ASSERT( + recalculated.Contains(ECGroupOomTrendWindow::LongWindow)); + longTrend = calculator.FindTrend(ECGroupOomTrendWindow::LongWindow); + UNIT_ASSERT_VALUES_EQUAL(longTrend->Stats->CurrentBytes, 8'320); + UNIT_ASSERT_VALUES_EQUAL(longTrend->TimeToOom.Seconds(), 840); + } + + Y_UNIT_TEST(RejectsNonLinearOomTrend) { + NActors::NDetail::TCGroupOomTrendCalculator calculator({ + .ShortWindow = TCGroupOomTrendWindowConfig{ + .Duration = TDuration::Minutes(1), + .MinimumRSquared = 0.99, + }, + }); + + calculator.AddSample(TMonotonic::Seconds(0), MakeMemoryStats(1'000, 10'000)); + calculator.AddSample(TMonotonic::Seconds(30), MakeMemoryStats(1'600, 10'000)); + UNIT_ASSERT(!calculator.HasResult(ECGroupOomTrendWindow::ShortWindow)); + calculator.AddSample(TMonotonic::Seconds(60), MakeMemoryStats(1'100, 10'000)); + + UNIT_ASSERT(calculator.HasResult(ECGroupOomTrendWindow::ShortWindow)); + UNIT_ASSERT(!calculator.FindTrend(ECGroupOomTrendWindow::ShortWindow)); + } + + Y_UNIT_TEST(OomControllerNotifiesAndReturnsTrend) { + TTempDir tempDir; + const TFsPath root = tempDir.Path(); + WriteFile(root, "proc/self/cgroup", "0::/\n"); + WriteFile(root, "sys/fs/cgroup/memory.current", "100\n"); + WriteFile(root, "sys/fs/cgroup/memory.max", "10000\n"); + + constexpr auto trendWindow = ECGroupOomTrendWindow::ShortWindow; + TCGroupOomConfig oomConfig; + oomConfig.ExecutorPoolId = SystemPoolId; + oomConfig.PollPeriod = TDuration::MilliSeconds(10); + oomConfig.MemoryUsageThreshold.reset(); + oomConfig.TrendWindows.ShortWindow = TCGroupOomTrendWindowConfig{ + .Duration = TDuration::MilliSeconds(500), + .RecalculationPeriod = TDuration::MilliSeconds(50), + .MinimumRSquared = 0, + }; + + auto actorSystem = MakeActorSystem( + MakeCGroupV2StatsSubSystem(MakeV2Config(root)), + {}, + MakeCGroupOomSubSystem(oomConfig)); + actorSystem->Start(); + + TResultWaiter<std::optional<TCGroupOomTrend>> activeResult; + TResultWaiter<std::optional<TCGroupOomTrend>> stoppedResult; + std::atomic<ui32> activeNotifications = 0; + std::atomic<ui32> stoppedNotifications = 0; + const TDuration timeToOomThreshold = TDuration::Seconds(10); + const TActorId subscriber = actorSystem->Register( + new TTrendSubscriberActor( + activeResult, + stoppedResult, + timeToOomThreshold, + activeNotifications, + stoppedNotifications), + TMailboxType::Simple, + SystemPoolId); + + auto& oom = GetCGroupOomSubSystem(*actorSystem); + oom.SubscribeToTrend( + subscriber, + trendWindow, + timeToOomThreshold); + + TResultWaiter<std::optional<TCGroupOomTrend>> noTrendResult; + RequestOomTrend(*actorSystem, oom, trendWindow, noTrendResult); + UNIT_ASSERT(!noTrendResult.Wait(TDuration::MilliSeconds(20))); + UNIT_ASSERT(noTrendResult.Wait(TDuration::Seconds(5))); + UNIT_ASSERT(!noTrendResult.GetValue()); + + for (ui64 currentBytes = 150; + currentBytes < 10'000 && + activeNotifications.load(std::memory_order_relaxed) == 0; + currentBytes += 50) { + WriteFile( + root, + "sys/fs/cgroup/memory.current", + ToString(currentBytes) + "\n"); + Sleep(TDuration::MilliSeconds(20)); + } + + UNIT_ASSERT(activeResult.Wait(TDuration::Seconds(5))); + const auto& notification = activeResult.GetValue(); + UNIT_ASSERT(notification); + UNIT_ASSERT(notification->Window == trendWindow); + UNIT_ASSERT_VALUES_EQUAL( + notification->WindowDuration, + TDuration::MilliSeconds(500)); + UNIT_ASSERT(notification->GrowthBytesPerSecond > 0); + UNIT_ASSERT(notification->TimeToOom <= TDuration::Seconds(10)); + + for (size_t attempt = 0; + attempt < 100 && + activeNotifications.load(std::memory_order_relaxed) < 2; + ++attempt) { + Sleep(TDuration::MilliSeconds(10)); + } + UNIT_ASSERT( + activeNotifications.load(std::memory_order_relaxed) >= 2); + + TResultWaiter<std::optional<TCGroupOomTrend>> latestResult; + RequestOomTrend(*actorSystem, oom, trendWindow, latestResult); + UNIT_ASSERT(latestResult.Wait(TDuration::Seconds(5))); + const auto& latest = latestResult.GetValue(); + UNIT_ASSERT(latest); + UNIT_ASSERT(latest->Window == trendWindow); + UNIT_ASSERT(latest->TimeToOom <= TDuration::Seconds(10)); + + WriteFile(root, "sys/fs/cgroup/memory.current", "100\n"); + UNIT_ASSERT(stoppedResult.Wait(TDuration::Seconds(5))); + Sleep(TDuration::MilliSeconds(50)); + UNIT_ASSERT_VALUES_EQUAL( + stoppedNotifications.load(std::memory_order_relaxed), + 1); + + oom.UnsubscribeFromTrend(subscriber); actorSystem->Stop(); } diff --git a/ydb/library/actors/helpers/actor_liveness_checker.cpp b/ydb/library/actors/helpers/actor_liveness_checker.cpp new file mode 100644 index 00000000000..45bae9cd4be --- /dev/null +++ b/ydb/library/actors/helpers/actor_liveness_checker.cpp @@ -0,0 +1,96 @@ +#include "actor_liveness_checker.h" + +#include <ydb/library/actors/core/hfunc.h> + +namespace NActors { + TActorLivenessChecker::TActorLivenessChecker(TVector<TActorLivenessCheckTarget> targets) { + for (const auto& target : targets) { + PendingTargets.emplace(target.ActorId, target.Cookie); + } + } + + void TActorLivenessChecker::Bootstrap() { + if (PendingTargets.empty()) { + return Complete(); + } + + Become(&TThis::StateFunc); + // Every local liveness probe eventually produces ActorAlive or ActorDead. + // Remote probes currently produce ActorLivenessUnsure. + for (const auto& [actorId, cookie] : PendingTargets) { + SendActorLivenessCheck(actorId, cookie); + } + } + + STFUNC(TActorLivenessChecker::StateFunc) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvents::TEvActorAlive, Handle) + hFunc(TEvents::TEvActorDead, Handle) + hFunc(TEvents::TEvActorLivenessUnsure, Handle) + } + } + + void TActorLivenessChecker::OnAlive(const TActorLivenessCheckTarget&) { + } + + void TActorLivenessChecker::OnDead(const TActorLivenessCheckTarget&) { + } + + void TActorLivenessChecker::OnUnsure(const TActorLivenessCheckTarget&) { + } + + void TActorLivenessChecker::OnFinish() { + } + + void TActorLivenessChecker::Handle(TEvents::TEvActorAlive::TPtr& ev) { + TActorLivenessCheckTarget target; + if (ExtractTarget(ev->Sender, target)) { + OnAlive(target); + CompleteIfDone(); + } + } + + void TActorLivenessChecker::Handle(TEvents::TEvActorDead::TPtr& ev) { + TActorLivenessCheckTarget target; + if (ExtractTarget(ev->Sender, target)) { + OnDead(target); + CompleteIfDone(); + } + } + + void TActorLivenessChecker::Handle(TEvents::TEvActorLivenessUnsure::TPtr& ev) { + TActorLivenessCheckTarget target; + if (ExtractTarget(ev->Sender, target)) { + OnUnsure(target); + CompleteIfDone(); + } + } + + bool TActorLivenessChecker::ExtractTarget( + const TActorId& actorId, + TActorLivenessCheckTarget& target) { + const auto it = PendingTargets.find(actorId); + if (it == PendingTargets.end()) { + return false; + } + + target = { + .ActorId = it->first, + .Cookie = it->second, + }; + PendingTargets.erase(it); + return true; + } + + void TActorLivenessChecker::CompleteIfDone() { + if (PendingTargets.empty()) { + Complete(); + } + } + + void TActorLivenessChecker::Complete() { + OnFinish(); + PassAway(); + } + +} diff --git a/ydb/library/actors/helpers/actor_liveness_checker.h b/ydb/library/actors/helpers/actor_liveness_checker.h new file mode 100644 index 00000000000..02465256cfb --- /dev/null +++ b/ydb/library/actors/helpers/actor_liveness_checker.h @@ -0,0 +1,47 @@ +#pragma once + +#include <ydb/library/actors/core/actor_bootstrapped.h> +#include <ydb/library/actors/core/events.h> + +#include <util/generic/hash.h> +#include <util/generic/vector.h> + +namespace NActors { + + struct TActorLivenessCheckTarget { + TActorId ActorId; + ui64 Cookie = 0; + }; + + class TActorLivenessChecker + : public TActorBootstrapped<TActorLivenessChecker> + { + public: + explicit TActorLivenessChecker(TVector<TActorLivenessCheckTarget> targets); + + void Bootstrap(); + + STFUNC(StateFunc); + + protected: + // Hooks run synchronously in the checker actor context. OnFinish is + // called exactly once after every target has produced a response. + virtual void OnAlive(const TActorLivenessCheckTarget& target); + virtual void OnDead(const TActorLivenessCheckTarget& target); + virtual void OnUnsure(const TActorLivenessCheckTarget& target); + virtual void OnFinish(); + + private: + void Handle(TEvents::TEvActorAlive::TPtr& ev); + void Handle(TEvents::TEvActorDead::TPtr& ev); + void Handle(TEvents::TEvActorLivenessUnsure::TPtr& ev); + + bool ExtractTarget(const TActorId& actorId, TActorLivenessCheckTarget& target); + void CompleteIfDone(); + void Complete(); + + private: + THashMap<TActorId, ui64> PendingTargets; + }; + +} diff --git a/ydb/library/actors/helpers/actor_liveness_checker_ut.cpp b/ydb/library/actors/helpers/actor_liveness_checker_ut.cpp new file mode 100644 index 00000000000..60a0795c19d --- /dev/null +++ b/ydb/library/actors/helpers/actor_liveness_checker_ut.cpp @@ -0,0 +1,154 @@ +#include "actor_liveness_checker.h" + +#include <ydb/library/actors/core/actor_bootstrapped.h> +#include <ydb/library/actors/testlib/test_runtime.h> + +#include <library/cpp/testing/unittest/registar.h> + +#include <utility> + +namespace NActors { +namespace { + + class TTargetActor + : public TActorBootstrapped<TTargetActor> + { + public: + void Bootstrap() { + Become(&TThis::StateFunc); + } + + private: + STRICT_STFUNC(StateFunc, + cFunc(TEvents::TSystem::Poison, PassAway) + ) + }; + + struct TCheckResult { + TVector<TActorLivenessCheckTarget> AliveTargets; + TVector<TActorLivenessCheckTarget> DeadTargets; + TVector<TActorLivenessCheckTarget> UnsureTargets; + size_t FinishCount = 0; + }; + + class TTestActorLivenessChecker + : public TActorLivenessChecker + { + public: + TTestActorLivenessChecker( + TVector<TActorLivenessCheckTarget> targets, + TCheckResult& result) + : TActorLivenessChecker(std::move(targets)) + , Result(result) + { + } + + private: + void OnAlive(const TActorLivenessCheckTarget& target) override { + Result.AliveTargets.push_back(target); + } + + void OnDead(const TActorLivenessCheckTarget& target) override { + Result.DeadTargets.push_back(target); + } + + void OnUnsure(const TActorLivenessCheckTarget& target) override { + Result.UnsureTargets.push_back(target); + } + + void OnFinish() override { + ++Result.FinishCount; + } + + private: + TCheckResult& Result; + }; + +} + +Y_UNIT_TEST_SUITE(TActorLivenessCheckerTest) { + Y_UNIT_TEST(KeepsLiveActor) { + TTestActorRuntimeBase runtime; + runtime.Initialize(); + + const TActorId target = runtime.Register(new TTargetActor()); + TCheckResult result; + runtime.Register(new TTestActorLivenessChecker( + {{ + .ActorId = target, + .Cookie = 10, + }}, + result)); + + TDispatchOptions options; + options.CustomFinalCondition = [&result] { + return result.FinishCount; + }; + runtime.DispatchEvents(options); + + UNIT_ASSERT_VALUES_EQUAL(result.FinishCount, 1); + UNIT_ASSERT_VALUES_EQUAL(result.AliveTargets.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(result.AliveTargets.front().ActorId, target); + UNIT_ASSERT_VALUES_EQUAL(result.AliveTargets.front().Cookie, 10); + UNIT_ASSERT(result.DeadTargets.empty()); + UNIT_ASSERT(result.UnsureTargets.empty()); + } + + Y_UNIT_TEST(ReportsDeadActorWithCookie) { + TTestActorRuntimeBase runtime; + runtime.Initialize(); + + const TActorId target(runtime.GetNodeId(), 0, Max<ui64>(), 0); + constexpr ui64 Cookie = 42; + TCheckResult result; + runtime.Register(new TTestActorLivenessChecker( + {{ + .ActorId = target, + .Cookie = Cookie, + }}, + result)); + + TDispatchOptions options; + options.CustomFinalCondition = [&result] { + return result.FinishCount; + }; + runtime.DispatchEvents(options); + + UNIT_ASSERT_VALUES_EQUAL(result.FinishCount, 1); + UNIT_ASSERT(result.AliveTargets.empty()); + UNIT_ASSERT_VALUES_EQUAL(result.DeadTargets.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(result.DeadTargets.front().ActorId, target); + UNIT_ASSERT_VALUES_EQUAL(result.DeadTargets.front().Cookie, Cookie); + UNIT_ASSERT(result.UnsureTargets.empty()); + } + + Y_UNIT_TEST(KeepsRemoteActorWhenLivenessIsUnsure) { + TTestActorRuntimeBase runtime; + runtime.Initialize(); + + const TActorId target(runtime.GetNodeId() + 1, 0, 1, 0); + constexpr ui64 Cookie = 10; + TCheckResult result; + runtime.Register(new TTestActorLivenessChecker( + {{ + .ActorId = target, + .Cookie = Cookie, + }}, + result)); + + TDispatchOptions options; + options.CustomFinalCondition = [&result] { + return result.FinishCount; + }; + runtime.DispatchEvents(options); + + UNIT_ASSERT_VALUES_EQUAL(result.FinishCount, 1); + UNIT_ASSERT(result.AliveTargets.empty()); + UNIT_ASSERT(result.DeadTargets.empty()); + UNIT_ASSERT_VALUES_EQUAL(result.UnsureTargets.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(result.UnsureTargets.front().ActorId, target); + UNIT_ASSERT_VALUES_EQUAL(result.UnsureTargets.front().Cookie, Cookie); + } +} + +} diff --git a/ydb/library/actors/helpers/ut/ya.make b/ydb/library/actors/helpers/ut/ya.make index 4ff76005089..76509caf89c 100644 --- a/ydb/library/actors/helpers/ut/ya.make +++ b/ydb/library/actors/helpers/ut/ya.make @@ -20,6 +20,7 @@ PEERDIR( ) SRCS( + actor_liveness_checker_ut.cpp selfping_actor_ut.cpp ) diff --git a/ydb/library/actors/helpers/ya.make b/ydb/library/actors/helpers/ya.make index 41dbece82c1..2b06263750c 100644 --- a/ydb/library/actors/helpers/ya.make +++ b/ydb/library/actors/helpers/ya.make @@ -3,6 +3,8 @@ LIBRARY() SRCS( activeactors.cpp activeactors.h + actor_liveness_checker.cpp + actor_liveness_checker.h collector_counters.cpp future_callback.h mon_histogram_helper.h @@ -20,4 +22,3 @@ END() RECURSE_FOR_TESTS( ut ) - diff --git a/ydb/library/actors/interconnect/events/events.h b/ydb/library/actors/interconnect/events/events.h index 540a5744447..de77e880573 100644 --- a/ydb/library/actors/interconnect/events/events.h +++ b/ydb/library/actors/interconnect/events/events.h @@ -75,6 +75,7 @@ namespace NActors { EvRegisterSession, EvUnregisterSession, EvStop, + EvMigrateDone, //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // nonlocal messages; their indices must be preserved in order to work properly while doing rolling update diff --git a/ydb/library/actors/interconnect/interconnect_common.h b/ydb/library/actors/interconnect/interconnect_common.h index efea50b752b..01e5bac6d97 100644 --- a/ydb/library/actors/interconnect/interconnect_common.h +++ b/ydb/library/actors/interconnect/interconnect_common.h @@ -84,17 +84,33 @@ namespace NActors { // 5s * 2^8 = 1280s, about 21 minutes with the current RDMA retry base delay. ui32 MaxRdmaRetryBackoffLevel = 8; bool CollectSubscriptionStackTrace = false; + TDuration SubscriberLivenessCheckInterval = TDuration::Hours(1); bool UseUring = false; bool EnableUringSQPOLL = false; // only effective when UseUring is set - // Enables negotiation and usage of TInterconnectSessionTCPv2 (no session continuation, no encryption). - // v2 is used only when both peers have this enabled and encryption is not in effect. - bool EnableInterconnectSessionV2 = false; - bool ChecksumInterconnectSessionV2 = false; - // Use io_uring SQPOLL mode for the v2 data-plane rings (kernel-side submission polling). - bool EnableSQPOLLv2 = true; - // Preserialize outgoing events on the session mailbox before handing them to the v2 engine (moves - // serialization cost off the engine's shard worker thread). - bool EnablePreserializeInV2 = false; + + struct { + // Enables negotiation and usage of TInterconnectSessionTCPv2 (no session continuation, no encryption). + // v2 is used only when both peers have this enabled and encryption is not in effect. + bool Enable = false; + bool ChecksumEvents = false; + // Use io_uring SQPOLL mode for the v2 data-plane rings (kernel-side submission polling). + // When the kernel poller is pegged (~100% CPU) while shard workers still have headroom, disable this + // so io_uring_submit/enter runs on the worker thread instead. + bool EnableSQPOLL = true; + // Preserialize outgoing events on the session mailbox before handing them to the v2 engine (moves + // serialization cost off the engine's shard worker thread). + bool EnablePreserializeEvents = false; + // Number of worker threads. + ui32 UringEngineThreads = 4; + // io_uring rings per v2 shard worker (default 1). Each ring may have its own SQPOLL thread, so this + // scales kernel submission-polling independently of the number of serialization workers. + ui32 UringEngineRingsPerShard = 1; + // SQPOLL kernel-thread idle window (ms) for v2 rings before it sleeps. Only used when EnableSQPOLLv2 + // is on. Matches TUringContext::SqThreadIdleMs by default. + ui32 UringEngineSqThreadIdleMs = 2000; + // Enable kernel threads sharing among different worker threads. + bool ShareRingsAmongThreads = false; + } V2; }; struct TWhiteboardSessionStatus { diff --git a/ydb/library/actors/interconnect/interconnect_handshake.cpp b/ydb/library/actors/interconnect/interconnect_handshake.cpp index 4d37e0c2eef..5df9b775e35 100644 --- a/ydb/library/actors/interconnect/interconnect_handshake.cpp +++ b/ydb/library/actors/interconnect/interconnect_handshake.cpp @@ -1033,7 +1033,7 @@ namespace NActors { // v2 session is incompatible with encryption and needs the io_uring data plane; only // request it when encryption is disabled locally and io_uring is available (buffer rings // are used when present, with a fallback to ordinary buffers on older kernels) - request.SetRequestSessionV2(Common->Settings.EnableInterconnectSessionV2 && + request.SetRequestSessionV2(Common->Settings.V2.Enable && Common->Settings.EncryptionMode == EEncryptionMode::DISABLED && TUringContext::IsAvailable()); request.SetHandshakeId(*HandshakeId); @@ -1420,7 +1420,7 @@ namespace NActors { // v2 session is used only when both peers enabled it, encryption is not in effect, and // this side has the io_uring data plane available Params.UseSessionV2 = request.GetRequestSessionV2() && - Common->Settings.EnableInterconnectSessionV2 && !Params.Encryption && + Common->Settings.V2.Enable && !Params.Encryption && TUringContext::IsAvailable(); if (Params.UseExternalDataChannel) { diff --git a/ydb/library/actors/interconnect/interconnect_tcp_session.cpp b/ydb/library/actors/interconnect/interconnect_tcp_session.cpp index 38d63517d8e..3cbb63c9f8b 100644 --- a/ydb/library/actors/interconnect/interconnect_tcp_session.cpp +++ b/ydb/library/actors/interconnect/interconnect_tcp_session.cpp @@ -2,6 +2,7 @@ #include "interconnect_tcp_session.h" #include "interconnect_handshake.h" #include "interconnect_zc_processor.h" +#include "subscriber_liveness_checker.h" #include <ydb/library/actors/core/probes.h> #include <ydb/library/actors/core/log.h> @@ -85,6 +86,10 @@ namespace NActors { Pool = std::make_unique<TEventHolderPool>(Proxy->Common, std::move(destroyCallback)); ChannelScheduler.ConstructInPlace(Proxy->PeerNodeId, Proxy->Common->ChannelsConfig, Proxy->Metrics, Proxy->Common->Settings.MaxSerializedEventSize, Params, Proxy->Common->RdmaMemPool); + if (const TDuration interval = Proxy->Common->Settings.SubscriberLivenessCheckInterval; + interval != TDuration::Zero()) { + Schedule(interval, new TEvCheckSubscriberLiveness); + } LOG_INFO(*TlsActivationContext, NActorsServices::INTERCONNECT_STATUS, "[%u] session created", Proxy->PeerNodeId); SetPrefix(Sprintf("Session %s [node %" PRIu32 "]", SelfId().ToString().data(), Proxy->PeerNodeId)); @@ -323,6 +328,16 @@ namespace NActors { } } + void TInterconnectSessionTCP::CheckSubscriberLiveness() { + const TDuration interval = Proxy->Common->Settings.SubscriberLivenessCheckInterval; + if (interval == TDuration::Zero()) { + return; + } + + RegisterSubscriberLivenessChecker(SelfId(), Subscribers); + Schedule(interval, new TEvCheckSubscriberLiveness); + } + void TInterconnectSessionTCP::UpdateSubscriber(const TActorId& actorId, ui64 cookie, ui32 activityIndex, TString eventTypeName, TString stackTrace) { auto updateInfo = [&] (TSubscriberInfo& info) { diff --git a/ydb/library/actors/interconnect/interconnect_tcp_session.h b/ydb/library/actors/interconnect/interconnect_tcp_session.h index 58f2bbf08aa..33c3f268d34 100644 --- a/ydb/library/actors/interconnect/interconnect_tcp_session.h +++ b/ydb/library/actors/interconnect/interconnect_tcp_session.h @@ -466,10 +466,13 @@ namespace NActors { EvRam, EvTerminate, EvFreeItems, + EvCheckSubscriberLiveness, }; struct TEvCheckCloseOnIdle : TEventLocal<TEvCheckCloseOnIdle, EvCheckCloseOnIdle> {}; struct TEvCheckLostConnection : TEventLocal<TEvCheckLostConnection, EvCheckLostConnection> {}; + struct TEvCheckSubscriberLiveness + : TEventLocal<TEvCheckSubscriberLiveness, EvCheckSubscriberLiveness> {}; struct TEvRam : TEventLocal<TEvRam, EvRam> { const bool Batching; @@ -563,6 +566,7 @@ namespace NActors { void ForwardDelayed(); void Subscribe(STATEFN_SIG); void Unsubscribe(STATEFN_SIG); + void CheckSubscriberLiveness(); void EnqueueForward(TAutoPtr<IEventHandle> ev); void UpdateSubscriber(const TActorId& actorId, ui64 cookie, ui32 activityIndex = Max<ui32>(), TString eventTypeName = {}, @@ -581,6 +585,7 @@ namespace NActors { fFunc(TEvInterconnect::TEvConnectNode::EventType, Subscribe) fFunc(TEvents::TEvSubscribe::EventType, Subscribe) fFunc(TEvents::TEvUnsubscribe::EventType, Unsubscribe) + cFunc(TEvCheckSubscriberLiveness::EventType, CheckSubscriberLiveness) cFunc(TEvFlush::EventType, HandleFlush) hFunc(TEvPollerReady, Handle) hFunc(TEvPollerRegisterResult, Handle) diff --git a/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp b/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp index b11b1a36866..5f7cf547c6d 100644 --- a/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp +++ b/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp @@ -1,5 +1,6 @@ #include "interconnect_tcp_session_v2.h" #include "interconnect_tcp_proxy.h" +#include "subscriber_liveness_checker.h" #include <util/stream/str.h> #include <util/string/cast.h> @@ -56,6 +57,10 @@ namespace NActors { Proxy->Metrics->SetPeerScopeId(Params.PeerScopeId); Proxy->Metrics->SetConnected(0); SetPrefix(Sprintf("SessionV2 %s [node %" PRIu32 "]", SelfId().ToString().data(), Proxy->PeerNodeId)); + if (const TDuration interval = Proxy->Common->Settings.SubscriberLivenessCheckInterval; + interval != TDuration::Zero()) { + Schedule(interval, new TEvPrivate::TEvCheckSubscriberLiveness); + } LOG_INFO_IC_SESSION("ICS90", "v2 session created"); } @@ -79,7 +84,7 @@ namespace NActors { }; SetNonBlock(*Socket, false); EngineHandle = Proxy->Common->UringEngineV2->Register(Socket, SelfId(), - Proxy->Common->Settings.ChecksumInterconnectSessionV2, Params.PeerScopeId, onDisconnectCallback, + Proxy->Common->Settings.V2.ChecksumEvents, Params.PeerScopeId, onDisconnectCallback, SelfId().NodeId() < Proxy->PeerNodeId, ClockSkew, PingRTT); if (!EngineHandle) { LOG_ERROR_IC_SESSION("ICS99", "v2 io_uring engine failed to register the connection"); @@ -107,8 +112,8 @@ namespace NActors { XdcSocket->Shutdown(SHUT_RDWR); } - for (const auto& [actorId, cookie] : Subscribers) { - Send(actorId, new TEvInterconnect::TEvNodeDisconnected(Proxy->PeerNodeId), 0, cookie); + for (const auto& [actorId, info] : Subscribers) { + Send(actorId, new TEvInterconnect::TEvNodeDisconnected(Proxy->PeerNodeId), 0, info.Cookie); } Subscribers.clear(); @@ -144,8 +149,11 @@ namespace NActors { Terminate(TDisconnectReason::UserRequest()); } - void TInterconnectSessionTCPv2::AddSubscriber(const TActorId& actorId, ui64 cookie) { - Subscribers[actorId] = cookie; + void TInterconnectSessionTCPv2::AddSubscriber(const TActorId& actorId, ui64 cookie, ui32 activityIndex) { + Subscribers[actorId] = { + .Cookie = cookie, + .ActivityIndex = activityIndex, + }; } IEventBase* TInterconnectSessionTCPv2::MakeNodeConnectedEvent() const { @@ -153,7 +161,7 @@ namespace NActors { } void TInterconnectSessionTCPv2::EnqueueOutgoing(TAutoPtr<IEventHandle> ev) { - if (Proxy->Common->Settings.EnablePreserializeInV2) { + if (Proxy->Common->Settings.V2.EnablePreserializeEvents) { ev->Preserialize(); } Proxy->Common->UringEngineV2->Send(EngineHandle, std::unique_ptr<IEventHandle>(ev.Release())); @@ -172,7 +180,7 @@ namespace NActors { Proxy->ValidateEvent(ev, "ForwardWithSubscribe"); auto msg = ev->Release<TEvForwardSubscribeSession>(); Y_ABORT_UNLESS(msg->Event); - AddSubscriber(msg->Event->Sender, msg->Event->Cookie); + AddSubscriber(msg->Event->Sender, msg->Event->Cookie, msg->ActivityIndex); Send(msg->Event->Sender, MakeNodeConnectedEvent(), 0, msg->Event->Cookie); EnqueueOutgoing(TAutoPtr<IEventHandle>(msg->Event.Release())); } @@ -188,6 +196,16 @@ namespace NActors { Subscribers.erase(ev->Sender); } + void TInterconnectSessionTCPv2::CheckSubscriberLiveness() { + const TDuration interval = Proxy->Common->Settings.SubscriberLivenessCheckInterval; + if (interval == TDuration::Zero()) { + return; + } + + RegisterSubscriberLivenessChecker(SelfId(), Subscribers); + Schedule(interval, new TEvPrivate::TEvCheckSubscriberLiveness); + } + void TInterconnectSessionTCPv2::HandlePoison() { Terminate(TDisconnectReason::UserRequest()); } @@ -204,6 +222,7 @@ namespace NActors { // str << "<tr><td>OutstandingWrites</td><td>" << PendingBatches.size() << "</td></tr>"; str << "<tr><td>BytesSent</td><td>" << BytesSent << "</td></tr>"; str << "<tr><td>BytesReceived</td><td>" << BytesReceived << "</td></tr>"; + str << "<tr><td>Subscribers.size()</td><td>" << Subscribers.size() << "</td></tr>"; str << "</table>"; str << "</div></div>"; TActivationContext::Send(new IEventHandle(ev->Recipient, ev->Sender, new NMon::TEvHttpInfoRes(str.Str()))); diff --git a/ydb/library/actors/interconnect/interconnect_tcp_session_v2.h b/ydb/library/actors/interconnect/interconnect_tcp_session_v2.h index 59a27aa7fee..535f0249902 100644 --- a/ydb/library/actors/interconnect/interconnect_tcp_session_v2.h +++ b/ydb/library/actors/interconnect/interconnect_tcp_session_v2.h @@ -72,6 +72,7 @@ namespace NActors { struct TEvPrivate { enum { EvTerminate = EventSpaceBegin(TEvents::ES_PRIVATE), + EvCheckSubscriberLiveness, }; struct TEvTerminate : TEventLocal<TEvTerminate, EvTerminate> { @@ -79,6 +80,9 @@ namespace NActors { TEvTerminate(TDisconnectReason reason) : Reason(reason) {} }; + + struct TEvCheckSubscriberLiveness + : TEventLocal<TEvCheckSubscriberLiveness, EvCheckSubscriberLiveness> {}; }; STATEFN(StateFunc) { @@ -88,6 +92,7 @@ namespace NActors { fFunc(TEvInterconnect::TEvConnectNode::EventType, HandleSubscribe) fFunc(TEvents::TEvSubscribe::EventType, HandleSubscribe) fFunc(TEvents::TEvUnsubscribe::EventType, HandleUnsubscribe) + cFunc(TEvPrivate::TEvCheckSubscriberLiveness::EventType, CheckSubscriberLiveness) cFunc(TEvents::TEvPoisonPill::EventType, HandlePoison) cFunc(TEvInterconnect::EvForwardDelayed, IgnoreForwardDelayed) hFunc(TEvPrivate::TEvTerminate, [&](auto& ev) { Terminate(ev->Get()->Reason); }); @@ -98,12 +103,13 @@ namespace NActors { void ForwardWithSubscribe(STATEFN_SIG); void HandleSubscribe(STATEFN_SIG); void HandleUnsubscribe(STATEFN_SIG); + void CheckSubscriberLiveness(); void HandlePoison(); void IgnoreForwardDelayed() {} void EnqueueOutgoing(TAutoPtr<IEventHandle> ev); - void AddSubscriber(const TActorId& actorId, ui64 cookie); + void AddSubscriber(const TActorId& actorId, ui64 cookie, ui32 activityIndex = Max<ui32>()); IEventBase* MakeNodeConnectedEvent() const; private: @@ -123,8 +129,13 @@ namespace NActors { ui64 BytesSent = 0; ui64 BytesReceived = 0; - // subscribers awaiting connection state notifications (actor id -> cookie) - THashMap<TActorId, ui64> Subscribers; + struct TSubscriberInfo { + ui64 Cookie = 0; + ui32 ActivityIndex = Max<ui32>(); + }; + + // subscribers awaiting connection state notifications + THashMap<TActorId, TSubscriberInfo> Subscribers; std::shared_ptr<std::atomic<int64_t>> ClockSkew = std::make_shared<std::atomic<int64_t>>(); std::shared_ptr<std::atomic<uint64_t>> PingRTT = std::make_shared<std::atomic<uint64_t>>(); diff --git a/ydb/library/actors/interconnect/interconnect_uring_engine.cpp b/ydb/library/actors/interconnect/interconnect_uring_engine.cpp index ed9628cede8..f84153a4002 100644 --- a/ydb/library/actors/interconnect/interconnect_uring_engine.cpp +++ b/ydb/library/actors/interconnect/interconnect_uring_engine.cpp @@ -1,10 +1,11 @@ #include "interconnect_uring_engine.h" #include "uring_recv_buffer_pool.h" -#include "uring_context.h" // for TUringContext::IsSupported() +#include "uring_context.h" // for TUringContext::IsSupported() / SqThreadIdleMs #include "v2_event_serializer.h" #include "interconnect_direct_session.h" +#include "interconnect_uring_event_queue.h" #include <ydb/library/actors/core/actorsystem.h> #include <ydb/library/actors/core/actor.h> @@ -21,6 +22,7 @@ #include <sys/socket.h> #include <sys/uio.h> #include <sys/timerfd.h> +#include <sys/eventfd.h> #include <cerrno> #include <deque> @@ -42,88 +44,26 @@ namespace NActors { constexpr size_t MaxSpansPerWrite = 64; constexpr size_t SerializeWindowSize = 65536; constexpr size_t MinSerializeWindowSize = 4096; + constexpr ui32 RebalanceTimerMs = 500; // drives MaybeOffload and also issues ping packet + constexpr ui32 OffloadBusyThreshold = 700000; // ppm + constexpr ui32 StealBusyThreshold = 300000; // ppm } - struct TEventPayload { - ui64 Conn; - TIntrusivePtr<IReceiveCallback> Callback; - }; - static_assert(sizeof(TEventPayload) <= sizeof(TActorId)); - - class TIncomingEventQueue { - IEventHandle Stub{0, 0, {}, {}, nullptr, 0}; - std::atomic<IEventHandle*> Head{&Stub}; - std::atomic<IEventHandle*> Tail{&Stub}; - - public: - TIncomingEventQueue() { - Stub.NextLinkPtr.store(0, std::memory_order_relaxed); - } - - bool Push(std::unique_ptr<IEventHandle> ev) { - IEventHandle *last = ev.release(); - last->NextLinkPtr.store(0, std::memory_order_relaxed); - IEventHandle *prev = Head.exchange(last, std::memory_order_acq_rel); - prev->NextLinkPtr.store(reinterpret_cast<uintptr_t>(last), std::memory_order_release); - return prev == &Stub; // if it was the first event - } - - bool IsEmpty() const { - IEventHandle *tail = Tail.load(std::memory_order_relaxed); - IEventHandle *next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); - IEventHandle *head = Head.load(std::memory_order_acquire); - return tail == &Stub && !next && tail == head; - } - - std::unique_ptr<IEventHandle> Pop() { - for (;;) { - IEventHandle *tail = Tail.load(std::memory_order_relaxed); - IEventHandle *next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); - IEventHandle *head; - - if (tail == &Stub) { - if (!next) { - if (head = Head.load(std::memory_order_acquire); tail != head) { - continue; - } else { - return nullptr; - } - } - Tail.store(next, std::memory_order_relaxed); - tail = next; - next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); - } - - if (next) { - Tail.store(next, std::memory_order_relaxed); - return std::unique_ptr<IEventHandle>(tail); - } - - head = Head.load(std::memory_order_acquire); - if (tail != head) { - continue; - } - - Stub.NextLinkPtr.store(0, std::memory_order_relaxed); - IEventHandle *prev = Head.exchange(&Stub, std::memory_order_acq_rel); - prev->NextLinkPtr.store(reinterpret_cast<uintptr_t>(&Stub), std::memory_order_release); - - next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); - if (next) { - Tail.store(next, std::memory_order_relaxed); - return std::unique_ptr<IEventHandle>(tail); - } - } - } - }; - class TUringEngine final : public IUringEngine { TActorSystem *ActorSystem = nullptr; // bound after construction via SetActorSystem() std::once_flag ActorSystemInitFlag; std::atomic_bool Stopping{false}; + enum class EMigrateState : ui8 { + None = 0, + Draining, + HandedOff, + }; + + // Low 3 bits of the session pointer are used as an io_uring user_data op tag; heap allocation + // alignment of this type is already >= 8 (actually 64 via base/members). struct TRegisteredSession : TEventDeserializer::IEventProcessor { - const ui32 ShardIdx; + std::atomic_uint32_t OwnerShard; const TIntrusivePtr<NInterconnect::TStreamSocket> Socket; const TActorId SessionId; const std::function<void(TDisconnectReason)> OnDisconnectCallback; @@ -141,6 +81,14 @@ namespace NActors { iovec Iov[MaxSpansPerWrite]; size_t IovLen = 0; size_t UnsentBytes = 0; + int ReadPendingRingIdx = -1; + ui32 PreferredRingIdx = 0; + std::atomic_uint64_t IncomingSeqNo{1}; + ui64 ExpectedSeqNo = 1; + + EMigrateState MigrateState = EMigrateState::None; + ui32 MigrateTargetShard = 0; + ui32 MigrateSourceShard = 0; const std::shared_ptr<std::atomic<int64_t>> ClockSkew; const std::shared_ptr<std::atomic<uint64_t>> PingRTT; @@ -148,11 +96,14 @@ namespace NActors { THashMap<TActorId, TIntrusivePtr<IReceiveCallback>> ReceiveCallbacks; NMonitoring::TDynamicCounters::TCounterPtr EventsReceived; - TRegisteredSession(ui32 shardIdx, TIntrusivePtr<NInterconnect::TStreamSocket> socket, TActorId sessionId, - bool checksumming, TScopeId peerScopeId, std::function<void(TDisconnectReason)> onDisconnectCallback, - TActorSystem *actorSystem, bool sendPings, std::shared_ptr<std::atomic<int64_t>> clockSkew, + std::vector<TIncomingEventQueue::TRecord> PendingRecordsHeap; + + TRegisteredSession(ui32 shardIdx, TIntrusivePtr<NInterconnect::TStreamSocket> socket, + TActorId sessionId, bool checksumming, TScopeId peerScopeId, + std::function<void(TDisconnectReason)> onDisconnectCallback, TActorSystem *actorSystem, + bool sendPings, std::shared_ptr<std::atomic<int64_t>> clockSkew, std::shared_ptr<std::atomic<uint64_t>> pingRTT) - : ShardIdx(shardIdx) + : OwnerShard(shardIdx) , Socket(std::move(socket)) , SessionId(sessionId) , OnDisconnectCallback(std::move(onDisconnectCallback)) @@ -244,7 +195,11 @@ namespace NActors { // Advance past exactly the bytes the kernel accepted. A writev can be short (e.g. under // backpressure or on a real network), so drop only fully-sent spans and trim the span that // straddles the boundary; the rest stay queued and are retried by the next writev. - for (size_t remaining = num; remaining && !OutgoingSpans.empty(); OutgoingSpans.pop_front()) { + for (auto& span : OutgoingSpans) { + NSan::CheckMemIsInitialized(span.data(), span.size()); + } + for (size_t remaining = num; remaining; OutgoingSpans.pop_front()) { + Y_DEBUG_ABORT_UNLESS(!OutgoingSpans.empty()); if (TContiguousSpan& front = OutgoingSpans.front(); front.size() <= remaining) { remaining -= front.size(); } else { @@ -338,34 +293,61 @@ namespace NActors { PingValues[2] = pingUs; PingRTT->store(Max(PingValues[0], PingValues[1], PingValues[2])); } + + bool IsMigratable() const { + return MigrateState == EMigrateState::None + && !UnregisterRequested + && !Terminated; + } + }; + + // In-process load signal consumed by rebalancing (not the 15s monitoring scrape path). + struct TShardLoad { + std::atomic_uint64_t Busy{0}; + std::atomic_uint64_t Total{0}; + + ui32 BusyFraction() const { + const ui64 total = Total.load(std::memory_order_relaxed); + if (!total) { + return 0; + } + return Busy.load(std::memory_order_relaxed) * 1'000'000 / total; + } }; class TShard { enum EOperationType { - kOpPipe = 1, + kOpEvent = 1, kOpRead, kOpWrite, kOpTimer, + kOpCancel, }; static const ui64 kOpMask = (1 << 3) - 1; + struct TRingSlot { + io_uring Ring{}; + i64 ItemsToSubmit = 0; + }; + + TUringEngine& Engine; + const ui32 ShardIdx; TIncomingEventQueue IncomingEventQueue; std::thread Worker; - io_uring Ring; - i64 ItemsToSubmit = 0; + std::vector<TRingSlot> Rings; + int EventFd = -1; + ui64 EventFdReadBuffer = 0; + int TimerFd = -1; + ui64 ReadTimerBuffer; std::atomic_bool WaitingForCQ{false}; - int ReadPipe; - int WritePipe; - char ReadPipeBuffer[256]; - - int TimerFd; - char ReadTimerBuffer[256]; + size_t OpShift = 0; struct TSessionHash { size_t operator()(const std::unique_ptr<TRegisteredSession>& p) const { return THash<void*>{}(p.get()); } size_t operator()(const TRegisteredSession *p) const { return THash<void*>{}(p); } + using is_transparent = void; }; struct TSessionEqual { @@ -373,9 +355,12 @@ namespace NActors { bool operator()(const T& x, const T& y) const { return x == y; } bool operator()(const TRegisteredSession *x, const T& y) const { return x == y.get(); } bool operator()(const T& x, const TRegisteredSession *y) const { return x.get() == y; } + using is_transparent = void; }; - THashSet<std::unique_ptr<TRegisteredSession>, TSessionHash, TSessionEqual> Sessions; + std::unordered_set<std::unique_ptr<TRegisteredSession>, TSessionHash, TSessionEqual> Sessions; + // conn -> target shard while OwnerShard still points here during handoff + THashMap<ui64, ui32> MigratingOut; NMonitoring::TDynamicCounters::TCounterPtr SessionsRegistered; NMonitoring::TDynamicCounters::TCounterPtr SessionsUnregistered; @@ -390,11 +375,15 @@ namespace NActors { NMonitoring::TDynamicCounters::TCounterPtr SQEAllocated; NMonitoring::TDynamicCounters::TCounterPtr SubmitCount; NMonitoring::TDynamicCounters::TCounterPtr CQEProcessed; - NMonitoring::TDynamicCounters::TCounterPtr PipeWakeups; + NMonitoring::TDynamicCounters::TCounterPtr EventWakeups; NMonitoring::TDynamicCounters::TCounterPtr PushedAsFirst; NMonitoring::TDynamicCounters::TCounterPtr PushedTotal; NMonitoring::TDynamicCounters::TCounterPtr ReadUnavail; NMonitoring::TDynamicCounters::TCounterPtr WriteUnavail; + NMonitoring::TDynamicCounters::TCounterPtr SessionsMigratedOut; + NMonitoring::TDynamicCounters::TCounterPtr SessionsMigratedIn; + NMonitoring::TDynamicCounters::TCounterPtr OutOfOrderCameIn; + NMonitoring::TDynamicCounters::TCounterPtr OutOfOrderProcessed; NMonitoring::TDynamicCounters::TCounterPtr OtherTotalTime; NMonitoring::TDynamicCounters::TCounterPtr CompleteWaitTotalTime; @@ -420,6 +409,8 @@ namespace NActors { std::vector<ui64> EventToWireTimeVec; + TShardLoad& Load; + private: class TActivityMeasure { TShard& Shard; @@ -449,14 +440,47 @@ namespace NActors { #define ACTIVITY(NAME) if (TActivityMeasure __measure{*this, NAME}; false); else + void InitRing(TRingSlot& slot, bool sqpoll, ui32 sqThreadIdleMs, TRingSlot *shareWith) { + auto tryIt = [&](std::optional<ui32> sqThreadIdleMs) { + io_uring_params params{}; + if (sqpoll) { + params.flags |= IORING_SETUP_SQPOLL; + if (sqThreadIdleMs) { + params.sq_thread_idle = *sqThreadIdleMs; + } + } + if (shareWith) { + params.flags |= IORING_SETUP_ATTACH_WQ; + params.wq_fd = shareWith->Ring.ring_fd; + } + return io_uring_queue_init_params(RingQueueDepth, &slot.Ring, ¶ms) == 0; + }; + if (!tryIt(sqThreadIdleMs) && !tryIt(std::nullopt)) { + Y_ABORT("failed to initialize ring"); + } + } + + void PublishLoadSample(ui64 busyDelta, ui64 totalDelta) { + // Keep a short EWMA-like window by decaying prior samples and adding the latest slice. + constexpr ui64 DecayNum = 3; + constexpr ui64 DecayDen = 4; + const ui64 prevBusy = Load.Busy.load(std::memory_order_relaxed); + const ui64 prevTotal = Load.Total.load(std::memory_order_relaxed); + Load.Busy.store(prevBusy * DecayNum / DecayDen + busyDelta, std::memory_order_relaxed); + Load.Total.store(prevTotal * DecayNum / DecayDen + totalDelta, std::memory_order_relaxed); + } + public: static NMonitoring::IHistogramCollectorPtr TimeCollector() { return NMonitoring::ExponentialHistogram(22, 2, 1000); } - TShard(const NMonitoring::TDynamicCounterPtr& shardCounters, bool sqpoll) + TShard(TUringEngine& engine, ui32 shardIdx, const NMonitoring::TDynamicCounterPtr& shardCounters, bool sqpoll, + ui32 ringsPerShard, ui32 sqThreadIdleMs, TShardLoad& load, TShard *shareRingsWith) #define COUNTER(NAME, DERIV) NAME(shardCounters->GetCounter(#NAME, DERIV)) - : COUNTER(SessionsRegistered, true) + : Engine(engine) + , ShardIdx(shardIdx) + , COUNTER(SessionsRegistered, true) , COUNTER(SessionsUnregistered, true) , COUNTER(EventsSent, true) , COUNTER(EventsReceived, true) @@ -469,11 +493,15 @@ namespace NActors { , COUNTER(SQEAllocated, true) , COUNTER(SubmitCount, true) , COUNTER(CQEProcessed, true) - , COUNTER(PipeWakeups, true) + , COUNTER(EventWakeups, true) , COUNTER(PushedAsFirst, true) , COUNTER(PushedTotal, true) , COUNTER(ReadUnavail, true) , COUNTER(WriteUnavail, true) + , COUNTER(SessionsMigratedOut, true) + , COUNTER(SessionsMigratedIn, true) + , COUNTER(OutOfOrderCameIn, true) + , COUNTER(OutOfOrderProcessed, true) #define TOTAL_TIME(NAME) NAME(shardCounters->GetCounter("TotalTime/" #NAME, true)) , TOTAL_TIME(OtherTotalTime) , TOTAL_TIME(CompleteWaitTotalTime) @@ -490,75 +518,122 @@ namespace NActors { , SerializeTime(shardCounters->GetNamedHistogram("sensor", "SerializeTime", TimeCollector())) , CompletionsProcessedAtOnce(shardCounters->GetNamedHistogram("sensor", "CompletionsProcessedAtOnce", NMonitoring::ExponentialHistogram(10, 2))) , SubmissionsProcessedAtOnce(shardCounters->GetNamedHistogram("sensor", "SubmissionsProcessedAtOnce", NMonitoring::ExponentialHistogram(12, 2))) + , Load(load) #undef TOTAL_TIME #undef COUNTER { - // initialize ring for this shard - ui32 flags = 0; - if (sqpoll) { - flags |= IORING_SETUP_SQPOLL; - } - if (io_uring_queue_init(RingQueueDepth, &Ring, flags) < 0) { - Y_ABORT("failed to initialize ring"); + EventFd = eventfd(0, 0); + if (EventFd == -1) { + Y_ABORT("eventfd() failed: %s", strerror(errno)); } - // create pipe to kick worker thread when first commands arrive - int fds[2]; - if (pipe(fds) == -1) { - Y_ABORT("pipe() failed: %s", strerror(errno)); + Rings.resize(ringsPerShard); + for (ui32 i = 0; i < Rings.size(); ++i) { + auto& slot = Rings[i]; + InitRing(slot, sqpoll, sqThreadIdleMs, shareRingsWith ? &shareRingsWith->Rings[i] : nullptr); + if (i > 0) { + if (int res = io_uring_register_eventfd(&slot.Ring, EventFd); res < 0) { + Y_ABORT("failed to register eventfd along with ring: %s", strerror(-res)); + } + } } - ReadPipe = fds[0]; - WritePipe = fds[1]; - // set timer TimerFd = timerfd_create(CLOCK_MONOTONIC, 0); Y_ABORT_UNLESS(TimerFd != -1); - // arm timer - itimerspec spec; - memset(&spec, 0, sizeof(spec)); - spec.it_interval.tv_sec = 2; // every two seconds - spec.it_value.tv_sec = 2; // initial expiration - timerfd_settime(TimerFd, 0, &spec, nullptr); - - // start worker thread - Worker = std::thread(std::bind(&TShard::WorkerThread, this)); + // Keep the timer armed for the shard lifetime. Disarming while an io_uring read is + // outstanding would leave a stuck SQE; idle CPU is controlled via sq_thread_idle instead. + itimerspec spec{}; + spec.it_interval.tv_sec = RebalanceTimerMs / 1000; + spec.it_interval.tv_nsec = 1'000'000 * (RebalanceTimerMs % 1000); + spec.it_value = spec.it_interval; + if (timerfd_settime(TimerFd, 0, &spec, nullptr) < 0) { + Y_ABORT("timerfd_settime failed: %s", strerror(errno)); + } } ~TShard() { Stop(); // joins the worker thread, so no completion will be dispatched after this point - io_uring_queue_exit(&Ring); // cancels any still-armed read/write without dispatching them + for (auto& slot : Rings) { + io_uring_queue_exit(&slot.Ring); + } DrainQueue(); // free commands that were enqueued after the worker stopped (teardown races) - close(ReadPipe); - close(WritePipe); + close(EventFd); close(TimerFd); // remaining registered sessions are freed as the Sessions container is destroyed } + void Start() { + Worker = std::thread(std::bind(&TShard::WorkerThread, this)); + } + void Register(std::unique_ptr<TRegisteredSession> session) { ++*SessionsRegistered; + + // this would be the session's first event, so its sequencing is not the problem + SendInternal(reinterpret_cast<ui64>(session.release()), static_cast<ui32>(ENetwork::EvRegisterSession), + {}, nullptr, false); + } + + void AcceptMigrated(std::unique_ptr<TRegisteredSession> session) { + ++*SessionsMigratedIn; + + // this event isn't the first one, but its sequence doesn't matter, because all further events are kept + // in order and forwarded to the new processor SendInternal(reinterpret_cast<ui64>(session.release()), static_cast<ui32>(ENetwork::EvRegisterSession), - {}, nullptr); + {}, nullptr, false); + } + + void Enqueue(TIncomingEventQueue::TRecord&& record) { + const bool first = IncomingEventQueue.Push(std::move(record)); + if (first) { + ++*PushedAsFirst; + } + ++*PushedTotal; + if (first && WaitingForCQ.load(std::memory_order_acquire)) { + // first command while waiting on CQ: kick the worker via the pipe on ring 0 + const ui64 value = 1; // this commands adds 1 to the counter stored in eventfd + ssize_t res; + while ((res = write(EventFd, &value, sizeof(value))) != sizeof(value)) { + if (res == -1 && errno == EINTR) { + continue; + } else { + Y_ABORT("write() to eventfd failed: %s", strerror(errno)); + } + } + ++*EventWakeups; + } } void Send(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback) { ++*EventsSent; - SendImpl(conn, std::move(ev), std::move(replyCallback)); + + // this event is strictly sequenced + SendImpl(conn, std::move(ev), std::move(replyCallback), true); } void Unregister(ui64 conn) { ++*SessionsUnregistered; - SendInternal(conn, static_cast<ui32>(ENetwork::EvUnregisterSession), {}, nullptr); + + // this event is sequenced too + SendInternal(conn, static_cast<ui32>(ENetwork::EvUnregisterSession), {}, nullptr, true); } void RegisterReceiveCallback(ui64 conn, TActorId localActorId, TIntrusivePtr<IReceiveCallback> callback) { ++*(callback ? DirectReceiveCallbacksRegistered : DirectReceiveCallbacksUnregistered); - SendInternal(conn, static_cast<ui32>(ENetwork::EvRegisterCallback), localActorId, std::move(callback)); + + // this event's ordering is important + SendInternal(conn, static_cast<ui32>(ENetwork::EvRegisterCallback), localActorId, std::move(callback), + true); + } + + void NotifyMigrateDone(ui64 conn) { + SendInternal(conn, static_cast<ui32>(ENetwork::EvMigrateDone), {}, nullptr, false); } void Stop() { if (Worker.joinable()) { - SendInternal(0, static_cast<ui32>(ENetwork::EvStop), {}, nullptr); + SendInternal(0, static_cast<ui32>(ENetwork::EvStop), {}, nullptr, false); Worker.join(); // The worker is stopped, so it is now safe to touch the sessions directly. Shut every // socket down so the peer observes the disconnect promptly instead of only when this shard @@ -577,79 +652,114 @@ namespace NActors { // the ownership handling of the worker loop: destroys the embedded TEventPayload and reclaims a // TRegisteredSession handed off via an unprocessed EvRegisterSession. void DrainQueue() { - while (std::unique_ptr<IEventHandle> ev{IncomingEventQueue.Pop()}) { - auto& payload = reinterpret_cast<TEventPayload&>(const_cast<TActorId&>(ev->InterconnectSession)); - if (ev->Type == static_cast<ui32>(ENetwork::EvRegisterSession)) { - std::unique_ptr<TRegisteredSession> reclaim(reinterpret_cast<TRegisteredSession*>(payload.Conn)); + while (auto record = IncomingEventQueue.Pop()) { + if (record->Ev->Type == static_cast<ui32>(ENetwork::EvRegisterSession)) { + delete reinterpret_cast<TRegisteredSession*>(record->Conn); } - payload.~TEventPayload(); } } - void SendImpl(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback) { - new(const_cast<TActorId*>(&ev->InterconnectSession)) TEventPayload{ - .Conn = conn, - .Callback = std::move(replyCallback), - }; - reinterpret_cast<ui64&>(const_cast<TScopeId&>(ev->OriginScopeId)) = GetCycleCountFast(); - const bool first = IncomingEventQueue.Push(std::move(ev)); - if (first) { - ++*PushedAsFirst; - } - ++*PushedTotal; - if (first && WaitingForCQ.load()) { - // this is the first command and we are currently waiting on CQ, so we have to push it through syscall - // writing to special fd - char temp = 0; - int res; - while ((res = write(WritePipe, &temp, 1)) != 1) { - if (res == -1) { - if (errno == EINTR) { - continue; - } else { - Y_ABORT("write() to pipe failed: %s", strerror(errno)); - } - } else { - Y_ABORT_UNLESS(res == 0); - Y_ABORT("write() to pipe failed: zero bytes written"); - } - } - ++*PipeWakeups; + void SendImpl(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback, + bool ensureSequence) { + ui64 seqNo = 0; + if (Y_LIKELY(ensureSequence)) { + Y_DEBUG_ABORT_UNLESS(conn); + auto& session = *reinterpret_cast<TRegisteredSession*>(conn); + seqNo = session.IncomingSeqNo.fetch_add(1); } + Enqueue(TIncomingEventQueue::TRecord{ + std::move(ev), + conn, + std::move(replyCallback), + GetCycleCountFast(), + seqNo, + }); } - void SendInternal(ui64 conn, ui32 type, TActorId sender, TIntrusivePtr<IReceiveCallback> callback) { - SendImpl(conn, std::make_unique<IEventHandle>(type, 0, TActorId(), sender, nullptr, 0), std::move(callback)); + void SendInternal(ui64 conn, ui32 type, TActorId sender, TIntrusivePtr<IReceiveCallback> callback, + bool ensureSequence) { + SendImpl(conn, std::make_unique<IEventHandle>(type, 0, TActorId(), sender, nullptr, 0), + std::move(callback), ensureSequence); + } + + bool ForwardIfMigratingOut(TIncomingEventQueue::TRecord *record) { + if (Y_LIKELY(MigratingOut.empty())) { + return false; + } + if (const auto it = MigratingOut.find(record->Conn); it != MigratingOut.end()) { + Engine.Shards[it->second]->Enqueue(std::move(*record)); + return true; + } + return false; } // GetSQE returns next available SQ entry, setting up ItemsToSubmit counter in order to commence submission - // on the end of the worker loop - io_uring_sqe *GetSQE(TRegisteredSession *session, EOperationType op) { - io_uring_sqe *sqe = io_uring_get_sqe(&Ring); - if (!sqe) { // submit queue is full: try to submit something to free it up - DoSubmit(); - sqe = io_uring_get_sqe(&Ring); + // on the end of the worker loop. Event/timer control ops always use ring 0. + io_uring_sqe *GetSQE(TRegisteredSession *session, EOperationType op, int ringIdx = -1) { + io_uring_sqe *sqe = nullptr; + + if (ringIdx != -1) { + TRingSlot& ring = Rings[ringIdx]; + sqe = io_uring_get_sqe(&ring.Ring); + if (!sqe) { // submit queue is full: try to submit something to free it up + DoSubmit(ring); + sqe = io_uring_get_sqe(&ring.Ring); + } + } else { + for (size_t i = 0; !sqe && i < Rings.size(); ++i) { + ringIdx = (i + OpShift) % Rings.size(); + TRingSlot& ring = Rings[ringIdx]; + sqe = io_uring_get_sqe(&ring.Ring); + } + for (size_t i = 0; !sqe && i < Rings.size(); ++i) { + ringIdx = (i + OpShift) % Rings.size(); + TRingSlot& ring = Rings[ringIdx]; + DoSubmit(ring); + sqe = io_uring_get_sqe(&ring.Ring); + } + if (sqe) { + ++OpShift; + } } + if (sqe) { - ++ItemsToSubmit; + Y_DEBUG_ABORT_UNLESS(ringIdx != -1); + if (op == kOpRead) { + Y_DEBUG_ABORT_UNLESS(session); + Y_DEBUG_ABORT_UNLESS(session->ReadPendingRingIdx == -1); + session->ReadPendingRingIdx = ringIdx; + } + ++Rings[ringIdx].ItemsToSubmit; uintptr_t sessionId = reinterpret_cast<uintptr_t>(session); Y_ABORT_UNLESS((sessionId & kOpMask) == 0); io_uring_sqe_set_data64(sqe, sessionId | op); - Y_DEBUG_ABORT_UNLESS(op == kOpPipe || op == kOpTimer ? session == nullptr : session != nullptr); + Y_DEBUG_ABORT_UNLESS(op == kOpEvent || op == kOpTimer ? session == nullptr : session != nullptr); ++*SQEAllocated; } return sqe; } + void PutEventReadRequest() { + io_uring_sqe *sqe = GetSQE(nullptr, kOpEvent, 0 /* first ring strict */); + Y_ABORT_UNLESS(sqe, "failed to obtain event SQE: SQ overflow"); + io_uring_prep_read(sqe, EventFd, &EventFdReadBuffer, sizeof(EventFdReadBuffer), -1); + } + + void PutTimer() { + io_uring_sqe *sqe = GetSQE(nullptr, kOpTimer); + Y_ABORT_UNLESS(sqe, "failed to obtain timer SQE: SQ overflow"); + io_uring_prep_read(sqe, TimerFd, &ReadTimerBuffer, sizeof(ReadTimerBuffer), -1); + } + // DoSubmit performs actual io_uring submit operation for all allocated entries during the worker loop - void DoSubmit() { + void DoSubmit(TRingSlot& slot) { ui64 enterTimestamp; ACTIVITY(&SubmitWaitTotalTime) { enterTimestamp = LastActivitySwitchTimestamp; for (;;) { - int res = io_uring_submit(&Ring); + int res = io_uring_submit(&slot.Ring); if (res == -EINTR) { continue; } @@ -662,128 +772,195 @@ namespace NActors { ++*SubmitCount; SubmitExecTime->Collect((LastActivitySwitchTimestamp - enterTimestamp) * Freq); - SubmissionsProcessedAtOnce->Collect(ItemsToSubmit, 1u); - ItemsToSubmit = 0; - } - - void PutPipeReadRequest() { - io_uring_sqe *sqe = GetSQE(nullptr, kOpPipe); - Y_ABORT_UNLESS(sqe, "failed to obtain pipe SQE: SQ overflow"); // TODO(alexvru): handle this somehow - io_uring_prep_read(sqe, ReadPipe, ReadPipeBuffer, sizeof(ReadPipeBuffer), -1); - } - - void PutTimer() { - io_uring_sqe *sqe = GetSQE(nullptr, kOpTimer); - Y_ABORT_UNLESS(sqe, "failed to obtain timer SQE: SQ overflow"); // TODO(alexvru): handle this somehow - io_uring_prep_read(sqe, TimerFd, ReadTimerBuffer, sizeof(ReadTimerBuffer), -1); + SubmissionsProcessedAtOnce->Collect(slot.ItemsToSubmit, 1u); + slot.ItemsToSubmit = 0; } void WorkerThread() { - LastActivitySwitchTimestamp = GetCycleCountFast(); - pthread_setname_np(pthread_self(), "IC_uring"); - // prepare read request in order for sender threads to wake this one up when waiting on CQ - PutPipeReadRequest(); + LastActivitySwitchTimestamp = GetCycleCountFast(); + ui64 loopStartTimestamp = LastActivitySwitchTimestamp; + + // Arm pipe + timer + PutEventReadRequest(); PutTimer(); for (;;) { // submit any pending SQ's (if we have any) - if (ItemsToSubmit) { - DoSubmit(); + for (auto& slot : Rings) { + if (slot.ItemsToSubmit) { + DoSubmit(slot); + } } - // wait for something to happen - WaitingForCQ.store(true); - if (IncomingEventQueue.IsEmpty()) { - io_uring_cqe *cqe; - ui64 enterTimestamp; - ACTIVITY(&CompleteWaitTotalTime) { - enterTimestamp = LastActivitySwitchTimestamp; - if (int res = io_uring_wait_cqe(&Ring, &cqe); res && res != -EINTR) { - Y_ABORT("io_uring_wait_cqe() failed: %s", strerror(-res)); - } - } - CompletionWaitTime->Collect((LastActivitySwitchTimestamp - enterTimestamp) * Freq); + // process pending CQ events from every ring + bool progress = ProcessCompletions(); + + // process pending events and commands + bool stopping = false; + progress |= ProcessPendingCommands(&stopping); + if (stopping) { + break; } - WaitingForCQ.store(false); - // process pending CQ events - io_uring_cqe *cqes[CqeBatchSize]; - i64 completionsProcessedAtOnce = 0; - while (const unsigned n = io_uring_peek_batch_cqe(&Ring, cqes, CqeBatchSize)) { - for (unsigned i = 0; i < n; ++i) { - DispatchCompletion(*cqes[i]); - } - io_uring_cq_advance(&Ring, n); - completionsProcessedAtOnce += n; - if (n < CqeBatchSize) { - break; + ui64 waitStartTimestamp = 0; + if (!progress) { // wait for something to happen -- no progress were made in this loop + WaitingForCQ.store(true, std::memory_order_release); + + // it is critical we first set WaitingForCQ, and then rechecking the queue + if (IncomingEventQueue.IsEmpty()) { + io_uring_cqe *cqe; + ui64 enterTimestamp; + ACTIVITY(&CompleteWaitTotalTime) { + waitStartTimestamp = enterTimestamp = LastActivitySwitchTimestamp; + // wait for the ring waiting for EventFd + if (int res = io_uring_wait_cqe(&Rings.front().Ring, &cqe); res && res != -EINTR) { + Y_ABORT("io_uring_wait_cqe() failed: %s", strerror(-res)); + } + } + CompletionWaitTime->Collect((LastActivitySwitchTimestamp - enterTimestamp) * Freq); } - } - if (completionsProcessedAtOnce) { - CompletionsProcessedAtOnce->Collect(completionsProcessedAtOnce, 1u); + + WaitingForCQ.store(false, std::memory_order_relaxed); } - // process pending events and commands - while (std::unique_ptr<IEventHandle> ev{IncomingEventQueue.Pop()}) { - auto& payload = reinterpret_cast<TEventPayload&>(const_cast<TActorId&>(ev->InterconnectSession)); - const ui64 conn = payload.Conn; - TIntrusivePtr<IReceiveCallback> callback = std::move(payload.Callback); - payload.~TEventPayload(); + const ui64 loopEndTimestamp = GetCycleCountFast(); + const ui64 total = loopEndTimestamp - loopStartTimestamp; + const ui64 wait = waitStartTimestamp ? loopEndTimestamp - waitStartTimestamp : 0; + const ui64 work = wait < total ? total - wait : 0; + PublishLoadSample(work, total); + loopStartTimestamp = loopEndTimestamp; + } + } - const ui64 cycleCountOnSend = reinterpret_cast<const ui64&>(ev->OriginScopeId); - const ui64 cycleCountOnEnter = GetCycleCountFast(); + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // command processing - switch (ev->Type) { - case static_cast<ui32>(ENetwork::EvRegisterCallback): - if (TRegisteredSession& session = GetSession(conn); callback) { - session.ReceiveCallbacks[ev->Sender] = std::move(callback); - } else { - session.ReceiveCallbacks.erase(ev->Sender); - } - break; + bool ProcessPendingCommands(bool *stopping) { + bool progress = false; + ui64 cycleCountOnEnter = 0; - case static_cast<ui32>(ENetwork::EvRegisterSession): { - std::unique_ptr<TRegisteredSession> session(reinterpret_cast<TRegisteredSession*>(conn)); - const auto [it, inserted] = Sessions.emplace(std::move(session)); - Y_ABORT_UNLESS(inserted); - (*it)->EventsReceived = EventsReceived; - IssueReadForSession(**it); - break; - } + while (auto record = IncomingEventQueue.Pop()) { + progress = true; + if (record->Ev->Type == static_cast<ui32>(ENetwork::EvStop)) { + *stopping = true; + break; + } - case static_cast<ui32>(ENetwork::EvUnregisterSession): { - TRegisteredSession& session = GetSession(conn); - // Do NOT free the session while it still has an armed recv or an in-flight - // writev: their io_uring completions carry a raw pointer to this object and - // would dereference freed memory. Mark it terminated (so no new ops are armed) - // and erase only once both are drained. The session actor has already shut the - // socket down before requesting unregistration, so the pending ops complete - // promptly (EOF/EPIPE). - session.Terminated = true; - session.UnregisterRequested = true; - MaybeEraseSession(session); - break; - } + if (!cycleCountOnEnter) { + cycleCountOnEnter = GetCycleCountFast(); + } - case static_cast<ui32>(ENetwork::EvStop): - return; + if (Y_LIKELY(record->SeqNo)) { + if (ForwardIfMigratingOut(&record.value())) { + // this record is just sent to the other thread + } else if (auto& session = GetSession(record->Conn); record->SeqNo != session.ExpectedSeqNo) { + Y_DEBUG_ABORT_UNLESS(session.ExpectedSeqNo < record->SeqNo); + session.PendingRecordsHeap.push_back(std::move(*record)); + std::ranges::push_heap(session.PendingRecordsHeap, std::greater<ui64>{}, + &TIncomingEventQueue::TRecord::SeqNo); + ++*OutOfOrderCameIn; + } else { + // process this event + const bool isUnregister = record->Ev->Type == static_cast<ui32>(ENetwork::EvUnregisterSession); + ProcessIncomingEvent(&record.value()); + if (!isUnregister) { + ++session.ExpectedSeqNo; - default: { - TRegisteredSession& session = GetSession(conn); - if (callback) { // register callback coming along with the message - session.ReceiveCallbacks[ev->Sender] = std::move(callback); + // check if there are other events in the process queue + if (auto& heap = session.PendingRecordsHeap; Y_UNLIKELY(!heap.empty())) { + while (!heap.empty() && heap.front().SeqNo == session.ExpectedSeqNo) { + std::ranges::pop_heap(heap, std::greater<ui64>{}, &TIncomingEventQueue::TRecord::SeqNo); + ProcessIncomingEvent(&heap.back()); + ++session.ExpectedSeqNo; + heap.pop_back(); + ++*OutOfOrderProcessed; + } + if (heap.empty()) { + heap.shrink_to_fit(); + } } - session.Serializer.Push(std::move(ev)); - IssueWritesForSession(session); - break; } } + } else { + // process unsequenced event + ProcessIncomingEvent(&record.value()); + } + + const ui64 cycleCountOnExit = GetCycleCountFast(); + CommandDeliveryTime->Collect((cycleCountOnEnter - record->ReceivedTimestamp) * Freq); + CommandExecTime->Collect((cycleCountOnExit - cycleCountOnEnter) * Freq); + cycleCountOnEnter = cycleCountOnExit; + } + + return progress; + } + + void ProcessIncomingEvent(TIncomingEventQueue::TRecord *record) { + switch (record->Ev->Type) { + case static_cast<ui32>(ENetwork::EvRegisterCallback): + if (TRegisteredSession& session = GetSession(record->Conn); record->Callback) { + session.ReceiveCallbacks[record->Ev->Sender] = std::move(record->Callback); + } else { + session.ReceiveCallbacks.erase(record->Ev->Sender); + } + break; + + case static_cast<ui32>(ENetwork::EvRegisterSession): { + std::unique_ptr<TRegisteredSession> session(reinterpret_cast<TRegisteredSession*>(record->Conn)); + if (session->MigrateState == EMigrateState::HandedOff) { + session->OwnerShard.store(ShardIdx, std::memory_order_release); // all new events will arrive here from now on + Engine.Shards[session->MigrateSourceShard]->NotifyMigrateDone(record->Conn); + session->MigrateState = EMigrateState::None; + } else { + Y_DEBUG_ABORT_UNLESS(session->MigrateState == EMigrateState::None); + } + session->PreferredRingIdx = OpShift++ % Rings.size(); + const auto [it, inserted] = Sessions.emplace(std::move(session)); + Y_ABORT_UNLESS(inserted); + (*it)->EventsReceived = EventsReceived; + IssueReadForSession(**it); + break; + } + + case static_cast<ui32>(ENetwork::EvUnregisterSession): { + TRegisteredSession& session = GetSession(record->Conn); + // Do NOT free the session while it still has an armed recv or an in-flight + // writev: their io_uring completions carry a raw pointer to this object and + // would dereference freed memory. Mark it terminated (so no new ops are armed) + // and erase only once both are drained. The session actor has already shut the + // socket down before requesting unregistration, so the pending ops complete + // promptly (EOF/EPIPE). + if (session.MigrateState != EMigrateState::None) { + session.MigrateState = EMigrateState::None; // unregister wins over migrate + } + session.Terminated = true; + session.UnregisterRequested = true; + if (session.ReadPending) { // cancel pending read in order to unregister the session + CancelOp(session, kOpRead, session.ReadPendingRingIdx); + } + MaybeEraseSession(session); + break; + } - const ui64 cycleCountOnExit = GetCycleCountFast(); - CommandDeliveryTime->Collect(NHPTimer::GetSeconds(cycleCountOnEnter - cycleCountOnSend) * 1e9); - CommandExecTime->Collect(NHPTimer::GetSeconds(cycleCountOnExit - cycleCountOnEnter) * 1e9); + case static_cast<ui32>(ENetwork::EvMigrateDone): { + const size_t num = MigratingOut.erase(record->Conn); + Y_DEBUG_ABORT_UNLESS(num == 1); + break; + } + + case static_cast<ui32>(ENetwork::EvStop): + Y_ABORT(); + + default: { + TRegisteredSession& session = GetSession(record->Conn); + if (record->Callback) { // register callback coming along with the message + session.ReceiveCallbacks[record->Ev->Sender] = std::move(record->Callback); + } + session.Serializer.Push(std::move(record->Ev)); + IssueWritesForSession(session); + break; } } } @@ -791,20 +968,45 @@ namespace NActors { //////////////////////////////////////////////////////////////////////////////////////////////////////////// // io_uring completion handlers - void DispatchCompletion(io_uring_cqe cqe) { + bool ProcessCompletions() { + bool progress = false; + i64 completionsProcessedAtOnce = 0; + for (auto& slot : Rings) { + io_uring_cqe *cqes[CqeBatchSize]; + while (const unsigned n = io_uring_peek_batch_cqe(&slot.Ring, cqes, CqeBatchSize)) { + progress = true; // we did something with the queues + for (unsigned i = 0; i < n; ++i) { + DispatchCompletion(*cqes[i]); + } + io_uring_cq_advance(&slot.Ring, n); + completionsProcessedAtOnce += n; + if (n < CqeBatchSize) { + break; + } + } + } + if (completionsProcessedAtOnce) { + CompletionsProcessedAtOnce->Collect(completionsProcessedAtOnce, 1u); + } + return progress; + } + + void DispatchCompletion(io_uring_cqe& cqe) { auto *session = reinterpret_cast<TRegisteredSession*>(uintptr_t(cqe.user_data) & ~uintptr_t(kOpMask)); Y_ABORT_UNLESS(!(cqe.flags & IORING_CQE_F_MORE)); // not expecting multiple completions - switch (static_cast<EOperationType>(cqe.user_data & kOpMask)) { - case kOpPipe: - // this operation is used just to break wait-CQE syscall and exit to process some commands; but - // we have to re-arm the pipe + const auto op = static_cast<EOperationType>(cqe.user_data & kOpMask); + switch (op) { + case kOpEvent: Y_DEBUG_ABORT_UNLESS(session == nullptr); - PutPipeReadRequest(); + Y_DEBUG_ABORT_UNLESS(cqe.res == sizeof(EventFdReadBuffer)); + PutEventReadRequest(); break; case kOpRead: Y_DEBUG_ABORT_UNLESS(session != nullptr); - DispatchRead(*session, cqe.res); // TODO(alexvru): maybe handle NONEMPTY (if it won't break equality) + Y_DEBUG_ABORT_UNLESS(session->ReadPendingRingIdx != -1); + session->ReadPendingRingIdx = -1; + DispatchRead(*session, cqe.res); break; case kOpWrite: @@ -817,17 +1019,98 @@ namespace NActors { DispatchTimer(); PutTimer(); break; + + case kOpCancel: + // Original op completes with -ECANCELED; the cancel SQE itself needs no action. + break; } ++*CQEProcessed; } void DispatchTimer() { for (auto& session : Sessions) { - if (!session->Terminated && session->SendPings && session->PingRequestSentTimestamp == 0) { + if (!session->Terminated && session->MigrateState == EMigrateState::None && session->SendPings && + session->PingRequestSentTimestamp == 0) { session->SendPingRequest(); IssueWritesForSession(*session); } } + + // Rebalance at most once per ping period to limit churn under short load spikes. + MaybeOffload(); + } + + void MaybeOffload() { + if (Engine.Shards.size() < 2 || Sessions.size() < 2 || Load.BusyFraction() < OffloadBusyThreshold) { + return; + } + + ui32 bestTarget = ShardIdx; + ui32 bestBusy = Max<ui32>(); + for (ui32 i = 0; i < Engine.Shards.size(); ++i) { + if (i == ShardIdx) { + continue; + } + if (const ui32 busy = Engine.ShardLoads[i].BusyFraction(); busy < bestBusy) { + bestBusy = busy; + bestTarget = i; + } + } + if (bestTarget == ShardIdx || bestBusy > StealBusyThreshold) { + return; + } + + TRegisteredSession *candidate = nullptr; + for (auto& session : Sessions) { + if (session->IsMigratable()) { + candidate = session.get(); + break; + } + } + if (!candidate) { + return; + } + + candidate->MigrateState = EMigrateState::Draining; + candidate->MigrateTargetShard = bestTarget; + candidate->MigrateSourceShard = ShardIdx; + Y_DEBUG_ABORT_UNLESS(candidate->MigrateTargetShard != candidate->MigrateSourceShard); + + if (candidate->ReadPending) { + CancelOp(*candidate, kOpRead, candidate->ReadPendingRingIdx); + } + + MaybeFinishMigrate(*candidate); + } + + void CancelOp(TRegisteredSession& session, EOperationType op, int ringIdx) { + Y_DEBUG_ABORT_UNLESS(ringIdx != -1); + io_uring_sqe *sqe = GetSQE(&session, kOpCancel, ringIdx); + Y_ABORT_UNLESS(sqe, "failed to obtain cancel SQE"); + const ui64 targetUserData = reinterpret_cast<uintptr_t>(&session) | op; + io_uring_prep_cancel64(sqe, targetUserData, 0); + } + + bool MaybeFinishMigrate(TRegisteredSession& session) { + if (session.MigrateState != EMigrateState::Draining || session.ReadPending || session.WritePending || + session.UnregisterRequested || session.Terminated) { + return false; + } + + const ui64 conn = reinterpret_cast<ui64>(&session); + const ui32 target = session.MigrateTargetShard; + session.MigrateState = EMigrateState::HandedOff; + + auto it = Sessions.find(&session); + Y_DEBUG_ABORT_UNLESS(it != Sessions.end()); + auto node = Sessions.extract(it); + Y_DEBUG_ABORT_UNLESS(node); + + MigratingOut[conn] = target; + ++*SessionsMigratedOut; + Engine.Shards[target]->AcceptMigrated(std::move(node.value())); + + return true; } void DispatchRead(TRegisteredSession& session, i32 res) { @@ -837,6 +1120,9 @@ namespace NActors { if (session.Terminated) { // teardown in progress: don't process further data or re-arm; just let the session drain // toward erasure below + } else if (res == -ECANCELED) { + // cancelled without migrate (should be rare); re-arm unless terminating + IssueReadForSession(session); } else if (res == -EAGAIN) { ++*ReadUnavail; IssueReadForSession(session); @@ -853,16 +1139,19 @@ namespace NActors { IssueWritesForSession(session); } + if (MaybeFinishMigrate(session)) { // NB: may free/move `session` + return; + } MaybeEraseSession(session); // NB: may free `session`; must be the last use } void IssueReadForSession(TRegisteredSession& session) { - if (session.Terminated) { + if (session.Terminated || session.MigrateState != EMigrateState::None) { return; } Y_DEBUG_ABORT_UNLESS(!session.ReadPending); TMutableContiguousSpan span = session.GetReadSpan(); - io_uring_sqe *sqe = GetSQE(&session, kOpRead); + io_uring_sqe *sqe = GetSQE(&session, kOpRead, session.PreferredRingIdx); Y_ABORT_UNLESS(sqe); io_uring_prep_read(sqe, *session.Socket, span.data(), span.size(), -1); session.ReadPending = true; @@ -875,6 +1164,8 @@ namespace NActors { if (session.Terminated) { // teardown in progress: don't retry the write or re-arm; just let the session drain // toward erasure below + } else if (res == -ECANCELED) { + IssueWritesForSession(session); } else if (res == -EAGAIN) { ++*WriteUnavail; SubmitIovec(session); @@ -894,11 +1185,15 @@ namespace NActors { IssueWritesForSession(session); } + if (MaybeFinishMigrate(session)) { + return; + } MaybeEraseSession(session); // NB: may free `session`; must be the last use } void IssueWritesForSession(TRegisteredSession& session) { - if (session.WritePending || session.Terminated || !session.Serializer.IsTrafficPending()) { + if (session.WritePending || session.Terminated || session.MigrateState != EMigrateState::None || + !session.Serializer.IsTrafficPending()) { return; } if (session.Serialize()) { @@ -917,7 +1212,10 @@ namespace NActors { } void SubmitIovec(TRegisteredSession& session) { - io_uring_sqe *sqe = GetSQE(&session, kOpWrite); + if (session.MigrateState != EMigrateState::None) { + return; + } + io_uring_sqe *sqe = GetSQE(&session, kOpWrite, session.PreferredRingIdx); Y_ABORT_UNLESS(sqe); io_uring_prep_writev(sqe, *session.Socket, session.Iov, session.IovLen, -1); session.WritePending = true; @@ -944,17 +1242,30 @@ namespace NActors { }; std::vector<std::unique_ptr<TShard>> Shards; + std::vector<TShardLoad> ShardLoads; std::atomic_uint64_t NextShardIdx; NMonitoring::TDynamicCounterPtr UringCounters; public: - TUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll) + TUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll, ui32 ringsPerShard, + ui32 sqThreadIdleMs, bool shareRingsAmongThreads) : UringCounters(std::move(counters)) { + ShardLoads = std::vector<TShardLoad>(numShards); Shards.reserve(numShards); for (ui32 i = 0; i < numShards; ++i) { - Shards.push_back(std::make_unique<TShard>(UringCounters->GetSubgroup("shard", "0" /*ToString(i)*/), sqpoll)); + Shards.push_back(std::make_unique<TShard>(*this, + i, // shardIdx + UringCounters->GetSubgroup("shard", "0" /*ToString(i)*/), + sqpoll, + ringsPerShard, + sqThreadIdleMs, + ShardLoads[i], + shareRingsAmongThreads && !Shards.empty() ? Shards.front().get() : nullptr)); + } + for (auto& shard : Shards) { + shard->Start(); } } @@ -978,7 +1289,17 @@ namespace NActors { return 0; // engine is shutting down; caller treats 0 as a failed registration and terminates } Y_ABORT_UNLESS(ActorSystem); - const ui32 shardIdx = NextShardIdx++ % Shards.size(); + + // Prefer the currently least-loaded shard (in-process signal); fall back to round-robin. + ui32 shardIdx = NextShardIdx++ % Shards.size(); + ui32 bestBusy = ShardLoads[shardIdx].BusyFraction(); + for (ui32 i = 0; i < ShardLoads.size(); ++i) { + if (const ui32 busy = ShardLoads[i].BusyFraction(); busy < bestBusy) { + bestBusy = busy; + shardIdx = i; + } + } + auto session = std::make_unique<TRegisteredSession>(shardIdx, std::move(socket), sessionActorId, checksumming, peerScopeId, std::move(onDisconnectCallback), ActorSystem, sendPings, std::move(clockSkew), std::move(pingRTT)); @@ -988,7 +1309,8 @@ namespace NActors { } TShard& GetShard(ui64 conn) const { - return *Shards.at(reinterpret_cast<TRegisteredSession*>(conn)->ShardIdx); + // OwnerShard is the indirection for dynamic mapping: stable conn handle, mutable owner. + return *Shards[reinterpret_cast<TRegisteredSession*>(conn)->OwnerShard.load(std::memory_order_acquire)]; } void Send(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback) override { @@ -1028,14 +1350,22 @@ namespace NActors { } }; - TUringEnginePtr CreateUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll) { + TUringEnginePtr CreateUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll, + ui32 ringsPerShard, ui32 sqThreadIdleMs, bool shareRingsAmongThreads) { if (!TUringContext::IsAvailable()) { return nullptr; } if (numShards < 1) { numShards = 1; } - return MakeIntrusive<TUringEngine>(numShards, std::move(counters), sqpoll); + if (ringsPerShard < 1) { + ringsPerShard = 1; + } + if (sqThreadIdleMs < 1) { + sqThreadIdleMs = TUringContext::SqThreadIdleMs; + } + return MakeIntrusive<TUringEngine>(numShards, std::move(counters), sqpoll, ringsPerShard, sqThreadIdleMs, + shareRingsAmongThreads); } } // namespace NActors diff --git a/ydb/library/actors/interconnect/interconnect_uring_engine.h b/ydb/library/actors/interconnect/interconnect_uring_engine.h index 72c58c4cfd9..629670ba9d7 100644 --- a/ydb/library/actors/interconnect/interconnect_uring_engine.h +++ b/ydb/library/actors/interconnect/interconnect_uring_engine.h @@ -48,10 +48,18 @@ namespace NActors { using TUringEnginePtr = TIntrusivePtr<IUringEngine>; - // Creates the engine with `numShards` rings/reaper threads (clamped to >= 1). The engine is created - // without an actor system (it does not exist yet at startup); bind it later via SetActorSystem before the - // first Register. Returns nullptr when io_uring is unavailable (non-Linux or a kernel that cannot create - // a ring). - TUringEnginePtr CreateUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll); + // Creates the engine with `numShards` worker threads (clamped to >= 1), each driving `ringsPerShard` + // io_uring rings (clamped to >= 1). With SQPOLL, each ring has its own kernel poll thread, so + // rings-per-shard scales submission-polling capacity independently of serialization workers. + // The engine is created without an actor system (it does not exist yet at startup); bind it later via + // SetActorSystem before the first Register. Returns nullptr when io_uring is unavailable (non-Linux or a + // kernel that cannot create a ring). + // + // When the kernel SQPOLL thread is saturated while shard workers still have headroom, try + // EnableSQPOLLv2=false (submit from the underutilized worker) or raise ringsPerShard. When workers are + // the bottleneck, raise numShards / lower ringsPerShard. + // sqThreadIdleMs is the SQPOLL kernel-thread idle timeout in milliseconds (ignored when sqpoll is false). + TUringEnginePtr CreateUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll, + ui32 ringsPerShard = 1, ui32 sqThreadIdleMs = 2000, bool shareRingsAmongThreads = false); } diff --git a/ydb/library/actors/interconnect/interconnect_uring_engine_stub.cpp b/ydb/library/actors/interconnect/interconnect_uring_engine_stub.cpp index 59b066426f6..e127ed13720 100644 --- a/ydb/library/actors/interconnect/interconnect_uring_engine_stub.cpp +++ b/ydb/library/actors/interconnect/interconnect_uring_engine_stub.cpp @@ -1,7 +1,7 @@ +#include "interconnect_uring_engine.h" + namespace NActors { - TUringEnginePtr CreateUringEngine(ui32, NMonitoring::TDynamicCounterPtr, bool) { + TUringEnginePtr CreateUringEngine(ui32, NMonitoring::TDynamicCounterPtr, bool, ui32, ui32) { return nullptr; } } - - diff --git a/ydb/library/actors/interconnect/interconnect_uring_event_queue.h b/ydb/library/actors/interconnect/interconnect_uring_event_queue.h new file mode 100644 index 00000000000..da1a88643f5 --- /dev/null +++ b/ydb/library/actors/interconnect/interconnect_uring_event_queue.h @@ -0,0 +1,125 @@ +#pragma once + +#include "interconnect_direct_session.h" + +namespace NActors { + + class TIncomingEventQueue { + IEventHandle Stub{0, 0, {}, {}, nullptr, 0}; + std::atomic<IEventHandle*> Head{&Stub}; + std::atomic<IEventHandle*> Tail{&Stub}; + + struct TEventPayload { + ui64 Conn; + TIntrusivePtr<IReceiveCallback> Callback; + }; + static_assert(sizeof(TEventPayload) <= sizeof(TActorId)); + + struct TEventPayload2 { + ui64 ReceivedTimestamp; // GetCycleCountFast() + ui64 SeqNo; + }; + static_assert(sizeof(TEventPayload2) <= sizeof(TScopeId)); + + public: + struct TRecord { + std::unique_ptr<IEventHandle> Ev; + ui64 Conn; + TIntrusivePtr<IReceiveCallback> Callback; + ui64 ReceivedTimestamp; + ui64 SeqNo; + }; + + public: + TIncomingEventQueue() { + Stub.NextLinkPtr.store(0, std::memory_order_relaxed); + } + + ~TIncomingEventQueue() { + Y_DEBUG_ABORT_UNLESS(IsEmpty()); // ensure this event queue has been properly drained by owner + } + + bool Push(TRecord&& record) { + IEventHandle *last = record.Ev.release(); + + // store some metadata in event's unmatching fields + new(const_cast<TActorId*>(&last->InterconnectSession)) TEventPayload{ + .Conn = record.Conn, + .Callback = std::move(record.Callback), + }; + new(const_cast<TScopeId*>(&last->OriginScopeId)) TEventPayload2{ + .ReceivedTimestamp = record.ReceivedTimestamp, + .SeqNo = record.SeqNo, + }; + + last->NextLinkPtr.store(0, std::memory_order_relaxed); + IEventHandle *prev = Head.exchange(last, std::memory_order_acq_rel); + prev->NextLinkPtr.store(reinterpret_cast<uintptr_t>(last), std::memory_order_release); + return prev == &Stub; // if it was the first event + } + + bool IsEmpty() const { + IEventHandle *tail = Tail.load(std::memory_order_relaxed); + IEventHandle *next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + IEventHandle *head = Head.load(std::memory_order_acquire); + return tail == &Stub && !next && tail == head; + } + + std::optional<TRecord> Pop() { + auto decompose = [&](IEventHandle *ev) { + auto& payload = reinterpret_cast<TEventPayload&>(const_cast<TActorId&>(ev->InterconnectSession)); + ui64 conn = payload.Conn; + TIntrusivePtr<IReceiveCallback> callback = std::move(payload.Callback); + payload.~TEventPayload(); + + auto& payload2 = reinterpret_cast<TEventPayload2&>(const_cast<TScopeId&>(ev->OriginScopeId)); + ui64 receivedTimestamp = payload2.ReceivedTimestamp; + ui64 seqNo = payload2.SeqNo; + payload2.~TEventPayload2(); + + return TRecord{std::unique_ptr<IEventHandle>(ev), conn, std::move(callback), receivedTimestamp, seqNo}; + }; + + for (;;) { + IEventHandle *tail = Tail.load(std::memory_order_relaxed); + IEventHandle *next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + IEventHandle *head; + + if (tail == &Stub) { + if (!next) { + if (head = Head.load(std::memory_order_acquire); tail != head) { + continue; + } else { + return std::nullopt; + } + } + Tail.store(next, std::memory_order_relaxed); + tail = next; + next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + } + + if (next) { + Tail.store(next, std::memory_order_relaxed); + return decompose(tail); + } + + head = Head.load(std::memory_order_acquire); + if (tail != head) { + continue; + } + + Stub.NextLinkPtr.store(0, std::memory_order_relaxed); + IEventHandle *prev = Head.exchange(&Stub, std::memory_order_acq_rel); + prev->NextLinkPtr.store(reinterpret_cast<uintptr_t>(&Stub), std::memory_order_release); + + next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + if (next) { + Tail.store(next, std::memory_order_relaxed); + return decompose(tail); + } + } + } + }; + + +} // NActors diff --git a/ydb/library/actors/interconnect/subscriber_liveness_checker.cpp b/ydb/library/actors/interconnect/subscriber_liveness_checker.cpp new file mode 100644 index 00000000000..6626447bbdf --- /dev/null +++ b/ydb/library/actors/interconnect/subscriber_liveness_checker.cpp @@ -0,0 +1,77 @@ +#include "subscriber_liveness_checker.h" + +#include <ydb/library/actors/core/actor_bootstrapped.h> +#include <ydb/library/actors/core/events.h> +#include <ydb/library/actors/core/hfunc.h> +#include <ydb/library/actors/core/log.h> +#include <ydb/library/actors/protos/services_common.pb.h> + +#include <util/generic/map.h> +#include <util/string/builder.h> + +namespace NActors { + namespace { + + TStringBuf FormatSubscriberActivityName(ui32 activityIndex) { + return activityIndex == Max<ui32>() ? TStringBuf("manual") : GetActivityTypeName(activityIndex); + } + + class TSubscriberLivenessChecker + : public TActorLivenessChecker + { + public: + TSubscriberLivenessChecker( + const TActorId& subscriptionOwner, + TVector<TActorLivenessCheckTarget> subscribers) + : TActorLivenessChecker(std::move(subscribers)) + , SubscriptionOwner(subscriptionOwner) + { + } + + private: + void OnDead(const TActorLivenessCheckTarget& target) override { + ++LeakedSubscribersByActivity[static_cast<ui32>(target.Cookie)]; + // The IC session identifies the subscription by event sender. + TActivationContext::Send(new IEventHandle( + SubscriptionOwner, + target.ActorId, + new TEvents::TEvUnsubscribe)); + } + + void OnFinish() override { + LogLeakedSubscribers(); + } + + void LogLeakedSubscribers() const { + if (LeakedSubscribersByActivity.empty()) { + return; + } + + TStringBuilder details; + bool first = true; + for (const auto& [activityIndex, count] : LeakedSubscribersByActivity) { + if (!first) { + details << ", "; + } + first = false; + details << "{activity# " << FormatSubscriberActivityName(activityIndex) + << " actors# " << count << '}'; + } + LOG_WARN_S(*TlsActivationContext, NActorsServices::INTERCONNECT_SESSION, + "Subscriber liveness check found leaked subscriptions: " << details); + } + + private: + const TActorId SubscriptionOwner; + TMap<ui32, ui64> LeakedSubscribersByActivity; + }; + + } + + IActor* CreateSubscriberLivenessChecker( + const TActorId& subscriptionOwner, + TVector<TActorLivenessCheckTarget> subscribers) { + return new TSubscriberLivenessChecker(subscriptionOwner, std::move(subscribers)); + } + +} diff --git a/ydb/library/actors/interconnect/subscriber_liveness_checker.h b/ydb/library/actors/interconnect/subscriber_liveness_checker.h new file mode 100644 index 00000000000..9d987297a83 --- /dev/null +++ b/ydb/library/actors/interconnect/subscriber_liveness_checker.h @@ -0,0 +1,36 @@ +#pragma once + +#include <ydb/library/actors/helpers/actor_liveness_checker.h> + +#include <util/generic/vector.h> + +#include <utility> + +namespace NActors { + + IActor* CreateSubscriberLivenessChecker( + const TActorId& subscriptionOwner, + TVector<TActorLivenessCheckTarget> subscribers); + + template <typename TSubscribers> + void RegisterSubscriberLivenessChecker( + const TActorId& subscriptionOwner, + const TSubscribers& subscribers) { + if (subscribers.empty()) { + return; + } + + TVector<TActorLivenessCheckTarget> subscriberInfos; + subscriberInfos.reserve(subscribers.size()); + for (const auto& item : subscribers) { + subscriberInfos.push_back({ + .ActorId = item.first, + .Cookie = item.second.ActivityIndex, + }); + } + TActivationContext::Register( + CreateSubscriberLivenessChecker(subscriptionOwner, std::move(subscriberInfos)), + subscriptionOwner); + } + +} diff --git a/ydb/library/actors/interconnect/ut/interconnect_ut.cpp b/ydb/library/actors/interconnect/ut/interconnect_ut.cpp index d82809d519f..50e0f53bea7 100644 --- a/ydb/library/actors/interconnect/ut/interconnect_ut.cpp +++ b/ydb/library/actors/interconnect/ut/interconnect_ut.cpp @@ -162,6 +162,37 @@ private: std::atomic<size_t> Received = 0; }; +class TConnectionSubscriberActor : public TActorBootstrapped<TConnectionSubscriberActor> { +public: + explicit TConnectionSubscriberActor(ui32 peerNodeId) + : PeerNodeId(peerNodeId) + {} + + void Bootstrap() { + Become(&TThis::StateFunc); + Send(TActivationContext::InterconnectProxy(PeerNodeId), new TEvents::TEvSubscribe); + } + + bool IsConnected() const { + return Connected.load(std::memory_order_acquire); + } + +private: + void Handle(TEvInterconnect::TEvNodeConnected::TPtr&) { + Connected.store(true, std::memory_order_release); + } + + STRICT_STFUNC(StateFunc, + hFunc(TEvInterconnect::TEvNodeConnected, Handle) + cFunc(TEvInterconnect::TEvNodeDisconnected::EventType, PassAway) + cFunc(TEvents::TSystem::Poison, PassAway) + ) + +private: + const ui32 PeerNodeId; + std::atomic<bool> Connected = false; +}; + class TBurstSenderActor : public TActorBootstrapped<TBurstSenderActor> { public: TBurstSenderActor(TActorId recipient, size_t messages, size_t payloadSize) @@ -525,6 +556,36 @@ private: std::shared_ptr<THandshakeFailureLogCounters> OutgoingHandshakeFailures; }; +struct TSubscriberLivenessLogState { + std::atomic<ui32> Warnings = 0; + TMutex Mutex; + TString LastWarning; +}; + +class TSubscriberLivenessLogBackend : public TLogBackend { +public: + explicit TSubscriberLivenessLogBackend(std::shared_ptr<TSubscriberLivenessLogState> state) + : State(std::move(state)) + {} + + void WriteData(const TLogRecord& rec) override { + const TStringBuf line(rec.Data, rec.Len); + if (rec.Priority == TLOG_WARNING && + line.Contains("Subscriber liveness check found leaked subscriptions")) { + with_lock (State->Mutex) { + State->LastWarning = line; + } + State->Warnings.fetch_add(1, std::memory_order_release); + } + } + + void ReopenLog() override { + } + +private: + std::shared_ptr<TSubscriberLivenessLogState> State; +}; + } // namespace class TSenderActor : public TActorBootstrapped<TSenderActor> { @@ -1081,6 +1142,79 @@ void RunKernelLivenessReconnectLocalFallbackNotApplied(bool withRdma) { UNIT_ASSERT_VALUES_EQUAL(WaitForSessionCounter(cluster, 2, 1, "Params.UseKernelLiveness"), 0ULL); } +void RunSubscriberLivenessCheck(bool useSessionV2, TDuration checkInterval) { + if (useSessionV2 && !TUringContext::IsAvailable()) { + Cerr << "io_uring not available; skipping" << Endl; + return; + } + + auto settingsCustomizer = [=](ui32, TInterconnectSettings& settings) { + settings.SubscriberLivenessCheckInterval = checkInterval; + settings.V2.Enable = useSessionV2; + }; + auto logState = std::make_shared<TSubscriberLivenessLogState>(); + auto loggerSettings = MakeIntrusive<NLog::TSettings>( + TActorId(0, "logger"), + static_cast<NLog::EComponent>(NActorsServices::LOGGER), + NLog::PRI_DEBUG, + NLog::PRI_DEBUG, + 0U); + loggerSettings->Append( + NActorsServices::EServiceCommon_MIN, + NActorsServices::EServiceCommon_MAX, + NActorsServices::EServiceCommon_Name); + loggerSettings->SetAllowDrop(false); + loggerSettings->SetThrottleDelay(TDuration::Zero()); + auto logBackendFactory = [logState] { + return TAutoPtr<TLogBackend>(new TSubscriberLivenessLogBackend(logState)); + }; + TTestICCluster cluster(2, TChannelsConfig(), nullptr, loggerSettings, + useSessionV2 ? TTestICCluster::EMPTY : TTestICCluster::DISABLE_RDMA, + {}, TDuration::Seconds(2), TNode::DefaultInflight(), settingsCustomizer, logBackendFactory); + + auto* subscriber = new TConnectionSubscriberActor(1); + const TActorId subscriberId = cluster.RegisterActor(subscriber, 2); + + WaitForCondition(TDuration::Seconds(10), [&] { + return subscriber->IsConnected(); + }, "subscriber connected"); + WaitForCondition(TDuration::Seconds(10), [&] { + try { + return GetSessionCounter(cluster, 2, 1, "Subscribers.size()") == 1; + } catch (const TPatternNotFound&) { + return false; + } + }, "live subscriber registered"); + + if (checkInterval != TDuration::Zero()) { + Sleep(3 * checkInterval); + UNIT_ASSERT_VALUES_EQUAL(GetSessionCounter(cluster, 2, 1, "Subscribers.size()"), 1); + UNIT_ASSERT_VALUES_EQUAL(logState->Warnings.load(std::memory_order_acquire), 0); + } + + cluster.KillActor(2, subscriberId); + if (checkInterval != TDuration::Zero()) { + WaitForCondition(TDuration::Seconds(10), [&] { + try { + return GetSessionCounter(cluster, 2, 1, "Subscribers.size()") == 0; + } catch (const TPatternNotFound&) { + return false; + } + }, "dead subscriber removed"); + WaitForCondition(TDuration::Seconds(10), [&] { + return logState->Warnings.load(std::memory_order_acquire) == 1; + }, "leaked subscriber warning"); + with_lock (logState->Mutex) { + UNIT_ASSERT_STRING_CONTAINS(logState->LastWarning, "activity# manual"); + UNIT_ASSERT_STRING_CONTAINS(logState->LastWarning, "actors# 1"); + } + } else { + Sleep(TDuration::MilliSeconds(300)); + UNIT_ASSERT_VALUES_EQUAL(GetSessionCounter(cluster, 2, 1, "Subscribers.size()"), 1); + UNIT_ASSERT_VALUES_EQUAL(logState->Warnings.load(std::memory_order_acquire), 0); + } +} + } // namespace Y_UNIT_TEST_SUITE(Interconnect) { @@ -1091,6 +1225,22 @@ Y_UNIT_TEST_SUITE(Interconnect) { RunScopeClassCounterRebindTest(TScopeId(2, 42), "other_tenant"); } + Y_UNIT_TEST(SubscriberLivenessCheck) { + RunSubscriberLivenessCheck(false, TDuration::MilliSeconds(100)); + } + + Y_UNIT_TEST(SubscriberLivenessCheckV2) { + RunSubscriberLivenessCheck(true, TDuration::MilliSeconds(100)); + } + + Y_UNIT_TEST(SubscriberLivenessCheckDisabled) { + RunSubscriberLivenessCheck(false, TDuration::Zero()); + } + + Y_UNIT_TEST(SubscriberLivenessCheckDisabledV2) { + RunSubscriberLivenessCheck(true, TDuration::Zero()); + } + Y_UNIT_TEST(RdmaRetryWatchdogPendingSessionsAggregated) { TTestActorRuntimeBase runtime; runtime.Initialize(); diff --git a/ydb/library/actors/interconnect/ut/lib/node.h b/ydb/library/actors/interconnect/ut/lib/node.h index 5c8cbca4d34..b3cf6a11f21 100644 --- a/ydb/library/actors/interconnect/ut/lib/node.h +++ b/ydb/library/actors/interconnect/ut/lib/node.h @@ -75,17 +75,24 @@ public: } setup.InterconnectCollectSubscriptionStackTrace = common->Settings.CollectSubscriptionStackTrace; - if (common->Settings.EnableInterconnectSessionV2) { + if (common->Settings.V2.Enable) { // Mirror production: create the shared v2 io_uring engine up front and publish it in Common; the // proxy binds it to the actor system on start (SetActorSystem). Shard count is overridable via // YDB_IC_V2_SHARDS so tests can force many connections onto a single ring. - ui32 uringShards = 4; + ui32 uringShards = common->Settings.V2.UringEngineThreads; if (const TString s = GetEnv("YDB_IC_V2_SHARDS"); !s.empty()) { uringShards = FromString<ui32>(s); } + ui32 ringsPerShard = common->Settings.V2.UringEngineRingsPerShard; + if (const TString s = GetEnv("YDB_IC_V2_RINGS_PER_SHARD"); !s.empty()) { + ringsPerShard = FromString<ui32>(s); + } common->UringEngineV2 = CreateUringEngine(uringShards, common->MonCounters->GetSubgroup("subsystem", "uring"), - common->Settings.EnableSQPOLLv2); + common->Settings.V2.EnableSQPOLL, + ringsPerShard, + common->Settings.V2.UringEngineSqThreadIdleMs, + common->Settings.V2.ShareRingsAmongThreads); setup.OnActorSystemCreated.push_back([engine = common->UringEngineV2](TActorSystem *actorSystem) { if (engine) { engine->SetActorSystem(actorSystem); diff --git a/ydb/library/actors/interconnect/ut/v2_session_ut.cpp b/ydb/library/actors/interconnect/ut/v2_session_ut.cpp index 8d93cc19d46..e6949208fc6 100644 --- a/ydb/library/actors/interconnect/ut/v2_session_ut.cpp +++ b/ydb/library/actors/interconnect/ut/v2_session_ut.cpp @@ -308,8 +308,8 @@ namespace { std::unique_ptr<TTestICCluster> MakeV2Cluster(ui32 tcpSocketBufferSize = 0) { auto customizer = [tcpSocketBufferSize](ui32, TInterconnectSettings& settings) { - settings.EnableInterconnectSessionV2 = true; - settings.ChecksumInterconnectSessionV2 = true; + settings.V2.Enable = true; + settings.V2.ChecksumEvents = true; if (tcpSocketBufferSize) { settings.TCPSocketBufferSize = tcpSocketBufferSize; } @@ -505,7 +505,7 @@ Y_UNIT_TEST_SUITE(InterconnectSessionV2) { constexpr ui32 kNodes = 5; auto customizer = [](ui32, TInterconnectSettings& settings) { - settings.EnableInterconnectSessionV2 = true; + settings.V2.Enable = true; }; auto cluster = std::make_unique<TTestICCluster>( kNodes, TChannelsConfig(), nullptr, nullptr, TTestICCluster::EMPTY, @@ -554,7 +554,7 @@ Y_UNIT_TEST_SUITE(InterconnectSessionV2) { constexpr ui32 kNodes = 5; auto customizer = [](ui32, TInterconnectSettings& settings) { - settings.EnableInterconnectSessionV2 = true; + settings.V2.Enable = true; }; auto cluster = std::make_unique<TTestICCluster>( kNodes, TChannelsConfig(), nullptr, nullptr, TTestICCluster::EMPTY, diff --git a/ydb/library/actors/interconnect/v2_event_serializer.cpp b/ydb/library/actors/interconnect/v2_event_serializer.cpp index 2bb23629cbb..0d21c2bf942 100644 --- a/ydb/library/actors/interconnect/v2_event_serializer.cpp +++ b/ydb/library/actors/interconnect/v2_event_serializer.cpp @@ -135,6 +135,7 @@ namespace NActors { memcpy(buffer.UnsafeGetDataMut(), span.data(), span.size()); span = {buffer.data(), span.size()}; buffer.TrimFront(buffer.size() - span.size()); + *bufferProduced = CumulativeProduced + span.size(); } } diff --git a/ydb/library/actors/interconnect/ya.make b/ydb/library/actors/interconnect/ya.make index 278b15da8d6..36f75e47534 100644 --- a/ydb/library/actors/interconnect/ya.make +++ b/ydb/library/actors/interconnect/ya.make @@ -57,6 +57,8 @@ SRCS( profiler.h rdma_sync_actor.cpp slowpoke_actor.h + subscriber_liveness_checker.cpp + subscriber_liveness_checker.h subscription_manager.cpp subscription_manager.h types.cpp diff --git a/ydb/library/services/services.proto b/ydb/library/services/services.proto index 250af7fd223..370b7b0761f 100644 --- a/ydb/library/services/services.proto +++ b/ydb/library/services/services.proto @@ -480,6 +480,8 @@ enum EServiceKikimr { // BlobChecker BLOB_CHECKER_ORCHESTRATOR = 4300; BLOB_CHECKER_WORKER = 4301; + + DBS_CONTROLLER = 4400; }; message TActivity { diff --git a/ydb/library/workload/vector/vector_command_index.cpp b/ydb/library/workload/vector/vector_command_index.cpp index 61d39c4b2c3..fd8607d60f1 100644 --- a/ydb/library/workload/vector/vector_command_index.cpp +++ b/ydb/library/workload/vector/vector_command_index.cpp @@ -54,9 +54,14 @@ int TWorkloadCommandBuildIndex::DoRun() { ddlQuery << "GLOBAL USING vector_kmeans_tree\n"; ddlQuery << "ON (embedding)\n"; ddlQuery << "WITH (\n"; - ddlQuery << " " << Params.GetDistanceDDL() << ",\n"; - ddlQuery << " vector_type=" << Params.VectorOpts.VectorType << ",\n"; - ddlQuery << " vector_dimension=" << Params.VectorOpts.VectorDimension; + ddlQuery << " " << Params.GetDistanceDDL(); + // When VectorDimension is 0, omit vector_type/vector_dimension so the server + // autodetects them from the table data. This is used for datasets imported + // from an external source (e.g. S3) whose dimension is not known in advance. + if (Params.VectorOpts.VectorDimension) { + ddlQuery << ",\n vector_type=" << Params.VectorOpts.VectorType; + ddlQuery << ",\n vector_dimension=" << Params.VectorOpts.VectorDimension; + } if (Params.KmeansTreeLevels) { ddlQuery << ",\n levels=" << Params.KmeansTreeLevels; } diff --git a/ydb/library/workload/vector/vector_recall_evaluator.cpp b/ydb/library/workload/vector/vector_recall_evaluator.cpp index 7733700813e..8b6f501a1b2 100644 --- a/ydb/library/workload/vector/vector_recall_evaluator.cpp +++ b/ydb/library/workload/vector/vector_recall_evaluator.cpp @@ -39,7 +39,7 @@ void TVectorRecallEvaluator::SelectReferenceResults(const TVectorSampler& sample << "SELECT s.id AS id" << ", " << (isAscending ? "BOTTOM_BY" : "TOP_BY") << "(" << MakeKeyExpression(Params, "m.") << ", Knn::" << functionName << "(m." << Params.EmbeddingColumn << ", s.embedding), " << Params.Limit << ") result_ids" - << " FROM " << Params.TableOpts.Name << " m" + << " FROM `" << Params.TableOpts.Name << "` m" << (Params.PrefixColumn ? " INNER JOIN " : " CROSS JOIN ") << "AS_TABLE($Samples) AS s"; if (Params.PrefixColumn) { refQueryBuilder << " ON s.prefix = m." << *Params.PrefixColumn; diff --git a/ydb/library/workload/vector/vector_sampler.cpp b/ydb/library/workload/vector/vector_sampler.cpp index 74e1cbc5a08..f5795a00942 100644 --- a/ydb/library/workload/vector/vector_sampler.cpp +++ b/ydb/library/workload/vector/vector_sampler.cpp @@ -19,7 +19,7 @@ TVectorSampler::TVectorSampler(const TVectorWorkloadParams& params) ui64 TVectorSampler::SelectOneId(bool min) { std::string query = std::format(R"_(--!syntax_v1 - SELECT Unwrap(CAST({1} AS uint64)) AS id FROM {0} ORDER BY {1} {2} LIMIT 1; + SELECT Unwrap(CAST({1} AS uint64)) AS id FROM `{0}` ORDER BY {1} {2} LIMIT 1; )_", Params.TableOpts.Name.c_str(), Params.KeyColumns[0].c_str(), min ? "" : "DESC"); // Execute the query @@ -50,7 +50,7 @@ void TVectorSampler::SelectPredefinedVectors() { if (Params.PrefixColumn) { vectorQueryBuilder << ", Unwrap(" << Params.PrefixColumn << ") as prefix_value"; } - vectorQueryBuilder << " FROM " << Params.QueryTableName + vectorQueryBuilder << " FROM `" << Params.QueryTableName << "`" << " ORDER BY "; for (size_t i = 0; i < Params.QueryTableKeyColumns.size(); i++) { if (i > 0) { @@ -146,13 +146,13 @@ void TVectorSampler::SampleExistingVectors() { if (Params.PrefixColumn) { vectorQuery = std::format(R"_(--!syntax_v1 SELECT Unwrap(CAST({0} as uint64)) as id, Unwrap({1}) as embedding, Unwrap({2}) as prefix_value - FROM {3} + FROM `{3}` WHERE {0} = {4}; )_", Params.KeyColumns[0].c_str(), Params.EmbeddingColumn.c_str(), Params.PrefixColumn->c_str(), Params.TableOpts.Name.c_str(), randomId); } else { vectorQuery = std::format(R"_(--!syntax_v1 SELECT Unwrap(CAST({0} as uint64)) as id, Unwrap({1}) as embedding - FROM {2} + FROM `{2}` WHERE {0} = {3}; )_", Params.KeyColumns[0].c_str(), Params.EmbeddingColumn.c_str(), Params.TableOpts.Name.c_str(), randomId); } diff --git a/ydb/library/workload/vector/vector_sql.cpp b/ydb/library/workload/vector/vector_sql.cpp index e406a2520f6..2e2658c6efa 100644 --- a/ydb/library/workload/vector/vector_sql.cpp +++ b/ydb/library/workload/vector/vector_sql.cpp @@ -64,7 +64,7 @@ std::string MakeSelect(const TVectorWorkloadParams& params, const TString& index if (params.PrefixColumn) ret << "DECLARE $PrefixValue as " << params.PrefixType << ";" << "\n"; ret << "pragma ydb.KMeansTreeSearchTopSize=\"" << params.KmeansTreeSearchClusters << "\";" << "\n"; - ret << "SELECT " << MakeKeyExpression(params, "") << " AS id FROM " << params.TableOpts.Name << "\n"; + ret << "SELECT " << MakeKeyExpression(params, "") << " AS id FROM `" << params.TableOpts.Name << "`\n"; if (!indexName.empty()) ret << "VIEW " << indexName << "\n"; if (params.PrefixColumn) diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp index f0b49e35fc0..fa8323abdf6 100644 --- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp +++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp @@ -1927,6 +1927,14 @@ public: // Stats.CodeGenFinalizeTime = f.GetCodeGenFinalizeTime(); // Stats.CodeGenModulePassTime = f.GetCodeGenModulePassTime(); + Stats.MkqlStats.clear(); + for (const auto& stat : protoStats.GetMkqlStats()) { + Stats.MkqlStats.emplace_back(TMkqlStat{ + TStatKey(stat.GetName(), stat.GetDeriv()), + stat.GetValue() + }); + } + for (const auto& input : protoStats.GetInputChannels()) { InputChannels[input.GetChannelId()]->FromProto(input); } diff --git a/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp b/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp index 9b529c720e6..d75f8e2a161 100644 --- a/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp +++ b/ydb/library/yql/providers/generic/actors/ut/yql_generic_lookup_actor_ut.cpp @@ -51,7 +51,21 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) { , Edge(edge) , FullscanLimit(fullscanLimit) { + } + TCallLookupActor( + std::shared_ptr<NKikimr::NMiniKQL::TScopedAlloc>&& alloc, + TLookupActorFactory&& lookupActorFactory, + NYql::NDqProto::StatusIds::StatusCode expectedError, + const NActors::TActorId& edge, + size_t fullscanLimit = 0) + : Alloc(std::move(alloc)) + , TypeEnv(std::make_shared<NKikimr::NMiniKQL::TTypeEnvironment>(*Alloc)) + , LookupActorFactory(std::move(lookupActorFactory)) + , ExpectedError(expectedError) + , Edge(edge) + , FullscanLimit(fullscanLimit) + { } void Bootstrap() { @@ -69,10 +83,12 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) { STFUNC(StateFunc) { switch (ev->GetTypeRewrite()) { hFunc(NYql::NDq::IDqAsyncLookupSource::TEvLookupResult, Handle); + hFunc(NYql::NDq::IDqComputeActorAsyncInput::TEvAsyncInputError, Handle); } } void Handle(NYql::NDq::IDqAsyncLookupSource::TEvLookupResult::TPtr& ev) { + UNIT_ASSERT(!ExpectedError); Callback(Alloc, ev); { auto guard = Guard(*Alloc); @@ -83,6 +99,12 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) { Send(Edge, new NActors::TEvents::TEvWakeup()); } + void Handle(NYql::NDq::IDqComputeActorAsyncInput::TEvAsyncInputError::TPtr& ev) { + UNIT_ASSERT(ExpectedError); + UNIT_ASSERT_EQUAL_C(ev->Get()->FatalCode, *ExpectedError, static_cast<ui64>(ev->Get()->FatalCode) << " != " << static_cast<ui64>(*ExpectedError) << ", issues = " << ev->Get()->Issues.ToOneLineString()); + Send(Edge, new NActors::TEvents::TEvWakeup()); + } + void Free() { if (Alloc) { auto guard = Guard(*Alloc); @@ -112,6 +134,7 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) { NActors::TActorId LookupActor; TLookupActorFactory LookupActorFactory; TCallback Callback; + std::optional<NYql::NDqProto::StatusIds::StatusCode> ExpectedError; NActors::TActorId Edge; size_t FullscanLimit; }; @@ -507,6 +530,159 @@ Y_UNIT_TEST_SUITE(GenericProviderLookupActor) { runtime.GrabEdgeEventRethrow<NActors::TEvents::TEvWakeup>(edge); } + Y_UNIT_TEST_TWIN(LookupWithFatalAuthErrors, ListSplitsOrReadError) { + auto alloc = std::make_shared<NKikimr::NMiniKQL::TScopedAlloc>(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), true, false); + NKikimr::NMiniKQL::TMemoryUsageInfo memUsage("TestMemUsage"); + NKikimr::NMiniKQL::THolderFactory holderFactory(alloc->Ref(), memUsage); + + auto loggerConfig = NYql::NProto::TLoggingConfig(); + loggerConfig.set_allcomponentslevel(::NYql::NProto::TLoggingConfig_ELevel::TLoggingConfig_ELevel_TRACE); + NYql::NLog::InitLogger(loggerConfig, false); + + TTestActorRuntimeBase runtime(1, 1, true); + runtime.Initialize(); + auto edge = runtime.AllocateEdgeActor(); + + NYql::TGenericDataSourceInstance dsi; + dsi.Setkind(NYql::EGenericDataSourceKind::YDB); + dsi.mutable_endpoint()->Sethost("some_host"); + dsi.mutable_endpoint()->Setport(2135); + dsi.Setdatabase("some_db"); + dsi.Setuse_tls(true); + dsi.set_protocol(::NYql::EGenericProtocol::NATIVE); + auto token = dsi.mutable_credentials()->mutable_token(); + token->Settype("IAM"); + token->Setvalue("token_value"); + + auto connectorMock = std::make_shared<NYql::NConnector::NTest::TConnectorClientMock>(); + + // clang-format off + // step 1: ListSplits + { + ::testing::InSequence seq; + { + auto listBuilder = connectorMock->ExpectListSplits(); + listBuilder + .Select() + .DataSourceInstance(dsi) + .What() + .Column("id", Ydb::Type::UINT64) + .NullableColumn("optional_id", Ydb::Type::UINT64) + .NullableColumn("string_value", Ydb::Type::STRING) + .Done() + .Table("lookup_test") + .Where() + .Filter() + .Disjunction() + .Operand() + .Conjunction() + .Operand().Equal().Column("id").Value<ui64>(2).Done().Done() + .Operand().Equal().Column("optional_id").OptionalValue<ui64>(102).Done().Done() + .Done() + .Done() + .Operand() + .Conjunction() + .Operand().Equal().Column("id").Value<ui64>(1).Done().Done() + .Operand().Equal().Column("optional_id").OptionalValue<ui64>(101).Done().Done() + .Done() + .Done() + .Operand() + .Conjunction() + .Operand().Equal().Column("id").Value<ui64>(0).Done().Done() + .Operand().Equal().Column("optional_id").OptionalValue<ui64>(100).Done().Done() + .Done() + .Done() + .Operand() + .Conjunction() + .Operand().Equal().Column("id").Value<ui64>(2).Done().Done() + .Operand().Equal().Column("optional_id").OptionalValue<ui64>(102).Done().Done() + .Done() + .Done() + .Done() + .Done() + .Done() + .Done() + .MaxSplitCount(1) + ; + if (ListSplitsOrReadError) { + listBuilder + .Status(NYdbGrpc::TGrpcStatus(grpc::StatusCode::UNAUTHENTICATED, "Mocked Error")) + ; + } else { + listBuilder + .Result() + .AddResponse(NewSuccess()) + .Description("Actual split info is not important") + ; + } + } + + if (!ListSplitsOrReadError) { + auto readBuilder = connectorMock->ExpectReadSplits(); + readBuilder + .DataSourceInstance(dsi) + .Filtering(NYql::NConnector::NApi::TReadSplitsRequest::FILTERING_MANDATORY) + .Split() + .Description("Actual split info is not important") + .Done() + .Status(NYdbGrpc::TGrpcStatus(grpc::StatusCode::UNAUTHENTICATED, "Mocked Error")) + ; + } + } + // clang-format on + + NYql::Generic::TLookupSource lookupSourceSettings; + *lookupSourceSettings.mutable_data_source_instance() = dsi; + lookupSourceSettings.Settable("lookup_test"); + lookupSourceSettings.SetTokenName("test_token"); + + google::protobuf::Any packedLookupSource; + Y_ABORT_UNLESS(packedLookupSource.PackFrom(lookupSourceSettings)); + + auto lookupActorFactory = [&holderFactory, connectorMock = std::move(connectorMock), lookupSourceSettings = std::move(lookupSourceSettings)](const NActors::TActorId& caller, std::shared_ptr<NKikimr::NMiniKQL::TScopedAlloc>& alloc, NKikimr::NMiniKQL::TTypeEnvironment& typeEnv) mutable { + NKikimr::NMiniKQL::TTypeBuilder typeBuilder(typeEnv); + + NKikimr::NMiniKQL::TStructTypeBuilder keyTypeBuilder{typeEnv}; + keyTypeBuilder.Add("id", typeBuilder.NewDataType(NYql::NUdf::EDataSlot::Uint64, false)); + keyTypeBuilder.Add("optional_id", typeBuilder.NewDataType(NYql::NUdf::EDataSlot::Uint64, true)); + NKikimr::NMiniKQL::TStructTypeBuilder outputypeBuilder{typeEnv}; + outputypeBuilder.Add("string_value", typeBuilder.NewDataType(NYql::NUdf::EDataSlot::String, true)); + + auto guard = Guard(*alloc.get()); + auto keyTypeHelper = std::make_shared<NYql::NDq::IDqAsyncLookupSource::TKeyTypeHelper>(keyTypeBuilder.Build()); + + auto [lookupSource, actor] = NYql::NDq::CreateGenericLookupActor( + std::move(connectorMock), + NKikimr::NKqp::NFederatedQueryTest::CreateCredentialsFactory("token_value"), + caller, + nullptr, + alloc, + keyTypeHelper, + std::move(lookupSourceSettings), + keyTypeBuilder.Build(), + outputypeBuilder.Build(), + typeEnv, + holderFactory, + 1'000'000, + {{"test_token", "{\"token\": \"token_value\"}"}}); + + auto request = std::make_shared<NYql::NDq::IDqAsyncLookupSource::TUnboxedValueMap>(3, keyTypeHelper->GetValueHash(), keyTypeHelper->GetValueEqual()); + for (size_t i = 0; i != 3; ++i) { + NYql::NUdf::TUnboxedValue* keyItems; + auto key = holderFactory.CreateDirectArrayHolder(2, keyItems); + keyItems[0] = NYql::NUdf::TUnboxedValuePod(ui64(i)); + keyItems[1] = NYql::NUdf::TUnboxedValuePod(ui64(100 + i)); + request->emplace(std::move(key), NYql::NUdf::TUnboxedValue{}); + } + + return std::pair { actor, std::move(request) }; + }; + + auto callLookupActor = new TCallLookupActor(std::move(alloc), std::move(lookupActorFactory), NYql::NDqProto::StatusIds::UNAUTHORIZED, edge); + runtime.Register(callLookupActor); + runtime.GrabEdgeEventRethrow<NActors::TEvents::TEvWakeup>(edge); + } + class TMockStructuredTokenCredentialsFactory : public NYql::IStructuredTokenCredentialsFactory { public: TMockStructuredTokenCredentialsFactory(const std::string yqlToken, const TVector<bool>& pattern) diff --git a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp index bac77c67a02..ae01bfe738f 100644 --- a/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp +++ b/ydb/library/yql/providers/generic/actors/yql_generic_lookup_actor.cpp @@ -281,17 +281,12 @@ namespace NYql::NDq { } void Handle(TEvError::TPtr ev) { - const auto error = ev->Get()->Error; - auto issues = NConnector::ErrorToIssues(error); - auto fatalCode = NDqProto::StatusIds::INTERNAL_ERROR; - - try { - fatalCode = NConnector::ErrorToDqStatus(error); - } catch (const std::exception& e) { - issues.AddIssue(TStringBuilder() << "Failed to convert YDB status code: " << e.what()); - } - - Send(ParentId, new IDqComputeActorAsyncInput::TEvAsyncInputError(-1, std::move(issues), fatalCode)); + const auto& error = ev->Get()->Error; + Send(ParentId, + new IDqComputeActorAsyncInput::TEvAsyncInputError( + /*inputId=*/-1, /* will be filled by LookupTransform actor */ + NConnector::ErrorToIssues(error), + NConnector::ErrorToDqStatus(error))); } void Handle(TEvLookupRetry::TPtr ev) { diff --git a/ydb/library/yql/providers/generic/connector/libcpp/error.cpp b/ydb/library/yql/providers/generic/connector/libcpp/error.cpp index 2ce7d8b5df0..c8695985d41 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/error.cpp +++ b/ydb/library/yql/providers/generic/connector/libcpp/error.cpp @@ -1,14 +1,15 @@ #include "error.h" #include <grpcpp/impl/codegen/status_code_enum.h> +#include <ydb/core/grpc_services/local_rpc/local_rpc.h> // for NKikimr::NRpcService::GrpcStatusToYdbStatus +#include <ydb/library/yql/dq/actors/dq.h> #include <yql/essentials/public/issue/yql_issue_message.h> #include <yql/essentials/utils/yql_panic.h> -#include <ydb/public/api/protos/ydb_status_codes.pb.h> namespace NYql::NConnector { NApi::TError NewSuccess() { NApi::TError error; - error.set_status(Ydb::StatusIds_StatusCode::StatusIds_StatusCode_SUCCESS); + error.set_status(Ydb::StatusIds::SUCCESS); return error; } @@ -28,30 +29,16 @@ namespace NYql::NConnector { } NDqProto::StatusIds::StatusCode ErrorToDqStatus(const NApi::TError& error) { - switch (error.status()) { - case ::Ydb::StatusIds::StatusCode::StatusIds_StatusCode_BAD_REQUEST: - return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_BAD_REQUEST; - case ::Ydb::StatusIds::StatusCode::StatusIds_StatusCode_INTERNAL_ERROR: - return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_INTERNAL_ERROR; - case ::Ydb::StatusIds::StatusCode::StatusIds_StatusCode_UNSUPPORTED: - return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_UNSUPPORTED; - case ::Ydb::StatusIds::StatusCode::StatusIds_StatusCode_NOT_FOUND: - return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_BAD_REQUEST; - case ::Ydb::StatusIds::StatusCode::StatusIds_StatusCode_SCHEME_ERROR: - return NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_SCHEME_ERROR; - default: - ythrow yexception() << "Unexpected YDB status code: " << ::Ydb::StatusIds::StatusCode_Name(error.status()); - } + return NYql::NDq::YdbStatusToDqStatus(error.status(), NYql::NDq::EStatusCompatibilityLevel::WithUnauthorized); } NApi::TError ErrorFromGRPCStatus(const NYdbGrpc::TGrpcStatus& status) { NApi::TError result; - if (status.GRpcStatusCode == grpc::OK) { - result.set_status(Ydb::StatusIds_StatusCode::StatusIds_StatusCode_SUCCESS); + if (status.Ok()) { + result.set_status(Ydb::StatusIds::SUCCESS); } else { - // FIXME: more appropriate error code for network error - result.set_status(Ydb::StatusIds_StatusCode::StatusIds_StatusCode_INTERNAL_ERROR); + result.set_status(status.InternalError ? Ydb::StatusIds::INTERNAL_ERROR : NKikimr::NRpcService::GrpcStatusToYdbStatus(static_cast<grpc::StatusCode>(status.GRpcStatusCode))); result.set_message(TString{status.Msg}); } diff --git a/ydb/library/yql/providers/generic/connector/libcpp/ya.make b/ydb/library/yql/providers/generic/connector/libcpp/ya.make index d7b7fd6fdf8..201bd912f0e 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/ya.make +++ b/ydb/library/yql/providers/generic/connector/libcpp/ya.make @@ -10,7 +10,9 @@ PEERDIR( contrib/libs/apache/arrow contrib/libs/grpc ydb/core/formats/arrow/serializer + ydb/core/grpc_services/local_rpc ydb/public/sdk/cpp/src/library/grpc/client + ydb/library/yql/dq/actors ydb/library/yql/dq/actors/protos yql/essentials/providers/common/proto yql/essentials/providers/common/proto diff --git a/ydb/library/yql/providers/generic/connector/tests/utils/ya.make b/ydb/library/yql/providers/generic/connector/tests/utils/ya.make index 0c2159ee704..5bbd0a6d5b6 100644 --- a/ydb/library/yql/providers/generic/connector/tests/utils/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/utils/ya.make @@ -22,6 +22,7 @@ ENDIF() PEERDIR( contrib/python/PyYAML yql/essentials/providers/common/proto + ydb/library/yql/providers/generic/connector/api/service ydb/library/yql/providers/generic/connector/tests/utils/types ydb/public/api/protos ) diff --git a/ydb/mvp/meta/support_links/grafana_dashboard_common.cpp b/ydb/mvp/meta/support_links/grafana_dashboard_common.cpp index 2ee68d32324..65da15f2704 100644 --- a/ydb/mvp/meta/support_links/grafana_dashboard_common.cpp +++ b/ydb/mvp/meta/support_links/grafana_dashboard_common.cpp @@ -29,25 +29,8 @@ void InsertOrReplaceDashboardVar(TCgiParameters& queryParameters, TStringBuf lab queryParameters.InsertUnescaped(varName, value); } -void ApplyGrafanaDashboardBindingPolicy( - TCgiParameters& queryParameters, - const THashMap<TString, TString>& clusterInfo, - const TCgiParameters& requestQueryParameters, - const TResolvedParamBindings& paramBindings) -{ - for (const auto& [name, value] : BuildNonIdentityRequestParamValues(requestQueryParameters)) { - InsertOrReplaceDashboardVar(queryParameters, name, value); - } - - for (const auto& [label, value] : BuildClusterInfoParamValues(clusterInfo, paramBindings.ClusterInfoMappings)) { - InsertOrReplaceDashboardVar(queryParameters, label, value); - } - - for (const auto& [label, value] : BuildStaticParamValues(paramBindings.StaticMappings)) { - InsertOrReplaceDashboardVar(queryParameters, label, value); - } - - for (const auto& [label, value] : BuildRequestParamValues(requestQueryParameters, paramBindings.RequestMappings)) { +void ApplyGrafanaDashboardBindingPolicy(TCgiParameters& queryParameters, const TVector<std::pair<TString, TString>>& parametersToAdd) { + for (const auto& [label, value] : parametersToAdd) { InsertOrReplaceDashboardVar(queryParameters, label, value); } } @@ -78,7 +61,7 @@ TString BuildGrafanaDashboardUrl( const TResolvedParamBindings& paramBindings) { auto [path, queryParameters] = BuildGrafanaDashboardUrlParts(grafanaEndpoint, url); - ApplyGrafanaDashboardBindingPolicy(queryParameters, clusterInfo, requestQueryParameters, paramBindings); + ApplyGrafanaDashboardBindingPolicy(queryParameters, BuildParametersToAdd(requestQueryParameters, clusterInfo, paramBindings)); return queryParameters.empty() ? path diff --git a/ydb/mvp/meta/support_links/grafana_logging_source.cpp b/ydb/mvp/meta/support_links/grafana_logging_source.cpp index 55c07029719..c4cc7c6a05b 100644 --- a/ydb/mvp/meta/support_links/grafana_logging_source.cpp +++ b/ydb/mvp/meta/support_links/grafana_logging_source.cpp @@ -91,25 +91,15 @@ bool TryBuildGrafanaLoggingUrl( } const TCgiParameters forwardedParameters = BuildForwardedParameters(input.Identity, input.AdditionalRequestParams); - TVector<std::pair<TString, TString>> bindings = BuildNonIdentityRequestParamValues(forwardedParameters); - for (auto& binding : BuildRequestParamValues(forwardedParameters, paramBindings.RequestMappings)) { - bindings.push_back(std::move(binding)); - } - for (auto& binding : BuildClusterInfoParamValues(input.ClusterInfo, paramBindings.ClusterInfoMappings)) { - bindings.push_back(std::move(binding)); - } - for (auto& binding : BuildStaticParamValues(paramBindings.StaticMappings)) { - bindings.push_back(std::move(binding)); - } - - TVector<std::pair<TString, TString>> resolvedBindings; - resolvedBindings.reserve(bindings.size()); - for (auto& binding : bindings) { - if (binding.first == "datasource") { - datasource = std::move(binding.second); + auto parametersToAdd = BuildParametersToAdd(forwardedParameters, input.ClusterInfo, paramBindings); + TVector<std::pair<TString, TString>> bindings; + bindings.reserve(parametersToAdd.size()); + for (auto& paramValue : parametersToAdd) { + if (paramValue.first == "datasource") { + datasource = std::move(paramValue.second); continue; } - resolvedBindings.push_back(std::move(binding)); + bindings.push_back(std::move(paramValue)); } if (datasource.empty()) { @@ -120,7 +110,7 @@ bool TryBuildGrafanaLoggingUrl( TCgiParameters queryParameters; queryParameters.InsertUnescaped("schemaVersion", "1"); queryParameters.InsertUnescaped("panes", NJson::WriteJson( - BuildGrafanaLoggingPanesJson(datasource, resolvedBindings), + BuildGrafanaLoggingPanesJson(datasource, bindings), false)); queryParameters.InsertUnescaped("orgId", "1"); diff --git a/ydb/mvp/meta/support_links/param_bindings.cpp b/ydb/mvp/meta/support_links/param_bindings.cpp index 80f1cc39e83..c566247672a 100644 --- a/ydb/mvp/meta/support_links/param_bindings.cpp +++ b/ydb/mvp/meta/support_links/param_bindings.cpp @@ -84,6 +84,28 @@ void ValidateParamsAreUnique(const TResolvedParamBindings& paramBindings, const } } +TVector<std::pair<TString, TString>> BuildParametersToAdd( + const TCgiParameters& requestParameters, + const THashMap<TString, TString>& clusterInfo, + const TResolvedParamBindings& paramBindings) +{ + TVector<std::pair<TString, TString>> parametersToAdd = BuildNonIdentityRequestParamValues(requestParameters); + + for (auto& paramValue : BuildRequestParamValues(requestParameters, paramBindings.RequestMappings)) { + parametersToAdd.push_back(std::move(paramValue)); + } + + for (auto& paramValue : BuildClusterInfoParamValues(clusterInfo, paramBindings.ClusterInfoMappings)) { + parametersToAdd.push_back(std::move(paramValue)); + } + + for (auto& paramValue : BuildStaticParamValues(paramBindings.StaticMappings)) { + parametersToAdd.push_back(std::move(paramValue)); + } + + return parametersToAdd; +} + TVector<std::pair<TString, TString>> BuildRequestParamValues( const TCgiParameters& requestParameters, const TVector<std::pair<TString, TString>>& requestMappings) diff --git a/ydb/mvp/meta/support_links/param_bindings.h b/ydb/mvp/meta/support_links/param_bindings.h index 293c709671f..b354f322dc5 100644 --- a/ydb/mvp/meta/support_links/param_bindings.h +++ b/ydb/mvp/meta/support_links/param_bindings.h @@ -16,6 +16,10 @@ struct TResolvedParamBindings { TResolvedParamBindings ResolveParamBindings(const TSupportLinkEntryConfig& config, const TResolvedParamBindings& defaultParamBindings); void ValidateParamsAreUnique(const TResolvedParamBindings& paramBindings, const TSupportLinkEntryConfig& config); +TVector<std::pair<TString, TString>> BuildParametersToAdd( + const TCgiParameters& requestParameters, + const THashMap<TString, TString>& clusterInfo, + const TResolvedParamBindings& paramBindings); TVector<std::pair<TString, TString>> BuildRequestParamValues( const TCgiParameters& requestParameters, const TVector<std::pair<TString, TString>>& requestMappings); diff --git a/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.cpp b/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.cpp index b44b0b99e8d..1061f739f29 100644 --- a/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.cpp +++ b/ydb/public/sdk/cpp/src/client/impl/internal/db_driver_state/state.cpp @@ -331,7 +331,12 @@ NThreading::TFuture<void> TDbDriverStateTracker::SendNotification( } } if (results.empty()) { - return NThreading::MakeFuture(); + // MakeFuture<void>() uses a process-wide singleton that may already be + // destroyed when driver shutdown is triggered by another singleton. + auto promise = NThreading::NewPromise<void>(); + auto future = promise.GetFuture(); + promise.SetValue(); + return future; } return NThreading::WaitExceptionOrAll(results); } diff --git a/ydb/public/sdk/cpp/src/client/topic/ut/basic_usage_ut.cpp b/ydb/public/sdk/cpp/src/client/topic/ut/basic_usage_ut.cpp index b97d12c47e7..f566d66b5ff 100644 --- a/ydb/public/sdk/cpp/src/client/topic/ut/basic_usage_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/topic/ut/basic_usage_ut.cpp @@ -28,6 +28,7 @@ #include <library/cpp/string_utils/base64/base64.h> #include <atomic> +#include <mutex> #include <util/digest/murmur.h> #include <util/stream/zlib.h> @@ -81,9 +82,19 @@ void ReadMessagesAndAssertOrderedBySeqNo(TTopicClient& client, ui64 SeqNo; std::string Data; }; - std::vector<TMessageInfo> messages; - messages.reserve(expectedCount); - NThreading::TPromise<void> donePromise = NThreading::NewPromise<void>(); + // Handlers may still run after Close(); keep shared ownership so late callbacks are safe. + struct TState { + std::mutex Lock; + std::vector<TMessageInfo> Messages; + NThreading::TPromise<void> DonePromise = NThreading::NewPromise<void>(); + + std::vector<TMessageInfo> CopyMessages() { + std::lock_guard guard(Lock); + return Messages; + } + }; + auto state = std::make_shared<TState>(); + state->Messages.reserve(expectedCount); TTopicReadSettings topicSettings(topicPath); topicSettings.ReadFromTimestamp(TInstant::Zero()); @@ -93,25 +104,31 @@ void ReadMessagesAndAssertOrderedBySeqNo(TTopicClient& client, .AutoPartitioningSupport(true) .AppendTopics(topicSettings); - readSettings.EventHandlers_.SimpleDataHandlers([&](TReadSessionEvent::TDataReceivedEvent& ev) { + readSettings.EventHandlers_.SimpleDataHandlers([state, expectedCount](TReadSessionEvent::TDataReceivedEvent& ev) { + std::lock_guard guard(state->Lock); for (auto& msg : ev.GetMessages()) { - messages.push_back(TMessageInfo{ + if (state->Messages.size() >= expectedCount) { + break; + } + state->Messages.push_back(TMessageInfo{ msg.GetPartitionSession()->GetPartitionId(), TString(msg.GetProducerId()), msg.GetSeqNo(), TString(msg.GetData()), }); } - if (messages.size() >= expectedCount) { - donePromise.SetValue(); + if (state->Messages.size() >= expectedCount) { + state->DonePromise.TrySetValue(); } }, true); auto readSession = client.CreateReadSession(readSettings); - UNIT_ASSERT_C(donePromise.GetFuture().Wait(timeout), - "Expected to read " << expectedCount << " messages within " << timeout << ", got " << messages.size()); + UNIT_ASSERT_C(state->DonePromise.GetFuture().Wait(timeout), + "Expected to read " << expectedCount << " messages within " << timeout); readSession->Close(TDuration::Seconds(5)); + const auto messages = state->CopyMessages(); + UNIT_ASSERT_VALUES_EQUAL_C(messages.size(), expectedCount, "Read message count mismatch: got " << messages.size() << ", expected " << expectedCount); diff --git a/ydb/public/sdk/cpp/src/library/grpc/client/grpc_common.cpp b/ydb/public/sdk/cpp/src/library/grpc/client/grpc_common.cpp index 0a89f90fd68..73ef789a627 100644 --- a/ydb/public/sdk/cpp/src/library/grpc/client/grpc_common.cpp +++ b/ydb/public/sdk/cpp/src/library/grpc/client/grpc_common.cpp @@ -8,7 +8,6 @@ #include <openssl/pem.h> #include <openssl/ssl.h> #include <openssl/x509.h> -#include <openssl/x509v3.h> #include <memory> #include <string> @@ -55,36 +54,26 @@ bool ValidateRootCertificates(const std::string& pemRootCerts, std::string& erro size_t certsParsed = 0; while (true) { std::unique_ptr<X509, decltype(&X509_free)> cert( - PEM_read_bio_X509(rootCertsBio, nullptr, nullptr, nullptr), + PEM_read_bio_X509_AUX(rootCertsBio, nullptr, nullptr, nullptr), &X509_free); if (!cert) { - const unsigned long errorCode = ERR_peek_last_error(); - if (errorCode == 0 || ERR_GET_REASON(errorCode) == PEM_R_NO_START_LINE) { - ERR_clear_error(); - break; + if (certsParsed == 0) { + errorMessage = "root CA PEM: failed to parse certificate #1: " + DrainOpenSslErrors(); + return false; } - errorMessage = "root CA PEM: " + DrainOpenSslErrors(); - return false; - } - std::unique_ptr<BASIC_CONSTRAINTS, decltype(&BASIC_CONSTRAINTS_free)> basicConstraints( - static_cast<BASIC_CONSTRAINTS*>(X509_get_ext_d2i(cert.get(), NID_basic_constraints, nullptr, nullptr)), - &BASIC_CONSTRAINTS_free); - const auto isCaCert = basicConstraints && basicConstraints->ca; - - if (!isCaCert) { + // gRPC treats a read error as the end of the bundle and uses all + // certificates parsed before it. ERR_clear_error(); - errorMessage = "root CA PEM: certificate is not a CA (BasicConstraints)"; - return false; + break; } + + // Match gRPC trust store semantics: with X509_V_FLAG_PARTIAL_CHAIN an + // explicitly trusted non-CA certificate may also be a trust anchor. ERR_clear_error(); ++certsParsed; } - if (certsParsed == 0) { - errorMessage = "root CA PEM: no certificates parsed"; - return false; - } return true; } diff --git a/ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp b/ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp index d69edfdb76d..149b794b021 100644 --- a/ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp +++ b/ydb/public/sdk/cpp/tests/unit/client/driver/driver_ut.cpp @@ -4,6 +4,7 @@ #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/types/exceptions/exceptions.h> #include <ydb/public/sdk/cpp/include/ydb-cpp-sdk/type_switcher.h> #include <ydb/public/sdk/cpp/src/client/impl/observability/constants.h> +#include <ydb/public/sdk/cpp/src/library/grpc/client/grpc_common.h> #include <ydb/public/sdk/cpp/tests/common/fake_metric_registry.h> #include <ydb/public/sdk/cpp/tests/common/fake_trace_provider.h> @@ -29,6 +30,17 @@ using namespace NYdb::NTable; namespace { + constexpr const char LegacyV1Certificate[] = R"(-----BEGIN CERTIFICATE----- +MIIBbTCCARMCFBthJdWIg/H6ITeelffnCYoK8fDFMAoGCCqGSM49BAMCMDkxCzAJ +BgNVBAYTAlJVMQwwCgYDVQQKDANZREIxHDAaBgNVBAMME0xlZ2FjeSBUZXN0IFJv +b3QgQ0EwHhcNMjYwNzI3MDk0NDU2WhcNMzYwNzI0MDk0NDU2WjA5MQswCQYDVQQG +EwJSVTEMMAoGA1UECgwDWURCMRwwGgYDVQQDDBNMZWdhY3kgVGVzdCBSb290IENB +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4zlS2ha5hOd20QJEh17FP/mjkzsO +PmwF7iY9zJ0HILwBjqxJSCGnNMMdT+A2d+Nry6de3WC6RkR72HTe6gffuTAKBggq +hkjOPQQDAgNIADBFAiEA/0rBKAconmtFcliTZ0i9HzIkQeG+E/zVMiUvlhwpylYC +IGfPhGBVwOMnr+uhwtpj4PAOIrlOQD/fBsaRtYuBRdg2 +-----END CERTIFICATE-----)"; + std::string ReadBuildInfo(grpc::ServerContext* context) { const auto& metadata = context->client_metadata(); const auto it = metadata.find(YDB_SDK_BUILD_INFO_HEADER); @@ -237,6 +249,35 @@ Y_UNIT_TEST_SUITE(CppGrpcClientSimpleTest) { UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::TRANSPORT_UNAVAILABLE); UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Client TLS credentials validation failed"); UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "root CA PEM:"); + UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "failed to parse certificate #1"); + } + + Y_UNIT_TEST(LegacyV1TrustAnchorPassesValidation) { + auto driver = TDriver( + TDriverConfig() + .SetEndpoint("localhost:100") + .UseSecureConnection(LegacyV1Certificate)); + auto client = NTable::TTableClient(driver); + + auto result = client.CreateSession().GetValueSync(); + auto issues = result.GetIssues().ToString(); + + UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::TRANSPORT_UNAVAILABLE); + UNIT_ASSERT(issues.find("Client TLS credentials validation failed") == std::string::npos); + } + + Y_UNIT_TEST(MalformedCertificateAfterValidRootPassesValidation) { + const std::string rootBundle = std::string(LegacyV1Certificate) + R"( +-----BEGIN CERTIFICATE----- +not-base64 +-----END CERTIFICATE-----)"; + grpc::SslCredentialsOptions sslOptions{ + .pem_root_certs = NYdb::TStringType{rootBundle}, + }; + std::string validationDetail; + + UNIT_ASSERT(NYdbGrpc::ValidateTlsCredentials(sslOptions, validationDetail)); + UNIT_ASSERT(validationDetail.empty()); } Y_UNIT_TEST(EmptyRootCertificateWithoutClientCredentialsKeepsBehavior) { diff --git a/ydb/services/scheme_secret/resolver.h b/ydb/services/scheme_secret/resolver.h index 509d1edc6cf..99c55a7ea9d 100644 --- a/ydb/services/scheme_secret/resolver.h +++ b/ydb/services/scheme_secret/resolver.h @@ -29,6 +29,8 @@ NThreading::TFuture<NKqp::TEvDescribeSecretsResponse::TDescription> DescribeSecr TDescribeSecretSettings settings = {} ); +bool IsSchemeSecret(const TString& secretName); + bool UseSchemaSecrets(const NKikimr::TFeatureFlags& flags, const TVector<TString>& secretNames); bool UseSchemaSecrets(const NKikimr::TFeatureFlags& flags, const TString& secretName); diff --git a/ydb/services/scheme_secret/service.cpp b/ydb/services/scheme_secret/service.cpp index c48eb1ca178..8b4903c480b 100644 --- a/ydb/services/scheme_secret/service.cpp +++ b/ydb/services/scheme_secret/service.cpp @@ -666,6 +666,17 @@ NThreading::TFuture<NKqp::TEvDescribeSecretsResponse::TDescription> DescribeSecr return promise.GetFuture(); } + if (AppData()->FeatureFlags.GetDisableOldSecrets()) { + // Just in case - when we disable old secrets, we'll make sure they are not needed any more + promise.SetValue( + NKqp::TEvDescribeSecretsResponse::TDescription( + Ydb::StatusIds::BAD_REQUEST, + { NYql::TIssue("Usage of old secrets is disabled now. Please use new secrets") } + ) + ); + return promise.GetFuture(); + } + actorSystem->Register(CreateDescribeSecretsActor(userToken ? userToken->GetUserSID() : "", secretNames, promise)); return promise.GetFuture(); } @@ -674,13 +685,17 @@ IActor* TDescribeSchemaSecretsServiceFactory::CreateService() { return new TDescribeSchemaSecretsService(); } +bool IsSchemeSecret(const TString& secretName) { + return secretName.StartsWith('/'); +} + bool UseSchemaSecrets(const NKikimr::TFeatureFlags& flags, const TVector<TString>& secretNames) { if (!flags.GetEnableSchemaSecrets()) { return false; } for (const auto& secretName : secretNames) { - if (!secretName.StartsWith('/')) { + if (!IsSchemeSecret(secretName)) { return false; } } @@ -689,7 +704,7 @@ bool UseSchemaSecrets(const NKikimr::TFeatureFlags& flags, const TVector<TString } bool UseSchemaSecrets(const NKikimr::TFeatureFlags& flags, const TString& secretName) { - return flags.GetEnableSchemaSecrets() && secretName.StartsWith('/'); + return flags.GetEnableSchemaSecrets() && IsSchemeSecret(secretName); } } // namespace NKikimr::NSecret diff --git a/ydb/services/sqs_topic/create_queue.cpp b/ydb/services/sqs_topic/create_queue.cpp index 91a92178771..8d88c8e777d 100644 --- a/ydb/services/sqs_topic/create_queue.cpp +++ b/ydb/services/sqs_topic/create_queue.cpp @@ -169,9 +169,9 @@ namespace NKikimr::NSqsTopic::V1 { topicRequest.mutable_supported_codecs()->add_codecs(Ydb::Topic::CODEC_RAW); topicRequest.set_content_based_deduplication(QueueAttributes.ContentBasedDeduplication.GetOrElse(false)); - if (QueueAttributes.ContentBasedDeduplication.GetOrElse(false)) { - topicRequest.set_partition_write_speed_messages_per_second(NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT); - topicRequest.set_partition_write_burst_messages(NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST); + if (QueueAttributes.FifoQueue) { + topicRequest.set_partition_write_speed_messages_per_second(NPQ::FIFO_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND); + topicRequest.set_partition_write_burst_messages(NPQ::FIFO_PARTITION_WRITE_BURST_MESSAGES); } AddConsumerToRequest(topicRequest.add_consumers()); diff --git a/ydb/services/sqs_topic/send_message.cpp b/ydb/services/sqs_topic/send_message.cpp index 18ec9a1d736..3ab08bd8933 100644 --- a/ydb/services/sqs_topic/send_message.cpp +++ b/ydb/services/sqs_topic/send_message.cpp @@ -45,7 +45,9 @@ #include <ydb/services/sqs_topic/statuses.h> #include <library/cpp/digest/md5/md5.h> +#include <library/cpp/openssl/crypto/sha.h> #include <util/generic/guid.h> +#include <util/string/hex.h> using namespace NActors; using namespace NKikimrClient; @@ -135,7 +137,10 @@ namespace NKikimr::NSqsTopic::V1 { .Index = item.BatchIndex, .MessageBody = std::move(item.MessageBody), .MessageGroupId = toOptional(std::move(item.MessageGroupId)), - .MessageDeduplicationId = toOptional(std::move(item.MessageDeduplicationId)), + // MessageDeduplicationId is supported for FIFO queues only. + .MessageDeduplicationId = QueueUrl_->Fifo + ? toOptional(std::move(item.MessageDeduplicationId)) + : std::nullopt, .Attributes = std::move(item.Attributes), .Delay = TDuration::Seconds(item.DelaySeconds), }); @@ -225,7 +230,7 @@ namespace NKikimr::NSqsTopic::V1 { void Handle(TEvents::TEvWakeup::TPtr& ev) { switch (static_cast<EWakeupTag>(ev->Get()->Tag)) { case EWakeupTag::RlAllowed: - CreateWriter(); + CreateWriterOrReply(); return; case EWakeupTag::RlNoResource: return this->ReplyWithError(MakeError(NSQS::NErrors::THROTTLING_EXCEPTION, "Request was throttled by the rate limiter")); @@ -274,6 +279,40 @@ namespace NKikimr::NSqsTopic::V1 { return true; } + void ApplyContentBasedDeduplication(bool enabled) { + ContentBasedDeduplication_ = enabled; + if (!Fifo_ || !WriterSettings_) { + return; + } + + TVector<NPQ::NMLP::TWriterSettings::TMessage> validMessages(Reserve(WriterSettings_->Messages.size())); + for (auto& message : WriterSettings_->Messages) { + if (!message.MessageDeduplicationId.has_value()) { + if (enabled) { + const auto digest = NOpenSsl::NSha256::Calc(message.MessageBody); + message.MessageDeduplicationId = HexEncode(TStringBuf( + reinterpret_cast<const char*>(digest.data()), + digest.size())); + } else { + Items[message.Index].ValidationError = MakeError( + NSQS::NErrors::MISSING_PARAMETER, + "No MessageDeduplicationId parameter."); + continue; + } + } + validMessages.push_back(std::move(message)); + } + WriterSettings_->Messages = std::move(validMessages); + } + + void CreateWriterOrReply() { + if (!WriterSettings_ || WriterSettings_->Messages.empty()) { + static_cast<TDerived*>(this)->ReplyAndDie(TlsActivationContext->AsActorContext()); + return; + } + CreateWriter(); + } + void HandleCacheNavigateResponse(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { // Second navigate: resolve the rate-limiter path from the database // serverless attributes, then proceed with the write. @@ -281,12 +320,17 @@ namespace NKikimr::NSqsTopic::V1 { if (auto rlContext = this->ExtractRlContext(ev)) { SetRlContext(*rlContext); if (IsQuotaRequired()) { - const ui64 ru = NBilling::CalcRu(CalcRuConsumption(PayloadSize_), NBilling::WRITE_BASE_COST, NBilling::WRITE_COST_PER_BLOCK, Fifo_, false); + const ui64 ru = NBilling::CalcRu( + CalcRuConsumption(PayloadSize_), + NBilling::WRITE_BASE_COST, + NBilling::WRITE_COST_PER_BLOCK, + Fifo_, + ContentBasedDeduplication_); Y_ABORT_UNLESS(MaybeRequestQuota(ru, EWakeupTag::RlAllowed, TlsActivationContext->AsActorContext())); return; } } - CreateWriter(); + CreateWriterOrReply(); return; } @@ -308,17 +352,21 @@ namespace NKikimr::NSqsTopic::V1 { TStringBuilder() << "Failed to describe topic: " << response.Status)); } + Y_ABORT_UNLESS(response.PQGroupInfo); + Fifo_ = QueueUrl_->Fifo; + ApplyContentBasedDeduplication( + Fifo_ && response.PQGroupInfo->Description.GetPQTabletConfig().GetContentBasedDeduplication()); + if (ShouldBeCharged_) { // Always put in request units metering mode SetMeteringMode(NKikimrPQ::TPQTabletConfig::METERING_MODE_REQUEST_UNITS); - Fifo_ = QueueUrl_->Fifo; // RU-metered topics need the rate-limiter path, which is not carried // by DoLocalRpc requests. Resolve it from the database attributes // before writing so the charge in Handle(TEvWriteResponse) can fire. this->SendRlPathNavigate(); } else { - CreateWriter(); + CreateWriterOrReply(); } } @@ -369,6 +417,7 @@ namespace NKikimr::NSqsTopic::V1 { TActorId WriterActor_; ui64 PayloadSize_{}; bool Fifo_{}; + bool ContentBasedDeduplication_ = false; TMaybe<NPQ::NMLP::TWriterSettings> WriterSettings_; }; diff --git a/ydb/services/sqs_topic/set_queue_attributes.cpp b/ydb/services/sqs_topic/set_queue_attributes.cpp index faede31ff72..8c5a429093f 100644 --- a/ydb/services/sqs_topic/set_queue_attributes.cpp +++ b/ydb/services/sqs_topic/set_queue_attributes.cpp @@ -6,7 +6,6 @@ #include "utils.h" #include <ydb/core/http_proxy/events.h> -#include <ydb/core/persqueue/public/constants.h> #include <ydb/core/protos/grpc_pq_old.pb.h> #include <ydb/core/ymq/base/limits.h> #include <ydb/core/ymq/error/error.h> @@ -188,13 +187,6 @@ namespace NKikimr::NSqsTopic::V1 { if (NewQueueAttributes.ContentBasedDeduplication.Defined()) { topicRequest.set_set_content_based_deduplication(*NewQueueAttributes.ContentBasedDeduplication); - if (*NewQueueAttributes.ContentBasedDeduplication) { - topicRequest.set_set_partition_write_speed_messages_per_second(NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_LIMIT); - topicRequest.set_set_partition_write_burst_messages(NPQ::CONTENT_BASED_DEDUPLICATION_MESSAGE_BURST); - } else { - topicRequest.set_set_partition_write_speed_messages_per_second(NPQ::DEFAULT_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND); - topicRequest.set_set_partition_write_burst_messages(NPQ::DEFAULT_PARTITION_WRITE_SPEED_MESSAGES_PER_SECOND); - } } auto* consumer = topicRequest.add_alter_consumers(); diff --git a/ydb/services/sqs_topic/ya.make b/ydb/services/sqs_topic/ya.make index d25a6329ce9..c3f3fd3f643 100644 --- a/ydb/services/sqs_topic/ya.make +++ b/ydb/services/sqs_topic/ya.make @@ -51,6 +51,7 @@ PEERDIR( ydb/core/ymq/base ydb/core/ymq/error library/cpp/json + library/cpp/openssl/crypto ) END() diff --git a/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/manager.cpp b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/manager.cpp index 4385b72fba8..602deaa82ca 100644 --- a/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/manager.cpp +++ b/ydb/services/workload_manager/metadata_subscription/resource_pool_classifier/manager.cpp @@ -59,6 +59,9 @@ NMetadata::NModifications::TOperationParsingResult TResourcePoolClassifierManage if (property == "resource_pool") { return TConclusionStatus::Fail("Cannot reset required property resource_pool"); } + if (property == "rank") { + return TConclusionStatus::Fail("Cannot reset property rank"); + } } else { continue; } diff --git a/ydb/services/workload_manager/ut/workload_service_ut.cpp b/ydb/services/workload_manager/ut/workload_service_ut.cpp index d48bc6a6fad..1bb372f97f6 100644 --- a/ydb/services/workload_manager/ut/workload_service_ut.cpp +++ b/ydb/services/workload_manager/ut/workload_service_ut.cpp @@ -997,9 +997,7 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifiersDdl) { ALTER RESOURCE POOL CLASSIFIER )" << classifierId << R"( RESET ( RANK ); - )"); - - WaitForFail(ydb, settings, poolId); + )", NYdb::EStatus::GENERIC_ERROR, "Cannot reset property rank"); } Y_UNIT_TEST(TestExplicitPoolId) { diff --git a/ydb/tests/fq/generic/utils/settings.py b/ydb/tests/fq/generic/utils/settings.py index 94603e213d2..18508fd587e 100644 --- a/ydb/tests/fq/generic/utils/settings.py +++ b/ydb/tests/fq/generic/utils/settings.py @@ -24,7 +24,7 @@ class Settings: endpoint: str hmac_secret_file: str - token_accessor_mock: TokenAccessorMock + token_accessor_mock: Optional[TokenAccessorMock] = None @dataclass class MdbMock: @@ -88,11 +88,12 @@ class Settings: grpc_host='localhost', grpc_port=endpoint_determiner.get_port('fq-connector-go', 2130), ), - token_accessor_mock=cls.TokenAccessorMock( + ) + if "TOKEN_ACCESSOR_MOCK_ENDPOINT" in environ.keys(): + s.token_accessor_mock = cls.TokenAccessorMock( endpoint=environ['TOKEN_ACCESSOR_MOCK_ENDPOINT'], hmac_secret_file=environ['TOKEN_ACCESSOR_HMAC_SECRET_FILE'], - ), - ) + ) if 'MDB_MOCK_ENDPOINT' in environ.keys(): s.mdb_mock = cls.MdbMock( diff --git a/ydb/tests/fq/streaming/generic/conftest.py b/ydb/tests/fq/streaming/generic/conftest.py new file mode 100644 index 00000000000..ee576b2f33b --- /dev/null +++ b/ydb/tests/fq/streaming/generic/conftest.py @@ -0,0 +1,34 @@ +import pytest +import random +import string +from typing import Final + +from ydb.tests.fq.streaming_common.common import Kikimr +from ydb.tests.fq.streaming_common.common import get_ydb_config +from ydb.tests.fq.streaming_common.common import set_test_env +from ydb.tests.fq.generic.utils.settings import Settings + +docker_compose_file_path: Final = "ydb/tests/fq/streaming/generic/docker-compose.yml" + + +def settings() -> Settings: + return Settings.from_env(docker_compose_file_path=docker_compose_file_path) + + +def kikimr(request, settings: Settings): + set_test_env(request) + kikimr = Kikimr(get_ydb_config(request, enable_fq_connector=settings)) + yield kikimr + kikimr.stop() + + +def entity_name(request): + suffix = ''.join(random.choices(string.ascii_letters + string.digits, k=8)) + + def entity_name_wrapper(name: str) -> str: + return f"{name}_{suffix}" + + return entity_name_wrapper diff --git a/ydb/tests/fq/streaming/generic/connector/Dockerfile b/ydb/tests/fq/streaming/generic/connector/Dockerfile new file mode 100644 index 00000000000..278ab0c7280 --- /dev/null +++ b/ydb/tests/fq/streaming/generic/connector/Dockerfile @@ -0,0 +1,12 @@ +ARG BASE_IMAGE=ghcr.io/ydb-platform/fq-connector-go:v0.11.0-rc.1@sha256:58a1e12de21ef3b403450b176b6220fd545d2802ce16dbc3b05ec4c91e075cdc +FROM $BASE_IMAGE + +COPY ./fq-connector-go.yaml /fq-connector-go.yaml +COPY ./entrypoint.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] + +HEALTHCHECK --interval=10s --timeout=300s --retries=30 \ + CMD cat /var/log/log.txt | grep 'server/service_connector.go:182\tstarting GRPC server' || exit 1 + + diff --git a/ydb/tests/fq/streaming/generic/connector/entrypoint.sh b/ydb/tests/fq/streaming/generic/connector/entrypoint.sh new file mode 100755 index 00000000000..9c93d884edf --- /dev/null +++ b/ydb/tests/fq/streaming/generic/connector/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# echo "$(dig tests-fq-streaming-generic-ydb +short) tests-fq-generic-streaming-ydb" >> /etc/hosts +sed '/\(127.0.0.1\|::1\).*localhost/s/^/# /' /etc/hosts >/tmp/hosts +cat /tmp/hosts > /etc/hosts +cat /etc/hosts + +/opt/ydb/bin/fq-connector-go server -c /fq-connector-go.yaml 2>&1 | tee /var/log/log.txt diff --git a/ydb/tests/fq/streaming/generic/connector/fq-connector-go.yaml b/ydb/tests/fq/streaming/generic/connector/fq-connector-go.yaml new file mode 100644 index 00000000000..851da84b60b --- /dev/null +++ b/ydb/tests/fq/streaming/generic/connector/fq-connector-go.yaml @@ -0,0 +1,60 @@ +connector_server: + endpoint: + host: "0.0.0.0" + port: 2130 + +logger: + log_level: DEBUG + enable_sql_query_logging: true + +metrics_server: + endpoint: + host: "0.0.0.0" + port: 8766 + +pprof_server: + endpoint: + host: "0.0.0.0" + port: 6060 + +paging: + bytes_per_page: 4194304 + prefetch_queue_capacity: 2 + +conversion: + use_unsafe_converters: true + +data_source_default: &data_source_default_var + open_connection_timeout: 5s + ping_connection_timeout: 5s + exponential_backoff: + initial_interval: 100ms + randomization_factor: 0.5 + multiplier: 1.5 + max_interval: 10s + max_elapsed_time: 10m + +datasources: + clickhouse: + <<: *data_source_default_var + + greenplum: + <<: *data_source_default_var + + ms_sql_server: + <<: *data_source_default_var + + mysql: + <<: *data_source_default_var + result_chan_capacity: 1024 + + postgresql: + <<: *data_source_default_var + + oracle: + <<: *data_source_default_var + + ydb: + <<: *data_source_default_var + use_underlay_network_for_dedicated_databases: true + mode: MODE_QUERY_SERVICE_NATIVE diff --git a/ydb/tests/fq/streaming/generic/docker-compose.yml b/ydb/tests/fq/streaming/generic/docker-compose.yml new file mode 100644 index 00000000000..6aab5bcdc86 --- /dev/null +++ b/ydb/tests/fq/streaming/generic/docker-compose.yml @@ -0,0 +1,10 @@ +services: + fq-connector-go: + build: ./connector + container_name: tests-fq-streaming-generic-fq-connector-go + ports: + - 2130 + extra_hosts: + - "localhost:host-gateway" + +version: "2.4" diff --git a/ydb/tests/fq/streaming/generic/test_iam_generic.py b/ydb/tests/fq/streaming/generic/test_iam_generic.py new file mode 100644 index 00000000000..f92a97f1de1 --- /dev/null +++ b/ydb/tests/fq/streaming/generic/test_iam_generic.py @@ -0,0 +1,244 @@ +import json +import pytest +import logging +import time +import datetime +from typing import Callable + +from ydb.tests.fq.streaming_common.common import Kikimr, StreamingTestBase +from ydb.tests.tools.datastreams_helpers.control_plane import Endpoint +import ydb.issues + +logger = logging.getLogger(__name__) + +# Random but stable service-account-id placeholder; real validation is done server-side. +FAKE_SERVICE_ACCOUNT_ID = "aje00000000000000000" + +# The token that the IAM emulator returns by default. +USER_TOKEN = "root@builtin" + + +class TestIamAuthGeneric(StreamingTestBase): + """Verify that a generic source works with AUTH_METHOD=IAM works end-to-end with connector.""" + + def create_iam_secret(self, kikimr: Kikimr, secret_name: str) -> None: + kikimr.ydb_client.query(f""" + CREATE SECRET `{secret_name}` WITH (value="{USER_TOKEN}"); + """) + + def set_cloud_id(self, kikimr: Kikimr, cloud_id: str = "test-cloud-id") -> None: + """Set cloud_id user attribute on the database root path (/Root). + + DescribeResourceId reads GetAttributes() of the database root, so we must + use ESchemeOpAlterUserAttributes rather than ALTER TABLE which only supports + table-level settings. + """ + kikimr.cluster.client.add_attr("/", "Root", {"cloud_id": cloud_id}, token="root@builtin") + + def create_iam_source( + self, + kikimr: Kikimr, + source_name: str, + secret_path: str, + endpoint: Endpoint, + shared_reading: bool = False, + service_account_id: str = FAKE_SERVICE_ACCOUNT_ID + ) -> None: + """Create an External Data Source that authenticates via IAM.""" + kikimr.ydb_client.query(f""" + CREATE EXTERNAL DATA SOURCE `{source_name}` WITH ( + SOURCE_TYPE = "Ydb", + LOCATION = "{endpoint.endpoint}", + DATABASE_NAME = "{endpoint.database}", + USE_TLS = "FALSE", + AUTH_METHOD = "IAM", + INITIAL_TOKEN_SECRET_PATH = "{secret_path}", + SERVICE_ACCOUNT_ID = "{service_account_id}", + SHARED_READING="{shared_reading}" + ); + """) + + def create_table( + self, + kikimr: Kikimr, + table_name: str, + ) -> None: + kikimr.ydb_client.query(f"CREATE TABLE `{table_name}` (a INT, b STRING, c Bool, d Timestamp, e Interval, PRIMARY KEY(a, b))") + kikimr.ydb_client.query(f"UPSERT INTO `{table_name}` (a, b, c, d, e) VALUES (1, 'abc', false, Timestamp('2025-08-21T11:22:33.456789Z'), Interval('PT1M'))") + kikimr.ydb_client.query(f"UPSERT INTO `{table_name}` (a, b, c, d, e) VALUES (2, 'abcdefghijklmnoprstuvwxyz', true, Timestamp('2025-08-21T22:33:44.567Z'), Interval('PT10S'))") + + @pytest.mark.parametrize( + "service_account_id", [FAKE_SERVICE_ACCOUNT_ID, "bad", "bad-token", "bad-skip-1", "bad-token-skip-1"] + ) + def test_generic_read_iam_auth( + self, + kikimr: Kikimr, + service_account_id: str, + entity_name: Callable[[str], str], + ) -> None: + """Creates and populates local table, and read it from query + via an IAM-auth external data source.""" + + endpoint = self.get_endpoint(kikimr, local_topics=True) + source_name = entity_name("iam_source") + table_name = entity_name("iam_table") + + # 1. Create the secret and set cloud_id on the database root. + secret_name = entity_name("iam_secret") + self.create_iam_secret(kikimr, secret_name) + self.set_cloud_id(kikimr) + time.sleep(1) + + # 2. Create and populate local table + self.create_table(kikimr, table_name) + + # 3. Create IAM-auth external data source. + self.create_iam_source(kikimr, source_name, secret_name, endpoint, service_account_id=service_account_id) + + tab = f"`{source_name}`.`{table_name}`" + + # 3. Read from IAM-auth source, verify results + try: + result = kikimr.ydb_client.query(f"SELECT * FROM {tab} ORDER BY a, b") + except ydb.issues.Error as ex: + assert service_account_id != FAKE_SERVICE_ACCOUNT_ID, ex + if service_account_id == 'bad': + assert 'Reject bad SA' in ex.message + if service_account_id == 'bad-token': + assert 'Access denied' in ex.message + logger.debug(ex) + kikimr.ydb_client.query(f"DROP EXTERNAL DATA SOURCE `{source_name}`;") + return + + assert service_account_id == FAKE_SERVICE_ACCOUNT_ID, "Unexpected success" + + assert result + rows = result[0].rows + logger.debug(rows) + # note: Interval type is currently not supported by fq connector and silently ignored + expected = [ + {'a': 1, 'b': b'abc', 'c': False, 'd': datetime.datetime(2025, 8, 21, 11, 22, 33, 456789)}, + {'a': 2, 'b': b'abcdefghijklmnoprstuvwxyz', 'c': True, 'd': datetime.datetime(2025, 8, 21, 22, 33, 44, 567000)}, + ] + assert rows == expected + + result = kikimr.ydb_client.query(f"SELECT * FROM {tab} WHERE a = 1 ORDER BY a, b") + expected = [*filter(lambda x: x["a"] == 1, expected)] + assert result + rows = result[0].rows + logger.debug(rows) + assert rows == expected + + kikimr.ydb_client.query(f"DROP TABLE `{table_name}`;") + kikimr.ydb_client.query(f"DROP EXTERNAL DATA SOURCE `{source_name}`;") + + @pytest.mark.parametrize( + "kikimr", + [ + {"enable_dq_source_stream_lookup_join": True}, + ], + indirect=["kikimr"], + ) + @pytest.mark.parametrize( + "service_account_id", [FAKE_SERVICE_ACCOUNT_ID, "bad", "bad-token", "bad-skip-1", "bad-token-skip-1"] + ) + def test_generic_lookup_iam_auth( + self, + kikimr: Kikimr, + service_account_id: str, + entity_name: Callable[[str], str], + ) -> None: + """Creates and populates local table, and read it from query + via an IAM-auth external data source.""" + + endpoint = self.get_endpoint(kikimr, local_topics=True) + source_name = entity_name("iam_source") + table_name = entity_name("iam_table") + query_name = entity_name("iam_query") + ttl = 1 + + # 1. Create the secret and set cloud_id on the database root. + secret_name = entity_name("iam_secret") + self.create_iam_secret(kikimr, secret_name) + self.set_cloud_id(kikimr) + time.sleep(1) + # 2. Create and populate local table + self.create_table(kikimr, table_name) + # 3. Create topics + self.init_topics(source_name, create_output=True, partitions_count=2, endpoint=endpoint) + # 4. Create IAM-auth external data source. + self.create_iam_source(kikimr, source_name, secret_name, endpoint, service_account_id=service_account_id) + tab = f"`{source_name}`.`{table_name}`" + inp = f"`{source_name}`.`{self.input_topic}`" + out = f"`{source_name}`.`{self.output_topic}`" + + # 5. Join data from topic and lookup using external source with IAM-auth + if service_account_id != FAKE_SERVICE_ACCOUNT_ID: + try: + messages = ['1', '2', '1', '2', '3'] + self.write_stream(messages, endpoint=endpoint) + kikimr.ydb_client.query(Rf""" + SELECT Data, a, b, c, CAST(d AS String) AS d + FROM {inp} AS i + LEFT JOIN /*+streamlookup(TTL {ttl} FullscanLimit 0)*/ ANY {tab} AS db + ON CAST(i.Data AS Int) = db.a + LIMIT 1 + """) + except ydb.issues.Error as ex: + if service_account_id == 'bad': + assert 'Reject bad SA' in ex.message + if service_account_id == 'bad-token': + assert 'Access denied' in ex.message + logger.info(ex) + logger.info(type(ex)) + logger.info(ex.args) + kikimr.ydb_client.query(f"DROP TABLE `{table_name}`;") + kikimr.ydb_client.query(f"DROP EXTERNAL DATA SOURCE `{source_name}`;") + return + assert False, "Unexpected success" + + kikimr.ydb_client.query(Rf""" + CREATE STREAMING QUERY `{query_name}` AS + DO BEGIN + INSERT INTO {out} SELECT UNWRAP(Yson2::SerializeJson(Yson2::From(TableRow()))) + FROM ( + SELECT Data, a, b, c, CAST(d AS String) AS d + FROM {inp} AS i + LEFT JOIN /*+streamlookup(TTL {ttl} FullscanLimit 0)*/ ANY {tab} AS db + ON CAST(i.Data AS Int) = db.a + ) + END DO + """) + + path = f"/Root/{query_name}" + self.wait_completed_checkpoints(kikimr, path) + messages = ['1', '2', '1', '2', '3'] + self.write_stream(messages, endpoint=endpoint) + expected = [ + '{"Data":"1","a":1,"b":"abc","c":false,"d":"2025-08-21T11:22:33.456789Z"}', + '{"Data":"2","a":2,"b":"abcdefghijklmnoprstuvwxyz","c":true,"d":"2025-08-21T22:33:44.567000Z"}', + ] * 2 + [ + '{"Data":"3","a":null,"b":null,"c":null,"d":null}', + ] + result = self.read_stream(len(expected), topic_path=self.output_topic, endpoint=endpoint) + logger.debug([*map(json.loads, sorted(result))]) + assert sorted(result) == sorted(expected) + + time.sleep(ttl) # at least TTL + + kikimr.ydb_client.query(f"UPDATE `{table_name}` SET c = not c") + expected = [ + '{"Data":"1","a":1,"b":"abc","c":true,"d":"2025-08-21T11:22:33.456789Z"}', + '{"Data":"2","a":2,"b":"abcdefghijklmnoprstuvwxyz","c":false,"d":"2025-08-21T22:33:44.567000Z"}', + ] * 2 + [ + '{"Data":"3","a":null,"b":null,"c":null,"d":null}', + ] + self.write_stream(messages, endpoint=endpoint) + result = self.read_stream(len(expected), topic_path=self.output_topic, endpoint=endpoint) + logger.debug([*map(json.loads, sorted(result))]) + assert sorted(result) == sorted(expected) + logger.debug(kikimr.ydb_client.query("SELECT * FROM `.sys/streaming_queries`")[0].rows) + + kikimr.ydb_client.query(f"DROP STREAMING QUERY `{query_name}`;") + kikimr.ydb_client.query(f"DROP TABLE `{table_name}`;") + kikimr.ydb_client.query(f"DROP EXTERNAL DATA SOURCE `{source_name}`;") diff --git a/ydb/tests/fq/streaming/generic/ya.make b/ydb/tests/fq/streaming/generic/ya.make new file mode 100644 index 00000000000..8b49475e90e --- /dev/null +++ b/ydb/tests/fq/streaming/generic/ya.make @@ -0,0 +1,51 @@ +PY3TEST() + +INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/fq_runner/ydb_runner_with_datastreams.inc) +INCLUDE(${ARCADIA_ROOT}/ydb/tests/fq/streaming_common/vm_metadata_emulator/recipe/recipe.inc) +INCLUDE(${ARCADIA_ROOT}/ydb/tests/fq/streaming_common/iam_grpc_emulator/recipe/recipe.inc) + +DATA(arcadia/ydb/library/yql/providers/generic/connector/tests/fq-connector-go) +ENV(COMPOSE_HTTP_TIMEOUT=1200) # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose +INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) + +TEST_SRCS( + test_iam_generic.py +) + +PY_SRCS( + conftest.py +) + +IF (SANITIZER_TYPE) + SIZE(LARGE) + INCLUDE(${ARCADIA_ROOT}/ydb/tests/large.inc) + REQUIREMENTS(ram:20) +ELSE() + REQUIREMENTS(ram:12) +ENDIF() + +PEERDIR( + ydb/tests/library + ydb/tests/library/test_meta + ydb/public/sdk/python + ydb/public/sdk/python/enable_v3_new_behavior + library/recipes/common + ydb/tests/olap/common + ydb/tests/tools/datastreams_helpers + ydb/tests/fq/streaming_common + yql/essentials/providers/common/proto + ydb/library/yql/providers/generic/connector/tests/utils + ydb/tests/fq/generic/utils + library/python/testing/recipe + library/python/testing/yatest_common + library/recipes/common + ydb/public/api/protos + contrib/python/pytest +) + +DEPENDS( + ydb/apps/ydb + ydb/tests/tools/pq_read +) + +END() diff --git a/ydb/tests/fq/streaming/ya.make b/ydb/tests/fq/streaming/ya.make index 3e2b1c6d5d2..2e75ce30fcc 100644 --- a/ydb/tests/fq/streaming/ya.make +++ b/ydb/tests/fq/streaming/ya.make @@ -51,3 +51,8 @@ DEPENDS( ) END() + +RECURSE_FOR_TESTS( + streaming_large + generic +) diff --git a/ydb/tests/fq/streaming_common/common.py b/ydb/tests/fq/streaming_common/common.py index 0ffa856006d..806d6d4ce11 100644 --- a/ydb/tests/fq/streaming_common/common.py +++ b/ydb/tests/fq/streaming_common/common.py @@ -29,7 +29,7 @@ def set_test_env(request): os.environ["YDB_TEST_ROW_DISPATCHER_REBALANCING_TIMEOUT_MS"] = rebalancing_timeout_ms -def get_ydb_config(request): +def get_ydb_config(request, enable_fq_connector=None): param = getattr(request, "param", {}) enable_watermarks = param.get("enable_watermarks", True) enable_watermarks_advanced = param.get("enable_watermarks_advanced", True) @@ -37,6 +37,7 @@ def get_ydb_config(request): enable_streaming_queries = param.get("enable_streaming_queries", True) enable_streaming_partition_balancing = param.get("use_partition_balancing", True) enable_user_attributes_in_topic_query = param.get("enable_user_attributes_in_topic_query", True) + enable_dq_source_stream_lookup_join = param.get("enable_dq_source_stream_lookup_join", True) extra_feature_flags = { "enable_external_data_sources", @@ -72,6 +73,7 @@ def get_ydb_config(request): "enable_streaming_partition_balancing": enable_streaming_partition_balancing, "enable_compile_cache_warmup": False, "enable_channel_memory_tracking": False, # Remove after fix https://github.com/ydb-platform/ydb/issues/46891 + "enable_dq_source_stream_lookup_join": enable_dq_source_stream_lookup_join, }, replication_config={ "iam_service_control": { @@ -86,6 +88,17 @@ def get_ydb_config(request): use_in_memory_pdisks=False, ) + if enable_fq_connector: + config.yaml_config["query_service_config"]["generic"] = { + "connector": { + "use_ssl": False, + "endpoint": { + "host": enable_fq_connector.connector.grpc_host, + "port": enable_fq_connector.connector.grpc_port, + }, + }, + } + config.yaml_config["log_config"]["default_level"] = 8 if "auth_config" not in config.yaml_config: config.yaml_config["auth_config"] = {} diff --git a/ydb/tests/fq/streaming_common/iam_grpc_emulator/bin/main.py b/ydb/tests/fq/streaming_common/iam_grpc_emulator/bin/main.py index 24ea510101d..a5dbd08172a 100644 --- a/ydb/tests/fq/streaming_common/iam_grpc_emulator/bin/main.py +++ b/ydb/tests/fq/streaming_common/iam_grpc_emulator/bin/main.py @@ -1,6 +1,6 @@ import argparse import logging -import random +# import random import time from concurrent import futures @@ -41,6 +41,8 @@ class IamTokenServicer(iam_token_service_pb2_grpc.IamTokenServiceServicer): def __init__(self, token, expires_in): self.token = token self.expires_in = expires_in + self.calls = 0 + self.token_calls = 0 def _pick_token(self): # if random.random() < 0.1: @@ -57,12 +59,45 @@ class IamTokenServicer(iam_token_service_pb2_grpc.IamTokenServiceServicer): return make_response(self._pick_token(), self.expires_in) def CreateForService(self, request, context): - token =self._pick_token() + token = self._pick_token() logger.debug( "IamTokenService.CreateForService called, service_id=%s microservice_id=%s resource_id=%s target_sa=%s token=%s", request.service_id, request.microservice_id, request.resource_id, request.target_service_account_id, token ) - return make_response(token, self.expires_in) + + target_sa = request.target_service_account_id + expires_in = self.expires_in + + if target_sa == 'bad': + context.set_code(grpc.StatusCode.PERMISSION_DENIED) + context.set_details("Reject bad SA") + return iam_token_service_pb2.CreateIamTokenResponse() + + if target_sa == 'bad-token': + return make_response("badtoken@builtin", expires_in) + + if target_sa.startswith('bad-skip-'): + skips = int(target_sa.split('-')[-1]) + self.calls += 1 + self.calls %= skips + 1 + expires_in = 0 + if self.calls == 0: + context.set_code(grpc.StatusCode.PERMISSION_DENIED) + context.set_details("Reject bad SA") + return iam_token_service_pb2.CreateIamTokenResponse() + + if target_sa.startswith('bad-token-skip-'): + skips = int(target_sa.split('-')[-1]) + self.token_calls += 1 + self.token_calls %= skips + 1 + expires_in = 0 + if self.token_calls == 0: + return make_response("badtoken@builtin", expires_in) + + if target_sa == 'bad-token': + return make_response("badtoken@builtin", expires_in) + + return make_response(token, expires_in) def main(): diff --git a/ydb/tests/fq/streaming_common/ya.make b/ydb/tests/fq/streaming_common/ya.make index e9fb4b5c950..1bcaf647524 100644 --- a/ydb/tests/fq/streaming_common/ya.make +++ b/ydb/tests/fq/streaming_common/ya.make @@ -13,3 +13,7 @@ PEERDIR( END() +RECURSE( + iam_grpc_emulator + vm_metadata_emulator +) diff --git a/ydb/tests/fq/ya.make b/ydb/tests/fq/ya.make index cf5aac05187..731a82476f8 100644 --- a/ydb/tests/fq/ya.make +++ b/ydb/tests/fq/ya.make @@ -18,5 +18,4 @@ RECURSE_FOR_TESTS( s3 streaming yds - streaming/streaming_large ) diff --git a/ydb/tests/functional/secrets/test_old_secrets_usage.py b/ydb/tests/functional/secrets/test_old_secrets_usage.py index e9622c85163..bb7cd703f8b 100644 --- a/ydb/tests/functional/secrets/test_old_secrets_usage.py +++ b/ydb/tests/functional/secrets/test_old_secrets_usage.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- import logging +import os +import boto3 import pytest +import requests from ydb.tests.library.common.wait_for import wait_for from ydb.tests.library.harness.kikimr_runner import KiKiMR @@ -11,6 +14,29 @@ from ydb.tests.oss.ydb_sdk_import import ydb logger = logging.getLogger(__name__) DATABASE = "/Root" +OLD_SECRETS_CREATION_DISABLED_MESSAGE = "Old secrets creation syntax is disabled now. Please use the new one" +CREATION_WITH_OLD_SECRETS_DISABLED_MESSAGE = ( + "Old secrets are disabled for creating new objects. Please use new secrets" +) +OLD_SECRETS_USAGE_DISABLED_MESSAGE = "Usage of old secrets is disabled now. Please use new secrets" + + +def setup_s3(): + s3_endpoint = os.getenv("S3_ENDPOINT") + s3_access_key = "minio" + s3_secret_key = "minio123" + s3_bucket = "test_bucket_old_secrets" + + resource = boto3.resource( + "s3", endpoint_url=s3_endpoint, aws_access_key_id=s3_access_key, aws_secret_access_key=s3_secret_key + ) + + bucket = resource.Bucket(s3_bucket) + bucket.create() + bucket.objects.all().delete() + bucket.put_object(Key="file.txt", Body="Hello S3!") + + return s3_endpoint, s3_access_key, s3_secret_key, s3_bucket class Utils: @@ -23,14 +49,19 @@ class Utils: self.driver = None self.session_pool = None + self.query_session_pool = None self._start_client() def _start_client(self): self.driver = ydb.Driver(endpoint=self.cluster.nodes[1].endpoint, database=DATABASE) self.driver.wait(5, fail_fast=True) self.session_pool = ydb.SessionPool(self.driver) + self.query_session_pool = ydb.QuerySessionPool(self.driver) def _stop_client(self): + if self.query_session_pool is not None: + self.query_session_pool.stop() + self.query_session_pool = None if self.session_pool is not None: self.session_pool.stop() self.session_pool = None @@ -42,9 +73,15 @@ class Utils: self._stop_client() self.cluster.stop() - def restart_cluster(self, disable_old_secret_creation, enable_schema_secrets=True): + def restart_cluster( + self, + disable_old_secret_creation=False, + disable_old_secrets=False, + enable_schema_secrets=True, + ): self._stop_client() self.config.yaml_config["feature_flags"]["disable_old_secret_creation"] = disable_old_secret_creation + self.config.yaml_config["feature_flags"]["disable_old_secrets"] = disable_old_secrets self.config.yaml_config["feature_flags"]["enable_schema_secrets"] = enable_schema_secrets self.cluster.update_configurator_and_restart(self.config) self._start_client_after_restart() @@ -68,34 +105,167 @@ class Utils: self.driver = driver_holder["driver"] self.session_pool = ydb.SessionPool(self.driver) + self.query_session_pool = ydb.QuerySessionPool(self.driver) - def create_old_secret(self, secret_name, value): + def execute_scheme(self, query): with self.session_pool.checkout() as session: - session.execute_scheme(f"CREATE OBJECT {secret_name} (TYPE SECRET) WITH value='{value}';") + session.execute_scheme(query) + + def execute_query(self, query): + return self.query_session_pool.execute_with_retries(query) + + def create_old_secret(self, secret_name, value): + self.execute_scheme(f"CREATE OBJECT {secret_name} (TYPE SECRET) WITH value='{value}';") def upsert_old_secret(self, secret_name, value): - with self.session_pool.checkout() as session: - session.execute_scheme(f"UPSERT OBJECT {secret_name} (TYPE SECRET) WITH value='{value}';") + self.execute_scheme(f"UPSERT OBJECT {secret_name} (TYPE SECRET) WITH value='{value}';") def alter_old_secret(self, secret_name, value): - with self.session_pool.checkout() as session: - session.execute_scheme(f"ALTER OBJECT {secret_name} (TYPE SECRET) SET value='{value}';") + self.execute_scheme(f"ALTER OBJECT {secret_name} (TYPE SECRET) SET value='{value}';") - def create_eds(self, eds_name, secret_name): - with self.session_pool.checkout() as session: - query = f""" - CREATE EXTERNAL DATA SOURCE `{eds_name}` WITH ( - SOURCE_TYPE="ObjectStorage", - LOCATION="my-bucket", - AUTH_METHOD="SERVICE_ACCOUNT", - SERVICE_ACCOUNT_ID="mysa", - SERVICE_ACCOUNT_SECRET_NAME="{secret_name}" - );""" - session.execute_scheme(query) + def create_eds(self, eds_name, secret_name, schema_secret=False, create_or_replace=False): + secret_setting = "SERVICE_ACCOUNT_SECRET_PATH" if schema_secret else "SERVICE_ACCOUNT_SECRET_NAME" + create_type = "CREATE OR REPLACE" if create_or_replace else "CREATE" + query = f""" + {create_type} EXTERNAL DATA SOURCE `{eds_name}` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="my-bucket", + AUTH_METHOD="SERVICE_ACCOUNT", + SERVICE_ACCOUNT_ID="mysa", + {secret_setting}="{secret_name}" + );""" + self.execute_scheme(query) + + def create_eds_aws(self, eds_name, access_key_secret, secret_key_secret, s3_location): + query = f""" + CREATE EXTERNAL DATA SOURCE `{eds_name}` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="{s3_location}", + AUTH_METHOD="AWS", + AWS_ACCESS_KEY_ID_SECRET_NAME="{access_key_secret}", + AWS_SECRET_ACCESS_KEY_SECRET_NAME="{secret_key_secret}", + AWS_REGION="ru-central-1" + );""" + self.execute_scheme(query) + + def alter_password_secret(self, object_type, object_name, secret_name, schema_secret=False): + secret_setting = "PASSWORD_SECRET_PATH" if schema_secret else "PASSWORD_SECRET_NAME" + self.execute_scheme(f""" + ALTER {object_type} `{object_name}` SET (STATE = "Paused"); + ALTER {object_type} `{object_name}` SET ({secret_setting} = "{secret_name}"); + ALTER {object_type} `{object_name}` SET (STATE = "StandBy"); + """) + + def read_from_eds(self, eds_name): + return self.execute_query(f""" + SELECT * FROM `{eds_name}`.`file.txt` WITH ( + FORMAT = "raw", + SCHEMA = ( Data String ) + );""") def create_schema_secret(self, secret_name, value): - with self.session_pool.checkout() as session: - session.execute_scheme(f"CREATE SECRET {secret_name} WITH (value='{value}');") + self.execute_scheme(f"CREATE SECRET `{secret_name}` WITH (value='{value}');") + + def create_table(self, table_name): + self.execute_scheme(f"CREATE TABLE `{table_name}` (Key Uint64, PRIMARY KEY (Key));") + + def create_transfer_table(self, table_name): + self.execute_scheme(f""" + CREATE TABLE `{table_name}` ( + partition Uint32 NOT NULL, + offset Uint64 NOT NULL, + message Utf8, + PRIMARY KEY (partition, offset) + );""") + + def create_topic(self, topic_name): + self.execute_scheme(f"CREATE TOPIC `{topic_name}`;") + + def create_async_replication(self, replication_name, table_name, replica_name, secret_name, schema_secret=False): + connection_string = f"grpc://{self.cluster.nodes[1].host}:{self.cluster.nodes[1].port}/?database={DATABASE}" + secret_setting = "PASSWORD_SECRET_PATH" if schema_secret else "PASSWORD_SECRET_NAME" + query = f""" + CREATE ASYNC REPLICATION `{replication_name}` FOR `{table_name}` AS `{replica_name}` WITH ( + CONNECTION_STRING="{connection_string}", + USER = "root", + {secret_setting} = "{secret_name}" + );""" + self.execute_scheme(query) + + def create_transfer(self, transfer_name, topic_name, table_name, secret_name, schema_secret=False): + connection_string = f"grpc://{self.cluster.nodes[1].host}:{self.cluster.nodes[1].port}/?database={DATABASE}" + secret_setting = "PASSWORD_SECRET_PATH" if schema_secret else "PASSWORD_SECRET_NAME" + query = f""" + $l = ($x) -> {{ + return [ + <| + partition:CAST($x._partition AS Uint32), + offset:CAST($x._offset AS Uint64), + message:CAST($x._data AS Utf8) + |> + ]; + }}; + + CREATE TRANSFER `{transfer_name}` + FROM `{topic_name}` TO `{table_name}` USING $l + WITH ( + CONNECTION_STRING="{connection_string}", + FLUSH_INTERVAL = Interval('PT1S'), + BATCH_SIZE_BYTES = 10, + USER = "root", + {secret_setting} = "{secret_name}" + );""" + self.execute_scheme(query) + + def mon_endpoint(self): + return f"http://{self.cluster.nodes[1].host}:{self.cluster.nodes[1].mon_port}" + + def viewer_describe(self, path): + response = requests.get( + f"{self.mon_endpoint()}/viewer/json/describe", + params={"database": DATABASE, "path": path}, + timeout=10, + ) + response.raise_for_status() + return response.json() + + def viewer_describe_replication(self, path): + response = requests.get( + f"{self.mon_endpoint()}/viewer/json/describe_replication", + params={"database": DATABASE, "path": path}, + timeout=10, + ) + response.raise_for_status() + return response.json() + + def viewer_describe_transfer(self, path): + response = requests.get( + f"{self.mon_endpoint()}/viewer/json/describe_transfer", + params={"database": DATABASE, "path": path}, + timeout=10, + ) + response.raise_for_status() + return response.json() + + def assert_path_ready(self, path): + description = self.viewer_describe(path) + assert description["Status"] == "StatusSuccess", description + assert description["PathDescription"]["Self"]["CreateFinished"] is True, description + return description + + def wait_object_error_state(self, describe_fn, path, expected_issue_substr): + def ready(): + description = describe_fn(path) + error = description.get("error") + if not error: + return False + return expected_issue_substr in str(error) + + if not wait_for(ready, timeout_seconds=60, step_seconds=1): + description = describe_fn(path) + raise AssertionError( + f"Object {path} didn't enter error state with {expected_issue_substr!r}, got: {description}" + ) @pytest.fixture @@ -105,30 +275,272 @@ def old_secrets_utils(): utils.stop() -def test_create_eds_with_old_secret_after_disabling_old_secret_creation(old_secrets_utils): - # can create old secrets by default +def test_old_secret_creation_is_disabled(old_secrets_utils): + # create old secret old_secrets_utils.create_old_secret("OldSecret", value="") - # can use old secrets by default - old_secrets_utils.create_eds("eds-before-restart", "OldSecret") - + # restart cluster with disabled old secret creation old_secrets_utils.restart_cluster(disable_old_secret_creation=True) - # can create schema secrets with old secrets disabled - old_secrets_utils.create_schema_secret("NewSecret", value="") - - # can use old secrets with old secrets disabled - old_secrets_utils.create_eds("eds-after-restart", "OldSecret") - - # can alter old secrets with old secrets disabled + # check that old secret can be altered old_secrets_utils.alter_old_secret("OldSecret", "NewValue") - # can not create old secrets with old secrets disabled + # check that old secret can't be created with pytest.raises(Exception) as exc_info: old_secrets_utils.create_old_secret("NewOldSecret", value="") - assert "Old secrets creation syntax is disabled now. Please use the new one" in str(exc_info.value) + assert OLD_SECRETS_CREATION_DISABLED_MESSAGE in str(exc_info.value) - # can not upsert old secrets with old secrets disabled + # check that old secret can't be upserted with pytest.raises(Exception) as exc_info: old_secrets_utils.upsert_old_secret("NewOldSecretUpsert", value="") - assert "Old secrets creation syntax is disabled now. Please use the new one" in str(exc_info.value) + assert OLD_SECRETS_CREATION_DISABLED_MESSAGE in str(exc_info.value) + + +def test_existing_objects_are_ok_if_old_secret_creation_is_disabled(old_secrets_utils): + # create all types of objects with old secrets + # secret + old_secrets_utils.create_old_secret("OldSecret", value="") + + # eds + old_secrets_utils.create_eds("eds-old-secret", "OldSecret") + + # replication + old_secrets_utils.create_table("repl_src") + old_secrets_utils.create_async_replication("replication-old-secret", "repl_src", "repl_dst", "OldSecret") + + # transfer + old_secrets_utils.create_topic("transfer_topic") + old_secrets_utils.create_transfer_table("transfer_dst") + old_secrets_utils.create_transfer("transfer-old-secret", "transfer_topic", "transfer_dst", "OldSecret") + + # restart cluster with disabled old secret creation + old_secrets_utils.restart_cluster(disable_old_secret_creation=True) + + # check that old objects are still OK + old_secrets_utils.assert_path_ready(f"{DATABASE}/eds-old-secret") + old_secrets_utils.assert_path_ready(f"{DATABASE}/replication-old-secret") + old_secrets_utils.assert_path_ready(f"{DATABASE}/transfer-old-secret") + + # check that existing objects can't be changed to use old secrets + with pytest.raises(Exception) as exc_info: + old_secrets_utils.create_eds( + "eds-old-secret", + "OldSecret", + create_or_replace=True, + ) + assert CREATION_WITH_OLD_SECRETS_DISABLED_MESSAGE in str(exc_info.value) + + with pytest.raises(Exception) as exc_info: + old_secrets_utils.alter_password_secret( + "ASYNC REPLICATION", + "replication-old-secret", + "OldSecret", + ) + assert CREATION_WITH_OLD_SECRETS_DISABLED_MESSAGE in str(exc_info.value) + + with pytest.raises(Exception) as exc_info: + old_secrets_utils.alter_password_secret( + "TRANSFER", + "transfer-old-secret", + "OldSecret", + ) + assert CREATION_WITH_OLD_SECRETS_DISABLED_MESSAGE in str(exc_info.value) + + +def test_existing_objects_cannot_use_old_secrets_if_old_secret_creation_is_disabled(old_secrets_utils): + # create all types of objects with old secrets + # secret + old_secrets_utils.create_old_secret("OldSecret", value="") + + # eds + old_secrets_utils.create_eds("eds-old-secret", "OldSecret") + + # replication + old_secrets_utils.create_table("repl_src") + old_secrets_utils.create_async_replication("replication-old-secret", "repl_src", "repl_dst", "OldSecret") + + # transfer + old_secrets_utils.create_topic("transfer_topic") + old_secrets_utils.create_transfer_table("transfer_dst") + old_secrets_utils.create_transfer("transfer-old-secret", "transfer_topic", "transfer_dst", "OldSecret") + + # restart cluster with disabled old secret creation + old_secrets_utils.restart_cluster(disable_old_secret_creation=True) + + # check that existing objects can't be changed to use old secrets + with pytest.raises(Exception) as exc_info: + old_secrets_utils.create_eds( + "eds-old-secret", + "OldSecret", + create_or_replace=True, + ) + assert CREATION_WITH_OLD_SECRETS_DISABLED_MESSAGE in str(exc_info.value) + + with pytest.raises(Exception) as exc_info: + old_secrets_utils.alter_password_secret( + "ASYNC REPLICATION", + "replication-old-secret", + "OldSecret", + ) + assert CREATION_WITH_OLD_SECRETS_DISABLED_MESSAGE in str(exc_info.value) + + with pytest.raises(Exception) as exc_info: + old_secrets_utils.alter_password_secret( + "TRANSFER", + "transfer-old-secret", + "OldSecret", + ) + assert CREATION_WITH_OLD_SECRETS_DISABLED_MESSAGE in str(exc_info.value) + + +def test_existing_objects_can_migrate_to_new_secrets_if_old_secret_creation_is_disabled(old_secrets_utils): + # create all types of objects with old secrets + old_secrets_utils.create_old_secret("OldSecret", value="") + old_secrets_utils.create_eds("eds-old-secret", "OldSecret") + old_secrets_utils.create_table("repl_src") + old_secrets_utils.create_async_replication("replication-old-secret", "repl_src", "repl_dst", "OldSecret") + old_secrets_utils.create_topic("transfer_topic") + old_secrets_utils.create_transfer_table("transfer_dst") + old_secrets_utils.create_transfer("transfer-old-secret", "transfer_topic", "transfer_dst", "OldSecret") + + # restart cluster with disabled old secret creation + old_secrets_utils.restart_cluster(disable_old_secret_creation=True) + + # non-secret ALTER (e.g. STATE) must still work for objects that use old secrets + old_secrets_utils.execute_scheme(""" + ALTER ASYNC REPLICATION `replication-old-secret` SET (STATE = "Paused"); + ALTER ASYNC REPLICATION `replication-old-secret` SET (STATE = "StandBy"); + """) + old_secrets_utils.execute_scheme(""" + ALTER TRANSFER `transfer-old-secret` SET (STATE = "Paused"); + ALTER TRANSFER `transfer-old-secret` SET (STATE = "StandBy"); + """) + + # migrate existing objects to new secrets + new_secret = "NewSecret" + old_secrets_utils.create_schema_secret(new_secret, value="") + + old_secrets_utils.create_eds( + "eds-old-secret", + new_secret, + schema_secret=True, + create_or_replace=True, + ) + old_secrets_utils.alter_password_secret( + "ASYNC REPLICATION", + "replication-old-secret", + new_secret, + schema_secret=True, + ) + old_secrets_utils.alter_password_secret( + "TRANSFER", + "transfer-old-secret", + new_secret, + schema_secret=True, + ) + + old_secrets_utils.assert_path_ready(f"{DATABASE}/eds-old-secret") + old_secrets_utils.assert_path_ready(f"{DATABASE}/replication-old-secret") + old_secrets_utils.assert_path_ready(f"{DATABASE}/transfer-old-secret") + + +def test_disable_old_secret_creation(old_secrets_utils): + old_secrets_utils.create_old_secret("OldSecret", value="") + + # restart cluster with disabled old secret creation + old_secrets_utils.restart_cluster(disable_old_secret_creation=True) + + # check that old secrets can't be used for eds + with pytest.raises(Exception) as exc_info: + old_secrets_utils.create_eds("eds-with-old-secret", "OldSecret") + assert CREATION_WITH_OLD_SECRETS_DISABLED_MESSAGE in str(exc_info.value) + + # check that old secrets can't be used for replication + with pytest.raises(Exception) as exc_info: + old_secrets_utils.create_async_replication("repl-with-old-secret", "repl_src", "repl_dst_old", "OldSecret") + assert CREATION_WITH_OLD_SECRETS_DISABLED_MESSAGE in str(exc_info.value) + + # check that old secrets can't be used for transfer + with pytest.raises(Exception) as exc_info: + old_secrets_utils.create_transfer("transfer-with-old-secret", "transfer_topic", "transfer_dst", "OldSecret") + assert CREATION_WITH_OLD_SECRETS_DISABLED_MESSAGE in str(exc_info.value) + + # check that new secrets can be still used + old_secrets_utils.create_schema_secret("NewSecret", value="") + + # create eds with new secret + old_secrets_utils.create_eds( + "eds-with-new-secret", + "NewSecret", + schema_secret=True, + ) + + # create replication with new secret + old_secrets_utils.create_table("repl_src_new") + old_secrets_utils.create_async_replication( + "repl-with-new-secret", "repl_src_new", "repl_dst_new", "NewSecret", schema_secret=True + ) + + # create transfer with new secret + old_secrets_utils.create_topic("transfer_topic_new") + old_secrets_utils.create_transfer_table("transfer_dst_new") + old_secrets_utils.create_transfer( + "transfer-with-new-secret", + "transfer_topic_new", + "transfer_dst_new", + "NewSecret", + schema_secret=True, + ) + + +def test_disable_old_secrets_dont_crash_existing_objects(old_secrets_utils): + s3_endpoint, s3_access_key, s3_secret_key, s3_bucket = setup_s3() + s3_location = f"{s3_endpoint}/{s3_bucket}" + + old_secrets_utils.create_old_secret("s3_access_key", value=s3_access_key) + old_secrets_utils.create_old_secret("s3_secret_key", value=s3_secret_key) + old_secrets_utils.create_old_secret("password", value="") + + # create eds with old secrets + eds_name = "eds-old-secret" + old_secrets_utils.create_eds_aws(eds_name, "s3_access_key", "s3_secret_key", s3_location) + old_secrets_utils.assert_path_ready(f"{DATABASE}/{eds_name}") + + # create replication with old secret + old_secrets_utils.create_table("repl_src") + old_secrets_utils.create_async_replication( + "replication-old-secret", + "repl_src", + "repl_dst", + "password", + ) + old_secrets_utils.assert_path_ready(f"{DATABASE}/replication-old-secret") + + # create transfer with old secret + old_secrets_utils.create_topic("transfer_topic") + old_secrets_utils.create_transfer_table("transfer_dst") + old_secrets_utils.create_transfer( + "transfer-old-secret", + "transfer_topic", + "transfer_dst", + "password", + ) + old_secrets_utils.assert_path_ready(f"{DATABASE}/transfer-old-secret") + + old_secrets_utils.restart_cluster(disable_old_secrets=True) + + with pytest.raises(Exception) as exc_info: + # After restart eds resolves secret only while reading + old_secrets_utils.read_from_eds(eds_name) + assert OLD_SECRETS_USAGE_DISABLED_MESSAGE in str(exc_info.value) + + old_secrets_utils.wait_object_error_state( + old_secrets_utils.viewer_describe_replication, + f"{DATABASE}/replication-old-secret", + OLD_SECRETS_USAGE_DISABLED_MESSAGE, + ) + old_secrets_utils.wait_object_error_state( + old_secrets_utils.viewer_describe_transfer, + f"{DATABASE}/transfer-old-secret", + OLD_SECRETS_USAGE_DISABLED_MESSAGE, + ) diff --git a/ydb/tests/functional/secrets/ya.make b/ydb/tests/functional/secrets/ya.make index 9facf4e8846..a7335cae6c9 100644 --- a/ydb/tests/functional/secrets/ya.make +++ b/ydb/tests/functional/secrets/ya.make @@ -25,6 +25,7 @@ DEPENDS( PEERDIR( contrib/python/boto3 + contrib/python/requests ydb/tests/functional/secrets/lib ydb/tests/library ydb/tests/library/fixtures diff --git a/ydb/tests/functional/sqs/messaging/test_generic_messaging.py b/ydb/tests/functional/sqs/messaging/test_generic_messaging.py index dc5af7f17c3..72cd1802f0b 100644 --- a/ydb/tests/functional/sqs/messaging/test_generic_messaging.py +++ b/ydb/tests/functional/sqs/messaging/test_generic_messaging.py @@ -250,6 +250,32 @@ class SqsGenericMessagingTest(KikimrSqsTestBase): self._wait_for_counter_value(receive_counter_labels, 1, default_value=0) @pytest.mark.parametrize(**TABLES_FORMAT_PARAMS) + def test_standard_queue_ignores_message_deduplication_id(self, tables_format): + self._init_with_params(is_fifo=False, tables_format=tables_format) + created_queue_url = self._create_queue_and_assert(self.queue_name, is_fifo=False) + + body = 'standard body with ignored deduplication id' + message_id = self._send_message_and_assert( + created_queue_url, body, seq_no='deduplication-id-1' + ) + read_message_result = self._read_while_not_empty(created_queue_url, 1) + + assert_that( + read_message_result, ReadResponseMatcher().with_message_ids([message_id]) + ) + + # MessageDeduplicationId is FIFO-only: ignored for standard queues and not returned on receive. + attributes = read_message_result[0].get('Attribute') + if attributes is not None: + attributes_by_name = {} + for a in attributes if isinstance(attributes, list) else [attributes]: + attributes_by_name[a['Name']] = a + assert_that( + attributes_by_name, + not_(has_item('MessageDeduplicationId')) + ) + + @pytest.mark.parametrize(**TABLES_FORMAT_PARAMS) def test_validates_message_attributes(self, tables_format): self._init_with_params(tables_format=tables_format) created_queue_url = self._create_queue_and_assert(self.queue_name) diff --git a/ydb/tests/functional/sqs_topic/test_change_message_visibility.py b/ydb/tests/functional/sqs_topic/test_change_message_visibility.py index c61c7626ec9..0cb7004f6b2 100644 --- a/ydb/tests/functional/sqs_topic/test_change_message_visibility.py +++ b/ydb/tests/functional/sqs_topic/test_change_message_visibility.py @@ -79,6 +79,7 @@ class TestSqsTopicChangeMessageVisibility(KikimrSqsTopicTestBase): QueueUrl=self._queue_url, MessageBody=message_body, MessageGroupId='message-group-1', + MessageDeduplicationId='deduplication-id-1', ) response = self._boto_client.receive_message( diff --git a/ydb/tests/functional/sqs_topic/test_change_message_visibility_batch.py b/ydb/tests/functional/sqs_topic/test_change_message_visibility_batch.py index a1eaf0d55f0..c04206af28c 100644 --- a/ydb/tests/functional/sqs_topic/test_change_message_visibility_batch.py +++ b/ydb/tests/functional/sqs_topic/test_change_message_visibility_batch.py @@ -89,6 +89,7 @@ class TestSqsTopicChangeMessageVisibilityBatch(KikimrSqsTopicTestBase): QueueUrl=self._queue_url, MessageBody=message_body, MessageGroupId='message-group-{}'.format(index), + MessageDeduplicationId='deduplication-id-{}'.format(index), ) response = self._boto_client.receive_message( diff --git a/ydb/tests/functional/sqs_topic/test_delete_message.py b/ydb/tests/functional/sqs_topic/test_delete_message.py index 2097b1686f2..3e8372519ab 100644 --- a/ydb/tests/functional/sqs_topic/test_delete_message.py +++ b/ydb/tests/functional/sqs_topic/test_delete_message.py @@ -164,6 +164,7 @@ class TestSqsTopicDeleteMessage(KikimrSqsTopicTestBase): QueueUrl=self._queue_url, MessageBody=message_body, MessageGroupId='message-group-1', + MessageDeduplicationId='deduplication-id-1', ) assert_that( @@ -207,6 +208,7 @@ class TestSqsTopicDeleteMessage(KikimrSqsTopicTestBase): QueueUrl=self._queue_url, MessageBody=message_body, MessageGroupId='message-group-{}'.format(index), + MessageDeduplicationId='deduplication-id-{}'.format(index), ) assert_that( diff --git a/ydb/tests/functional/sqs_topic/test_purge_queue.py b/ydb/tests/functional/sqs_topic/test_purge_queue.py index f44ed42c6b0..5f1f4099b62 100644 --- a/ydb/tests/functional/sqs_topic/test_purge_queue.py +++ b/ydb/tests/functional/sqs_topic/test_purge_queue.py @@ -42,6 +42,7 @@ class TestSqsTopicPurgeQueue(KikimrSqsTopicTestBase): QueueUrl=self._queue_url, MessageBody='hello from fifo sqs', MessageGroupId='message-group-1', + MessageDeduplicationId='deduplication-id-1', ) assert_that( diff --git a/ydb/tests/functional/sqs_topic/test_receive_message.py b/ydb/tests/functional/sqs_topic/test_receive_message.py index 8cecdab6793..ac8692b8733 100644 --- a/ydb/tests/functional/sqs_topic/test_receive_message.py +++ b/ydb/tests/functional/sqs_topic/test_receive_message.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from hamcrest import assert_that, equal_to, has_length, not_none, not_ +from hamcrest import assert_that, equal_to, has_key, has_length, is_not, not_none, not_ from ydb.tests.library.sqs_topic.test_base import KikimrSqsTopicTestBase @@ -36,6 +36,7 @@ class TestSqsTopicReceiveMessage(KikimrSqsTopicTestBase): QueueUrl=self._queue_url, MessageBody=message_body, MessageGroupId='message-group-1', + MessageDeduplicationId='deduplication-id-1', ) response = self._boto_client.receive_message( @@ -102,6 +103,33 @@ class TestSqsTopicReceiveMessage(KikimrSqsTopicTestBase): assert_that(messages[0]['Body'], equal_to(message_body)) assert_that(messages[0]['Attributes']['MessageGroupId'], equal_to(message_group_id)) + def test_receive_message_standard_ignores_message_deduplication_id(self): + queue_name = self._make_queue_name('receive_message_standard_ignores_message_deduplication_id') + self._queue_url = self._boto_client.create_queue(QueueName=queue_name)['QueueUrl'] + + message_body = 'hello from std sqs' + deduplication_id = 'deduplication-id-1' + self._boto_client.send_message( + QueueUrl=self._queue_url, + MessageBody=message_body, + MessageDeduplicationId=deduplication_id, + ) + + response = self._boto_client.receive_message( + QueueUrl=self._queue_url, + WaitTimeSeconds=20, + MaxNumberOfMessages=1, + AttributeNames=['All'], + ) + + messages = response.get('Messages') + assert_that(messages, not_none()) + assert_that(messages, has_length(1)) + assert_that(messages[0]['Body'], equal_to(message_body)) + # MessageDeduplicationId is FIFO-only: ignored for standard queues and not returned on receive. + attributes = messages[0].get('Attributes', {}) + assert_that(attributes, is_not(has_key('MessageDeduplicationId'))) + def test_receive_message_fifo_with_message_deduplication_id(self): self._create_fifo_queue('receive_message_fifo_with_message_deduplication_id') diff --git a/ydb/tests/functional/sqs_topic/test_send_message.py b/ydb/tests/functional/sqs_topic/test_send_message.py index 1500fba1b58..463ade7ff85 100644 --- a/ydb/tests/functional/sqs_topic/test_send_message.py +++ b/ydb/tests/functional/sqs_topic/test_send_message.py @@ -1,7 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from hamcrest import assert_that, equal_to, not_none +import botocore + +from hamcrest import assert_that, equal_to, not_, not_none, raises from ydb.tests.library.sqs_topic.test_base import KikimrSqsTopicTestBase @@ -30,6 +32,7 @@ class TestSqsTopicSendMessage(KikimrSqsTopicTestBase): QueueUrl=self._queue_url, MessageBody=message_body, MessageGroupId='message-group-1', + MessageDeduplicationId='deduplication-id-1', ) assert_that(response['MessageId'], not_none()) @@ -85,3 +88,76 @@ class TestSqsTopicSendMessage(KikimrSqsTopicTestBase): ) assert_that(duplicate_response['MessageId'], equal_to(response['MessageId'])) + + def test_send_message_fifo_with_content_based_deduplication(self): + queue_name = self._make_fifo_queue_name('send_message_fifo_with_content_based_deduplication') + self._queue_url = self._boto_client.create_queue( + QueueName=queue_name, + Attributes={ + 'FifoQueue': 'true', + 'ContentBasedDeduplication': 'true', + }, + )['QueueUrl'] + + message_body = 'hello from fifo sqs' + response = self._boto_client.send_message( + QueueUrl=self._queue_url, + MessageBody=message_body, + MessageGroupId='message-group-1', + ) + + assert_that(response['MessageId'], not_none()) + assert_that(response['SequenceNumber'], not_none()) + + # Same body without MessageDeduplicationId: SHA-256 of body is used, so duplicate is dropped. + duplicate_response = self._boto_client.send_message( + QueueUrl=self._queue_url, + MessageBody=message_body, + MessageGroupId='message-group-1', + ) + + assert_that(duplicate_response['MessageId'], equal_to(response['MessageId'])) + + # Different body produces a different content hash and is not deduplicated. + other_response = self._boto_client.send_message( + QueueUrl=self._queue_url, + MessageBody='other body', + MessageGroupId='message-group-1', + ) + + assert_that(other_response['MessageId'], not_(equal_to(response['MessageId']))) + + # Explicit MessageDeduplicationId overrides content-based hash. + explicit_response = self._boto_client.send_message( + QueueUrl=self._queue_url, + MessageBody='yet another body', + MessageGroupId='message-group-1', + MessageDeduplicationId='explicit-deduplication-id', + ) + explicit_duplicate_response = self._boto_client.send_message( + QueueUrl=self._queue_url, + MessageBody='completely different body', + MessageGroupId='message-group-1', + MessageDeduplicationId='explicit-deduplication-id', + ) + + assert_that(explicit_duplicate_response['MessageId'], equal_to(explicit_response['MessageId'])) + + def test_send_message_fifo_without_content_based_deduplication(self): + self._create_fifo_queue('send_message_fifo_without_content_based_deduplication') + + def send_without_message_deduplication_id(): + self._boto_client.send_message( + QueueUrl=self._queue_url, + MessageBody='hello from fifo sqs', + MessageGroupId='message-group-1', + ) + + # Without ContentBasedDeduplication, MessageDeduplicationId is required. + assert_that( + send_without_message_deduplication_id, + raises( + botocore.exceptions.ClientError, + pattern='MissingParameter', + ), + ) diff --git a/ydb/tests/functional/sqs_topic/test_send_message_batch.py b/ydb/tests/functional/sqs_topic/test_send_message_batch.py index 5097816fbe5..4a1a664b425 100644 --- a/ydb/tests/functional/sqs_topic/test_send_message_batch.py +++ b/ydb/tests/functional/sqs_topic/test_send_message_batch.py @@ -76,6 +76,7 @@ class TestSqsTopicSendMessageBatch(KikimrSqsTopicTestBase): 'Id': str(index), 'MessageBody': message_body, 'MessageGroupId': 'message-group-1', + 'MessageDeduplicationId': 'deduplication-id-{}'.format(index), } for index, message_body in enumerate(message_bodies) ], diff --git a/ydb/tests/functional/tenants/test_remove_storage_groups.py b/ydb/tests/functional/tenants/test_remove_storage_groups.py index f5a996ec2c5..195c1854821 100644 --- a/ydb/tests/functional/tenants/test_remove_storage_groups.py +++ b/ydb/tests/functional/tenants/test_remove_storage_groups.py @@ -78,8 +78,6 @@ def test_remove_storage_group(ydb_cluster, ydb_root, ydb_safe_test_name, ydb_cli storage_units_to_remove={'hdd': 2}, ) - ydb_cluster.restart_slots() - def get_storage_units(): status = ydb_cluster.get_database_status(database) units = sum([unit.count for unit in status.allocated_resources.storage_units]) @@ -95,5 +93,13 @@ def test_remove_storage_group(ydb_cluster, ydb_root, ydb_safe_test_name, ydb_cli status = ydb_cluster.get_database_status(database) assert_that(status.state, equal_to(cms_pb.GetDatabaseStatusResult.RUNNING)) - ydb_cluster.unregister_and_stop_slots(database_nodes) + # Check data integrity + with ydb.QuerySessionPool(driver, size=1) as pool: + result_sets = pool.execute_with_retries("SELECT * FROM " + table_name) + assert len(result_sets) == 1 + assert len(result_sets[0].rows) == 1 + assert result_sets[0].rows[0].key == 1 + assert result_sets[0].rows[0].value == b"value1" + ydb_cluster.remove_database(database) + ydb_cluster.unregister_and_stop_slots(database_nodes) diff --git a/ydb/tests/functional/udf_store/upload_udf/ya.make b/ydb/tests/functional/udf_store/upload_udf/ya.make index 7bf85be0153..2012e9e4d81 100644 --- a/ydb/tests/functional/udf_store/upload_udf/ya.make +++ b/ydb/tests/functional/udf_store/upload_udf/ya.make @@ -9,6 +9,10 @@ PEERDIR( ydb/tests/oss/ydb_sdk_import ) +NO_CHECK_IMPORTS( + ydb.tests.oss.canonical.* +) + DEPENDS( ydb/tests/stress/kv_volume_tool ) diff --git a/ydb/tests/library/sqs/requests_client.py b/ydb/tests/library/sqs/requests_client.py index c7cf0f2c240..904f958b28d 100644 --- a/ydb/tests/library/sqs/requests_client.py +++ b/ydb/tests/library/sqs/requests_client.py @@ -371,8 +371,9 @@ class SqsHttpApi(object): delay_seconds=None, attributes=None, deduplication_id=None, group_id=None ): if not to_bytes(queue_url).endswith(to_bytes('.fifo')): - if deduplication_id is not None or group_id is not None: - raise ValueError("Deduplication id and Group id parameters may be set for FIFO queues only") + if group_id is not None: + raise ValueError("Group id parameter may be set for FIFO queues only") + # MessageDeduplicationId is accepted for standard queues but ignored by the server. params = { 'QueueUrl': queue_url, 'MessageBody': message_body, diff --git a/ydb/tests/stress/compare_index_performance/tests/README.md b/ydb/tests/stress/compare_index_performance/tests/README.md index 87e54f4ffdf..ef54b76ec28 100644 --- a/ydb/tests/stress/compare_index_performance/tests/README.md +++ b/ydb/tests/stress/compare_index_performance/tests/README.md @@ -73,6 +73,13 @@ sanity check) or two different refs — **without a local build**. | `compare_flamegraph` | `` | `1`/`true` → collect CPU flamegraphs (see below) | | `compare_perf_sudo` | `` | `1`/`true` → run `perf` under `sudo` | | `compare_perf_freq` | `50` | `perf record -F` sampling frequency | +| `compare_dataset_source` | `generate` | `generate` (random data) or `s3` (import fixed dataset from S3) | +| `compare_s3_endpoint` | `` | S3 endpoint URL (required when `dataset_source=s3`) | +| `compare_s3_bucket` | `` | S3 bucket name (required when `dataset_source=s3`) | +| `compare_s3_query_destination` | `` | Database destination path for the queries table (optional, defaults to `{database}/vector_query_table`) | +| `compare_s3_destination` | `` | Database destination path for import (required when `dataset_source=s3`) | +| `compare_s3_query_source` | `` | S3 object key prefix for a pre-computed queries table (optional, skips dynamic creation) | +| `compare_s3_query_destination` | `` | Database destination path for the queries table (optional, defaults to `{database}/vector_query_table`) `table_service_config` values like `enable_vector_index_read=true` are parsed into booleans; everything else is kept as a string. Feature flags are appended @@ -131,6 +138,31 @@ the toolkit in `contrib/tools/flame-graph` (shipped to the sandbox via `DATA`): --test-param compare_current_table_service_config=enable_vector_index_read=true \ --test-param compare_flamegraph=1 \ --test-param compare_perf_sudo=1 + +# Import data from S3 instead of auto-generating (uses a fixed dataset). +./ya make --build relwithdebinfo -tA \ + ydb/tests/stress/compare_index_performance/tests \ + --test-param compare_dataset_source=s3 \ + --test-param compare_s3_endpoint=https://storage.yandexcloud.net \ + --test-param compare_s3_bucket=vector-index \ + --test-param compare_s3_source=wikipedia \ + --test-param compare_s3_destination=/Root/testdb/wikipedia \ + --test-param compare_iterations=3 \ + --test-param compare_duration=60 + +# Import both dataset and pre-computed queries table from S3 (exact query +# vectors are reused across iterations and across sides for perfect reproducibility). +./ya make --build relwithdebinfo -tA \ + ydb/tests/stress/compare_index_performance/tests \ + --test-param compare_dataset_source=s3 \ + --test-param compare_s3_endpoint=https://storage.yandexcloud.net \ + --test-param compare_s3_bucket=vector-index \ + --test-param compare_s3_source=wikipedia \ + --test-param compare_s3_destination=/Root/testdb/wikipedia \ + --test-param compare_s3_query_source=wikipedia_queries \ + --test-param compare_s3_query_destination=/Root/vector_query_table \ + --test-param compare_iterations=3 \ + --test-param compare_duration=60 ``` ## Results diff --git a/ydb/tests/stress/compare_index_performance/tests/test_compare.py b/ydb/tests/stress/compare_index_performance/tests/test_compare.py index 9a0db0506c7..71432d4c2b5 100644 --- a/ydb/tests/stress/compare_index_performance/tests/test_compare.py +++ b/ydb/tests/stress/compare_index_performance/tests/test_compare.py @@ -33,10 +33,14 @@ # if perf cannot profile, the test fails. perf adds overhead, so measured # Txs/Sec are perturbed when on. +import contextlib import os import signal import statistics import subprocess +import threading +import time +import traceback import urllib.request import pytest @@ -201,6 +205,11 @@ class TestCompareIndexPerformance: self.rows = yatest.common.get_param('compare_rows', default='10000') self.threads = yatest.common.get_param('compare_threads', default='10') self.targets = yatest.common.get_param('compare_targets', default='1000') + # Vector index structure params; None → server auto-detects + _clusters = yatest.common.get_param('compare_vector_clusters', default='') + _levels = yatest.common.get_param('compare_vector_levels', default='') + self.vector_clusters = _clusters if _clusters else None + self.vector_levels = _levels if _levels else None # Per-cluster config self.baseline_flags = _parse_feature_flags( @@ -212,7 +221,31 @@ class TestCompareIndexPerformance: self.current_tsc = _parse_table_service_config( yatest.common.get_param('compare_current_table_service_config', default='')) - self.workload = yatest.common.get_param('compare_workload', default='all') + self.workload = yatest.common.get_param('compare_workload', default='vector') + if self.workload not in ('vector', 'fulltext'): + print(f"WARNING: unrecognized compare_workload={self.workload!r}; " + "all workloads will be skipped (expected 'vector' or 'fulltext')") + + # Dataset source: 'generate' (default) or 's3' + self.dataset_source = yatest.common.get_param('compare_dataset_source', default='generate') + # S3 import params (used when dataset_source == 's3') + self.s3_endpoint = yatest.common.get_param('compare_s3_endpoint', default='') + self.s3_bucket = yatest.common.get_param('compare_s3_bucket', default='') + self.s3_source = yatest.common.get_param('compare_s3_source', default='') + self.s3_destination = yatest.common.get_param('compare_s3_destination', default='') + # Optional S3 queries table (when dataset_source == 's3' and queries pre-computed) + self.s3_query_source = yatest.common.get_param('compare_s3_query_source', default='') + self.s3_query_destination = yatest.common.get_param('compare_s3_query_destination', default='') + + if self.dataset_source == 's3': + missing = [name for name, val in [ + ('compare_s3_endpoint', self.s3_endpoint), + ('compare_s3_bucket', self.s3_bucket), + ('compare_s3_source', self.s3_source), + ('compare_s3_destination', self.s3_destination), + ] if not val] + if missing: + pytest.fail(f"compare_dataset_source=s3 requires: {', '.join(missing)}") # Flamegraph collection (off by default). When on, each workload run is # profiled with `perf` and a CPU flamegraph SVG is produced per side per @@ -268,44 +301,113 @@ class TestCompareIndexPerformance: ) cluster = KiKiMR(config) cluster.start() + # A stray SIGINT keeps aborting the flamegraph run during teardown even + # though this process no longer sends any signal (perf is stopped via a + # sentinel file). _sigint_debug logs where it comes from and SWALLOWS it + # (which, if signal masking works here at all, is also the fix). + shield = self._sigint_debug(label) if self.flamegraph else contextlib.nullcontext() + with shield: + try: + endpoint = "grpc://localhost:%s" % cluster.nodes[1].port + log_file = yatest.common.output_path(log_name) + err_file = yatest.common.output_path(log_name + ".err") + print(f"--- Running {label} workload against {ydbd_path} ---") + # stdout and stderr go to separate files: the Txs/Sec line is parsed + # from stdout, so stderr noise must not interleave into it. + with open(log_file, "w") as out, open(err_file, "w") as err: + perf = self._perf_start(cluster.nodes[1].pid, svg_name) if self.flamegraph else None + workload_ok = False + try: + run_workload(endpoint, out, err) + workload_ok = True + finally: + if perf is not None: + # Convert/validate the profile only if the workload + # itself succeeded; otherwise just stop perf so its + # error does not mask the original workload failure. + self._perf_finish(perf, svg_name, validate=workload_ok) + return extract_total_txs_sec(log_file) + finally: + cluster.stop() + + @contextlib.contextmanager + def _sigint_debug(self, label): + # Diagnose (and, if possible, suppress) the stray SIGINT that aborts the + # flamegraph run. Writes findings to a published sigint_diag.log: + # - whether we are on the main thread (signal.signal only works there), + # - whether installing a SIGINT handler succeeds, + # - for every SIGINT: a timestamp and the full Python stack (which frame + # / subprocess call it interrupted). + # The handler SWALLOWS the signal (returns without raising), so if signal + # masking is effective in this py3test the run survives — making this both + # the probe and the fix. If signal.signal() raises (not the main thread), + # that is logged and tells us in-process masking is impossible here. + diag = yatest.common.output_path("sigint_diag.log") + + def log(msg): + with open(diag, "a") as f: + f.write("[%.3f][%s] %s\n" % (time.time(), label, msg)) + + on_main = threading.current_thread() is threading.main_thread() + log("enter: on_main_thread=%s pid=%s" % (on_main, os.getpid())) + hits = [] + + def handler(signum, frame): + hits.append(signum) + log("SIGINT #%d (signum=%d) SWALLOWED; interrupted stack:\n%s" + % (len(hits), signum, "".join(traceback.format_stack(frame)))) + + prev, installed = None, False try: - endpoint = "grpc://localhost:%s" % cluster.nodes[1].port - log_file = yatest.common.output_path(log_name) - err_file = yatest.common.output_path(log_name + ".err") - print(f"--- Running {label} workload against {ydbd_path} ---") - # stdout and stderr go to separate files: the Txs/Sec line is parsed - # from stdout, so stderr noise must not interleave into it. - with open(log_file, "w") as out, open(err_file, "w") as err: - perf = self._perf_start(cluster.nodes[1].pid, svg_name) if self.flamegraph else None - workload_ok = False - try: - run_workload(endpoint, out, err) - workload_ok = True - finally: - if perf is not None: - # Convert/validate the profile only if the workload - # itself succeeded; otherwise just stop perf so its error - # does not mask the original workload failure. - self._perf_finish(perf, svg_name, validate=workload_ok) - return extract_total_txs_sec(log_file) + prev = signal.signal(signal.SIGINT, handler) + installed = True + log("signal.signal(SIGINT, handler) OK") + except (ValueError, OSError) as exc: + log("signal.signal FAILED (%r) -> cannot mask SIGINT in-process" % (exc,)) + try: + yield finally: - cluster.stop() + if installed: + try: + signal.signal(signal.SIGINT, prev) + except (ValueError, OSError): + pass + log("exit: sigint_hits=%d" % len(hits)) # --- flamegraph collection (perf record -> stackcollapse -> flamegraph) --- def _perf_start(self, pid, svg_name): - # Profile the live ydbd process for the whole workload run (until SIGINT). - # Run perf in its own session/process group so we can interrupt it - # reliably, including when wrapped in `sudo` (which would otherwise not - # forward our SIGINT to the perf child). - # perf.data is an intermediate; keep it in the work dir (not the - # published output dir). Under sudo it is written owned by root, which - # would make ya's output collection fail with PermissionError if it lived - # under output_path — it is chowned back to us in _perf_stop. + # Profile the live ydbd process for the whole workload run. + # + # `perf record` finalizes perf.data only when it receives SIGINT/SIGTERM, + # so it MUST be signaled to stop. We deliberately do NOT signal it from + # this (pytest) process: under sudo on the CI runner that SIGINT leaked + # back into pytest and aborted the session as a KeyboardInterrupt, and + # in-process signal masking proved ineffective here (signal.signal() is a + # no-op off the main thread / gets reset around yatest.common.execute). + # + # Instead perf runs inside a small shell wrapper that waits for a sentinel + # file and then sends `kill -INT` to perf ITSELF. _perf_stop just creates + # that file — pytest never sends a signal, so none can leak to it. The + # wrapper runs in its own session (start_new_session), so the internal + # SIGINT stays fully contained. perf.data is an intermediate; keep it in + # the work dir (not the published output dir). Under sudo it is written + # owned by root, which would make ya's output collection fail with + # PermissionError under output_path — it is chowned back in _perf_stop. perf_data = yatest.common.work_path(svg_name + ".perf.data") perf_log = yatest.common.output_path(svg_name + ".perf.log") + stop_file = yatest.common.work_path(svg_name + ".perf.stop") + if os.path.exists(stop_file): + os.remove(stop_file) + # Positional args ($1..$4) avoid any dependency on sudo env passing. + wrapper = ( + 'perf record -F "$1" --call-graph dwarf -g --proc-map-timeout=10000 ' + '--pid "$2" -o "$3" & p=$!; ' + 'while [ ! -e "$4" ]; do sleep 0.2; done; ' + 'kill -INT "$p"; wait "$p"' + ) cmd = (["sudo"] if self.perf_sudo else []) + [ - "perf", "record", "-F", str(self.perf_freq), "--call-graph", "dwarf", - "-g", "--proc-map-timeout=10000", "--pid", str(pid), "-o", perf_data, + "sh", "-c", wrapper, "perfwrap", + str(self.perf_freq), str(pid), perf_data, stop_file, ] log = open(perf_log, "w") try: @@ -315,24 +417,29 @@ class TestCompareIndexPerformance: log.close() pytest.fail(f"failed to start perf for flamegraph {svg_name}: {exc}; " f"perf log: {perf_log}") - return {"proc": proc, "log": log, "data": perf_data, "perf_log": perf_log} + return {"proc": proc, "log": log, "data": perf_data, + "perf_log": perf_log, "stop": stop_file} def _perf_stop(self, perf): - # Send SIGINT to perf's process group (perf flushes perf.data on SIGINT). - # With sudo, perf is not our direct child, so signal the whole group via - # `sudo kill` to reach the privileged perf process. + # Ask perf to stop by creating the sentinel file the wrapper is waiting + # on (see _perf_start); the wrapper then sends perf its SIGINT internally. + # pytest itself sends NO signal, so nothing can leak back into it. proc = perf["proc"] try: - if self.perf_sudo: - subprocess.call(["sudo", "kill", "-INT", f"-{proc.pid}"]) - else: - os.killpg(proc.pid, signal.SIGINT) + open(perf["stop"], "w").close() except OSError: - proc.send_signal(signal.SIGINT) + pass try: proc.wait(timeout=120) except subprocess.TimeoutExpired: - proc.kill() + # Wrapper never noticed the sentinel; kill the whole perf session + # group. It is isolated (start_new_session) so a negative-pgid kill + # cannot reach pytest. Under sudo the group is root-owned. + pgid = os.getpgid(proc.pid) + if self.perf_sudo: + subprocess.call(["sudo", "kill", "-KILL", "--", f"-{pgid}"]) + else: + os.killpg(pgid, signal.SIGKILL) proc.wait() perf["log"].close() # perf ran as root under sudo, so perf.data is root-owned. Chown it back @@ -531,50 +638,96 @@ class TestCompareIndexPerformance: # --- tests --- def test_vector(self): - if self.workload not in ("all", "vector"): + if self.workload != "vector": pytest.skip(f"compare_workload={self.workload} excludes vector") baseline_ydbd = self._baseline_ydbd() current_ydbd = self._current_ydbd() - data_dir = yatest.common.output_path("vector_data") main_values = [] current_values = [] - # First iteration: baseline runs in generate mode (creates + dumps the - # query table). Subsequent baseline runs and all current runs use load - # mode, reusing the dumped query table. - for i in range(1, self.iterations + 1): - print(f"=== Vector iteration {i}/{self.iterations} ===") - baseline_mode = "generate" if i == 1 else "load" + vector_index_args = [] + if self.vector_clusters is not None: + vector_index_args += ["--clusters", self.vector_clusters] + if self.vector_levels is not None: + vector_index_args += ["--levels", self.vector_levels] - def baseline_workload(endpoint, out, err, mode=baseline_mode): - self._exec_workload("YDB_VECTOR_WORKLOAD_PATH", endpoint, out, err, [ - "--mode", mode, "--data-dir", data_dir, - "--targets", self.targets, "--warmup", self.warmup, - "--rows", self.rows, "--threads", self.threads, - ]) + if self.dataset_source == "s3": + # S3 mode: import the same fixed dataset from S3 on every iteration + # instead of generating random data. + s3_args = [ + "--s3-endpoint", self.s3_endpoint, + "--s3-bucket", self.s3_bucket, + "--s3-source", self.s3_source, + "--s3-destination", self.s3_destination, + ] + if self.s3_query_source: + s3_args += ["--s3-query-source", self.s3_query_source] + if self.s3_query_destination: + s3_args += ["--s3-query-destination", self.s3_query_destination] + for i in range(1, self.iterations + 1): + print(f"=== Vector iteration {i}/{self.iterations} (dataset_source=s3) ===") - def current_workload(endpoint, out, err): - self._exec_workload("YDB_VECTOR_WORKLOAD_PATH", endpoint, out, err, [ - "--mode", "load", "--data-dir", data_dir, - "--targets", self.targets, "--warmup", self.warmup, - "--rows", self.rows, "--threads", self.threads, - ]) + def s3_baseline_workload(endpoint, out, err): + self._exec_workload("YDB_VECTOR_WORKLOAD_PATH", endpoint, out, err, [ + "--mode", "s3", + "--targets", self.targets, "--warmup", self.warmup, + "--rows", self.rows, "--threads", self.threads, + ] + vector_index_args + s3_args) - collect_value(main_values, self._run_one( - self.ref, baseline_ydbd, self.baseline_flags, self.baseline_tsc, - baseline_workload, f"vector_main_{i}.log", f"vector_main_{i}.svg")) - collect_value(current_values, self._run_one( - "current", current_ydbd, self.current_flags, self.current_tsc, - current_workload, f"vector_current_{i}.log", f"vector_current_{i}.svg")) - if self.flamegraph: - self._flamegraph_diff("vector", i) + def s3_current_workload(endpoint, out, err): + self._exec_workload("YDB_VECTOR_WORKLOAD_PATH", endpoint, out, err, [ + "--mode", "s3", + "--targets", self.targets, "--warmup", self.warmup, + "--rows", self.rows, "--threads", self.threads, + ] + vector_index_args + s3_args) + + collect_value(main_values, self._run_one( + self.ref, baseline_ydbd, self.baseline_flags, self.baseline_tsc, + s3_baseline_workload, f"vector_main_{i}.log", f"vector_main_{i}.svg")) + collect_value(current_values, self._run_one( + "current", current_ydbd, self.current_flags, self.current_tsc, + s3_current_workload, f"vector_current_{i}.log", f"vector_current_{i}.svg")) + if self.flamegraph: + self._flamegraph_diff("vector", i) + else: + # Generate mode (default): first iteration generates data + dumps query table, + # subsequent iterations reuse the dumped query table. + data_dir = yatest.common.output_path("vector_data") + + for i in range(1, self.iterations + 1): + print(f"=== Vector iteration {i}/{self.iterations} (dataset_source=generate) ===") + + baseline_mode = "generate" if i == 1 else "load" + + def baseline_workload(endpoint, out, err, mode=baseline_mode): + self._exec_workload("YDB_VECTOR_WORKLOAD_PATH", endpoint, out, err, [ + "--mode", mode, "--data-dir", data_dir, + "--targets", self.targets, "--warmup", self.warmup, + "--rows", self.rows, "--threads", self.threads, + ] + vector_index_args) + + def current_workload(endpoint, out, err): + self._exec_workload("YDB_VECTOR_WORKLOAD_PATH", endpoint, out, err, [ + "--mode", "load", "--data-dir", data_dir, + "--targets", self.targets, "--warmup", self.warmup, + "--rows", self.rows, "--threads", self.threads, + ] + vector_index_args) + + collect_value(main_values, self._run_one( + self.ref, baseline_ydbd, self.baseline_flags, self.baseline_tsc, + baseline_workload, f"vector_main_{i}.log", f"vector_main_{i}.svg")) + collect_value(current_values, self._run_one( + "current", current_ydbd, self.current_flags, self.current_tsc, + current_workload, f"vector_current_{i}.log", f"vector_current_{i}.svg")) + if self.flamegraph: + self._flamegraph_diff("vector", i) self._report("vector", "vector select", self._summarize(main_values, current_values)) def test_fulltext(self): - if self.workload not in ("all", "fulltext"): + if self.workload != "fulltext": pytest.skip(f"compare_workload={self.workload} excludes fulltext") baseline_ydbd = self._baseline_ydbd() diff --git a/ydb/tests/stress/compare_index_performance/tests/ya.make b/ydb/tests/stress/compare_index_performance/tests/ya.make index 72ef6366f92..a497d09c9ac 100644 --- a/ydb/tests/stress/compare_index_performance/tests/ya.make +++ b/ydb/tests/stress/compare_index_performance/tests/ya.make @@ -9,7 +9,7 @@ TEST_SRCS( test_compare.py ) -REQUIREMENTS(ram:32 cpu:4) +REQUIREMENTS(ram:64 cpu:4) TAG(ya:external) SIZE(LARGE) diff --git a/ydb/tests/stress/vector_workload/__main__.py b/ydb/tests/stress/vector_workload/__main__.py index bdff9c08bbf..84dd6e6afd0 100644 --- a/ydb/tests/stress/vector_workload/__main__.py +++ b/ydb/tests/stress/vector_workload/__main__.py @@ -10,19 +10,30 @@ if __name__ == '__main__': parser.add_argument('--endpoint', default='grpc://localhost:2135', help="YDB endpoint") parser.add_argument('--database', default=None, required=True, help='A database to connect') parser.add_argument('--duration', default=120, type=int, help='A duration of workload in seconds') - parser.add_argument('--mode', default='standalone', choices=['standalone', 'generate', 'load'], - help='Mode: standalone (default), generate (generate + dump), load (restore + run)') + parser.add_argument('--mode', default='standalone', choices=['standalone', 'generate', 'load', 's3'], + help='Mode: standalone (default), generate (generate + dump), load (restore + run), s3 (import from S3)') parser.add_argument('--data-dir', default=None, help='Directory for dump/restore data (required for generate/load modes)') parser.add_argument('--targets', default=1000, type=int, help='Number of query vectors for run select (default: 1000)') parser.add_argument('--warmup', default=0, type=int, help='Warmup duration in seconds before measured run (default: 0, disabled)') parser.add_argument('--rows', default=10000, type=int, help='Number of rows in generated database (default: 10000)') parser.add_argument('--threads', default=10, type=int, help='Number of threads for load testing (default: 10)') + parser.add_argument('--clusters', default=None, type=int, help='Number of clusters in kmeans tree (default: server auto-detect)') + parser.add_argument('--levels', default=None, type=int, help='Number of levels in kmeans tree (default: server auto-detect)') + parser.add_argument('--s3-endpoint', default=None, help='S3 endpoint for dataset import (required for --mode=s3)') + parser.add_argument('--s3-bucket', default=None, help='S3 bucket for dataset import (required for --mode=s3)') + parser.add_argument('--s3-source', default=None, help='S3 object key prefix for dataset import (required for --mode=s3)') + parser.add_argument('--s3-destination', default=None, help='Database path for dataset import (required for --mode=s3)') + parser.add_argument('--s3-query-source', default=None, help='S3 object key prefix for query table import (optional for --mode=s3)') + parser.add_argument('--s3-query-destination', default=None, help='Database path for query table import (optional for --mode=s3)') parser.add_argument('--log_file', default=None, help='Append log into specified file') args = parser.parse_args() if args.mode in ('generate', 'load') and not args.data_dir: parser.error(f"--data-dir is required for --mode={args.mode}") + if args.mode == 's3': + if not all([args.s3_endpoint, args.s3_bucket, args.s3_source, args.s3_destination]): + parser.error("--s3-endpoint, --s3-bucket, --s3-source, and --s3-destination are required for --mode=s3") if args.log_file: logging.basicConfig( @@ -35,6 +46,11 @@ if __name__ == '__main__': workload = YdbVectorWorkload(args.endpoint, args.database, duration=args.duration, mode=args.mode, data_dir=args.data_dir, targets=args.targets, - warmup=args.warmup, rows=args.rows, threads=args.threads) + warmup=args.warmup, rows=args.rows, threads=args.threads, + clusters=args.clusters, levels=args.levels, + s3_endpoint=args.s3_endpoint, s3_bucket=args.s3_bucket, + s3_source=args.s3_source, s3_destination=args.s3_destination, + s3_query_source=args.s3_query_source, + s3_query_destination=args.s3_query_destination) workload.start() workload.join() diff --git a/ydb/tests/stress/vector_workload/workload/__init__.py b/ydb/tests/stress/vector_workload/workload/__init__.py index 1be27e0dd2c..d32fe1bf50e 100644 --- a/ydb/tests/stress/vector_workload/workload/__init__.py +++ b/ydb/tests/stress/vector_workload/workload/__init__.py @@ -1,3 +1,4 @@ +import json import logging import os import shutil @@ -15,7 +16,8 @@ QUERY_TABLE_NAME = "vector_query_table" class YdbVectorWorkload(WorkloadBase): - def __init__(self, endpoint, database, duration, mode="standalone", data_dir=None, targets=1000, warmup=0, rows=10000, threads=10): + def __init__(self, endpoint, database, duration, mode="standalone", data_dir=None, targets=1000, warmup=0, rows=10000, threads=10, clusters=None, levels=None, + s3_endpoint=None, s3_bucket=None, s3_source=None, s3_destination=None, s3_query_source=None, s3_query_destination=None): super().__init__(None, '', 'vector_workload', None) self.endpoint = endpoint self.database = database @@ -26,9 +28,37 @@ class YdbVectorWorkload(WorkloadBase): self.warmup = str(warmup) self.rows = str(rows) self.threads = str(threads) + self.clusters = str(clusters) if clusters is not None else None + self.levels = str(levels) if levels is not None else None + self.s3_endpoint = s3_endpoint + self.s3_bucket = s3_bucket + self.s3_source = s3_source + self.s3_destination = s3_destination + self.s3_query_source = s3_query_source + self.s3_query_destination = s3_query_destination + # Table the workload operates on. In s3 mode the data is imported to a + # user-provided destination path, otherwise the default workload table. + self.table_name = "vector_index_workload" + self.query_table_name = QUERY_TABLE_NAME + if self.mode == "s3": + if self.s3_destination: + self.table_name = self._rel_to_database(self.s3_destination) + if self.s3_query_source: + query_dest = self.s3_query_destination or f"{self.database}/{QUERY_TABLE_NAME}" + self.query_table_name = self._rel_to_database(query_dest) self.tempdir = None self._unpack_resource('ydb_cli') + def _rel_to_database(self, path): + """Convert an absolute database path to a name relative to the database.""" + prefix = self.database.rstrip('/') + '/' + if path.startswith(prefix): + return path[len(prefix):] + if path.rstrip('/') == self.database.rstrip('/'): + raise ValueError( + f"path must be under the database, not the database itself: {path}") + return path.lstrip('/') + def __del__(self): if self.tempdir is not None: self.tempdir.cleanup() @@ -78,18 +108,77 @@ class YdbVectorWorkload(WorkloadBase): print(f"Attempt {attempt}/{retries} failed, retrying in {delay}s...") time.sleep(delay) + def _wait_for_table_stats(self, timeout=90, interval=3): + """Poll until the table's row-count statistic is non-zero. + + After an index build the row-count estimate is computed asynchronously. + The vector workload's Init() reads it via DescribeTable().GetTableRows() + and aborts ("statistics is not calculated yet") if it starts too early. + We poll the same underlying datashard stat through the .sys/partition_stats + system view and return as soon as it lands — usually far quicker than a + fixed sleep. This is best-effort: if the view is not queryable we fall + back to a short fixed wait, and on timeout we proceed anyway (the select's + own cmd_run_with_retry is the backstop).""" + full_path = f"{self.database.rstrip('/')}/{self.table_name}" + query = ( + "SELECT COALESCE(SUM(RowCount), 0u) AS rows " + f"FROM `.sys/partition_stats` WHERE Path = '{full_path}';" + ) + cmd = self.get_cli_prefix() + ['yql', '-s', query, '--format', 'json-unicode'] + print(f"Waiting for table statistics on {full_path}...") + deadline = time.time() + timeout + query_ok = False + while time.time() < deadline: + rows = None + try: + proc = subprocess.run(cmd, check=True, text=True, capture_output=True) + query_ok = True + for line in proc.stdout.splitlines(): + line = line.strip() + if line: + val = json.loads(line).get('rows') + if val is not None: + rows = int(val) + except (subprocess.CalledProcessError, ValueError) as e: + if not query_ok: + # System view not queryable here: don't spin, fall back to a + # brief fixed wait and let the select's retry cover the rest. + print(f"Cannot query table statistics ({e}); falling back to fixed wait") + time.sleep(30) + return + print("Stats poll query failed, retrying...") + if rows: + print(f"Table statistics ready: {full_path} has ~{rows} rows") + return + time.sleep(interval) + print(f"Timed out after {timeout}s waiting for statistics on {full_path}; proceeding") + + def _build_index_subcmds(self): + """Subcommands to build the index on the workload table.""" + subcmds = ['build-index', '--distance', 'cosine', '--table', self.table_name] + if self.mode == "s3": + # An imported dataset has a fixed, externally-defined vector dimension + # and type. Pass 0 so the CLI omits them from the DDL and the server + # autodetects both from the data. + subcmds += ['--vector-dimension', '0'] + if self.clusters is not None: + subcmds += ['--kmeans-tree-clusters', self.clusters] + if self.levels is not None: + subcmds += ['--kmeans-tree-levels', self.levels] + return subcmds + def _create_query_table(self): """Create a query table with N rows from the main table.""" - print(f"Creating query table {QUERY_TABLE_NAME} with {self.targets} rows...") + print(f"Creating query table {self.query_table_name} with {self.targets} rows...") create_sql = ( - f"CREATE TABLE `{QUERY_TABLE_NAME}` " + f"CREATE TABLE `{self.query_table_name}` " f"(id Uint64 NOT NULL, embedding String, PRIMARY KEY(id));" ) self.cmd_run(self.get_cli_prefix() + ['yql', '-s', create_sql]) populate_sql = ( - f"UPSERT INTO `{QUERY_TABLE_NAME}` " - f"SELECT id, embedding FROM `vector_index_workload` " + f"UPSERT INTO `{self.query_table_name}` " + f"SELECT id, embedding FROM `{self.table_name}` " f"ORDER BY id LIMIT {self.targets};" ) self.cmd_run(self.get_cli_prefix() + ['yql', '-s', populate_sql]) @@ -102,7 +191,7 @@ class YdbVectorWorkload(WorkloadBase): self.cmd_run( self.get_tools_prefix(subcmds=[ 'dump', - '-p', self.database + '/' + QUERY_TABLE_NAME, + '-p', self.database + '/' + self.query_table_name, '-o', self.data_dir, ]) ) @@ -134,14 +223,11 @@ class YdbVectorWorkload(WorkloadBase): ) # Build index explicitly and wait for completion self.cmd_run_with_retry( - self.get_command_prefix(subcmds=[ - 'build-index', - '--distance', 'cosine', - ]) + self.get_command_prefix(subcmds=self._build_index_subcmds()) ) - # Wait for table statistics to be calculated after index build - print("Waiting for table statistics to be calculated...") - time.sleep(30) + # Wait until table statistics (row-count estimate) are computed after the + # index build; the select workload aborts if it starts before they land. + self._wait_for_table_stats() def _get_select_subcmds(self, seconds): subcmds = [ @@ -149,9 +235,10 @@ class YdbVectorWorkload(WorkloadBase): '--seconds', str(seconds), '--threads', self.threads, '--targets', self.targets, + '--table', self.table_name, ] - if self.mode in ('generate', 'load'): - subcmds.extend(['--query-table', QUERY_TABLE_NAME]) + if self.mode in ('generate', 'load', 's3'): + subcmds.extend(['--query-table', self.query_table_name]) return subcmds def _run_select(self): @@ -192,10 +279,45 @@ class YdbVectorWorkload(WorkloadBase): self.get_command_prefix(subcmds=['clean']) ) + def _import_s3_data(self): + """Import data from S3 using ydb import s3, optionally including the queries table.""" + item_main = f"Source={self.s3_source},Destination={self.s3_destination}" + cmd = self.get_cli_prefix() + [ + "import", "s3", + "--s3-endpoint", self.s3_endpoint, + "--bucket", self.s3_bucket, + "--item", item_main, + ] + if self.s3_query_source: + query_dest = self.s3_query_destination or f"{self.database}/{QUERY_TABLE_NAME}" + item_query = f"Source={self.s3_query_source},Destination={query_dest}" + cmd += ["--item", item_query] + print(f"Importing queries table from S3: {self.s3_query_source} -> {query_dest}") + self.cmd_run(cmd) + # Build index explicitly and wait for completion + self.cmd_run_with_retry( + self.get_command_prefix(subcmds=self._build_index_subcmds()) + ) + # Wait until table statistics (row-count estimate) are computed after the + # index build; the select workload aborts if it starts before they land. + self._wait_for_table_stats() + + def __loop_s3(self): + """S3 mode: import data from S3, optionally import queries table, build index, run select, clean.""" + self._import_s3_data() + if not self.s3_query_source: + self._create_query_table() + self._run_select() + self.cmd_run( + self.get_command_prefix(subcmds=['clean']) + ) + def get_workload_thread_funcs(self): if self.mode == "generate": return [self.__loop_generate] elif self.mode == "load": return [self.__loop_load] + elif self.mode == "s3": + return [self.__loop_s3] else: return [self.__loop_standalone] diff --git a/yt/yt/client/federated/cache.cpp b/yt/yt/client/federated/cache.cpp index 7c126ece2e2..60ce865ddcc 100644 --- a/yt/yt/client/federated/cache.cpp +++ b/yt/yt/client/federated/cache.cpp @@ -59,7 +59,7 @@ private: THROW_ERROR_EXCEPTION_UNLESS( clusters.size() == seenClusters.size(), - "Numbers of desired and configured clusters do not match (Desired: %v, Configured: %v)", + "Numbers of desired and configured clusters do not match: desired %v, configured %v", clusters, seenClusters); diff --git a/yt/yt/client/kafka/requests.cpp b/yt/yt/client/kafka/requests.cpp index f1826cd2795..7c599d0a014 100644 --- a/yt/yt/client/kafka/requests.cpp +++ b/yt/yt/client/kafka/requests.cpp @@ -284,7 +284,7 @@ void TRecordBatch::Deserialize(IKafkaProtocolReader* reader) } } if (reader->GetReadBytesCount() != Length) { - THROW_ERROR_EXCEPTION("Unexpected record batch length (Expected: %v, Actual: %v)", Length, reader->GetReadBytesCount()); + THROW_ERROR_EXCEPTION("Unexpected record batch length: expected %v, actual %v", Length, reader->GetReadBytesCount()); } } else { THROW_ERROR_EXCEPTION("Unsupported MagicByte %v in RecordBatch deserialization", static_cast<int>(MagicByte)); diff --git a/yt/yt/core/concurrency/notify_manager.cpp b/yt/yt/core/concurrency/notify_manager.cpp index 75be9b75ead..f66a89cfbb3 100644 --- a/yt/yt/core/concurrency/notify_manager.cpp +++ b/yt/yt/core/concurrency/notify_manager.cpp @@ -219,14 +219,15 @@ void TNotifyManager::NotifyOne(TCpuInstant cpuInstant) CpuInstantToInstant(GetMinEnqueuedAt())); EventCount_->NotifyOne(); } else { - auto lockedInstant = LockedInstant_.load(); - auto waitTime = CpuDurationToDuration(cpuInstant - lockedInstant); - if (waitTime > WaitTimeWarningThreshold) { - // Notifications are locked during more than 30 seconds. - YT_LOG_WARNING("Action is probably stuck (MinEnqueuedAt: %v, LockedInstant: %v, WaitTime: %v)", - CpuInstantToInstant(GetMinEnqueuedAt()), - lockedInstant, - waitTime); + static const auto WaitTimeWarningThresholdCpu = DurationToCpuDuration(WaitTimeWarningThreshold); + + auto minEnqueuedAt = GetMinEnqueuedAt(); + if (minEnqueuedAt != SentinelMinEnqueuedAt && + cpuInstant - minEnqueuedAt > WaitTimeWarningThresholdCpu) + { + YT_LOG_WARNING("Action is probably stuck (MinEnqueuedAt: %v, WaitTime: %v)", + CpuInstantToInstant(minEnqueuedAt), + CpuDurationToDuration(cpuInstant - minEnqueuedAt)); } } } diff --git a/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp b/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp index dd8f6596551..afee3288996 100644 --- a/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp +++ b/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp @@ -167,7 +167,7 @@ TEST_P(TBoundedConcurrencyInvokerParametrizedReconfigureTest, SetMaxConcurrentIn auto guard = Guard(lock); auto concurrentInvocations = runnedCallbacks - finishedCallbacks; - THROW_ERROR_EXCEPTION_UNLESS(concurrentInvocations <= maxConcurrentInvocations, "Number of concurrent invocations exceeds maximum (ConcurrentInvocations: %v, MaxConcurrentInvocations: %v)", + THROW_ERROR_EXCEPTION_UNLESS(concurrentInvocations <= maxConcurrentInvocations, "Number of concurrent invocations %v exceeds maximum %v", concurrentInvocations, maxConcurrentInvocations); if (callbackIndex > maxConcurrentInvocations) { diff --git a/yt/yt/core/net/address.cpp b/yt/yt/core/net/address.cpp index 16eee1cdb5b..dfe29b605c7 100644 --- a/yt/yt/core/net/address.cpp +++ b/yt/yt/core/net/address.cpp @@ -1222,7 +1222,7 @@ TIP6Address TMtnAddress::ToIP6Address() const ui64 TMtnAddress::GetBytesRangeValue(int leftIndex, int rightIndex) const { if (leftIndex > rightIndex) { - THROW_ERROR_EXCEPTION("Left index is greater than right index (LeftIndex: %v, RightIndex: %v)", + THROW_ERROR_EXCEPTION("Left index %v is greater than right index %v", leftIndex, rightIndex); } @@ -1239,17 +1239,17 @@ ui64 TMtnAddress::GetBytesRangeValue(int leftIndex, int rightIndex) const void TMtnAddress::SetBytesRangeValue(int leftIndex, int rightIndex, ui64 value) { if (leftIndex > rightIndex) { - THROW_ERROR_EXCEPTION("Left index is greater than right index (LeftIndex: %v, RightIndex: %v)", + THROW_ERROR_EXCEPTION("Left index %v is greater than right index %v", leftIndex, rightIndex); } auto bytesInRange = rightIndex - leftIndex; if (value >= (1ull << (8 * bytesInRange))) { - THROW_ERROR_EXCEPTION("Value is too large to be set in [leftIndex; rightIndex) interval (LeftIndex: %v, RightIndex: %v, Value %v)", + THROW_ERROR_EXCEPTION("Value %v is too large to be set in interval [%v, %v)", + value, leftIndex, - rightIndex, - value); + rightIndex); } auto* addressBytes = Address_.GetRawBytes(); diff --git a/yt/yt/core/ytree/polymorphic_yson_struct-inl.h b/yt/yt/core/ytree/polymorphic_yson_struct-inl.h index 595f8fafd70..067956ad5db 100644 --- a/yt/yt/core/ytree/polymorphic_yson_struct-inl.h +++ b/yt/yt/core/ytree/polymorphic_yson_struct-inl.h @@ -236,7 +236,7 @@ void TPolymorphicYsonStruct<TMapping>::MergeWith(const TPolymorphicYsonStruct& o THROW_ERROR_EXCEPTION_UNLESS( GetType() == other.GetType(), - "Can't merge polymorphic yson structs with different types stored (ThisType: %v, OtherType: %v)", + "Cannot merge polymorphic YSON structs with different stored types: %Qlv vs %Qlv", GetType(), other.GetType()); diff --git a/yt/yt/library/formats/arrow_writer.cpp b/yt/yt/library/formats/arrow_writer.cpp index 54756b7cff6..713c23c602f 100644 --- a/yt/yt/library/formats/arrow_writer.cpp +++ b/yt/yt/library/formats/arrow_writer.cpp @@ -898,7 +898,9 @@ void SerializeDateColumn( }, [&] (auto value) { if (value > std::numeric_limits<i32>::max()) { - THROW_ERROR_EXCEPTION("Date value cannot be represented in arrow (Value: %v, MaxAllowedValue: %v)", value, std::numeric_limits<i32>::max()); + THROW_ERROR_EXCEPTION("Date value %v cannot be represented in arrow: maximum allowed value is %v", + value, + std::numeric_limits<i32>::max()); } *currentOutput++ = value; }); @@ -1007,7 +1009,7 @@ void SerializeTimestampColumn( }, [&] (auto value) { if (value > std::numeric_limits<i64>::max()) { - THROW_ERROR_EXCEPTION("Timestamp value cannot be represented in arrow (Value: %v, MaxAllowedValue: %v)", value, std::numeric_limits<i64>::max()); + THROW_ERROR_EXCEPTION("Timestamp value %v cannot be represented in arrow: maximum allowed value is %v", value, std::numeric_limits<i64>::max()); } *currentOutput++ = value; }); |
