diff options
author | mshulb <mshulb@yandex-team.com> | 2023-11-15 16:25:30 +0300 |
---|---|---|
committer | mshulb <mshulb@yandex-team.com> | 2023-11-15 18:06:54 +0300 |
commit | e78b894c23fbad4cc2adaffd0fbd0462d0f8ecba (patch) | |
tree | eb72c6e6756ebbd8db61c450d2d7d61eab5496c5 | |
parent | 7d29550ba7d942914ec96a5101e6309148ebce50 (diff) | |
download | ydb-e78b894c23fbad4cc2adaffd0fbd0462d0f8ecba.tar.gz |
KIKIMR-19957 Implement UNIFORM_PARTITIONS for UUID.
Note that UUID is stored and compared as byte array with following byte
order:
```
UUID 00 11 22 33-44 55-66 77-88 99-AA BB CC DD EE FF
bytes 33 22 11 00 55 44 77 66 88 99 AA BB CC DD EE FF
byte order 3 2 1 0 5 4 7 6 8 9 10 11 12 13 14 15
```
Thus UUID ordering gives surprising results (not that anyone should
use it on UUIDs).
-rw-r--r-- | ydb/core/engine/mkql_proto.cpp | 34 | ||||
-rw-r--r-- | ydb/core/engine/mkql_proto.h | 3 | ||||
-rw-r--r-- | ydb/core/engine/mkql_proto_ut.cpp | 3 | ||||
-rw-r--r-- | ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 175 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp | 3 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/schemeshard_impl.cpp | 21 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp | 3 | ||||
-rw-r--r-- | ydb/core/tx/tx_proxy/datareq.cpp | 3 | ||||
-rw-r--r-- | ydb/core/tx/tx_proxy/read_table_impl.cpp | 3 | ||||
-rw-r--r-- | ydb/core/tx/tx_proxy/resolvereq.cpp | 3 | ||||
-rw-r--r-- | ydb/library/mkql_proto/mkql_proto.h | 27 | ||||
-rw-r--r-- | ydb/library/uuid/uuid.h | 25 | ||||
-rw-r--r-- | ydb/public/sdk/cpp/client/ydb_value/value.cpp | 1 |
13 files changed, 258 insertions, 46 deletions
diff --git a/ydb/core/engine/mkql_proto.cpp b/ydb/core/engine/mkql_proto.cpp index e14042352b..0a3867d876 100644 --- a/ydb/core/engine/mkql_proto.cpp +++ b/ydb/core/engine/mkql_proto.cpp @@ -18,7 +18,8 @@ bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType, const TConstArrayRef<NScheme::TTypeInfo>& types, bool allowCastFromString, TVector<TCell>& key, - TString& errStr) + TString& errStr, + TVector<TString>& memoryOwner) { #define CHECK_OR_RETURN_ERROR(cond, descr) \ @@ -157,18 +158,37 @@ bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType, if (v.HasBytes()) { c = TCell(v.GetBytes().data(), v.GetBytes().size()); } else if (v.HasText()) { - auto typeDesc = types[i].GetTypeDesc(); + auto typeDesc = types[i].GetTypeDesc(); auto convert = NPg::PgNativeBinaryFromNativeText(v.GetText(), NPg::PgTypeIdFromTypeDesc(typeDesc)); if (convert.Error) { CHECK_OR_RETURN_ERROR(false, Sprintf("Cannot parse value of type Pg: %s in tuple at position %" PRIu32, convert.Error->data(), i)); } else { - c = TCell(convert.Str.data(), convert.Str.size()); + auto &data = memoryOwner.emplace_back(convert.Str); + c = TCell(data); } } else { CHECK_OR_RETURN_ERROR(false, Sprintf("Cannot parse value of type Pg in tuple at position %" PRIu32, i)); } break; } + case NScheme::NTypeIds::Uuid: + { + if (v.HasLow128()) { + if (!v.HasHi128()) { + CHECK_OR_RETURN_ERROR(false, Sprintf("UUID has Low128 but not Hi128 at position: %" PRIu32, i)); + } + auto &data = memoryOwner.emplace_back(); + data.resize(NUuid::UUID_LEN); + NUuid::UuidHalfsToBytes(data.Detach(), data.size(), v.GetHi128(), v.GetLow128()); + c = TCell(data); + } else if (v.HasBytes()) { + Y_ABORT_UNLESS(v.GetBytes().size() == NUuid::UUID_LEN); + c = TCell(v.GetBytes().data(), v.GetBytes().size()); + } else { + CHECK_OR_RETURN_ERROR(false, Sprintf("Cannot parse value of type Uuid in tuple at position %" PRIu32, i)); + } + break; + } default: CHECK_OR_RETURN_ERROR(false, Sprintf("Unsupported typeId %" PRIu16 " at index %" PRIu32, typeId, i)); break; @@ -280,6 +300,14 @@ bool CellToValue(NScheme::TTypeInfo type, const TCell& c, NKikimrMiniKQL::TValue break; } + case NScheme::NTypeIds::Uuid: { + ui64 high = 0, low = 0; + NUuid::UuidBytesToHalfs(c.Data(), c.Size(), high, low); + val.MutableOptional()->SetHi128(high); + val.MutableOptional()->SetLow128(low); + break; + } + default: errStr = "Unknown type: " + ToString(typeId); return false; diff --git a/ydb/core/engine/mkql_proto.h b/ydb/core/engine/mkql_proto.h index 1c316119c5..6e78806227 100644 --- a/ydb/core/engine/mkql_proto.h +++ b/ydb/core/engine/mkql_proto.h @@ -18,7 +18,8 @@ bool CellsFromTuple(const NKikimrMiniKQL::TType* tupleType, const TConstArrayRef<NScheme::TTypeInfo>& expectedTypes, bool allowCastFromString, TVector<TCell>& key, - TString& errStr); + TString& errStr, + TVector<TString>& memoryOwner); bool CellToValue(NScheme::TTypeInfo type, const TCell& c, NKikimrMiniKQL::TValue& val, TString& errStr); diff --git a/ydb/core/engine/mkql_proto_ut.cpp b/ydb/core/engine/mkql_proto_ut.cpp index 4e6a76ee43..eda9583be7 100644 --- a/ydb/core/engine/mkql_proto_ut.cpp +++ b/ydb/core/engine/mkql_proto_ut.cpp @@ -509,8 +509,9 @@ Y_UNIT_TEST(TestExportVariantStructTypeYdb) { UNIT_ASSERT_C(parseOk, paramsProto); TVector<TCell> cells; + TVector<TString> memoryOwner; TString errStr; - bool res = CellsFromTuple(¶ms.GetType(), params.GetValue(), types, true, cells, errStr); + bool res = CellsFromTuple(¶ms.GetType(), params.GetValue(), types, true, cells, errStr, memoryOwner); UNIT_ASSERT_VALUES_EQUAL_C(res, errStr.empty(), paramsProto); return errStr; diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 83946d31e3..cff91a5cac 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -9,6 +9,8 @@ #include <ydb/core/testlib/cs_helper.h> #include <ydb/core/testlib/common_helper.h> #include <ydb/core/formats/arrow/serializer/full.h> +#include <ydb/library/uuid/uuid.h> + #include <library/cpp/threading/local_executor/local_executor.h> #include <util/generic/serialized_enum.h> @@ -1083,7 +1085,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto describeResult = session.DescribeTable("/Root/moved").GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); } { @@ -1096,7 +1098,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto describeResult = session.DescribeTable("/Root/table").GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); } { @@ -1124,11 +1126,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) { { auto describeResult = session.DescribeTable("/Root/moved").GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); } { auto describeResult = session.DescribeTable("/Root/movedsecond").GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); } } @@ -1165,12 +1167,12 @@ Y_UNIT_TEST_SUITE(KqpScheme) { { auto describeResult = session.DescribeTable("/Root/second").GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); } { auto describeResult = session.DescribeTable("/Root/table").GetValueSync(); - UNIT_ASSERT_C(!describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(!describeResult.IsSuccess(), describeResult.GetIssues().ToString()); } } @@ -1622,7 +1624,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings().WithTableStatistics(true)).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetPartitionsCount(), 4); AlterTableSetttings(session, tableName, {{"UNIFORM_PARTITIONS", "8"}}, compat, @@ -1658,7 +1660,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto describeResult = session.DescribeTable(tableName, TDescribeTableSettings().WithTableStatistics(true).WithKeyShardBoundary(true)).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetPartitionsCount(), 5); auto extractValue = [](const TValue& val) { @@ -1724,7 +1726,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto describeResult = session.DescribeTable(tableName, TDescribeTableSettings().WithTableStatistics(true).WithKeyShardBoundary(true)).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetPartitionsCount(), 4); auto extractValue = [](const TValue& val) { @@ -1779,7 +1781,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto describeResult = session.DescribeTable(tableName, TDescribeTableSettings().WithTableStatistics(true).WithKeyShardBoundary(true)).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); UNIT_ASSERT_VALUES_EQUAL(describeResult.GetTableDescription().GetPartitionsCount(), 4); auto extractValue = [](const TValue& val) { @@ -1854,6 +1856,141 @@ Y_UNIT_TEST_SUITE(KqpScheme) { "Partition at keys has 2 key values while there are only 1 key columns", "Unexpected error message"); } + struct testData { + TString condition; + ui64 resultRows; + ui64 touchedPartitions; + }; + + void uuidInsertAndCheck(TSession &session, TString tableName, TVector<testData> expectedPartitions) { + TVector<TString> uuids { + {"AAAAAA00-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAA11-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAA22-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAA33-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAA44-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAA55-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAA66-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAA77-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAA88-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAA99-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAAAA-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAABB-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAACC-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAADD-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAAEE-0000-458F-ABE9-4A0CD520903B"}, + {"AAAAAAFF-0000-458F-ABE9-4A0CD520903B"}, + }; + { + TStringBuilder builder; + builder << "REPLACE INTO `" << tableName << "` (Key, Value) VALUES "; + for (ui32 i = 0; i < uuids.size() - 1; ++i) { + builder << "(Uuid(\"" << uuids[i] << "\"), " << i << "),"; + } + builder << "(Uuid(\"" << uuids[uuids.size() - 1] << "\"), " << uuids.size() - 1 << ");"; + TString query = builder; + + auto replaceResult = session.ExecuteDataQuery(query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(replaceResult.GetStatus(), EStatus::SUCCESS, + replaceResult.GetIssues().ToString()); + } + + for (auto &test: expectedPartitions) { + TString query = Sprintf("SELECT Key as cnt from `%s` WHERE %s;", tableName.data(), test.condition.data()); + + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.CollectQueryStats(ECollectQueryStatsMode::Profile); + + auto selectResult = session.ExecuteDataQuery(query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(), + execSettings).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(selectResult.GetStatus(), EStatus::SUCCESS, + selectResult.GetIssues().ToString()); + + UNIT_ASSERT_VALUES_EQUAL(selectResult.GetResultSets().size(), 1); + + auto& stats = NYdb::TProtoAccessor::GetProto(*selectResult.GetStats()); + UNIT_ASSERT_VALUES_EQUAL(selectResult.GetResultSet(0).RowsCount(), test.resultRows); + UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).partitions_count(), test.touchedPartitions); + } + } + + Y_UNIT_TEST(CreateTableWithPartitionAtKeysUuid) { + TKikimrSettings kikimrSettings; + TKikimrRunner kikimr(kikimrSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithPartitionAtKeysSimpleUuid"; + + { + auto builder = TTableBuilder() + .AddNonNullableColumn("Key", EPrimitiveType::Uuid) + .AddNullableColumn("Value", EPrimitiveType::Int32) + .SetPrimaryKeyColumn("Key"); + + // Ordering is not lexicographic as UUID is stored in binary form with the following byte order + // from original hex pairs: [3 2 1 0 5 4 7 6 8 9 a b c d e f] + // String UUID (with spaces added) 00 11 22 33-44 55-66 77-88 99-AA BB CC DD EE FF + // becomes 33 22 11 00 55 44 77 66 88 99 AA BB CC DD EE FF + const TVector <TUuidValue> expectedRanges = { + TUuidValue("FFFFFF11-C00F-458F-ABE9-4A0CD520903B"), + TUuidValue("FFFFFFDD-AF48-428B-9D13-893C220118C4") + }; + auto explicitPartitions = TExplicitPartitions(); + for (ui32 i = 0; i < expectedRanges.size(); i++) { + explicitPartitions.AppendSplitPoints( + TValueBuilder().BeginTuple().AddElement() + .OptionalUuid(expectedRanges[i]).EndTuple().Build() + ); + } + auto result = session.CreateTable(tableName, + builder.Build(), + TCreateTableSettings() + .PartitioningPolicy(TPartitioningPolicy().ExplicitPartitions(explicitPartitions)) + ).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + // See comment above for comparison explanation. + TVector<testData> inputs = { + {"Key > Cast(\"00000000-FFFF-FFFF-ABE9-4A0CD520903B\" as Uuid)",16, 3}, + {"Key > Cast(\"FFFFFF11-C00F-458F-ABE9-4A0CD520903B\" as Uuid)", 14, 2}, + {"Key < Cast(\"FFFFFF11-C00F-458F-ABE9-4A0CD520903B\" as Uuid)", 2, 1} + }; + uuidInsertAndCheck(session, tableName, inputs); + } + + Y_UNIT_TEST(CreateTableWithUniformPartitionsUuid) { + TKikimrSettings kikimrSettings; + TKikimrRunner kikimr(kikimrSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + TString tableName = "/Root/TableWithPartitionAtKeysSimpleUuid"; + + { + auto builder = TTableBuilder() + .AddNonNullableColumn("Key", EPrimitiveType::Uuid) + .AddNullableColumn("Value", EPrimitiveType::Int32) + .SetPrimaryKeyColumn("Key"); + + auto result = session.CreateTable(tableName, + builder.Build(), + TCreateTableSettings() + .PartitioningPolicy(TPartitioningPolicy().UniformPartitions(4)) + ).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + } + // See comment above for comparison explanation. + TVector<testData> inputs = { + {"Key > Cast(\"00000000-FFFF-FFFF-ABE9-4A0CD520903B\" as Uuid)",16, 4}, + {"Key < Cast(\"0000003F-C00F-458F-ABE9-4A0CD520903B\" as Uuid)", 4, 1}, + {"Key < Cast(\"0000007F-C00F-458F-ABE9-4A0CD520903B\" as Uuid)", 8, 2}, + {"Key < Cast(\"000000BF-C00F-458F-ABE9-4A0CD520903B\" as Uuid)", 12, 3}, + {"Key < Cast(\"000000FF-C00F-458F-ABE9-4A0CD520903B\" as Uuid)", 15, 4} + }; + uuidInsertAndCheck(session, tableName, inputs); + } + Y_UNIT_TEST(CreateTableWithFamiliesRegular) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); @@ -1876,7 +2013,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings()).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); const auto& columnFamilies = describeResult.GetTableDescription().GetColumnFamilies(); UNIT_ASSERT_VALUES_EQUAL(columnFamilies.size(), 3); for (const auto& family : columnFamilies) { @@ -1915,7 +2052,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { { auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings()).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); const auto& columnFamilies = describeResult.GetTableDescription().GetColumnFamilies(); UNIT_ASSERT_VALUES_EQUAL(columnFamilies.size(), 2); for (const auto& family : columnFamilies) { @@ -1947,7 +2084,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { { auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings()).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); const auto& columnFamilies = describeResult.GetTableDescription().GetColumnFamilies(); UNIT_ASSERT_VALUES_EQUAL(columnFamilies.size(), 3); for (const auto& family : columnFamilies) { @@ -1992,7 +2129,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto describeResult = session.DescribeTable(tableName).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); UNIT_ASSERT(describeResult.GetTableDescription().GetStorageSettings().GetStoreExternalBlobs().GetOrElse(false)); } @@ -3189,7 +3326,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto describeResult = session.DescribeTable(tableName, NYdb::NTable::TDescribeTableSettings()).GetValueSync(); - UNIT_ASSERT_C(describeResult.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); const auto tableDesc = session.DescribeTable(tableName).GetValueSync().GetTableDescription(); TVector<TTableColumn> columns = tableDesc.GetTableColumns(); UNIT_ASSERT_VALUES_EQUAL(columns.size(), 3); @@ -3831,7 +3968,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } + } { TString query = R"( @@ -3898,7 +4035,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } + } } Y_UNIT_TEST(DefaultValuesForTable3) { @@ -3923,7 +4060,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Default expr Key is nullable or optional, but column has not null constraint"); - } + } } Y_UNIT_TEST(DefaultValuesForTable4) { @@ -3949,7 +4086,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::GENERIC_ERROR, result.GetIssues().ToString()); - } + } } Y_UNIT_TEST(ChangefeedRetentionPeriod) { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp index 63c23985c1..f22f6e8dbc 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_pq.cpp @@ -121,8 +121,9 @@ TTopicInfo::TPtr CreatePersQueueGroup(TOperationContext& context, if (op.PartitionBoundariesSize()) { TVector<TCell> cells; TString error; + TVector<TString> memoryOwner; if (!NMiniKQL::CellsFromTuple(nullptr, op.GetPartitionBoundaries(i), pqGroupInfo->KeySchema, false, - cells, error)) { + cells, error, memoryOwner)) { status = NKikimrScheme::StatusSchemeError; errStr = Sprintf("Invalid partition boundary at position: %u, error: %s", i, error.data()); return nullptr; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 8d92948bcc..c4972c3533 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -14,6 +14,8 @@ #include <ydb/core/statistics/stat_service.h> #include <ydb/core/scheme/scheme_types_proto.h> #include <ydb/library/yql/minikql/mkql_type_ops.h> +#include <util/system/byteorder.h> +#include <util/system/unaligned_mem.h> namespace NKikimr { namespace NSchemeShard { @@ -6229,10 +6231,11 @@ bool TSchemeShard::FillSplitPartitioning(TVector<TString>& rangeEnds, const TCon auto& boundary = boundaries.Get(i); TVector<TCell> rangeEnd; TSerializedCellVec prefix; + TVector<TString> memoryOwner; if (boundary.HasSerializedKeyPrefix()) { prefix.Parse(boundary.GetSerializedKeyPrefix()); rangeEnd = TVector<TCell>(prefix.GetCells().begin(), prefix.GetCells().end()); - } else if (!NMiniKQL::CellsFromTuple(nullptr, boundary.GetKeyPrefix(), keyColTypes, false, rangeEnd, errStr)) { + } else if (!NMiniKQL::CellsFromTuple(nullptr, boundary.GetKeyPrefix(), keyColTypes, false, rangeEnd, errStr, memoryOwner)) { errStr = Sprintf("Error at split boundary %d: %s", i, errStr.data()); return false; } @@ -6405,6 +6408,22 @@ bool TSchemeShard::FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 key maxVal = Max<ui64>(); valSz = 8; break; + case NScheme::NTypeIds::Uuid: { + maxVal = Max<ui64>(); + valSz = 16; + char buffer[16] = {}; + + for (ui32 i = 1; i < partitionCount; ++i) { + ui64 val = maxVal * (double(i) / partitionCount); + // Make sure most significant byte is at the start of the byte buffer for UUID comparison. + val = HostToInet(val); + WriteUnaligned<ui64>(buffer, val); + rangeEnd[0] = TCell(buffer, valSz); + rangeEnds.push_back(TSerializedCellVec::Serialize(rangeEnd)); + } + + return true; + } default: errStr = TStringBuilder() << "Unsupported first key column type " << NScheme::TypeName(firstKeyColType) << ", only Uint32 and Uint64 are supported"; return false; diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index d0de3aa5e5..881813906a 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -484,7 +484,8 @@ void CheckBoundaries(const NKikimrScheme::TEvDescribeSchemeResult &record) { for (ui32 i = 0; i < descr.GetTable().SplitBoundarySize(); ++i) { const auto& b = descr.GetTable().GetSplitBoundary(i); TVector<TCell> cells; - NMiniKQL::CellsFromTuple(nullptr, b.GetKeyPrefix(), keyColTypes, false, cells, errStr); + TVector<TString> memoryOwner; + NMiniKQL::CellsFromTuple(nullptr, b.GetKeyPrefix(), keyColTypes, false, cells, errStr, memoryOwner); UNIT_ASSERT_VALUES_EQUAL(errStr, ""); TString serialized = TSerializedCellVec::Serialize(cells); diff --git a/ydb/core/tx/tx_proxy/datareq.cpp b/ydb/core/tx/tx_proxy/datareq.cpp index c5b49e633c..cc4fc17df5 100644 --- a/ydb/core/tx/tx_proxy/datareq.cpp +++ b/ydb/core/tx/tx_proxy/datareq.cpp @@ -3024,6 +3024,7 @@ bool TDataReq::ParseRangeKey(const NKikimrMiniKQL::TParams &proto, EParseRangeKeyExp exp) { TVector<TCell> key; + TVector<TString> memoryOwner; if (proto.HasValue()) { if (!proto.HasType()) { UnresolvedKeys.push_back("No type was specified in the range key tuple"); @@ -3033,7 +3034,7 @@ bool TDataReq::ParseRangeKey(const NKikimrMiniKQL::TParams &proto, auto& value = proto.GetValue(); auto& type = proto.GetType(); TString errStr; - bool res = NMiniKQL::CellsFromTuple(&type, value, keyType, true, key, errStr); + bool res = NMiniKQL::CellsFromTuple(&type, value, keyType, true, key, errStr, memoryOwner); if (!res) { UnresolvedKeys.push_back("Failed to parse range key tuple: " + errStr); return false; diff --git a/ydb/core/tx/tx_proxy/read_table_impl.cpp b/ydb/core/tx/tx_proxy/read_table_impl.cpp index 6645208e76..e5577d058a 100644 --- a/ydb/core/tx/tx_proxy/read_table_impl.cpp +++ b/ydb/core/tx/tx_proxy/read_table_impl.cpp @@ -96,6 +96,7 @@ namespace { TVector<TString>& unresolvedKeys) { TVector<TCell> key; + TVector<TString> memoryOwner; if (proto.HasValue()) { if (!proto.HasType()) { unresolvedKeys.push_back("No type was specified in the range key tuple"); @@ -105,7 +106,7 @@ namespace { auto& value = proto.GetValue(); auto& type = proto.GetType(); TString errStr; - bool res = NMiniKQL::CellsFromTuple(&type, value, keyTypes, true, key, errStr); + bool res = NMiniKQL::CellsFromTuple(&type, value, keyTypes, true, key, errStr, memoryOwner); if (!res) { unresolvedKeys.push_back("Failed to parse range key tuple: " + errStr); return false; diff --git a/ydb/core/tx/tx_proxy/resolvereq.cpp b/ydb/core/tx/tx_proxy/resolvereq.cpp index 01353f98e1..eb916abc89 100644 --- a/ydb/core/tx/tx_proxy/resolvereq.cpp +++ b/ydb/core/tx/tx_proxy/resolvereq.cpp @@ -25,6 +25,7 @@ namespace { TVector<TString>& unresolvedKeys) { TVector<TCell> key; + TVector<TString> memoryOwner; if (proto.HasValue()) { if (!proto.HasType()) { unresolvedKeys.push_back("No type was specified in the range key tuple"); @@ -34,7 +35,7 @@ namespace { auto& value = proto.GetValue(); auto& type = proto.GetType(); TString errStr; - bool res = NMiniKQL::CellsFromTuple(&type, value, keyTypes, true, key, errStr); + bool res = NMiniKQL::CellsFromTuple(&type, value, keyTypes, true, key, errStr, memoryOwner); if (!res) { unresolvedKeys.push_back("Failed to parse range key tuple: " + errStr); return false; diff --git a/ydb/library/mkql_proto/mkql_proto.h b/ydb/library/mkql_proto/mkql_proto.h index 52817eea90..e9ed92b320 100644 --- a/ydb/library/mkql_proto/mkql_proto.h +++ b/ydb/library/mkql_proto/mkql_proto.h @@ -4,6 +4,7 @@ #include <ydb/library/yql/minikql/mkql_node.h> #include <ydb/library/mkql_proto/protos/minikql.pb.h> #include <ydb/public/api/protos/ydb_value.pb.h> +#include <ydb/library/uuid/uuid.h> namespace NKikimr::NMiniKQL { @@ -31,25 +32,19 @@ TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TType& type, const NKiki TRuntimeNode ImportValueFromProto(const NKikimrMiniKQL::TParams& params, const TTypeEnvironment& env); inline void UuidToMkqlProto(const char* str, size_t sz, NKikimrMiniKQL::TValue& res) { - union { - ui64 half[2]; - char bytes[sizeof(ui64) * 2]; - } buf; - Y_ABORT_UNLESS(sizeof(buf) == sz); - memcpy(buf.bytes, str, sizeof(buf)); - res.SetLow128(buf.half[0]); - res.SetHi128(buf.half[1]); + ui64 high = 0, low = 0; + NUuid::UuidBytesToHalfs(str, sz, high, low); + + res.SetLow128(low); + res.SetHi128(high); } inline void UuidToYdbProto(const char* str, size_t sz, Ydb::Value& res) { - union { - ui64 half[2]; - char bytes[sizeof(ui64) * 2]; - } buf; - Y_ABORT_UNLESS(sizeof(buf) == sz); - memcpy(buf.bytes, str, sizeof(buf)); - res.set_low_128(buf.half[0]); - res.set_high_128(buf.half[1]); + ui64 high = 0, low = 0; + NUuid::UuidBytesToHalfs(str, sz, high, low); + + res.set_low_128(low); + res.set_high_128(high); } } diff --git a/ydb/library/uuid/uuid.h b/ydb/library/uuid/uuid.h index 4c89090371..b1bb1912bd 100644 --- a/ydb/library/uuid/uuid.h +++ b/ydb/library/uuid/uuid.h @@ -1,5 +1,6 @@ #pragma once #include <util/system/types.h> +#include <util/system/yassert.h> #include <cctype> #include <utility> @@ -9,6 +10,8 @@ class IOutputStream; namespace NKikimr { namespace NUuid { +static constexpr ui32 UUID_LEN = 16; + void UuidToString(ui16 dw[8], IOutputStream& out); void UuidHalfsToByteString(ui64 low, ui64 hi, IOutputStream& out); @@ -92,5 +95,27 @@ bool ParseUuidToArray(const T& buf, ui16* dw, bool shortForm) { return true; } +inline void UuidHalfsToBytes(char *dst, size_t dstSize, ui64 hi, ui64 low) { + union { + char bytes[UUID_LEN]; + ui64 half[2]; + } buf; + Y_ABORT_UNLESS(UUID_LEN == dstSize); + buf.half[0] = low; + buf.half[1] = hi; + memcpy(dst, buf.bytes, sizeof(buf)); +} + +inline void UuidBytesToHalfs(const char *str, size_t sz, ui64 &high, ui64 &low) { + union { + char bytes[UUID_LEN]; + ui64 half[2]; + } buf; + Y_ABORT_UNLESS(UUID_LEN == sz); + memcpy(buf.bytes, str, sizeof(buf)); + low = buf.half[0]; + high = buf.half[1]; +} + } } diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.cpp b/ydb/public/sdk/cpp/client/ydb_value/value.cpp index ef7a6179b9..0b2e1e24c8 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value.cpp @@ -1021,6 +1021,7 @@ TUuidValue::TUuidValue(const TString& uuidString) { ThrowFatalError(TStringBuilder() << "Unable to parse string as uuid"); } static_assert(sizeof(dw) == sizeof(Buf_.Bytes)); + // TODO: check output on big-endian machines here and everywhere. std::memcpy(Buf_.Bytes, dw, sizeof(dw)); } |