diff options
| author | monster <monster@ydb.tech> | 2023-02-10 15:42:00 +0300 |
|---|---|---|
| committer | monster <monster@ydb.tech> | 2023-02-10 15:42:00 +0300 |
| commit | f1892ba708c7e291bd6809366b4b7c0bccc36d3e (patch) | |
| tree | 516690bccde15878922c91db0971e296bb6b52f7 | |
| parent | c5c1a23f428643e0fb6675b84dc4d7f00abf1ead (diff) | |
| download | ydb-f1892ba708c7e291bd6809366b4b7c0bccc36d3e.tar.gz | |
pg types parametrization
85 files changed, 1351 insertions, 613 deletions
diff --git a/ydb/core/grpc_services/resolve_local_db_table.cpp b/ydb/core/grpc_services/resolve_local_db_table.cpp index 3f1aeb74a58..81ae3d12a0c 100644 --- a/ydb/core/grpc_services/resolve_local_db_table.cpp +++ b/ydb/core/grpc_services/resolve_local_db_table.cpp @@ -45,7 +45,7 @@ namespace NGRpcService { const NTable::TScheme::TTableInfo* tableInfo = scheme.Tables.FindPtr(*ti); for (const auto& col : tableInfo->Columns) { - entry.Columns[col.first] = TSysTables::TTableColumnInfo(col.second.Name, col.first, col.second.PType, col.second.KeyOrder); + entry.Columns[col.first] = TSysTables::TTableColumnInfo(col.second.Name, col.first, col.second.PType, col.second.PTypeMod, col.second.KeyOrder); } } diff --git a/ydb/core/grpc_services/rpc_kh_describe.cpp b/ydb/core/grpc_services/rpc_kh_describe.cpp index 1ceb70b9e1e..c5131ce14c5 100644 --- a/ydb/core/grpc_services/rpc_kh_describe.cpp +++ b/ydb/core/grpc_services/rpc_kh_describe.cpp @@ -178,7 +178,10 @@ private: auto& typeInfo = col.second.PType; auto* item = colMeta->mutable_type(); if (typeInfo.GetTypeId() == NScheme::NTypeIds::Pg) { - item->mutable_pg_type()->set_oid(NPg::PgTypeIdFromTypeDesc(typeInfo.GetTypeDesc())); + auto* typeDesc = typeInfo.GetTypeDesc(); + auto* pg = item->mutable_pg_type(); + pg->set_type_name(NPg::PgTypeNameFromTypeDesc(typeDesc)); + pg->set_oid(NPg::PgTypeIdFromTypeDesc(typeDesc)); } else { item->mutable_optional_type()->mutable_item() ->set_type_id((Ydb::Type::PrimitiveTypeId)typeInfo.GetTypeId()); diff --git a/ydb/core/grpc_services/rpc_load_rows.cpp b/ydb/core/grpc_services/rpc_load_rows.cpp index 8092d80428e..3e76492491f 100644 --- a/ydb/core/grpc_services/rpc_load_rows.cpp +++ b/ydb/core/grpc_services/rpc_load_rows.cpp @@ -92,7 +92,7 @@ bool CheckValueData(NScheme::TTypeInfo type, const TCell& cell, TString& err) { } if (!ok) { - err = Sprintf("Invalid %s value", NScheme::TypeName(type)); + err = Sprintf("Invalid %s value", NScheme::TypeName(type).c_str()); } return ok; @@ -195,7 +195,7 @@ public: {} private: - static bool CellFromProtoVal(NScheme::TTypeInfo type, const Ydb::Value* vp, + static bool CellFromProtoVal(NScheme::TTypeInfo type, i32 typmod, const Ydb::Value* vp, TCell& c, TString& err, TMemoryPool& valueDataPool) { if (vp->Hasnull_flag_value()) { @@ -272,21 +272,53 @@ private: break; } case NScheme::NTypeIds::Pg : { + TString binary; + bool isText = false; TString text = val.Gettext_value(); if (!text.empty()) { + isText = true; auto desc = type.GetTypeDesc(); auto id = NPg::PgTypeIdFromTypeDesc(desc); - auto result = NPg::PgNativeBinaryFromNativeText(text, id); - if (!result.Error.empty()) { + auto res = NPg::PgNativeBinaryFromNativeText(text, id); + if (res.Error) { err = TStringBuilder() << "Invalid text value for " - << NPg::PgTypeNameFromTypeDesc(desc) << ": " << result.Error; + << NPg::PgTypeNameFromTypeDesc(desc) << ": " << *res.Error; return false; } - const auto valueInPool = valueDataPool.AppendString(TStringBuf(result.Str)); - c = TCell(valueInPool.data(), valueInPool.size()); + binary = res.Str; } else { - TString binary = val.Getbytes_value(); - c = TCell(binary.data(), binary.size()); + binary = val.Getbytes_value(); + } + auto* desc = type.GetTypeDesc(); + if (typmod != -1 && NPg::TypeDescNeedsCoercion(desc)) { + auto res = NPg::PgNativeBinaryCoerce(TStringBuf(binary), desc, typmod); + if (res.Error) { + err = TStringBuilder() << "Unable to coerce value for " + << NPg::PgTypeNameFromTypeDesc(desc) << ": " << *res.Error; + return false; + } + if (res.NewValue) { + const auto valueInPool = valueDataPool.AppendString(TStringBuf(*res.NewValue)); + c = TCell(valueInPool.data(), valueInPool.size()); + } else if (isText) { + const auto valueInPool = valueDataPool.AppendString(TStringBuf(binary)); + c = TCell(valueInPool.data(), valueInPool.size()); + } else { + c = TCell(binary.data(), binary.size()); + } + } else { + auto error = NPg::PgNativeBinaryValidate(TStringBuf(binary), desc); + if (error) { + err = TStringBuilder() << "Invalid binary value for " + << NPg::PgTypeNameFromTypeDesc(desc) << ": " << *error; + return false; + } + if (isText) { + const auto valueInPool = valueDataPool.AppendString(TStringBuf(binary)); + c = TCell(valueInPool.data(), valueInPool.size()); + } else { + c = TCell(binary.data(), binary.size()); + } } break; } @@ -311,7 +343,7 @@ private: return false; } cells.push_back({}); - if (!CellFromProtoVal(fd.Type, &proto.Getitems(fd.PositionInStruct), cells.back(), err, valueDataPool)) { + if (!CellFromProtoVal(fd.Type, fd.Typmod, &proto.Getitems(fd.PositionInStruct), cells.back(), err, valueDataPool)) { return false; } diff --git a/ydb/core/grpc_services/rpc_log_store.cpp b/ydb/core/grpc_services/rpc_log_store.cpp index 432a74c9dd0..7c2d62aee92 100644 --- a/ydb/core/grpc_services/rpc_log_store.cpp +++ b/ydb/core/grpc_services/rpc_log_store.cpp @@ -95,10 +95,11 @@ bool ConvertSchemaFromPublicToInternal(const Ydb::LogStore::Schema& from, NKikim auto* col = to.AddColumns(); col->SetName(column.name()); NScheme::TTypeInfo typeInfo; - if (!ExtractColumnTypeInfo(typeInfo, column.type(), status, error)) { + TString typeMod; + if (!ExtractColumnTypeInfo(typeInfo, typeMod, column.type(), status, error)) { return false; } - auto typeName = NScheme::TypeName(typeInfo); + auto typeName = NScheme::TypeName(typeInfo, typeMod); col->SetType(typeName); if (key.count(column.name())) { col->SetNotNull(true); diff --git a/ydb/core/grpc_services/rpc_long_tx.cpp b/ydb/core/grpc_services/rpc_long_tx.cpp index b50d99fe428..230ec01ed0f 100644 --- a/ydb/core/grpc_services/rpc_long_tx.cpp +++ b/ydb/core/grpc_services/rpc_long_tx.cpp @@ -37,9 +37,9 @@ std::shared_ptr<arrow::Schema> ExtractArrowSchema(const NKikimrSchemeOp::TColumn TVector<std::pair<TString, NScheme::TTypeInfo>> columns; for (auto& col : schema.GetColumns()) { Y_VERIFY(col.HasTypeId()); - auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(col.GetTypeId(), col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); - columns.emplace_back(col.GetName(), typeInfo); + columns.emplace_back(col.GetName(), typeInfoMod.TypeInfo); } return NArrow::MakeArrowSchema(columns); diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp index 892fe134244..c5e53f48863 100644 --- a/ydb/core/grpc_services/rpc_read_columns.cpp +++ b/ydb/core/grpc_services/rpc_read_columns.cpp @@ -270,7 +270,7 @@ private: KeyColumnTypes[ci.second.KeyOrder] = ci.second.PType; columns.resize(Max<size_t>(columns.size(), ci.second.KeyOrder + 1)); - columns[ci.second.KeyOrder] = {ci.second.Id, ci.second.PType}; + columns[ci.second.KeyOrder] = {ci.second.Id, ci.second.PType, ci.second.PTypeMod}; } } @@ -283,7 +283,7 @@ private: } auto ci = entry.Columns.find(id->second); - columns.push_back({ci->second.Id, ci->second.PType}); + columns.push_back({ci->second.Id, ci->second.PType, ci->second.PTypeMod}); valueColumnNamesAndTypes.push_back({ci->second.Name, ci->second.PType}); ValueColumnTypes.push_back(ci->second.PType); diff --git a/ydb/core/kqp/common/kqp_resolve.h b/ydb/core/kqp/common/kqp_resolve.h index 7d9c3e3b2d3..542e9e03384 100644 --- a/ydb/core/kqp/common/kqp_resolve.h +++ b/ydb/core/kqp/common/kqp_resolve.h @@ -24,6 +24,7 @@ public: struct TColumn { ui32 Id; NScheme::TTypeInfo Type; + TString TypeMod; }; struct TTable { diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp index 5cc59095c86..aa40b41e95c 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_actor.cpp @@ -303,7 +303,7 @@ private: ev->Record.SetLocalPathId(ScanData->TableId.PathId.LocalPathId); for (auto& column: ScanData->GetColumns()) { ev->Record.AddColumnTags(column.Tag); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.Type, column.TypeMod); ev->Record.AddColumnTypes(columnType.TypeId); if (columnType.TypeInfo) { *ev->Record.AddColumnTypeInfos() = *columnType.TypeInfo; diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp index 0b5e8cefcab..82c1b23ebe2 100644 --- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp @@ -1555,7 +1555,7 @@ private: for (auto& column : read.Columns) { auto* protoColumn = protoReadMeta->AddColumns(); protoColumn->SetId(column.Id); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.Type, column.TypeMod); protoColumn->SetType(columnType.TypeId); if (columnType.TypeInfo) { *protoColumn->MutableTypeInfo() = *columnType.TypeInfo; @@ -1578,7 +1578,7 @@ private: auto& protoColumn = *protoColumnWrite.MutableColumn(); protoColumn.SetId(columnWrite.Column.Id); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(columnWrite.Column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(columnWrite.Column.Type, columnWrite.Column.TypeMod); protoColumn.SetType(columnType.TypeId); if (columnType.TypeInfo) { *protoColumn.MutableTypeInfo() = *columnType.TypeInfo; @@ -1601,7 +1601,7 @@ private: const auto& tableInfo = TableKeys.GetTable(stageInfo.Meta.TableId); for (const auto& keyColumnName : tableInfo.KeyColumns) { const auto& keyColumn = tableInfo.Columns.at(keyColumnName); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(keyColumn.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(keyColumn.Type, keyColumn.TypeMod); protoTaskMeta.AddKeyColumnTypes(columnType.TypeId); if (columnType.TypeInfo) { *protoTaskMeta.AddKeyColumnTypeInfos() = *columnType.TypeInfo; @@ -1618,7 +1618,7 @@ private: for (auto& column : task.Meta.Reads->front().Columns) { auto* protoColumn = protoTaskMeta.AddColumns(); protoColumn->SetId(column.Id); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.Type, column.TypeMod); protoColumn->SetType(columnType.TypeId); if (columnType.TypeInfo) { *protoColumn->MutableTypeInfo() = *columnType.TypeInfo; diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.h b/ydb/core/kqp/executer_actor/kqp_executer_impl.h index 75f50425cfd..686207b901a 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_impl.h +++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.h @@ -705,7 +705,7 @@ protected: FillTableMeta(stageInfo, settings.MutableTable()); for (auto& keyColumn : keyTypes) { - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(keyColumn); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(keyColumn, ""); if (columnType.TypeInfo) { *settings.AddKeyColumnTypeInfos() = *columnType.TypeInfo; } else { @@ -717,7 +717,7 @@ protected: for (auto& column : columns) { auto* protoColumn = settings.AddColumns(); protoColumn->SetId(column.Id); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.Type, column.TypeMod); protoColumn->SetType(columnType.TypeId); if (columnType.TypeInfo) { *protoColumn->MutableTypeInfo() = *columnType.TypeInfo; diff --git a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp index a80a87d6f64..2737b45ccf9 100644 --- a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp @@ -553,7 +553,7 @@ private: const auto& tableInfo = TableKeys.GetTable(stageInfo.Meta.TableId); for (const auto& keyColumnName : tableInfo.KeyColumns) { const auto& keyColumn = tableInfo.Columns.at(keyColumnName); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(keyColumn.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(keyColumn.Type, keyColumn.TypeMod); protoTaskMeta.AddKeyColumnTypes(columnType.TypeId); if (columnType.TypeInfo) { *protoTaskMeta.AddKeyColumnTypeInfos() = *columnType.TypeInfo; @@ -603,7 +603,7 @@ private: for (auto columnType : task.Meta.ReadInfo.ResultColumnsTypes) { auto* protoResultColumn = protoTaskMeta.AddResultColumns(); protoResultColumn->SetId(0); - auto protoColumnType = NScheme::ProtoColumnTypeFromTypeInfo(columnType); + auto protoColumnType = NScheme::ProtoColumnTypeFromTypeInfoMod(columnType, ""); protoResultColumn->SetType(protoColumnType.TypeId); if (protoColumnType.TypeInfo) { *protoResultColumn->MutableTypeInfo() = *protoColumnType.TypeInfo; @@ -624,7 +624,7 @@ private: for (auto& column : task.Meta.Reads->front().Columns) { auto* protoColumn = protoTaskMeta.AddColumns(); protoColumn->SetId(column.Id); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.Type, ""); protoColumn->SetType(columnType.TypeId); if (columnType.TypeInfo) { *protoColumn->MutableTypeInfo() = *columnType.TypeInfo; diff --git a/ydb/core/kqp/executer_actor/kqp_tasks_graph.h b/ydb/core/kqp/executer_actor/kqp_tasks_graph.h index 3a961d1d9b8..860d88bda42 100644 --- a/ydb/core/kqp/executer_actor/kqp_tasks_graph.h +++ b/ydb/core/kqp/executer_actor/kqp_tasks_graph.h @@ -107,6 +107,7 @@ struct TTaskMeta { struct TColumn { ui32 Id = 0; NScheme::TTypeInfo Type; + TString TypeMod; TString Name; }; @@ -180,8 +181,10 @@ TVector<TTaskMeta::TColumn> BuildKqpColumns(const Proto& op, const TKqpTableKeys for (const auto& column : op.GetColumns()) { TTaskMeta::TColumn c; + const auto& tableColumn = table.Columns.at(column.GetName()); c.Id = column.GetId(); - c.Type = table.Columns.at(column.GetName()).Type; + c.Type = tableColumn.Type; + c.TypeMod = tableColumn.TypeMod; c.Name = column.GetName(); columns.emplace_back(std::move(c)); diff --git a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp index ac5ae77a5c5..adadd1a8577 100644 --- a/ydb/core/kqp/gateway/kqp_metadata_loader.cpp +++ b/ydb/core/kqp/gateway/kqp_metadata_loader.cpp @@ -156,12 +156,12 @@ TTableMetadataResult GetLoadTableMetadataResult(const NSchemeCache::TSchemeCache } else { Y_VERIFY(columnDesc.PType.GetTypeDesc(), "no pg type descriptor"); Y_VERIFY(!notNull, "pg not null types are not allowed"); - typeName = NPg::PgTypeNameFromTypeDesc(columnDesc.PType.GetTypeDesc()); + typeName = NPg::PgTypeNameFromTypeDesc(columnDesc.PType.GetTypeDesc(), columnDesc.PTypeMod); } tableMeta->Columns.emplace( columnDesc.Name, NYql::TKikimrColumnMetadata( - columnDesc.Name, columnDesc.Id, typeName, notNull, columnDesc.PType + columnDesc.Name, columnDesc.Id, typeName, notNull, columnDesc.PType, columnDesc.PTypeMod ) ); if (columnDesc.KeyOrder >= 0) { diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp index d251aad27e9..5bc3dc8fa24 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.cpp @@ -214,8 +214,12 @@ void SetColumnType(Ydb::Type& protoType, const TString& typeName, bool notNull) auto* typeDesc = NKikimr::NPg::TypeDescFromPgTypeName(typeName); if (typeDesc) { Y_VERIFY(!notNull, "It is not allowed to create NOT NULL pg columns"); - auto pg = protoType.mutable_pg_type(); + auto* pg = protoType.mutable_pg_type(); + pg->set_type_name(NKikimr::NPg::PgTypeNameFromTypeDesc(typeDesc)); + pg->set_type_modifier(NKikimr::NPg::TypeModFromPgTypeName(typeName)); pg->set_oid(NKikimr::NPg::PgTypeIdFromTypeDesc(typeDesc)); + pg->set_typlen(0); + pg->set_typmod(0); return; } diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index b8688ebe175..6145d2c9f75 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -182,16 +182,19 @@ struct TKikimrColumnMetadata { TString Type; bool NotNull = false; NKikimr::NScheme::TTypeInfo TypeInfo; + TString TypeMod; TVector<TString> Families; TKikimrColumnMetadata() = default; - TKikimrColumnMetadata(const TString& name, ui32 id, const TString& type, bool notNull, NKikimr::NScheme::TTypeInfo typeInfo = {}) + TKikimrColumnMetadata(const TString& name, ui32 id, const TString& type, bool notNull, + NKikimr::NScheme::TTypeInfo typeInfo = {}, const TString& typeMod = {}) : Name(name) , Id(id) , Type(type) , NotNull(notNull) , TypeInfo(typeInfo) + , TypeMod(typeMod) {} explicit TKikimrColumnMetadata(const NKikimrKqp::TKqpColumnMetadataProto* message) @@ -201,8 +204,10 @@ struct TKikimrColumnMetadata { , NotNull(message->GetNotNull()) , Families(message->GetFamily().begin(), message->GetFamily().end()) { - TypeInfo = NKikimr::NScheme::TypeInfoFromProtoColumnType(message->GetTypeId(), + auto typeInfoMod = NKikimr::NScheme::TypeInfoModFromProtoColumnType(message->GetTypeId(), message->HasTypeInfo() ? &message->GetTypeInfo() : nullptr); + TypeInfo = typeInfoMod.TypeInfo; + TypeMod = typeInfoMod.TypeMod; } void ToMessage(NKikimrKqp::TKqpColumnMetadataProto* message) const { @@ -210,7 +215,7 @@ struct TKikimrColumnMetadata { message->SetId(Id); message->SetType(Type); message->SetNotNull(NotNull); - auto columnType = NKikimr::NScheme::ProtoColumnTypeFromTypeInfo(TypeInfo); + auto columnType = NKikimr::NScheme::ProtoColumnTypeFromTypeInfoMod(TypeInfo, TypeMod); message->SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *message->MutableTypeInfo() = *columnType.TypeInfo; diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp index 47f6e8a122d..763da848c72 100644 --- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp +++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp @@ -176,6 +176,7 @@ bool TKikimrTableDescription::Load(TExprContext& ctx, bool withSystemColumns) { type = ctx.MakeType<TPgExprType>(NKikimr::NPg::PgTypeIdFromTypeDesc(column.TypeInfo.GetTypeDesc())); } } + if (!column.NotNull && column.TypeInfo.GetTypeId() != NKikimr::NScheme::NTypeIds::Pg) { type = ctx.MakeType<TOptionalExprType>(type); } diff --git a/ydb/core/kqp/runtime/kqp_compute.h b/ydb/core/kqp/runtime/kqp_compute.h index ae90fd8e466..b3cfc21d2e6 100644 --- a/ydb/core/kqp/runtime/kqp_compute.h +++ b/ydb/core/kqp/runtime/kqp_compute.h @@ -15,6 +15,7 @@ public: struct TColumn { NTable::TTag Tag; NScheme::TTypeInfo Type; + TString TypeMod; }; // used only at then building of a computation graph, to inject taskId in runtime nodes diff --git a/ydb/core/kqp/runtime/kqp_scan_data.cpp b/ydb/core/kqp/runtime/kqp_scan_data.cpp index 4ae2eb8b2fb..8205487ad47 100644 --- a/ydb/core/kqp/runtime/kqp_scan_data.cpp +++ b/ydb/core/kqp/runtime/kqp_scan_data.cpp @@ -447,8 +447,10 @@ TKqpScanComputeContext::TScanData::TScanData(const NKikimrTxDataShard::TKqpTrans for (const auto& column : meta.GetColumns()) { NMiniKQL::TKqpScanComputeContext::TColumn c; c.Tag = column.GetId(); - c.Type = NScheme::TypeInfoFromProtoColumnType(column.GetType(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(column.GetType(), column.HasTypeInfo() ? &column.GetTypeInfo() : nullptr); + c.Type = typeInfoMod.TypeInfo; + c.TypeMod = typeInfoMod.TypeMod; if (!IsSystemColumn(c.Tag)) { Columns.emplace_back(std::move(c)); @@ -465,8 +467,10 @@ TKqpScanComputeContext::TScanData::TScanData(const NKikimrTxDataShard::TKqpTrans for (const auto& resColumn : meta.GetResultColumns()) { NMiniKQL::TKqpScanComputeContext::TColumn c; c.Tag = resColumn.GetId(); - c.Type = NScheme::TypeInfoFromProtoColumnType(resColumn.GetType(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(resColumn.GetType(), resColumn.HasTypeInfo() ? &resColumn.GetTypeInfo() : nullptr); + c.Type = typeInfoMod.TypeInfo; + c.TypeMod = typeInfoMod.TypeMod; if (!IsSystemColumn(c.Tag)) { ResultColumns.emplace_back(std::move(c)); diff --git a/ydb/core/kqp/runtime/kqp_scan_data_ut.cpp b/ydb/core/kqp/runtime/kqp_scan_data_ut.cpp index aae2d0c669f..90eab85f740 100644 --- a/ydb/core/kqp/runtime/kqp_scan_data_ut.cpp +++ b/ydb/core/kqp/runtime/kqp_scan_data_ut.cpp @@ -14,26 +14,26 @@ using TTypeInfo = NScheme::TTypeInfo; struct TDataRow { TSmallVec<TKqpComputeContextBase::TColumn> Columns() { return { - {0, TTypeInfo(NTypeIds::Bool)}, - {1, TTypeInfo(NTypeIds::Int8)}, - {2, TTypeInfo(NTypeIds::Int16)}, - {3, TTypeInfo(NTypeIds::Int32)}, - {4, TTypeInfo(NTypeIds::Int64)}, - {5, TTypeInfo(NTypeIds::Uint8)}, - {6, TTypeInfo(NTypeIds::Uint16)}, - {7, TTypeInfo(NTypeIds::Uint32)}, - {8, TTypeInfo(NTypeIds::Uint64)}, - {9, TTypeInfo(NTypeIds::Float)}, - {10, TTypeInfo(NTypeIds::Double)}, - {11, TTypeInfo(NTypeIds::String)}, - {12, TTypeInfo(NTypeIds::Utf8)}, - {13, TTypeInfo(NTypeIds::Json)}, - {14, TTypeInfo(NTypeIds::Yson)}, - {15, TTypeInfo(NTypeIds::Date)}, - {16, TTypeInfo(NTypeIds::Datetime)}, - {17, TTypeInfo(NTypeIds::Timestamp)}, - {18, TTypeInfo(NTypeIds::Interval)}, - {19, TTypeInfo(NTypeIds::Decimal)}, + {0, TTypeInfo(NTypeIds::Bool), ""}, + {1, TTypeInfo(NTypeIds::Int8), ""}, + {2, TTypeInfo(NTypeIds::Int16), ""}, + {3, TTypeInfo(NTypeIds::Int32), ""}, + {4, TTypeInfo(NTypeIds::Int64), ""}, + {5, TTypeInfo(NTypeIds::Uint8), ""}, + {6, TTypeInfo(NTypeIds::Uint16), ""}, + {7, TTypeInfo(NTypeIds::Uint32), ""}, + {8, TTypeInfo(NTypeIds::Uint64), ""}, + {9, TTypeInfo(NTypeIds::Float), ""}, + {10, TTypeInfo(NTypeIds::Double), ""}, + {11, TTypeInfo(NTypeIds::String), ""}, + {12, TTypeInfo(NTypeIds::Utf8), ""}, + {13, TTypeInfo(NTypeIds::Json), ""}, + {14, TTypeInfo(NTypeIds::Yson), ""}, + {15, TTypeInfo(NTypeIds::Date), ""}, + {16, TTypeInfo(NTypeIds::Datetime), ""}, + {17, TTypeInfo(NTypeIds::Timestamp), ""}, + {18, TTypeInfo(NTypeIds::Interval), ""}, + {19, TTypeInfo(NTypeIds::Decimal), ""}, }; } diff --git a/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp b/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp index e1386cff6f0..95d0908192e 100644 --- a/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp +++ b/ydb/core/kqp/runtime/kqp_stream_lookup_actor.cpp @@ -43,6 +43,7 @@ public: keyColumn.GetName(), keyColumn.GetId(), NScheme::TTypeInfo{static_cast<NScheme::TTypeId>(keyColumn.GetTypeId())}, + "", keyOrder++ } ); diff --git a/ydb/core/kqp/ut/pg/CMakeLists.darwin.txt b/ydb/core/kqp/ut/pg/CMakeLists.darwin.txt index 5c0c5568cf8..47c4c850150 100644 --- a/ydb/core/kqp/ut/pg/CMakeLists.darwin.txt +++ b/ydb/core/kqp/ut/pg/CMakeLists.darwin.txt @@ -61,7 +61,7 @@ set_yunittest_property( ydb-core-kqp-ut-pg PROPERTY LABELS - SMALL + MEDIUM ) set_yunittest_property( TEST diff --git a/ydb/core/kqp/ut/pg/CMakeLists.linux-aarch64.txt b/ydb/core/kqp/ut/pg/CMakeLists.linux-aarch64.txt index c1cab438236..3908d66312e 100644 --- a/ydb/core/kqp/ut/pg/CMakeLists.linux-aarch64.txt +++ b/ydb/core/kqp/ut/pg/CMakeLists.linux-aarch64.txt @@ -64,7 +64,7 @@ set_yunittest_property( ydb-core-kqp-ut-pg PROPERTY LABELS - SMALL + MEDIUM ) set_yunittest_property( TEST diff --git a/ydb/core/kqp/ut/pg/CMakeLists.linux.txt b/ydb/core/kqp/ut/pg/CMakeLists.linux.txt index bf122a5ede1..200982bc5b4 100644 --- a/ydb/core/kqp/ut/pg/CMakeLists.linux.txt +++ b/ydb/core/kqp/ut/pg/CMakeLists.linux.txt @@ -66,7 +66,7 @@ set_yunittest_property( ydb-core-kqp-ut-pg PROPERTY LABELS - SMALL + MEDIUM ) set_yunittest_property( TEST diff --git a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp index 9c8fbe6266d..3d063c15c09 100644 --- a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp +++ b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp @@ -13,19 +13,17 @@ extern "C" { namespace { struct TPgTypeTestSpec { + ui32 TypeId; bool IsKey; std::function<TString(size_t)> TextIn, TextOut; - std::function<TString(TString)> ArrayPrint; - TPgTypeTestSpec() = default; - TPgTypeTestSpec( - bool isKey, - std::function<TString(size_t)> in, - std::function<TString(size_t)> out, - std::function<TString(TString)> print = [] (auto s) { return Sprintf("{%s,%s}", s.c_str(), s.c_str()); }) - : IsKey(isKey) - , TextIn(in) - , TextOut(out) - , ArrayPrint(print) {} + std::function<TString(TString)> ArrayPrint = [] (auto s) { return Sprintf("{%s,%s}", s.c_str(), s.c_str()); }; + }; + + struct TPgTypeCoercionTestSpec { + ui32 TypeId; + TString TypeMod; + bool ShouldPass; + std::function<TString()> TextIn, TextOut; }; } @@ -35,8 +33,9 @@ namespace NKqp { using namespace NYdb; using namespace NYdb::NTable; -NYdb::NScripting::TExecuteYqlResult -ExecutePgSelect(NKikimr::NKqp::TKikimrRunner& kikimr, const TString& tableName) { +NYdb::NScripting::TExecuteYqlResult ExecutePgSelect( + NKikimr::NKqp::TKikimrRunner& kikimr, const TString& tableName) +{ NYdb::NScripting::TScriptingClient client(kikimr.GetDriver()); auto result = client.ExecuteYqlScript( TStringBuilder() << R"( @@ -44,23 +43,22 @@ ExecutePgSelect(NKikimr::NKqp::TKikimrRunner& kikimr, const TString& tableName) SELECT * FROM ")" << tableName << "\"" ).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); return result; } void ExecutePgInsert( NKikimr::NKqp::TKikimrRunner& kikimr, const TString& tableName, - ui32 id, - const TPgTypeTestSpec& spec -) { + const TPgTypeTestSpec& spec) +{ NYdb::NScripting::TScriptingClient client(kikimr.GetDriver()); - auto valType = NYql::NPg::LookupType(id).Name; + auto valType = NYql::NPg::LookupType(spec.TypeId).Name; auto keyType = (spec.IsKey) ? valType : "int2"; - if (id == BITOID) { + if (spec.TypeId == BITOID) { valType.append("(4)"); } - for (size_t i = 0; i < ((id == BOOLOID) ? 2 : 3); i++) { + for (size_t i = 0; i < ((spec.TypeId == BOOLOID) ? 2 : 3); i++) { auto keyIn = (spec.IsKey) ? spec.TextIn(i) : ToString(i); TString req = Sprintf("\ --!syntax_pg\n\ @@ -69,23 +67,22 @@ void ExecutePgInsert( )", tableName.Data(), keyIn.Data(), keyType.Data(), spec.TextIn(i).Data(), valType.Data()); Cerr << req << Endl; auto result = client.ExecuteYqlScript(req).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } } void ExecutePgArrayInsert( NKikimr::NKqp::TKikimrRunner& kikimr, const TString& tableName, - ui32 id, - const TPgTypeTestSpec& spec -) { + const TPgTypeTestSpec& spec) +{ NYdb::NScripting::TScriptingClient client(kikimr.GetDriver()); - auto valType = NYql::NPg::LookupType(id).Name; - if (id == BITOID) { + auto valType = NYql::NPg::LookupType(spec.TypeId).Name; + if (spec.TypeId == BITOID) { valType.append("(4)"); } - for (size_t i = 0; i < ((id == BOOLOID) ? 2 : 3); i++) { - auto keyEntry = Sprintf("'%d'::int2", i); + for (size_t i = 0; i < ((spec.TypeId == BOOLOID) ? 2 : 3); i++) { + auto keyEntry = Sprintf("'%u'::int2", i); auto valueEntry = Sprintf( "ARRAY ['%s'::%s, '%s'::%s]", spec.TextIn(i).Data(), @@ -100,7 +97,7 @@ void ExecutePgArrayInsert( );", tableName.Data(), keyEntry.Data(), valueEntry.Data()); Cerr << req << Endl; auto result = client.ExecuteYqlScript(req).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } } @@ -125,293 +122,540 @@ void ValidatePgYqlResult(const NYdb::NScripting::TExecuteYqlResult& result, cons Y_UNIT_TEST_SUITE(KqpPg) { - auto makePgType = [] (ui32 oid, i32 typlen = -1) { return TPgType(oid, typlen, -1); }; - - TMap< - ui32, - TPgTypeTestSpec - > typeSpecs ={ - { BOOLOID, { + TVector<TPgTypeTestSpec> typeSpecs = { + { + BOOLOID, true, [] (auto i) { return TString(i ? "true" : "false"); }, [] (auto i) { return TString(i ? "t" : "f"); } - } }, - { CHAROID, { + { + CHAROID, true, [] (auto i) { return Sprintf("%c", (char)(i + '0')); }, [] (auto i) { return Sprintf("%c", (char)(i + '0')); } - } }, - { INT2OID, { + { + INT2OID, true, [] (auto i) { return Sprintf("%u", i); }, [] (auto i) { return Sprintf("%u", i); } - } }, - { INT4OID, { + { + INT4OID, true, [] (auto i) { return Sprintf("%u", i); }, [] (auto i) { return Sprintf("%u", i); } - } }, - { INT8OID, { + { + INT8OID, true, [] (auto i) { return Sprintf("%u", i); }, [] (auto i) { return Sprintf("%u", i); } - } }, - { FLOAT4OID, { + { + FLOAT4OID, true, [] (auto i) { return Sprintf("%g", i + 0.5f); }, [] (auto i) { return Sprintf("%g", i + 0.5f); } - } }, - { FLOAT8OID, { + { + FLOAT8OID, true, [] (auto i) { return Sprintf("%lg", i + 0.5); }, [] (auto i) { return Sprintf("%lg", i + 0.5); } - } }, - { TEXTOID, { + { + TEXTOID, true, [] (auto i) { return Sprintf("text %u", i); }, [] (auto i) { return Sprintf("text %u", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { BPCHAROID, { + { + BPCHAROID, true, [] (auto i) { return Sprintf("bpchar %u", i); }, [] (auto i) { return Sprintf("bpchar %u", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { VARCHAROID, { + { + VARCHAROID, false, [] (auto i) { return Sprintf("varchar %u", i); }, [] (auto i) { return Sprintf("varchar %u", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { NAMEOID, { + { + NAMEOID, true, [] (auto i) { return Sprintf("name %u", i); }, [] (auto i) { return Sprintf("name %u", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { NUMERICOID, { + { + NUMERICOID, true, [] (auto i) { return Sprintf("%lg", i + 0.12345); }, [] (auto i) { return Sprintf("%lg", i + 0.12345); } - } }, - { MONEYOID, { - true, + { + MONEYOID, + false, // no HashProcId [] (auto i) { return Sprintf("%lg", i + i / 100.); }, [] (auto i) { return Sprintf("$%.2lf", i + i / 100.); } - } }, - { DATEOID, { + { + DATEOID, true, [] (auto i) { return Sprintf("1970-01-%02u", i + 1); }, [] (auto i) { return Sprintf("1970-01-%02u", i + 1); } - } }, - { TIMEOID, { + { + TIMEOID, true, [] (auto i) { return Sprintf("%02u:01:02.345", i); }, [] (auto i) { return Sprintf("%02u:01:02.345", i); } - } }, - { TIMESTAMPOID, { + { + TIMESTAMPOID, true, [] (auto i) { return Sprintf("1970-01-01 %02u:01:02.345", i); }, [] (auto i) { return Sprintf("1970-01-01 %02u:01:02.345", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { TIMETZOID, { + { + TIMETZOID, true, [] (auto i) { return Sprintf("%02u:01:02.345-03", i); }, [] (auto i) { return Sprintf("%02u:01:02.345-03", i); } - } }, - { TIMESTAMPTZOID, { + { + TIMESTAMPTZOID, true, [] (auto i) { return Sprintf("1970-01-01 %02u:01:02.345 -3:00", i); }, [] (auto i) { return Sprintf("1970-01-01 %02u:01:02.345+00", i + 3); }, // TODO: investigate [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { INTERVALOID, { + { + INTERVALOID, true, [] (auto i) { return Sprintf("P01-02-03T04:05:%02u", i); }, [] (auto i) { return Sprintf("1 year 2 mons 3 days 04:05:%02u", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { BITOID, { - true, + { + BITOID, + false, // no HashProcId [] (auto i) { return Sprintf("%c%c%c%c", (i&8)?'1':'0', (i&4)?'1':'0', (i&2)?'1':'0', (i&1)?'1':'0'); }, [] (auto i) { return Sprintf("%c%c%c%c", (i&8)?'1':'0', (i&4)?'1':'0', (i&2)?'1':'0', (i&1)?'1':'0'); } - } }, - { VARBITOID, { - true, + { + VARBITOID, + false, // no HashProcId [] (auto i) { return Sprintf("%c%c%c%c", (i&8)?'1':'0', (i&4)?'1':'0', (i&2)?'1':'0', (i&1)?'1':'0'); }, [] (auto i) { return Sprintf("%c%c%c%c", (i&8)?'1':'0', (i&4)?'1':'0', (i&2)?'1':'0', (i&1)?'1':'0'); } - } }, - { POINTOID, { + { + POINTOID, false, [] (auto i) { return Sprintf("(10, %u)", i); }, [] (auto i) { return Sprintf("(10,%u)", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { LINEOID, { + { + LINEOID, false, [] (auto i) { return Sprintf("{1, 2, %u}", i); }, [] (auto i) { return Sprintf("{1,2,%u}", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { LSEGOID, { + { + LSEGOID, false, [] (auto i) { return Sprintf("[(0, 0), (1, %u)]", i); }, [] (auto i) { return Sprintf("[(0,0),(1,%u)]", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { BOXOID, { + { + BOXOID, false, [] (auto i) { return Sprintf("(1, %u), (0, 0)", i + 1); }, [] (auto i) { return Sprintf("(1,%u),(0,0)", i + 1); }, [] (auto s) { return Sprintf("{%s;%s}", s.c_str(), s.c_str()); } - } }, - { PATHOID, { + { + PATHOID, false, [] (auto i) { return Sprintf("((0, 1), (2, 3), (4, %u))", i); }, [] (auto i) { return Sprintf("((0,1),(2,3),(4,%u))", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { POLYGONOID, { + { + POLYGONOID, false, [] (auto i) { return Sprintf("((0, 1), (2, 3), (4, %u))", i); }, [] (auto i) { return Sprintf("((0,1),(2,3),(4,%u))", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { CIRCLEOID, { + { + CIRCLEOID, false, [] (auto i) { return Sprintf("<(0, 1), %u>", i); }, [] (auto i) { return Sprintf("<(0,1),%u>", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { INETOID, { + { + INETOID, false, [] (auto i) { return Sprintf("128.%u.0.0/16", i); }, [] (auto i) { return Sprintf("128.%u.0.0/16", i); } - } }, - { CIDROID, { + { + CIDROID, false, [] (auto i) { return Sprintf("128.%u.0.0/16", i); }, [] (auto i) { return Sprintf("128.%u.0.0/16", i); } - } }, - { MACADDROID, { + { + MACADDROID, false, [] (auto i) { return Sprintf("08:00:2b:01:02:%02u", i); }, [] (auto i) { return Sprintf("08:00:2b:01:02:%02u", i); } - } }, - { MACADDR8OID, { + { + MACADDR8OID, false, [] (auto i) { return Sprintf("08:00:2b:01:02:03:04:%02u", i); }, [] (auto i) { return Sprintf("08:00:2b:01:02:03:04:%02u", i); } - } }, - { UUIDOID, { + { + UUIDOID, false, [] (auto i) { return Sprintf("00000000-0000-0000-0000-0000000000%02u", i); }, [] (auto i) { return Sprintf("00000000-0000-0000-0000-0000000000%02u", i); } - } }, - { JSONOID, { + { + JSONOID, false, [] (auto i) { return Sprintf("[%u]", i); }, [] (auto i) { return Sprintf("[%u]", i); } - } }, - { JSONBOID, { + { + JSONBOID, false, [] (auto i) { return Sprintf("[%u]", i); }, [] (auto i) { return Sprintf("[%u]", i); } - } }, - { JSONPATHOID, { + { + JSONPATHOID, false, [] (auto i) { return Sprintf("$[%u]", i); }, [] (auto i) { return Sprintf("$[%u]", i); } - } }, - { XMLOID, { + { + XMLOID, false, [] (auto i) { return Sprintf("<a>%u</a>", i); }, [] (auto i) { return Sprintf("<a>%u</a>", i); } - } }, - { TSQUERYOID, { + { + TSQUERYOID, false, [] (auto i) { return Sprintf("a&b%u", i); }, [] (auto i) { return Sprintf("'a' & 'b%u'", i); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { TSVECTOROID, { + { + TSVECTOROID, false, [] (auto i) { return Sprintf("a:1 b:%u", i + 2); }, [] (auto i) { return Sprintf("'a':1 'b':%u", i + 2); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } }, - { INT2VECTOROID, { + { + INT2VECTOROID, false, [] (auto i) { return Sprintf("%u %u %u", i, i + 1, i + 2); }, [] (auto i) { return Sprintf("%u %u %u", i, i + 1, i + 2); }, [] (auto s) { return Sprintf("{\"%s\",\"%s\"}", s.c_str(), s.c_str()); } - } } }; + TPgTypeTestSpec typeByteaSpec{ + BYTEAOID, + true, + [] (auto i) { return Sprintf("bytea %u", i); }, + [] (auto i) { return Sprintf("\\x627974656120%x", i + 48); } + }; + + TPgTypeTestSpec typeByteaArraySpec{ + BYTEAARRAYOID, + true, + [] (auto i) { return Sprintf("{a%u, b%u}", i, i + 10); }, + [] (auto i) { return Sprintf("{\"\\\\x61%x\",\"\\\\x6231%x\"}", i + 48, i + 48); } + }; + + +#define SUCCESS true +#define FAIL false + + TVector<TPgTypeCoercionTestSpec> typeCoercionSpecs = { + { + BPCHAROID, "2", + FAIL, + [] () { return TString("abcd"); }, + [] () { return TString(""); } + }, + { + BPCHAROID, "4", + SUCCESS, + [] () { return TString("abcd"); }, + [] () { return TString("abcd"); } + }, + { + BPCHAROID, "6", + SUCCESS, + [] () { return TString("abcd"); }, + [] () { return TString("abcd "); } + }, + + { + VARCHAROID, "2", + FAIL, + [] () { return TString("abcd"); }, + [] () { return TString(""); } + }, + { + VARCHAROID, "4", + SUCCESS, + [] () { return TString("abcd"); }, + [] () { return TString("abcd"); } + }, + { + VARCHAROID, "6", + SUCCESS, + [] () { return TString("abcd"); }, + [] () { return TString("abcd"); } + }, + + { + BITOID, "2", + FAIL, + [] () { return TString("1111"); }, + [] () { return TString(""); } + }, + { + BITOID, "4", + SUCCESS, + [] () { return TString("1111"); }, + [] () { return TString("1111"); } + }, + { + BITOID, "6", + FAIL, + [] () { return TString("1111"); }, + [] () { return TString(""); } + }, + + { + VARBITOID, "2", + FAIL, + [] () { return TString("1111"); }, + [] () { return TString(""); } + }, + { + VARBITOID, "4", + SUCCESS, + [] () { return TString("1111"); }, + [] () { return TString("1111"); } + }, + { + VARBITOID, "6", + SUCCESS, + [] () { return TString("1111"); }, + [] () { return TString("1111"); } + }, + + { + NUMERICOID, "2", + FAIL, + [] () { return TString("9999"); }, + [] () { return TString(""); } + }, + { + NUMERICOID, "4", + SUCCESS, + [] () { return TString("9999.1234"); }, + [] () { return TString("9999"); } + }, + { + NUMERICOID, "4", + SUCCESS, + [] () { return TString("9999"); }, + [] () { return TString("9999"); } + }, + { + NUMERICOID, "6", + SUCCESS, + [] () { return TString("9999"); }, + [] () { return TString("9999"); } + }, + { + NUMERICOID, "20,2", + SUCCESS, + [] () { return TString("99.1234"); }, + [] () { return TString("99.12"); } + }, + { + NUMERICOID, "20,4", + SUCCESS, + [] () { return TString("99.1234"); }, + [] () { return TString("99.1234"); } + }, + { + NUMERICOID, "20,6", + SUCCESS, + [] () { return TString("99.1234"); }, + [] () { return TString("99.123400"); } + }, + + { + TIMEOID, "2", + SUCCESS, + [] () { return TString("01:02:03.1234"); }, + [] () { return TString("01:02:03.12"); } + }, + { + TIMEOID, "4", + SUCCESS, + [] () { return TString("01:02:03.1234"); }, + [] () { return TString("01:02:03.1234"); } + }, + { + TIMEOID, "6", + SUCCESS, + [] () { return TString("01:02:03.1234"); }, + [] () { return TString("01:02:03.1234"); } + }, + + { + TIMETZOID, "2", + SUCCESS, + [] () { return TString("01:02:03.1234+00"); }, + [] () { return TString("01:02:03.12+00"); } + }, + { + TIMETZOID, "4", + SUCCESS, + [] () { return TString("01:02:03.1234+00"); }, + [] () { return TString("01:02:03.1234+00"); } + }, + { + TIMETZOID, "6", + SUCCESS, + [] () { return TString("01:02:03.1234+00"); }, + [] () { return TString("01:02:03.1234+00"); } + }, + + { + TIMESTAMPOID, "2", + SUCCESS, + [] () { return TString("2001-01-01 01:02:03.1234"); }, + [] () { return TString("2001-01-01 01:02:03.12"); } + }, + { + TIMESTAMPOID, "4", + SUCCESS, + [] () { return TString("2001-01-01 01:02:03.1234"); }, + [] () { return TString("2001-01-01 01:02:03.1234"); } + }, + { + TIMESTAMPOID, "6", + SUCCESS, + [] () { return TString("2001-01-01 01:02:03.1234"); }, + [] () { return TString("2001-01-01 01:02:03.1234"); } + }, + + { + TIMESTAMPTZOID, "2", + SUCCESS, + [] () { return TString("2001-01-01 01:02:03.1234+00"); }, + [] () { return TString("2001-01-01 01:02:03.12+00"); } + }, + { + TIMESTAMPTZOID, "4", + SUCCESS, + [] () { return TString("2001-01-01 01:02:03.1234+00"); }, + [] () { return TString("2001-01-01 01:02:03.1234+00"); } + }, + { + TIMESTAMPTZOID, "6", + SUCCESS, + [] () { return TString("2001-01-01 01:02:03.1234+00"); }, + [] () { return TString("2001-01-01 01:02:03.1234+00"); } + }, + + { + INTERVALOID, "day", + SUCCESS, + [] () { return TString("100 01:02:03.1234"); }, + [] () { return TString("100 days"); } + }, + { + INTERVALOID, "day to minute", + SUCCESS, + [] () { return TString("100 01:02:03.1234"); }, + [] () { return TString("100 days 01:02:00"); } + }, + { + INTERVALOID, "day to second,2", + SUCCESS, + [] () { return TString("100 01:02:03.1234"); }, + [] () { return TString("100 days 01:02:03.12"); } + }, + { + INTERVALOID, "day to second,6", + SUCCESS, + [] () { return TString("100 01:02:03.1234"); }, + [] () { return TString("100 days 01:02:03.1234"); } + }, + { + INTERVALOID, "day to second,6", + SUCCESS, + [] () { return TString("100 01:02:03.1234"); }, + [] () { return TString("100 days 01:02:03.1234"); } + }, + }; + +#undef SUCCESS +#undef FAIL + auto createTable = [] ( NYdb::NTable::TTableClient& db, NYdb::NTable::TSession& session, - ui32 id, + ui32 typeId, bool isKey, bool isText, std::function<TString(size_t)> textIn, TString setTableName = "", ui16 rowCount = 10, - TVector <TString> colNames = {"key", "value"} + TVector<TString> colNames = {"key", "value"} ) { + auto* typeDesc = NPg::TypeDescFromPgTypeId(typeId); + auto typeName = NPg::PgTypeNameFromTypeDesc(typeDesc); + TTableBuilder builder; if (isKey) { - builder.AddNullableColumn(colNames[0], makePgType(id)); + builder.AddNullableColumn(colNames[0], TPgType(typeName)); } else { - builder.AddNullableColumn(colNames[0], makePgType(INT2OID)); + builder.AddNullableColumn(colNames[0], TPgType("pgint2")); } - builder.AddNullableColumn(colNames[1], makePgType(id)); + builder.AddNullableColumn(colNames[1], TPgType(typeName)); builder.SetPrimaryKeyColumn(colNames[0]); auto tableName = (setTableName.empty()) ? - Sprintf("/Root/Pg%u_%s", id, isText ? "t" : "b") : setTableName; + Sprintf("/Root/Pg%u_%s", typeId, isText ? "t" : "b") : setTableName; auto result = session.CreateTable(tableName, builder.Build()).GetValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); @@ -419,20 +663,20 @@ Y_UNIT_TEST_SUITE(KqpPg) { NYdb::TValueBuilder rows; rows.BeginList(); for (size_t i = 0; i < rowCount; ++i) { - auto str = isText ? textIn(i) : NPg::PgNativeBinaryFromNativeText(textIn(i), id).Str; + auto str = isText ? textIn(i) : NPg::PgNativeBinaryFromNativeText(textIn(i), typeId).Str; auto mode = isText ? TPgValue::VK_TEXT : TPgValue::VK_BINARY; if (isKey) { rows.AddListItem() .BeginStruct() - .AddMember(colNames[0]).Pg(TPgValue(mode, str, makePgType(id))) - .AddMember(colNames[1]).Pg(TPgValue(mode, str, makePgType(id))) + .AddMember(colNames[0]).Pg(TPgValue(mode, str, TPgType(typeName))) + .AddMember(colNames[1]).Pg(TPgValue(mode, str, TPgType(typeName))) .EndStruct(); } else { auto int2Str = NPg::PgNativeBinaryFromNativeText(Sprintf("%u", i), INT2OID).Str; rows.AddListItem() .BeginStruct() - .AddMember(colNames[0]).Pg(TPgValue(TPgValue::VK_BINARY, int2Str, makePgType(INT2OID))) - .AddMember(colNames[1]).Pg(TPgValue(mode, str, makePgType(id))) + .AddMember(colNames[0]).Pg(TPgValue(TPgValue::VK_BINARY, int2Str, TPgType("pgint2"))) + .AddMember(colNames[1]).Pg(TPgValue(mode, str, TPgType(typeName))) .EndStruct(); } } @@ -450,16 +694,71 @@ Y_UNIT_TEST_SUITE(KqpPg) { return tableName; }; + auto createCoercionTable = [] ( + NYdb::NTable::TTableClient& db, + NYdb::NTable::TSession& session, + ui32 typeId, + const TString& typeMod, + std::function<TString()> textIn, + size_t rowCount = 1 + ) { + auto* typeDesc = NPg::TypeDescFromPgTypeId(typeId); + auto typeName = NPg::PgTypeNameFromTypeDesc(typeDesc); + + TTableBuilder builder; + builder.AddNullableColumn("key", TPgType("pgint2")); + builder.AddNullableColumn("value", TPgType(typeName, typeMod)); + builder.SetPrimaryKeyColumn("key"); + + auto paramsHash = THash<TString>()(typeMod); + auto tableName = Sprintf("/Root/Coerce_%s_%" PRIu64, typeName.c_str(), paramsHash); + auto createResult = session.CreateTable(tableName, builder.Build()).GetValueSync(); + UNIT_ASSERT_C(createResult.IsSuccess(), createResult.GetIssues().ToString()); + + auto describeResult = session.DescribeTable(tableName).GetValueSync(); + UNIT_ASSERT_C(describeResult.IsSuccess(), describeResult.GetIssues().ToString()); + auto tableColumns = describeResult.GetTableDescription().GetTableColumns(); + for (const auto& column : tableColumns) { + const auto& name = column.Name; + const auto& type = column.Type; + if (name == "value") { + TTypeParser parser(type); + auto pgType = parser.GetPg(); + UNIT_ASSERT(pgType.TypeName == typeName); + UNIT_ASSERT(pgType.TypeModifier == typeMod); + } + } + + NYdb::TValueBuilder rows; + rows.BeginList(); + for (size_t i = 0; i < rowCount; ++i) { + auto str = NPg::PgNativeBinaryFromNativeText(textIn(), typeId).Str; + auto int2Str = NPg::PgNativeBinaryFromNativeText(Sprintf("%u", i), INT2OID).Str; + rows.AddListItem() + .BeginStruct() + .AddMember("key").Pg(TPgValue(TPgValue::VK_BINARY, int2Str, TPgType("pgint2"))) + .AddMember("value").Pg(TPgValue(TPgValue::VK_BINARY, str, TPgType(typeName, typeMod))) + .EndStruct(); + } + rows.EndList(); + + auto upsertResult = db.BulkUpsert(tableName, rows.Build()).GetValueSync(); + if (!upsertResult.IsSuccess()) { + Cerr << upsertResult.GetIssues().ToString() << Endl; + return std::make_pair(tableName, false); + } + + return std::make_pair(tableName, true); + }; + + Y_UNIT_TEST(CreateTableBulkUpsertAndRead) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); - auto testSingleType = [&kikimr] (ui32 id, bool isKey, bool isText, - std::function<TString(size_t)> textIn, - std::function<TString(size_t)> textOut) - { + auto testSingleType = [&kikimr] (const TPgTypeTestSpec& spec, bool isText) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - auto tableName = createTable(db, session, id, isKey, isText, textIn); + auto tableName = createTable(db, session, spec.TypeId, spec.IsKey, isText, spec.TextIn); auto readSettings = TReadTableSettings() .AppendColumns("key") @@ -479,15 +778,15 @@ Y_UNIT_TEST_SUITE(KqpPg) { auto resultSet = part.ExtractPart(); TResultSetParser parser(resultSet); for (size_t i = 0; parser.TryNextRow(); ++i) { - auto check = [&parser, &id, &i] (const TString& column, const TString& expected) { + auto check = [&parser, &spec, &i] (const TString& column, const TString& expected) { auto& c = parser.ColumnParser(column); - auto result = NPg::PgNativeTextFromNativeBinary(c.GetPg().Content_, id); - UNIT_ASSERT_C(result.Error.empty(), result.Error); + auto result = NPg::PgNativeTextFromNativeBinary(c.GetPg().Content_, spec.TypeId); + UNIT_ASSERT_C(!result.Error, *result.Error); UNIT_ASSERT_VALUES_EQUAL(expected, result.Str); Cerr << expected << Endl; }; - auto expected = textOut(i); - if (isKey) { + auto expected = spec.TextOut(i); + if (spec.IsKey) { check("key", expected); } check("value", expected); @@ -497,55 +796,94 @@ Y_UNIT_TEST_SUITE(KqpPg) { session.Close().GetValueSync(); }; - auto testType = [&] (ui32 id, const TPgTypeTestSpec& typeSpec) - { - testSingleType(id, typeSpec.IsKey, false, typeSpec.TextIn, typeSpec.TextOut); - testSingleType(id, typeSpec.IsKey, true, typeSpec.TextIn, typeSpec.TextOut); - - auto arrayId = NYql::NPg::LookupType(id).ArrayTypeId; - - auto textInArray = [&typeSpec] (auto i) { - auto str = typeSpec.TextIn(i); - return typeSpec.ArrayPrint(str); + auto testType = [&] (const TPgTypeTestSpec& spec) { + auto textInArray = [&spec] (auto i) { + auto str = spec.TextIn(i); + return spec.ArrayPrint(str); }; - auto textOutArray = [&typeSpec] (auto i) { - auto str = typeSpec.TextOut(i); - return typeSpec.ArrayPrint(str); + auto textOutArray = [&spec] (auto i) { + auto str = spec.TextOut(i); + return spec.ArrayPrint(str); }; - testSingleType(arrayId, typeSpec.IsKey, false, textInArray, textOutArray); - testSingleType(arrayId, typeSpec.IsKey, true, textInArray, textOutArray); + auto arrayTypeId = NYql::NPg::LookupType(spec.TypeId).ArrayTypeId; + TPgTypeTestSpec arraySpec{arrayTypeId, spec.IsKey, textInArray, textOutArray}; + + testSingleType(spec, false); + testSingleType(spec, true); + testSingleType(arraySpec, false); + testSingleType(arraySpec, true); }; auto testByteaType = [&] () { - testSingleType(BYTEAOID, true, false, - [] (auto i) { return Sprintf("bytea %u", i); }, - [] (auto i) { return Sprintf("\\x627974656120%x", i + 48); }); - - testSingleType(BYTEAOID, true, true, - [] (auto i) { return Sprintf("bytea %u", i); }, - [] (auto i) { return Sprintf("\\x627974656120%x", i + 48); }); - - testSingleType(BYTEAARRAYOID, true, false, - [] (auto i) { return Sprintf("{a%u, b%u}", i, i + 10); }, - [] (auto i) { return Sprintf("{\"\\\\x61%x\",\"\\\\x6231%x\"}", i + 48, i + 48); }); - - testSingleType(BYTEAARRAYOID, true, true, - [] (auto i) { return Sprintf("{a%u, b%u}", i, i + 10); }, - [] (auto i) { return Sprintf("{\"\\\\x61%x\",\"\\\\x6231%x\"}", i + 48, i + 48); }); + testSingleType(typeByteaSpec, false); + testSingleType(typeByteaSpec, true); + testSingleType(typeByteaArraySpec, false); + testSingleType(typeByteaArraySpec, true); }; testByteaType(); - for (const auto& [oid, spec] : typeSpecs) { - testType(oid, spec); + for (const auto& spec : typeSpecs) { + testType(spec); } // TODO: varchar as a key // TODO: native range/multirange types (use get_range_io_data()) } + Y_UNIT_TEST(TypeCoercionBulkUpsert) { + TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); + + auto testType = [&kikimr] (const TPgTypeCoercionTestSpec& spec) { + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + TString tableName; + bool success; + std::tie(tableName, success) = createCoercionTable(db, session, spec.TypeId, spec.TypeMod, spec.TextIn); + + UNIT_ASSERT_VALUES_EQUAL(success, spec.ShouldPass); + if (!success) { + return; + } + + auto readSettings = TReadTableSettings() + .AppendColumns("key") + .AppendColumns("value"); + + auto it = session.ReadTable(tableName, readSettings).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + + bool eos = false; + while (!eos) { + auto part = it.ReadNext().ExtractValueSync(); + if (!part.IsSuccess()) { + eos = true; + Y_ENSURE(part.EOS()); + continue; + } + auto resultSet = part.ExtractPart(); + TResultSetParser parser(resultSet); + for (size_t i = 0; parser.TryNextRow(); ++i) { + auto expected = spec.TextOut(); + auto& c = parser.ColumnParser("value"); + auto result = NPg::PgNativeTextFromNativeBinary(c.GetPg().Content_, spec.TypeId); + UNIT_ASSERT_C(!result.Error, *result.Error); + UNIT_ASSERT_VALUES_EQUAL(expected, result.Str); + Cerr << expected << Endl; + } + } + + session.Close().GetValueSync(); + }; + + for (const auto& spec : typeCoercionSpecs) { + testType(spec); + } + } + Y_UNIT_TEST(EmptyQuery) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); NYdb::NScripting::TScriptingClient client(kikimr.GetDriver()); @@ -554,7 +892,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { --!syntax_pg )").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); Y_ENSURE(result.GetResultSets().empty()); } @@ -571,7 +909,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { ) AS t (int8, varchar); )").GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); CompareYson(R"([ ["1";"one"]; @@ -582,55 +920,46 @@ Y_UNIT_TEST_SUITE(KqpPg) { Y_UNIT_TEST(TableSelect) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); - auto testSingleType = [&kikimr] (ui32 id, const TPgTypeTestSpec& spec) - { + + auto testSingleType = [&kikimr] (const TPgTypeTestSpec& spec) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - auto tableName = createTable(db, session, id, spec.IsKey, false, spec.TextIn); + auto tableName = createTable(db, session, spec.TypeId, spec.IsKey, false, spec.TextIn); session.Close().GetValueSync(); + NYdb::NScripting::TScriptingClient client(kikimr.GetDriver()); auto result = ExecutePgSelect(kikimr, tableName); ValidatePgYqlResult(result, spec); }; - - auto testType = [&] (ui32 id, const TPgTypeTestSpec& typeSpec) - { - testSingleType(id, typeSpec); - - auto arrayId = NYql::NPg::LookupType(id).ArrayTypeId; - auto textInArray = [&typeSpec] (auto i) { - auto str = typeSpec.TextIn(i); - return typeSpec.ArrayPrint(str); + auto testType = [&] (const TPgTypeTestSpec& spec) { + auto textInArray = [&spec] (auto i) { + auto str = spec.TextIn(i); + return spec.ArrayPrint(str); }; - auto textOutArray = [&typeSpec] (auto i) { - auto str = typeSpec.TextOut(i); - return typeSpec.ArrayPrint(str); + + auto textOutArray = [&spec] (auto i) { + auto str = spec.TextOut(i); + return spec.ArrayPrint(str); }; - TPgTypeTestSpec arraySpec(typeSpec.IsKey, textInArray, textOutArray); - testSingleType(arrayId, arraySpec); + auto arrayTypeId = NYql::NPg::LookupType(spec.TypeId).ArrayTypeId; + TPgTypeTestSpec arraySpec{arrayTypeId, spec.IsKey, textInArray, textOutArray}; + + testSingleType(spec); + testSingleType(arraySpec); }; auto testByteaType = [&] () { - TPgTypeTestSpec byteaSpec( - true, - [] (auto i) { return Sprintf("bytea %u", i); }, - [] (auto i) { return Sprintf("\\x627974656120%x", i + 48); } - ); - - TPgTypeTestSpec bytearrSpec( - false, - [] (auto i) { return Sprintf("{a%u, b%u}", i, i + 10); }, - [] (auto i) { return Sprintf("{\"\\\\x61%x\",\"\\\\x6231%x\"}", i + 48, i + 48); } - ); - testSingleType(BYTEAOID, byteaSpec); - testSingleType(BYTEAARRAYOID, bytearrSpec); + testSingleType(typeByteaSpec); + testSingleType(typeByteaArraySpec); }; + testByteaType(); - for (const auto& [oid, spec] : typeSpecs) { - Cerr << oid << Endl; - testType(oid, spec); + + for (const auto& spec : typeSpecs) { + Cerr << spec.TypeId << Endl; + testType(spec); } } @@ -644,88 +973,56 @@ Y_UNIT_TEST_SUITE(KqpPg) { Y_UNIT_TEST(TableInsert) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); - auto testSingleType = [&kikimr] (ui32 id, const TPgTypeTestSpec& spec) { + auto testSingleType = [&kikimr] (const TPgTypeTestSpec& spec) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - auto tableName = createTable( - db, - session, - id, - spec.IsKey, - false, - spec.TextIn, - "", - 0 - ); + auto tableName = createTable(db, session, spec.TypeId, spec.IsKey, false, spec.TextIn, "", 0); session.Close().GetValueSync(); - ExecutePgInsert(kikimr, tableName, id, spec); + ExecutePgInsert(kikimr, tableName, spec); auto result = ExecutePgSelect(kikimr, tableName); ValidatePgYqlResult(result, spec); }; - auto testType = [&] (ui32 id, const TPgTypeTestSpec& typeSpec) - { - testSingleType(id, typeSpec); - }; auto testByteaType = [&] () { - TPgTypeTestSpec byteaSpec( - true, - [] (auto i) { return Sprintf("bytea %u", i); }, - [] (auto i) { return Sprintf("\\x627974656120%x", i + 48); } - ); - - TPgTypeTestSpec bytearrSpec( - false, - [] (auto i) { return Sprintf("{a%u, b%u}", i, i + 10); }, - [] (auto i) { return Sprintf("{\"\\\\x61%x\",\"\\\\x6231%x\"}", i + 48, i + 48); } - ); - testSingleType(BYTEAOID, byteaSpec); - testSingleType(BYTEAARRAYOID, bytearrSpec); + testSingleType(typeByteaSpec); + + TPgTypeTestSpec typeByteaArraySpecForInsert{ + BYTEAARRAYOID, false, typeByteaArraySpec.TextIn, typeByteaArraySpec.TextOut}; + + testSingleType(typeByteaArraySpecForInsert); }; + testByteaType(); - for (auto [oid, spec] : typeSpecs) { - Cerr << oid << Endl; - if (oid == CHAROID) { + + for (const auto& spec : typeSpecs) { + Cerr << spec.TypeId << Endl; + if (spec.TypeId == CHAROID) { continue; // I cant come up with a query with explicit char conversion. // ::char, ::character casts to pg_bpchar } - if (oid == MONEYOID || oid == BITOID || oid == VARBITOID) { - spec.IsKey = false; - // Those types do not have HashProcId, so are not hashable, - // And we can not validate their uniqueness as keys in INSERT. - } - testType(oid, spec); + testSingleType(spec); } } Y_UNIT_TEST(TableArrayInsert) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); - auto testSingleType = [&kikimr] (ui32 id, const TPgTypeTestSpec& spec) { + auto testSingleType = [&kikimr] (const TPgTypeTestSpec& spec) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - auto arrayId = NYql::NPg::LookupType(id).ArrayTypeId; - auto tableName = createTable( - db, - session, - arrayId, - spec.IsKey/*false*/, - false, - spec.TextIn, - "", - 0 - ); + auto arrayId = NYql::NPg::LookupType(spec.TypeId).ArrayTypeId; + auto tableName = createTable(db, session, arrayId, spec.IsKey, false, spec.TextIn, "", 0); session.Close().GetValueSync(); - ExecutePgArrayInsert(kikimr, tableName, id, spec); + ExecutePgArrayInsert(kikimr, tableName, spec); auto result = ExecutePgSelect(kikimr, tableName); TResultSetParser parser(result.GetResultSetParser(0)); for (size_t i = 0; parser.TryNextRow(); ++i) { - auto check = [&parser, &id, &spec] (const TString& column, const TString& expected) { + auto check = [&parser] (const TString& column, const TString& expected) { auto& c = parser.ColumnParser(column); UNIT_ASSERT_VALUES_EQUAL(expected, c.GetPg().Content_); }; @@ -734,45 +1031,37 @@ Y_UNIT_TEST_SUITE(KqpPg) { } }; - - auto testType = [&] (ui32 id, const TPgTypeTestSpec& typeSpec) { - auto textOutArray = [&typeSpec] (auto i) { - auto str = typeSpec.TextOut(i); - return typeSpec.ArrayPrint(str); + auto testType = [&] (const TPgTypeTestSpec& spec) { + auto textOutArray = [&spec] (auto i) { + auto str = spec.TextOut(i); + return spec.ArrayPrint(str); }; - TPgTypeTestSpec arraySpec(false, typeSpec.TextIn, textOutArray); - testSingleType(id, arraySpec); + TPgTypeTestSpec arraySpec{spec.TypeId, false, spec.TextIn, textOutArray}; + testSingleType(arraySpec); }; - for (auto [oid, spec] : typeSpecs) { - Cerr << oid << Endl; - if (oid == CHAROID) { + for (const auto& spec : typeSpecs) { + Cerr << spec.TypeId << Endl; + if (spec.TypeId == CHAROID) { continue; // I cant come up with a query with explicit char conversion. // ::char, ::character casts to pg_bpchar } - if (oid == MONEYOID || oid == BITOID || oid == VARBITOID) { - spec.IsKey = false; - // Those types do not have HashProcId, so are not hashable, - // And we can not validate their uniqueness as keys in INSERT. - } - testType(oid, spec); + testType(spec); } } Y_UNIT_TEST(InsertFromSelect) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); - auto testSingleType = [&kikimr] (ui32 id, bool isKey, - std::function<TString(size_t)> textIn, - std::function<TString(size_t)> textOut) - { + auto testSingleType = [&kikimr] (const TPgTypeTestSpec& spec) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - auto tableName = createTable(db, session, id, isKey, false, textIn, "", 10, {"key1", "value1"}); - TString emptyTableName = "/Root/PgEmpty" + ToString(id); - createTable(db, session, id, isKey, false, textIn, emptyTableName, 0); + auto tableName = createTable(db, session, spec.TypeId, spec.IsKey, false, spec.TextIn, "", 10, {"key1", "value1"}); + TString emptyTableName = "/Root/PgEmpty" + ToString(spec.TypeId); + createTable(db, session, spec.TypeId, spec.IsKey, false, spec.TextIn, emptyTableName, 0); session.Close().GetValueSync(); + NYdb::NScripting::TScriptingClient client(kikimr.GetDriver()); auto result = client.ExecuteYqlScript( TStringBuilder() << R"( @@ -782,22 +1071,25 @@ Y_UNIT_TEST_SUITE(KqpPg) { UNIT_ASSERT_EQUAL(result.GetStatus(), EStatus::INTERNAL_ERROR); }; - auto testType = [&] (ui32 id, const TPgTypeTestSpec& typeSpec) - { - testSingleType(id, typeSpec.IsKey, typeSpec.TextIn, typeSpec.TextOut); - }; - - testType(INT2OID, typeSpecs[INT2OID]); + for (const auto& spec : typeSpecs) { + Cerr << spec.TypeId << Endl; + if (spec.TypeId == CHAROID) { + continue; + // I cant come up with a query with explicit char conversion. + // ::char, ::character casts to pg_bpchar + } + testSingleType(spec); + } } Y_UNIT_TEST(CreateTable) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); - auto testSingleType = [&kikimr] (ui32 id, const TPgTypeTestSpec& spec, bool isArray) - { + + auto testSingleType = [&kikimr] (const TPgTypeTestSpec& spec, bool isArray) { NYdb::NScripting::TScriptingClient client(kikimr.GetDriver()); - auto tableName = "/Root/Pg" + ToString(id) + (isArray ? "array" : ""); - auto typeName = ((isArray) ? "_pg" : "pg") + NYql::NPg::LookupType(id).Name; + auto tableName = "/Root/Pg" + ToString(spec.TypeId) + (isArray ? "array" : ""); + auto typeName = ((isArray) ? "_pg" : "pg") + NYql::NPg::LookupType(spec.TypeId).Name; auto keyEntry = spec.IsKey ? ("key "+ typeName) : "key pgint2"; auto valueEntry = "value " + typeName; auto req = Sprintf("\ @@ -808,17 +1100,18 @@ Y_UNIT_TEST_SUITE(KqpPg) { );", tableName.Data(), keyEntry.Data(), valueEntry.Data()); Cerr << req << Endl; auto result = client.ExecuteYqlScript(req).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + if (!isArray) { - ExecutePgInsert(kikimr, tableName, id, spec); + ExecutePgInsert(kikimr, tableName, spec); result = ExecutePgSelect(kikimr, tableName); ValidatePgYqlResult(result, spec); } else { - ExecutePgArrayInsert(kikimr, tableName, id, spec); + ExecutePgArrayInsert(kikimr, tableName, spec); result = ExecutePgSelect(kikimr, tableName); TResultSetParser parser(result.GetResultSetParser(0)); for (size_t i = 0; parser.TryNextRow(); ++i) { - auto check = [&parser, &id, &spec] (const TString& column, const TString& expected) { + auto check = [&parser, &spec] (const TString& column, const TString& expected) { auto& c = parser.ColumnParser(column); UNIT_ASSERT_VALUES_EQUAL(expected, c.GetPg().Content_); }; @@ -828,34 +1121,26 @@ Y_UNIT_TEST_SUITE(KqpPg) { } }; - auto testType = [&] (ui32 id, const TPgTypeTestSpec& typeSpec) - { - testSingleType(id, typeSpec, false); - - auto textOutArray = [&typeSpec] (auto i) { - auto str = typeSpec.TextOut(i); - return typeSpec.ArrayPrint(str); + auto testType = [&] (const TPgTypeTestSpec& spec) { + auto textOutArray = [&spec] (auto i) { + auto str = spec.TextOut(i); + return spec.ArrayPrint(str); }; - TPgTypeTestSpec arraySpec(false, typeSpec.TextIn, textOutArray); - testSingleType(id, arraySpec, true); + TPgTypeTestSpec arraySpec{spec.TypeId, false, spec.TextIn, textOutArray}; + testSingleType(spec, false); + testSingleType(arraySpec, true); }; - for (auto [oid, spec] : typeSpecs) { - if (oid == CHAROID) { - continue; - } - if (oid == CHAROID) { + + for (const auto& spec : typeSpecs) { + Cerr << spec.TypeId << Endl; + if (spec.TypeId == CHAROID) { continue; // I cant come up with a query with explicit char conversion. // ::char, ::character casts to pg_bpchar } - if (oid == MONEYOID || oid == BITOID || oid == VARBITOID) { - spec.IsKey = false; - // Those types do not have HashProcId, so are not hashable, - // And we can not validate their uniqueness as keys in INSERT. - } - testType(oid, spec); + testType(spec); } } @@ -863,7 +1148,7 @@ Y_UNIT_TEST_SUITE(KqpPg) { TKikimrRunner kikimr(NKqp::TKikimrSettings().SetWithSampleTables(false)); TTableBuilder builder; - UNIT_ASSERT_EXCEPTION(builder.AddNonNullableColumn("key", makePgType(INT2OID)), yexception); + UNIT_ASSERT_EXCEPTION(builder.AddNonNullableColumn("key", TPgType("pgint2")), yexception); NYdb::NScripting::TScriptingClient client(kikimr.GetDriver()); auto req = TStringBuilder() << R"( diff --git a/ydb/core/persqueue/pq_impl.cpp b/ydb/core/persqueue/pq_impl.cpp index 58d35a74199..fc14f81d847 100644 --- a/ydb/core/persqueue/pq_impl.cpp +++ b/ydb/core/persqueue/pq_impl.cpp @@ -634,9 +634,9 @@ void TPersQueue::ApplyNewConfigAndReply(const TActorContext& ctx) KeySchema.clear(); KeySchema.reserve(Config.PartitionKeySchemaSize()); for (const auto& component : Config.GetPartitionKeySchema()) { - auto typeInfo = NScheme::TypeInfoFromProtoColumnType(component.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(component.GetTypeId(), component.HasTypeInfo() ? &component.GetTypeInfo() : nullptr); - KeySchema.push_back(typeInfo); + KeySchema.push_back(typeInfoMod.TypeInfo); } Y_VERIFY(TopicName.size(), "Need topic name here"); @@ -759,9 +759,9 @@ void TPersQueue::ReadConfig(const NKikimrClient::TKeyValueResponse::TReadResult& KeySchema.clear(); KeySchema.reserve(Config.PartitionKeySchemaSize()); for (const auto& component : Config.GetPartitionKeySchema()) { - auto typeInfo = NScheme::TypeInfoFromProtoColumnType(component.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(component.GetTypeId(), component.HasTypeInfo() ? &component.GetTypeInfo() : nullptr); - KeySchema.push_back(typeInfo); + KeySchema.push_back(typeInfoMod.TypeInfo); } ui32 cacheSize = CACHE_SIZE; diff --git a/ydb/core/protos/type_info.proto b/ydb/core/protos/type_info.proto index 669e0f5ee7b..3ce4b6ed831 100644 --- a/ydb/core/protos/type_info.proto +++ b/ydb/core/protos/type_info.proto @@ -3,4 +3,5 @@ option java_package = "ru.yandex.kikimr.proto"; message TTypeInfo { optional uint32 PgTypeId = 1; + optional string PgTypeMod = 2; } diff --git a/ydb/core/scheme/scheme_type_info.cpp b/ydb/core/scheme/scheme_type_info.cpp index f655cd5f192..5631b6de550 100644 --- a/ydb/core/scheme/scheme_type_info.cpp +++ b/ydb/core/scheme/scheme_type_info.cpp @@ -5,9 +5,9 @@ namespace NKikimr::NScheme { -const char* TypeName(const TTypeInfo typeInfo) { +::TString TypeName(const TTypeInfo typeInfo, const ::TString& typeMod) { if (typeInfo.GetTypeId() == NScheme::NTypeIds::Pg) { - return NPg::PgTypeNameFromTypeDesc(typeInfo.GetTypeDesc()); + return NPg::PgTypeNameFromTypeDesc(typeInfo.GetTypeDesc(), typeMod); } else { return TypeName(typeInfo.GetTypeId()); } diff --git a/ydb/core/scheme/scheme_type_info.h b/ydb/core/scheme/scheme_type_info.h index f36f98112a6..307486ffdb8 100644 --- a/ydb/core/scheme/scheme_type_info.h +++ b/ydb/core/scheme/scheme_type_info.h @@ -4,6 +4,6 @@ namespace NKikimr::NScheme { -const char* TypeName(const TTypeInfo typeInfo); +::TString TypeName(const TTypeInfo typeInfo, const ::TString& typeMod = {}); } // NKikimr::NScheme diff --git a/ydb/core/scheme/scheme_types_proto.cpp b/ydb/core/scheme/scheme_types_proto.cpp index 02bfbda2551..97faf06ddf1 100644 --- a/ydb/core/scheme/scheme_types_proto.cpp +++ b/ydb/core/scheme/scheme_types_proto.cpp @@ -3,24 +3,32 @@ namespace NKikimr::NScheme { -TProtoColumnType ProtoColumnTypeFromTypeInfo(const TTypeInfo typeInfo) { +TProtoColumnType ProtoColumnTypeFromTypeInfoMod(const TTypeInfo typeInfo, const ::TString& typeMod) { TProtoColumnType columnType; columnType.TypeId = (ui32)typeInfo.GetTypeId(); if (typeInfo.GetTypeId() == NTypeIds::Pg) { Y_VERIFY(typeInfo.GetTypeDesc(), "no pg type descriptor"); columnType.TypeInfo = NKikimrProto::TTypeInfo(); columnType.TypeInfo->SetPgTypeId(NPg::PgTypeIdFromTypeDesc(typeInfo.GetTypeDesc())); + if (typeMod) { + columnType.TypeInfo->SetPgTypeMod(typeMod); + } } return columnType; } -TTypeInfo TypeInfoFromProtoColumnType(ui32 typeId, const NKikimrProto::TTypeInfo* typeInfo) { +TTypeInfoMod TypeInfoModFromProtoColumnType(ui32 typeId, const NKikimrProto::TTypeInfo* typeInfo) { auto type = (TTypeId)typeId; if (type == NTypeIds::Pg) { - Y_VERIFY(typeInfo); - return TTypeInfo(type, NPg::TypeDescFromPgTypeId(typeInfo->GetPgTypeId())); + Y_VERIFY(typeInfo, "no type info for pg type"); + TTypeInfoMod res; + res.TypeInfo = TTypeInfo(type, NPg::TypeDescFromPgTypeId(typeInfo->GetPgTypeId())); + if (typeInfo->HasPgTypeMod()) { + res.TypeMod = typeInfo->GetPgTypeMod(); + } + return res; } - return TTypeInfo(type); + return {TTypeInfo(type), ""}; } } // namespace NKikimr::NScheme diff --git a/ydb/core/scheme/scheme_types_proto.h b/ydb/core/scheme/scheme_types_proto.h index 43c8789579c..ce46ccad116 100644 --- a/ydb/core/scheme/scheme_types_proto.h +++ b/ydb/core/scheme/scheme_types_proto.h @@ -10,8 +10,13 @@ struct TProtoColumnType { std::optional<NKikimrProto::TTypeInfo> TypeInfo; }; -TProtoColumnType ProtoColumnTypeFromTypeInfo(const TTypeInfo typeInfo); +TProtoColumnType ProtoColumnTypeFromTypeInfoMod(const TTypeInfo typeInfo, const ::TString& typeMod); -TTypeInfo TypeInfoFromProtoColumnType(ui32 typeId, const NKikimrProto::TTypeInfo* typeInfo); +struct TTypeInfoMod { + TTypeInfo TypeInfo; + ::TString TypeMod; +}; + +TTypeInfoMod TypeInfoModFromProtoColumnType(ui32 typeId, const NKikimrProto::TTypeInfo* typeInfo); } // namespace NKikimr::NScheme diff --git a/ydb/core/sys_view/common/schema.cpp b/ydb/core/sys_view/common/schema.cpp index 78d7c1bf81a..80ef81d4dec 100644 --- a/ydb/core/sys_view/common/schema.cpp +++ b/ydb/core/sys_view/common/schema.cpp @@ -114,7 +114,7 @@ private: static void Fill(TSchema& schema) { schema.Columns[Column::ColumnId] = TSysTables::TTableColumnInfo( Table::template TableColumns<Column>::GetColumnName(), - Column::ColumnId, NScheme::TTypeInfo(Column::ColumnType), -1); + Column::ColumnId, NScheme::TTypeInfo(Column::ColumnType), "", -1); } }; diff --git a/ydb/core/tablet_flat/flat_cxx_database.h b/ydb/core/tablet_flat/flat_cxx_database.h index 021a520859b..f15905864b0 100644 --- a/ydb/core/tablet_flat/flat_cxx_database.h +++ b/ydb/core/tablet_flat/flat_cxx_database.h @@ -2197,7 +2197,7 @@ struct TStaticSchemaFiller { schema.Columns[Column::ColumnId] = NTable::TColumn( TTable::template TableColumns<Column>::GetColumnName(), Column::ColumnId, - NScheme::TTypeInfo(Column::ColumnType)); + NScheme::TTypeInfo(Column::ColumnType), ""); } }; diff --git a/ydb/core/tablet_flat/flat_dbase_apply.cpp b/ydb/core/tablet_flat/flat_dbase_apply.cpp index 32ec4f02df6..dc23becc890 100644 --- a/ydb/core/tablet_flat/flat_dbase_apply.cpp +++ b/ydb/core/tablet_flat/flat_dbase_apply.cpp @@ -1,6 +1,7 @@ #include "flat_dbase_apply.h" #include <ydb/core/base/localdb.h> +#include <ydb/core/scheme/scheme_types_proto.h> namespace NKikimr { namespace NTable { @@ -32,9 +33,11 @@ bool TSchemeModifier::Apply(const TAlterRecord &delta) null = TCell(raw.data(), raw.size()); } - ui32 pgTypeId = delta.HasColumnTypeInfo() ? delta.GetColumnTypeInfo().GetPgTypeId() : 0; + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(delta.GetColumnType(), + delta.HasColumnTypeInfo() ? &delta.GetColumnTypeInfo() : nullptr); + ui32 pgTypeId = NPg::PgTypeIdFromTypeDesc(typeInfoMod.TypeInfo.GetTypeDesc()); changes = AddPgColumn(table, delta.GetColumnName(), delta.GetColumnId(), - delta.GetColumnType(), pgTypeId, delta.GetNotNull(), null); + delta.GetColumnType(), pgTypeId, typeInfoMod.TypeMod, delta.GetNotNull(), null); } else if (action == TAlterRecord::DropColumn) { changes = DropColumn(table, delta.GetColumnId()); } else if (action == TAlterRecord::AddColumnToKey) { @@ -224,10 +227,10 @@ bool TSchemeModifier::DropTable(ui32 id) bool TSchemeModifier::AddColumn(ui32 tid, const TString &name, ui32 id, ui32 type, bool notNull, TCell null) { Y_VERIFY(type != (ui32)NScheme::NTypeIds::Pg, "No pg type data"); - return AddPgColumn(tid, name, id, type, 0, notNull, null); + return AddPgColumn(tid, name, id, type, 0, "", notNull, null); } -bool TSchemeModifier::AddPgColumn(ui32 tid, const TString &name, ui32 id, ui32 type, ui32 pgType, bool notNull, TCell null) +bool TSchemeModifier::AddPgColumn(ui32 tid, const TString &name, ui32 id, ui32 type, ui32 pgType, const TString& pgTypeMod, bool notNull, TCell null) { auto *table = Table(tid); @@ -259,9 +262,10 @@ bool TSchemeModifier::AddPgColumn(ui32 tid, const TString &name, ui32 id, ui32 t Y_VERIFY_S(itName->second == id, describeFailure()); // Sanity check that this column exists and types match Y_VERIFY(it != table->Columns.end() && it->second.Name == name); - Y_VERIFY_S(it->second.PType == typeInfo, + Y_VERIFY_S(it->second.PType == typeInfo && it->second.PTypeMod == pgTypeMod, "Table " << tid << " '" << table->Name << "' column " << id << " '" << name - << "' expected type " << NScheme::TypeName(typeInfo) << ", existing type " << NScheme::TypeName(it->second.PType)); + << "' expected type " << NScheme::TypeName(typeInfo, pgTypeMod) + << ", existing type " << NScheme::TypeName(it->second.PType, it->second.PTypeMod)); return false; } @@ -269,16 +273,17 @@ bool TSchemeModifier::AddPgColumn(ui32 tid, const TString &name, ui32 id, ui32 t // We assume column is renamed when the same id already exists if (it != table->Columns.end()) { - Y_VERIFY_S(it->second.PType == typeInfo, + Y_VERIFY_S(it->second.PType == typeInfo && it->second.PTypeMod == pgTypeMod, "Table " << tid << " '" << table->Name << "' column " << id << " '" << it->second.Name << "' renamed to '" << name << "'" - << " with type " << NScheme::TypeName(typeInfo) << ", existing type " << NScheme::TypeName(it->second.PType)); + << " with type " << NScheme::TypeName(typeInfo, pgTypeMod) + << ", existing type " << NScheme::TypeName(it->second.PType, it->second.PTypeMod)); table->ColumnNames.erase(it->second.Name); it->second.Name = name; table->ColumnNames.emplace(name, id); return true; } - auto pr = table->Columns.emplace(id, TColumn(name, id, typeInfo, notNull)); + auto pr = table->Columns.emplace(id, TColumn(name, id, typeInfo, pgTypeMod, notNull)); Y_VERIFY(pr.second); it = pr.first; table->ColumnNames.emplace(name, id); diff --git a/ydb/core/tablet_flat/flat_dbase_apply.h b/ydb/core/tablet_flat/flat_dbase_apply.h index b9008b73abb..c90113c804c 100644 --- a/ydb/core/tablet_flat/flat_dbase_apply.h +++ b/ydb/core/tablet_flat/flat_dbase_apply.h @@ -33,7 +33,7 @@ namespace NTable { bool AddTable(const TString& name, ui32 id); bool DropTable(ui32 id); bool AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null = { }); - bool AddPgColumn(ui32 table, const TString& name, ui32 id, ui32 type, ui32 pgType, bool notNull, TCell null = { }); + bool AddPgColumn(ui32 table, const TString& name, ui32 id, ui32 type, ui32 pgType, const TString& pgTypeMod, bool notNull, TCell null = { }); bool DropColumn(ui32 table, ui32 id); bool AddColumnToFamily(ui32 table, ui32 column, ui32 family); bool AddColumnToKey(ui32 table, ui32 column); diff --git a/ydb/core/tablet_flat/flat_dbase_scheme.cpp b/ydb/core/tablet_flat/flat_dbase_scheme.cpp index 2c973fa9044..0292e12d4cf 100644 --- a/ydb/core/tablet_flat/flat_dbase_scheme.cpp +++ b/ydb/core/tablet_flat/flat_dbase_scheme.cpp @@ -29,7 +29,8 @@ TAutoPtr<TSchemeChanges> TScheme::GetSnapshot() const { const auto &col = it.second; delta.AddPgColumn(table, col.Name, it.first, col.PType.GetTypeId(), - NPg::PgTypeIdFromTypeDesc(col.PType.GetTypeDesc()), col.NotNull, col.Null); + NPg::PgTypeIdFromTypeDesc(col.PType.GetTypeDesc()), + col.PTypeMod, col.NotNull, col.Null); delta.AddColumnToFamily(table, it.first, col.Family); } @@ -99,10 +100,10 @@ TAlter& TAlter::DropTable(ui32 id) TAlter& TAlter::AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null) { Y_VERIFY(type != (ui32)NScheme::NTypeIds::Pg, "No pg type data"); - return AddPgColumn(table, name, id, type, 0, notNull, null); + return AddPgColumn(table, name, id, type, 0, "", notNull, null); } -TAlter& TAlter::AddPgColumn(ui32 table, const TString& name, ui32 id, ui32 type, ui32 pgType, bool notNull, TCell null) +TAlter& TAlter::AddPgColumn(ui32 table, const TString& name, ui32 id, ui32 type, ui32 pgType, const TString& pgTypeMod, bool notNull, TCell null) { TAlterRecord& delta = *Log.AddDelta(); delta.SetDeltaType(TAlterRecord::AddColumn); @@ -112,6 +113,9 @@ TAlter& TAlter::AddPgColumn(ui32 table, const TString& name, ui32 id, ui32 type, delta.SetColumnType(type); if (pgType != 0) { delta.MutableColumnTypeInfo()->SetPgTypeId(pgType); + if (!pgTypeMod.empty()) { + delta.MutableColumnTypeInfo()->SetPgTypeMod(pgTypeMod); + } } delta.SetNotNull(notNull); diff --git a/ydb/core/tablet_flat/flat_dbase_scheme.h b/ydb/core/tablet_flat/flat_dbase_scheme.h index 56cc65ec94d..85098da115e 100644 --- a/ydb/core/tablet_flat/flat_dbase_scheme.h +++ b/ydb/core/tablet_flat/flat_dbase_scheme.h @@ -247,7 +247,7 @@ public: TAlter& AddTable(const TString& name, ui32 id); TAlter& DropTable(ui32 id); TAlter& AddColumn(ui32 table, const TString& name, ui32 id, ui32 type, bool notNull, TCell null = { }); - TAlter& AddPgColumn(ui32 table, const TString& name, ui32 id, ui32 type, ui32 pgType, bool notNull, TCell null = { }); + TAlter& AddPgColumn(ui32 table, const TString& name, ui32 id, ui32 type, ui32 pgType, const TString& pgTypeMod, bool notNull, TCell null = { }); TAlter& DropColumn(ui32 table, ui32 id); TAlter& AddColumnToFamily(ui32 table, ui32 column, ui32 family); TAlter& AddFamily(ui32 table, ui32 family, ui32 room); diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index 1aded82071f..b02ada11bf7 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -3883,7 +3883,6 @@ void TExecutor::RenderHtmlCounters(NMon::TEvRemoteHttpInfo::TPtr &ev) const { void TExecutor::RenderHtmlPage(NMon::TEvRemoteHttpInfo::TPtr &ev) const { auto cgi = ev->Get()->Cgi(); TStringStream str; - const NScheme::TTypeRegistry& tr = *AppData()->TypeRegistry; if (cgi.Has("force_compaction")) { bool ok; @@ -3971,7 +3970,7 @@ void TExecutor::RenderHtmlPage(NMon::TEvRemoteHttpInfo::TPtr &ev) const { TABLER() { TABLED() {str << col.Name;} TABLED() {str << col.Id;} - TABLED() {str << tr.GetTypeName(col.PType.GetTypeId());} + TABLED() {str << NScheme::TypeName(col.PType, col.PTypeMod);} TABLED() {str << (isKey ? ToString(col.KeyOrder) : "");} } } diff --git a/ydb/core/tablet_flat/flat_table_column.h b/ydb/core/tablet_flat/flat_table_column.h index 49f8fb7a9d0..ac2a4095d4b 100644 --- a/ydb/core/tablet_flat/flat_table_column.h +++ b/ydb/core/tablet_flat/flat_table_column.h @@ -15,9 +15,11 @@ namespace NTable { TColumn() = default; - TColumn(const TString& name, TTag tag, NScheme::TTypeInfo type, bool notNull = false) + TColumn(const TString& name, TTag tag, NScheme::TTypeInfo type, + const TString& typeMod, bool notNull = false) : Id(tag) , PType(type) + , PTypeMod(typeMod) , Name(name) , NotNull(notNull) { @@ -29,6 +31,7 @@ namespace NTable { return Id == col.Id && PType == col.PType + && PTypeMod == col.PTypeMod && KeyOrder == col.KeyOrder && Name == col.Name && Family == col.Family @@ -48,6 +51,7 @@ namespace NTable { NTable::TTag Id = Max<TTag>(); NScheme::TTypeInfo PType; + TString PTypeMod; TString Name; ui32 Family = LeaderFamily; NTable::TPos KeyOrder = Max<TPos>(); diff --git a/ydb/core/tablet_flat/flat_table_part.cpp b/ydb/core/tablet_flat/flat_table_part.cpp index ff042b266ff..405b231f68b 100644 --- a/ydb/core/tablet_flat/flat_table_part.cpp +++ b/ydb/core/tablet_flat/flat_table_part.cpp @@ -53,8 +53,8 @@ TIntrusiveConstPtr<TPartScheme> TPartScheme::Parse(TArrayRef<const char> raw, bo cols.emplace_back(); cols.back().Tag = one.GetTag(); - cols.back().TypeInfo = NScheme::TypeInfoFromProtoColumnType(one.GetType(), - one.HasTypeInfo() ? &one.GetTypeInfo() : nullptr); + cols.back().TypeInfo = NScheme::TypeInfoModFromProtoColumnType(one.GetType(), + one.HasTypeInfo() ? &one.GetTypeInfo() : nullptr).TypeInfo; cols.back().Pos = cols.size() - 1; cols.back().Group = one.GetGroup(); @@ -193,7 +193,7 @@ TSharedData TPartScheme::Serialize() const for (const auto& col : AllColumns) { auto* pb = proto.AddColumns(); pb->SetTag(col.Tag); - auto protoType = NScheme::ProtoColumnTypeFromTypeInfo(col.TypeInfo); + auto protoType = NScheme::ProtoColumnTypeFromTypeInfoMod(col.TypeInfo, ""); pb->SetType(protoType.TypeId); if (protoType.TypeInfo) { *pb->MutableTypeInfo() = *protoType.TypeInfo; diff --git a/ydb/core/tablet_flat/test/libs/rows/layout.h b/ydb/core/tablet_flat/test/libs/rows/layout.h index 6657b4fd08a..4e854a64896 100644 --- a/ydb/core/tablet_flat/test/libs/rows/layout.h +++ b/ydb/core/tablet_flat/test/libs/rows/layout.h @@ -23,7 +23,7 @@ namespace NTest{ Tags_.push_back(tag); // pg types are not supported - Cols.emplace_back("", tag, NScheme::TTypeInfo(type)); + Cols.emplace_back("", tag, NScheme::TTypeInfo(type), ""); Cols.back().Family = group; Cols.back().SetDefault(null); diff --git a/ydb/core/tablet_flat/ut_pg/flat_database_pg_ut.cpp b/ydb/core/tablet_flat/ut_pg/flat_database_pg_ut.cpp index 4abd5e70191..5130aaeb4da 100644 --- a/ydb/core/tablet_flat/ut_pg/flat_database_pg_ut.cpp +++ b/ydb/core/tablet_flat/ut_pg/flat_database_pg_ut.cpp @@ -36,16 +36,16 @@ Y_UNIT_TEST_SUITE(TFlatDatabasePgTest) { db.Begin(); db->Alter() .AddTable("TestTable", tableId) - .AddPgColumn(tableId, "boolean", IdBool, NScheme::NTypeIds::Pg, BOOLOID, false) - .AddPgColumn(tableId, "char", IdChar, NScheme::NTypeIds::Pg, CHAROID, false) - .AddPgColumn(tableId, "int2", IdInt2, NScheme::NTypeIds::Pg, INT2OID, false) - .AddPgColumn(tableId, "int4", IdInt4, NScheme::NTypeIds::Pg, INT4OID, false) - .AddPgColumn(tableId, "int8", IdInt8, NScheme::NTypeIds::Pg, INT8OID, false) - .AddPgColumn(tableId, "float4", IdFloat4, NScheme::NTypeIds::Pg, FLOAT4OID, false) - .AddPgColumn(tableId, "float8", IdFloat8, NScheme::NTypeIds::Pg, FLOAT8OID, false) - .AddPgColumn(tableId, "text", IdText, NScheme::NTypeIds::Pg, TEXTOID, false) - .AddPgColumn(tableId, "bytea", IdBytea, NScheme::NTypeIds::Pg, BYTEAOID, false) - .AddPgColumn(tableId, "bpchar", IdBpchar, NScheme::NTypeIds::Pg, BPCHAROID, false) + .AddPgColumn(tableId, "boolean", IdBool, NScheme::NTypeIds::Pg, BOOLOID, "", false) + .AddPgColumn(tableId, "char", IdChar, NScheme::NTypeIds::Pg, CHAROID, "", false) + .AddPgColumn(tableId, "int2", IdInt2, NScheme::NTypeIds::Pg, INT2OID, "", false) + .AddPgColumn(tableId, "int4", IdInt4, NScheme::NTypeIds::Pg, INT4OID, "", false) + .AddPgColumn(tableId, "int8", IdInt8, NScheme::NTypeIds::Pg, INT8OID, "", false) + .AddPgColumn(tableId, "float4", IdFloat4, NScheme::NTypeIds::Pg, FLOAT4OID, "", false) + .AddPgColumn(tableId, "float8", IdFloat8, NScheme::NTypeIds::Pg, FLOAT8OID, "", false) + .AddPgColumn(tableId, "text", IdText, NScheme::NTypeIds::Pg, TEXTOID, "", false) + .AddPgColumn(tableId, "bytea", IdBytea, NScheme::NTypeIds::Pg, BYTEAOID, "", false) + .AddPgColumn(tableId, "bpchar", IdBpchar, NScheme::NTypeIds::Pg, BPCHAROID, "", false) .AddColumnToKey(tableId, IdText) .AddColumnToKey(tableId, IdBytea) .AddColumnToKey(tableId, IdInt2) diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 7cb866d4e73..d45c08eec84 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -989,9 +989,9 @@ NOlap::TIndexInfo TColumnShard::ConvertSchema(const NKikimrSchemeOp::TColumnTabl for (const auto& col : schema.GetColumns()) { const ui32 id = col.GetId(); const TString& name = col.GetName(); - auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(col.GetTypeId(), col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); - indexInfo.Columns[id] = NTable::TColumn(name, id, typeInfo); + indexInfo.Columns[id] = NTable::TColumn(name, id, typeInfoMod.TypeInfo, typeInfoMod.TypeMod); indexInfo.ColumnNames[name] = id; } diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/columnshard_ut_common.cpp index 872d4df5bd1..f9572611454 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/columnshard_ut_common.cpp @@ -130,7 +130,7 @@ void ScanIndexStats(TTestBasicRuntime& runtime, TActorId& sender, const TVector< auto ydbSchema = PrimaryIndexStatsSchema; for (const auto& col : ydbSchema.Columns) { record.AddColumnTags(col.second.Id); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(col.second.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(col.second.PType, col.second.PTypeMod); record.AddColumnTypes(columnType.TypeId); if (columnType.TypeInfo) { *record.AddColumnTypeInfos() = *columnType.TypeInfo; diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.h b/ydb/core/tx/columnshard/columnshard_ut_common.h index afb459a5de3..c557a367684 100644 --- a/ydb/core/tx/columnshard/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/columnshard_ut_common.h @@ -173,7 +173,7 @@ struct TTestSchema { NKikimrSchemeOp::TOlapColumnDescription col; col.SetId(id); col.SetName(name); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(type, ""); col.SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *col.MutableTypeInfo() = *columnType.TypeInfo; diff --git a/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp index 423318b75fe..02252c098b1 100644 --- a/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp @@ -180,7 +180,7 @@ TIndexInfo TestTableInfo(const TVector<std::pair<TString, TTypeInfo>>& ydbSchema auto& name = ydbSchema[i].first; auto& type = ydbSchema[i].second; - indexInfo.Columns[id] = NTable::TColumn(name, id, type); + indexInfo.Columns[id] = NTable::TColumn(name, id, type, ""); indexInfo.ColumnNames[name] = id; } diff --git a/ydb/core/tx/datashard/build_index.cpp b/ydb/core/tx/datashard/build_index.cpp index 1eb08345a05..2be7f1cb11a 100644 --- a/ydb/core/tx/datashard/build_index.cpp +++ b/ydb/core/tx/datashard/build_index.cpp @@ -64,7 +64,10 @@ static TTags BuildTags(const TColumnsTags& allTags, const TVector<TString>& inde static void ProtoYdbTypeFromTypeInfo(Ydb::Type* type, const NScheme::TTypeInfo typeInfo) { if (typeInfo.GetTypeId() == NScheme::NTypeIds::Pg) { - type->mutable_pg_type()->set_oid(NPg::PgTypeIdFromTypeDesc(typeInfo.GetTypeDesc())); + auto* typeDesc = typeInfo.GetTypeDesc(); + auto* pg = type->mutable_pg_type(); + pg->set_type_name(NPg::PgTypeNameFromTypeDesc(typeDesc)); + pg->set_oid(NPg::PgTypeIdFromTypeDesc(typeDesc)); } else { type->set_type_id((Ydb::Type::PrimitiveTypeId)typeInfo.GetTypeId()); } diff --git a/ydb/core/tx/datashard/datashard_kqp.cpp b/ydb/core/tx/datashard/datashard_kqp.cpp index 6839de09103..a41e12da5e6 100644 --- a/ydb/core/tx/datashard/datashard_kqp.cpp +++ b/ydb/core/tx/datashard/datashard_kqp.cpp @@ -314,9 +314,10 @@ using TWriteOpMeta = NKikimrTxDataShard::TKqpTransaction::TDataTaskMeta::TWriteO using TColumnMeta = NKikimrTxDataShard::TKqpTransaction::TColumnMeta; NTable::TColumn GetColumn(const TColumnMeta& columnMeta) { - auto typeInfo = NScheme::TypeInfoFromProtoColumnType(columnMeta.GetType(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(columnMeta.GetType(), columnMeta.HasTypeInfo() ? &columnMeta.GetTypeInfo() : nullptr); - return NTable::TColumn(columnMeta.GetName(), columnMeta.GetId(), typeInfo); + return NTable::TColumn(columnMeta.GetName(), columnMeta.GetId(), + typeInfoMod.TypeInfo, typeInfoMod.TypeMod); } TVector<NTable::TColumn> GetColumns(const TReadOpMeta& readMeta) { diff --git a/ydb/core/tx/datashard/datashard_user_table.cpp b/ydb/core/tx/datashard/datashard_user_table.cpp index 8a8016c475b..2a3558d909e 100644 --- a/ydb/core/tx/datashard/datashard_user_table.cpp +++ b/ydb/core/tx/datashard/datashard_user_table.cpp @@ -219,9 +219,9 @@ void TUserTable::ParseProto(const NKikimrSchemeOp::TTableDescription& descr) for (const auto& col : descr.GetColumns()) { TUserColumn& column = Columns[col.GetId()]; if (column.Name.empty()) { - auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(col.GetTypeId(), col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); - column = TUserColumn(typeInfo, col.GetName()); + column = TUserColumn(typeInfoMod.TypeInfo, typeInfoMod.TypeMod, col.GetName()); } column.Family = col.GetFamily(); column.NotNull = col.GetNotNull(); @@ -334,7 +334,7 @@ void TUserTable::AlterSchema() { auto descr = schema.AddColumns(); descr->SetName(column.Name); descr->SetId(col.first); - auto protoType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + auto protoType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.Type, column.TypeMod); descr->SetTypeId(protoType.TypeId); if (protoType.TypeInfo) { *descr->MutableTypeInfo() = *protoType.TypeInfo; @@ -397,9 +397,9 @@ void TUserTable::DoApplyCreate( ui32 columnId = col.first; const TUserColumn& column = col.second; - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.Type, column.TypeMod); ui32 pgTypeId = columnType.TypeInfo ? columnType.TypeInfo->GetPgTypeId() : 0; - alter.AddPgColumn(tid, column.Name, columnId, columnType.TypeId, pgTypeId, column.NotNull); + alter.AddPgColumn(tid, column.Name, columnId, columnType.TypeId, pgTypeId, column.TypeMod, column.NotNull); alter.AddColumnToFamily(tid, columnId, column.Family); } @@ -501,9 +501,9 @@ void TUserTable::ApplyAlter( if (!oldTable.Columns.contains(colId)) { for (ui32 tid : tids) { - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.Type); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.Type, column.TypeMod); ui32 pgTypeId = columnType.TypeInfo ? columnType.TypeInfo->GetPgTypeId() : 0; - alter.AddPgColumn(tid, column.Name, colId, columnType.TypeId, pgTypeId, column.NotNull); + alter.AddPgColumn(tid, column.Name, colId, columnType.TypeId, pgTypeId, column.TypeMod, column.NotNull); } } diff --git a/ydb/core/tx/datashard/datashard_user_table.h b/ydb/core/tx/datashard/datashard_user_table.h index 409e6773886..42e7dd5ae1d 100644 --- a/ydb/core/tx/datashard/datashard_user_table.h +++ b/ydb/core/tx/datashard/datashard_user_table.h @@ -233,13 +233,15 @@ struct TUserTable : public TThrRefBase { struct TUserColumn { NScheme::TTypeInfo Type; + TString TypeMod; TString Name; bool IsKey; ui32 Family = 0; bool NotNull = false; - TUserColumn(NScheme::TTypeInfo type, TString name, bool isKey = false) + TUserColumn(NScheme::TTypeInfo type, TString typeMod, TString name, bool isKey = false) : Type(type) + , TypeMod(typeMod) , Name(name) , IsKey(isKey) {} diff --git a/ydb/core/tx/datashard/datashard_ut_common.cpp b/ydb/core/tx/datashard/datashard_ut_common.cpp index 5ade7fe5bf3..9b921ca7739 100644 --- a/ydb/core/tx/datashard/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/datashard_ut_common.cpp @@ -199,9 +199,9 @@ void TTester::RegisterTableInResolver(const TString& schemeText) keyIdx = ki; } } - auto typeInfo = NScheme::TypeInfoFromProtoColumnType(c.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(c.GetTypeId(), c.HasTypeInfo() ? &c.GetTypeInfo() : nullptr); - table.Columns.insert(std::make_pair(c.GetName(), TColumn{c.GetId(), keyIdx, typeInfo, 0, EColumnTypeConstraint::Nullable})); + table.Columns.insert(std::make_pair(c.GetName(), TColumn{c.GetId(), keyIdx, typeInfoMod.TypeInfo, 0, EColumnTypeConstraint::Nullable})); } DbSchemeResolver.AddTable(table); } diff --git a/ydb/core/tx/datashard/import_common.h b/ydb/core/tx/datashard/import_common.h index 7ae51dfbfb2..241246a85e9 100644 --- a/ydb/core/tx/datashard/import_common.h +++ b/ydb/core/tx/datashard/import_common.h @@ -81,10 +81,11 @@ public: return ColumnNameIndex.contains(name); } - NScheme::TTypeInfo GetColumnType(const TString& name) const { + std::pair<NScheme::TTypeInfo, TString> GetColumnType(const TString& name) const { auto it = ColumnNameIndex.find(name); Y_VERIFY(it != ColumnNameIndex.end()); - return it->second->second.Type; + auto& column = it->second->second; + return {column.Type, column.TypeMod}; } const TVector<ui32>& GetKeyColumnIds() const { diff --git a/ydb/core/tx/datashard/import_s3.cpp b/ydb/core/tx/datashard/import_s3.cpp index cb91713f93a..14778c2cae4 100644 --- a/ydb/core/tx/datashard/import_s3.cpp +++ b/ydb/core/tx/datashard/import_s3.cpp @@ -527,9 +527,9 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader> { columnOrderTypes.reserve(Scheme.GetColumns().size()); for (const auto& column : Scheme.GetColumns()) { - auto typeInfo = NScheme::TypeInfoFromProtoColumnType(column.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(column.GetTypeId(), column.HasTypeInfo() ? &column.GetTypeInfo() : nullptr); - columnOrderTypes.emplace_back(TableInfo.KeyOrder(column.GetName()), typeInfo); + columnOrderTypes.emplace_back(TableInfo.KeyOrder(column.GetName()), typeInfoMod.TypeInfo); } TVector<TCell> keys; @@ -627,14 +627,14 @@ class TS3Downloader: public TActorBootstrapped<TS3Downloader> { << ": name# " << column.GetName()); } - const auto typeInfo = TableInfo.GetColumnType(column.GetName()); - auto columnTypeInfo = NScheme::TypeInfoFromProtoColumnType(column.GetTypeId(), + const auto type = TableInfo.GetColumnType(column.GetName()); + auto schemeType = NScheme::TypeInfoModFromProtoColumnType(column.GetTypeId(), column.HasTypeInfo() ? &column.GetTypeInfo() : nullptr); - if (typeInfo != columnTypeInfo) { + if (type.first != schemeType.TypeInfo || type.second != schemeType.TypeMod) { return finish(TStringBuilder() << "Scheme mismatch: column type mismatch" << ": name# " << column.GetName() - << ", expected# " << NScheme::TypeName(typeInfo) - << ", got# " << NScheme::TypeName(columnTypeInfo)); + << ", expected# " << NScheme::TypeName(type.first, type.second) + << ", got# " << NScheme::TypeName(schemeType.TypeInfo, schemeType.TypeMod)); } } diff --git a/ydb/core/tx/datashard/read_table_scan.cpp b/ydb/core/tx/datashard/read_table_scan.cpp index efd5640606a..9b12fe7ac74 100644 --- a/ydb/core/tx/datashard/read_table_scan.cpp +++ b/ydb/core/tx/datashard/read_table_scan.cpp @@ -123,9 +123,9 @@ public: , ResultStream(ResultString) { for (auto &col : request.GetColumns()) { - auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(col.GetTypeId(), col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); - ColTypes.push_back(typeInfo); + ColTypes.push_back(typeInfoMod.TypeInfo); } } @@ -257,12 +257,17 @@ private: auto *meta = res.add_columns(); meta->set_name(col.GetName()); - auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(col.GetTypeId(), col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); if (col.GetTypeId() == NScheme::NTypeIds::Pg) { - auto pgType = meta->mutable_type()->mutable_pg_type(); - pgType->set_oid(NPg::PgTypeIdFromTypeDesc(typeInfo.GetTypeDesc())); + auto* pg = meta->mutable_type()->mutable_pg_type(); + auto* typeDesc = typeInfoMod.TypeInfo.GetTypeDesc(); + pg->set_type_name(NPg::PgTypeNameFromTypeDesc(typeDesc)); + pg->set_type_modifier(typeInfoMod.TypeMod); + pg->set_oid(NPg::PgTypeIdFromTypeDesc(typeDesc)); + pg->set_typlen(0); + pg->set_typmod(0); } else { auto id = static_cast<NYql::NProto::TypeIds>(col.GetTypeId()); if (id == NYql::NProto::Decimal) { diff --git a/ydb/core/tx/datashard/sys_tables.h b/ydb/core/tx/datashard/sys_tables.h index ff02286561b..51abbc525d7 100644 --- a/ydb/core/tx/datashard/sys_tables.h +++ b/ydb/core/tx/datashard/sys_tables.h @@ -11,14 +11,16 @@ struct TSysTables { TString Name; ui32 Id = 0; NScheme::TTypeInfo PType; + TString PTypeMod; i32 KeyOrder = -1; TTableColumnInfo() = default; - TTableColumnInfo(TString name, ui32 colId, NScheme::TTypeInfo type, i32 keyOrder = -1) + TTableColumnInfo(TString name, ui32 colId, NScheme::TTypeInfo type, const TString& typeMod = {}, i32 keyOrder = -1) : Name(name) , Id(colId) , PType(type) + , PTypeMod(typeMod) , KeyOrder(keyOrder) {} }; @@ -118,8 +120,8 @@ struct TSysTables { auto type = NScheme::TTypeInfo(NScheme::TUint64::TypeId); auto typeUi32 = NScheme::TTypeInfo(NScheme::TUint32::TypeId); - columns[0] = TTableColumnInfo(GetColName(EColumns::LockId), (ui32)EColumns::LockId, type, 0); - columns[1] = TTableColumnInfo(GetColName(EColumns::DataShard), (ui32)EColumns::DataShard, type, 1); + columns[0] = TTableColumnInfo(GetColName(EColumns::LockId), (ui32)EColumns::LockId, type, "", 0); + columns[1] = TTableColumnInfo(GetColName(EColumns::DataShard), (ui32)EColumns::DataShard, type, "", 1); columns[2] = TTableColumnInfo(GetColName(EColumns::Generation), (ui32)EColumns::Generation, typeUi32); columns[3] = TTableColumnInfo(GetColName(EColumns::Counter), (ui32)EColumns::Counter, type); @@ -127,8 +129,8 @@ struct TSysTables { keyTypes.push_back(type); if (v2) { - columns[4] = TTableColumnInfo(GetColName(EColumns::SchemeShard), (ui32)EColumns::SchemeShard, type, 2); - columns[5] = TTableColumnInfo(GetColName(EColumns::PathId), (ui32)EColumns::PathId, type, 3); + columns[4] = TTableColumnInfo(GetColName(EColumns::SchemeShard), (ui32)EColumns::SchemeShard, type, "", 2); + columns[5] = TTableColumnInfo(GetColName(EColumns::PathId), (ui32)EColumns::PathId, type, "", 3); keyTypes.push_back(type); keyTypes.push_back(type); } diff --git a/ydb/core/tx/scheme_board/cache.cpp b/ydb/core/tx/scheme_board/cache.cpp index b0c7b7fe857..e0ab9b6470f 100644 --- a/ydb/core/tx/scheme_board/cache.cpp +++ b/ydb/core/tx/scheme_board/cache.cpp @@ -746,8 +746,11 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { auto& column = Columns[columnDesc.GetId()]; column.Id = columnDesc.GetId(); column.Name = columnDesc.GetName(); - column.PType = NScheme::TypeInfoFromProtoColumnType(columnDesc.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(columnDesc.GetTypeId(), columnDesc.HasTypeInfo() ? &columnDesc.GetTypeInfo() : nullptr); + column.PType = typeInfoMod.TypeInfo; + column.PTypeMod = typeInfoMod.TypeMod; + if (columnDesc.GetNotNull()) { NotNullColumns.insert(columnDesc.GetName()); } @@ -807,8 +810,10 @@ class TSchemeCache: public TMonitorableActor<TSchemeCache> { auto& column = Columns[columnDesc.GetId()]; column.Id = columnDesc.GetId(); column.Name = columnDesc.GetName(); - column.PType = NScheme::TypeInfoFromProtoColumnType(columnDesc.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(columnDesc.GetTypeId(), columnDesc.HasTypeInfo() ? &columnDesc.GetTypeInfo() : nullptr); + column.PType = typeInfoMod.TypeInfo; + column.PTypeMod = typeInfoMod.TypeMod; nameToId[column.Name] = column.Id; if (columnDesc.GetNotNull()) { NotNullColumns.insert(columnDesc.GetName()); diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 78524389b70..d2972a87cbd 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -388,7 +388,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { return true; } - typedef std::tuple<TPathId, ui32, TString, NScheme::TTypeInfo, ui32, ui64, ui64, ui32, ETableColumnDefaultKind, TString, bool> TColumnRec; + typedef std::tuple<TPathId, ui32, TString, NScheme::TTypeInfo, TString, ui32, ui64, ui64, ui32, ETableColumnDefaultKind, TString, bool> TColumnRec; typedef TDeque<TColumnRec> TColumnRows; bool LoadColumns(NIceDb::TNiceDb& db, TColumnRows& columnRows) const { @@ -413,17 +413,18 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto defaultValue = rowSet.GetValue<Schema::Columns::DefaultValue>(); auto notNull = rowSet.GetValueOrDefault<Schema::Columns::NotNull>(false); - NScheme::TTypeInfo typeInfo; + NScheme::TTypeInfoMod typeInfoMod; if (typeData) { NKikimrProto::TTypeInfo protoType; Y_VERIFY(ParseFromStringNoSizeLimit(protoType, typeData)); - typeInfo = NScheme::TypeInfoFromProtoColumnType(typeId, &protoType); + typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(typeId, &protoType); } else { - typeInfo = NScheme::TTypeInfo(typeId); + typeInfoMod.TypeInfo = NScheme::TTypeInfo(typeId); } columnRows.emplace_back(pathId, colId, - colName, typeInfo, keyOrder, createVersion, deleteVersion, + colName, typeInfoMod.TypeInfo, typeInfoMod.TypeMod, + keyOrder, createVersion, deleteVersion, family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { @@ -455,17 +456,18 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto defaultValue = rowSet.GetValue<Schema::MigratedColumns::DefaultValue>(); auto notNull = rowSet.GetValueOrDefault<Schema::MigratedColumns::NotNull>(false); - NScheme::TTypeInfo typeInfo; + NScheme::TTypeInfoMod typeInfoMod; if (typeData) { NKikimrProto::TTypeInfo protoType; Y_VERIFY(ParseFromStringNoSizeLimit(protoType, typeData)); - typeInfo = NScheme::TypeInfoFromProtoColumnType(typeId, &protoType); + typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(typeId, &protoType); } else { - typeInfo = NScheme::TTypeInfo(typeId); + typeInfoMod.TypeInfo = NScheme::TTypeInfo(typeId); } columnRows.emplace_back(pathId, colId, - colName, typeInfo, keyOrder, createVersion, deleteVersion, + colName, typeInfoMod.TypeInfo, typeInfoMod.TypeMod, + keyOrder, createVersion, deleteVersion, family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { @@ -499,17 +501,18 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto defaultValue = rowSet.GetValue<Schema::ColumnAlters::DefaultValue>(); auto notNull = rowSet.GetValueOrDefault<Schema::ColumnAlters::NotNull>(false); - NScheme::TTypeInfo typeInfo; + NScheme::TTypeInfoMod typeInfoMod; if (typeData) { NKikimrProto::TTypeInfo protoType; Y_VERIFY(ParseFromStringNoSizeLimit(protoType, typeData)); - typeInfo = NScheme::TypeInfoFromProtoColumnType(typeId, &protoType); + typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(typeId, &protoType); } else { - typeInfo = NScheme::TTypeInfo(typeId); + typeInfoMod.TypeInfo = NScheme::TTypeInfo(typeId); } columnRows.emplace_back(pathId, colId, - colName, typeInfo, keyOrder, createVersion, deleteVersion, + colName, typeInfoMod.TypeInfo, typeInfoMod.TypeMod, + keyOrder, createVersion, deleteVersion, family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { @@ -541,17 +544,18 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { auto defaultValue = rowSet.GetValue<Schema::MigratedColumnAlters::DefaultValue>(); auto notNull = rowSet.GetValueOrDefault<Schema::MigratedColumnAlters::NotNull>(false); - NScheme::TTypeInfo typeInfo; + NScheme::TTypeInfoMod typeInfoMod; if (typeData) { NKikimrProto::TTypeInfo protoType; Y_VERIFY(ParseFromStringNoSizeLimit(protoType, typeData)); - typeInfo = NScheme::TypeInfoFromProtoColumnType(typeId, &protoType); + typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(typeId, &protoType); } else { - typeInfo = NScheme::TTypeInfo(typeId); + typeInfoMod.TypeInfo = NScheme::TTypeInfo(typeId); } columnRows.emplace_back(pathId, colId, - colName, typeInfo, keyOrder, createVersion, deleteVersion, + colName, typeInfoMod.TypeInfo, typeInfoMod.TypeMod, + keyOrder, createVersion, deleteVersion, family, defaultKind, defaultValue, notNull); if (!rowSet.Next()) { @@ -1919,19 +1923,20 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 colId = std::get<1>(rec); TString colName = std::get<2>(rec); NScheme::TTypeInfo typeInfo = std::get<3>(rec); - ui32 keyOrder = std::get<4>(rec); - ui64 createVersion = std::get<5>(rec); - ui64 deleteVersion = std::get<6>(rec); - ui32 family = std::get<7>(rec); - auto defaultKind = std::get<8>(rec); - auto defaultValue = std::get<9>(rec); - auto notNull = std::get<10>(rec); + TString typeMod = std::get<4>(rec); + ui32 keyOrder = std::get<5>(rec); + ui64 createVersion = std::get<6>(rec); + ui64 deleteVersion = std::get<7>(rec); + ui32 family = std::get<8>(rec); + auto defaultKind = std::get<9>(rec); + auto defaultValue = std::get<10>(rec); + auto notNull = std::get<11>(rec); Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); Y_VERIFY_S(Self->PathsById.at(pathId)->IsTable() || Self->PathsById.at(pathId)->IsExternalTable(), "Path is not a table or external table, pathId: " << pathId); Y_VERIFY_S(Self->Tables.FindPtr(pathId) || Self->ExternalTables.FindPtr(pathId), "Table or external table don't exist, pathId: " << pathId); - TTableInfo::TColumn colInfo(colName, colId, typeInfo); + TTableInfo::TColumn colInfo(colName, colId, typeInfo, typeMod); colInfo.KeyOrder = keyOrder; colInfo.CreateVersion = createVersion; colInfo.DeleteVersion = deleteVersion; @@ -1976,13 +1981,14 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { ui32 colId = std::get<1>(rec); TString colName = std::get<2>(rec); NScheme::TTypeInfo typeInfo = std::get<3>(rec); - ui32 keyOrder = std::get<4>(rec); - ui64 createVersion = std::get<5>(rec); - ui64 deleteVersion = std::get<6>(rec); - ui32 family = std::get<7>(rec); - auto defaultKind = std::get<8>(rec); - auto defaultValue = std::get<9>(rec); - auto notNull = std::get<10>(rec); + TString typeMod = std::get<4>(rec); + ui32 keyOrder = std::get<5>(rec); + ui64 createVersion = std::get<6>(rec); + ui64 deleteVersion = std::get<7>(rec); + ui32 family = std::get<8>(rec); + auto defaultKind = std::get<9>(rec); + auto defaultValue = std::get<10>(rec); + auto notNull = std::get<11>(rec); Y_VERIFY_S(Self->PathsById.contains(pathId), "Path doesn't exist, pathId: " << pathId); Y_VERIFY_S(Self->PathsById.at(pathId)->IsTable(), "Path is not a table, pathId: " << pathId); @@ -1995,7 +2001,7 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { tableInfo->AlterData->NextColumnId = colId + 1; // calc next NextColumnId } - TTableInfo::TColumn colInfo(colName, colId, typeInfo); + TTableInfo::TColumn colInfo(colName, colId, typeInfo, typeMod); colInfo.KeyOrder = keyOrder; colInfo.CreateVersion = createVersion; colInfo.DeleteVersion = deleteVersion; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp index 2ceee89224f..1cf1c24aa1d 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_alter_olap_table.cpp @@ -96,9 +96,9 @@ TColumnTableInfo::TPtr ParseParams( for (const auto& col : tableSchema->GetColumns()) { ui32 id = col.GetId(); TString name = col.GetName(); - auto typeInfo = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(col.GetTypeId(), col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); - columns[id] = TOlapSchema::TColumn{id, name, typeInfo, Max<ui32>()}; + columns[id] = TOlapSchema::TColumn{id, name, typeInfoMod.TypeInfo, Max<ui32>()}; columnsByName[name] = id; // TODO: add checks for compatibility with new schema after we allow such changes diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp index de9d29af56c..95c1c2f32ec 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_cdc_stream.cpp @@ -740,7 +740,7 @@ TVector<ISubOperationBase::TPtr> CreateNewCdcStream(TOperationId opId, const TTx auto& keyComponent = *pqConfig.AddPartitionKeySchema(); keyComponent.SetName(column.Name); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.PType, column.PTypeMod); keyComponent.SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *keyComponent.MutableTypeInfo() = *columnType.TypeInfo; diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp index 7621265af9a..1da1e4eacf0 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_external_table.cpp @@ -93,7 +93,7 @@ TExternalTableInfo::TPtr CreateExternalTable(const NKikimrSchemeOp::TExternalTab nextColumnId = colId + 1 > nextColumnId ? colId + 1 : nextColumnId; TTableInfo::TColumn& column = externalTableInfo->Columns[colId]; - column = TTableInfo::TColumn(colName, colId, typeInfo); + column = TTableInfo::TColumn(colName, colId, typeInfo, ""); // TODO: do we need typeMod here? column.NotNull = col.GetNotNull(); } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp index c2317e4661e..290b39106e7 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_table.cpp @@ -112,7 +112,7 @@ bool DoInitPartitioning(TTableInfo::TPtr tableInfo, if (!IsAllowedKeyType(type)) { errStr = Sprintf("Column %s has wrong key type %s", - tableInfo->Columns[ki].Name.c_str(), NScheme::TypeName(type)); + tableInfo->Columns[ki].Name.c_str(), NScheme::TypeName(type).c_str()); return false; } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp index 9f67b76510c..48ade77c5aa 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_upgrade_subdomain.cpp @@ -220,7 +220,7 @@ public: auto colDescr = descr.AddColumns(); colDescr->SetId(columnId); colDescr->SetName(column.Name); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.PType, column.PTypeMod); colDescr->SetColType(columnType.TypeId); if (columnType.TypeInfo) { *colDescr->MutableColTypeInfo() = *columnType.TypeInfo; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 5f807a5cfff..aebd5be3062 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -2376,7 +2376,7 @@ void TSchemeShard::PersistTableAltered(NIceDb::TNiceDb& db, const TPathId pathId ui32 colId = col.first; const TTableInfo::TColumn& cinfo = col.second; TString typeData; - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod); if (columnType.TypeInfo) { Y_VERIFY(columnType.TypeInfo->SerializeToString(&typeData)); } @@ -2434,7 +2434,7 @@ void TSchemeShard::PersistAddAlterTable(NIceDb::TNiceDb& db, TPathId pathId, con ui32 colId = col.first; const TTableInfo::TColumn& cinfo = col.second; TString typeData; - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod); if (columnType.TypeInfo) { Y_VERIFY(columnType.TypeInfo->SerializeToString(&typeData)); } @@ -2575,7 +2575,7 @@ void TSchemeShard::PersistExternalTable(NIceDb::TNiceDb &db, TPathId pathId, con ui32 colId = col.first; const TTableInfo::TColumn& cinfo = col.second; TString typeData; - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod); if (columnType.TypeInfo) { Y_VERIFY(columnType.TypeInfo->SerializeToString(&typeData)); } @@ -5944,7 +5944,7 @@ TString TSchemeShard::FillAlterTableTxBody(TPathId pathId, TShardIdx shardIdx, T auto descr = proto->AddDropColumns(); descr->SetName(colInfo.Name); descr->SetId(colInfo.Id); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(colInfo.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(colInfo.PType, colInfo.PTypeMod); descr->SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *descr->MutableTypeInfo() = *columnType.TypeInfo; @@ -5953,7 +5953,7 @@ TString TSchemeShard::FillAlterTableTxBody(TPathId pathId, TShardIdx shardIdx, T auto descr = proto->AddColumns(); descr->SetName(colInfo.Name); descr->SetId(colInfo.Id); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(colInfo.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(colInfo.PType, colInfo.PTypeMod); descr->SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *descr->MutableTypeInfo() = *columnType.TypeInfo; @@ -6137,6 +6137,7 @@ void TSchemeShard::FillTableDescription(TPathId tableId, ui32 partitionIdx, ui64 } bool TSchemeShard::FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 keySize, NScheme::TTypeInfo firstKeyColType, ui32 partitionCount, const NScheme::TTypeRegistry* typeRegistry, TString& errStr) { + Y_UNUSED(typeRegistry); if (partitionCount > 1) { // RangeEnd key will have first cell with non-NULL value and rest of the cells with NULLs TVector<TCell> rangeEnd(keySize); @@ -6155,7 +6156,7 @@ bool TSchemeShard::FillUniformPartitioning(TVector<TString>& rangeEnds, ui32 key valSz = 8; break; default: - errStr = TStringBuilder() << "Unsupported first key column type " << typeRegistry->GetTypeName(typeId) << ", only Uint32 and Uint64 are supported"; + errStr = TStringBuilder() << "Unsupported first key column type " << NScheme::TypeName(firstKeyColType) << ", only Uint32 and Uint64 are supported"; return false; } diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index 5366f47b72e..e5c75e9e907 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -123,6 +123,7 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( } NScheme::TTypeInfo typeInfo; + TString typeMod; if (type) { // Only allow YQL types if (!NScheme::NTypeIds::IsYqlType(type->GetTypeId())) { @@ -137,6 +138,7 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( return nullptr; } typeInfo = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, typeDesc); + typeMod = NPg::TypeModFromPgTypeName(typeName); } ui32 colId = col.HasId() ? col.GetId() : alterData->NextColumnId; @@ -154,7 +156,7 @@ TTableInfo::TAlterDataPtr TTableInfo::CreateAlterData( colName2Id[colName] = colId; TTableInfo::TColumn& column = alterData->Columns[colId]; - column = TTableInfo::TColumn(colName, colId, typeInfo); + column = TTableInfo::TColumn(colName, colId, typeInfo, typeMod); column.Family = columnFamily ? columnFamily->GetId() : 0; column.NotNull = col.GetNotNull(); if (source) @@ -319,7 +321,7 @@ TVector<ui32> TTableInfo::FillDescriptionCache(TPathElement::TPtr pathInfo) { auto colDescr = TableDescription.AddColumns(); colDescr->SetName(column.Name); colDescr->SetId(column.Id); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.PType, column.PTypeMod); colDescr->SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *colDescr->MutableTypeInfo() = *columnType.TypeInfo; @@ -2045,7 +2047,7 @@ bool TOlapSchema::UpdateProto(NKikimrSchemeOp::TColumnTableSchema& proto, TStrin #endif } - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(typeInfo); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(typeInfo, ""); colProto.SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *colProto.MutableTypeInfo() = *columnType.TypeInfo; @@ -2155,7 +2157,8 @@ bool TOlapSchema::Parse(const NKikimrSchemeOp::TColumnTableSchema& proto, TStrin } if (colProto.HasTypeInfo()) { - col.Type = NScheme::TypeInfoFromProtoColumnType(colProto.GetTypeId(), &colProto.GetTypeInfo()); + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(colProto.GetTypeId(), &colProto.GetTypeInfo()); + col.Type = typeInfoMod.TypeInfo; } else { col.Type = NScheme::TTypeInfo(colProto.GetTypeId()); } @@ -2327,8 +2330,9 @@ TOlapStoreInfo::TOlapStoreInfo( auto& col = preset.Columns[colProto.GetId()]; col.Id = colProto.GetId(); col.Name = colProto.GetName(); - col.Type = NScheme::TypeInfoFromProtoColumnType(colProto.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(colProto.GetTypeId(), colProto.HasTypeInfo() ? &colProto.GetTypeInfo() : nullptr); + col.Type = typeInfoMod.TypeInfo; preset.ColumnsByName[col.Name] = col.Id; } for (const auto& keyName : presetProto.GetSchema().GetKeyColumnNames()) { diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index b156aee3658..9dbf42fa513 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -315,8 +315,8 @@ struct TTableInfo : public TSimpleRefCount<TTableInfo> { TString DefaultValue; bool NotNull = false; - TColumn(const TString& name, ui32 id, NScheme::TTypeInfo type) - : NTable::TScheme::TColumn(name, id, type) + TColumn(const TString& name, ui32 id, NScheme::TTypeInfo type, const TString& typeMod) + : NTable::TScheme::TColumn(name, id, type, typeMod) , CreateVersion(0) , DeleteVersion(Max<ui64>()) {} diff --git a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp index bb04164791b..f98fab70c6b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_path_describer.cpp @@ -779,7 +779,8 @@ void TPathDescriber::DescribeBlobDepot(const TPath& path) { } void TPathDescriber::DescribeExternalTable(const TActorContext& ctx, TPathId pathId, TPathElement::TPtr pathEl) { - const NScheme::TTypeRegistry* typeRegistry = AppData(ctx)->TypeRegistry; + Y_UNUSED(ctx); + auto it = Self->ExternalTables.FindPtr(pathId); Y_VERIFY(it, "ExternalTable is not found"); TExternalTableInfo::TPtr externalTableInfo = *it; @@ -800,8 +801,8 @@ void TPathDescriber::DescribeExternalTable(const TActorContext& ctx, TPathId pat auto colDescr = entry->AddColumns(); colDescr->SetName(cinfo.Name); - colDescr->SetType(typeRegistry->GetTypeName(cinfo.PType.GetTypeId())); // TODO: no pg type details in string type - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType); + colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod)); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod); colDescr->SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *colDescr->MutableTypeInfo() = *columnType.TypeInfo; @@ -982,6 +983,7 @@ THolder<TEvSchemeShard::TEvDescribeSchemeResultBuilder> DescribePath( void TSchemeShard::DescribeTable(const TTableInfo::TPtr tableInfo, const NScheme::TTypeRegistry* typeRegistry, bool fillConfig, bool fillBoundaries, NKikimrSchemeOp::TTableDescription* entry) const { + Y_UNUSED(typeRegistry); THashMap<ui32, TString> familyNames; bool familyNamesBuilt = false; @@ -994,8 +996,8 @@ void TSchemeShard::DescribeTable(const TTableInfo::TPtr tableInfo, const NScheme auto colDescr = entry->AddColumns(); colDescr->SetName(cinfo.Name); - colDescr->SetType(typeRegistry->GetTypeName(cinfo.PType.GetTypeId())); // TODO: no pg type details in string type - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(cinfo.PType); + colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod)); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod); colDescr->SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *colDescr->MutableTypeInfo() = *columnType.TypeInfo; diff --git a/ydb/core/tx/schemeshard/schemeshard_utils.cpp b/ydb/core/tx/schemeshard/schemeshard_utils.cpp index 23241b143a4..84c6fd44d86 100644 --- a/ydb/core/tx/schemeshard/schemeshard_utils.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_utils.cpp @@ -295,8 +295,6 @@ NKikimrSchemeOp::TTableDescription CalcImplTableDesc( implKeyToImplColumn[implTableColumns.Keys[keyId]] = keyId; } - const TAppData* appData = AppData(); - result.ClearColumns(); for (auto& iter: baseTableInfo->Columns) { const NSchemeShard::TTableInfo::TColumn& column = iter.second; @@ -307,11 +305,7 @@ NKikimrSchemeOp::TTableDescription CalcImplTableDesc( if (implTableColumns.Columns.contains(column.Name)) { auto item = result.AddColumns(); item->SetName(column.Name); - - // TODO: support pg types - Y_VERIFY(column.PType.GetTypeId() != NScheme::NTypeIds::Pg); - item->SetType(appData->TypeRegistry->GetTypeName(column.PType.GetTypeId())); - + item->SetType(NScheme::TypeName(column.PType, column.PTypeMod)); ui32 order = Max<ui32>(); if (implKeyToImplColumn.contains(column.Name)) { order = implKeyToImplColumn.at(column.Name); diff --git a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp index a47cebcf4a9..f2918d23027 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp @@ -434,8 +434,9 @@ void CheckBoundaries(const NKikimrScheme::TEvDescribeSchemeResult &record) { const NKikimrSchemeOp::TPathDescription& descr = record.GetPathDescription(); THashMap<ui32, NScheme::TTypeInfo> colTypes; for (const auto& col : descr.GetTable().GetColumns()) { - colTypes[col.GetId()] = NScheme::TypeInfoFromProtoColumnType(col.GetTypeId(), + auto typeInfoMod = NScheme::TypeInfoModFromProtoColumnType(col.GetTypeId(), col.HasTypeInfo() ? &col.GetTypeInfo() : nullptr); + colTypes[col.GetId()] = typeInfoMod.TypeInfo; } TVector<NScheme::TTypeInfo> keyColTypes; for (const auto& ki : descr.GetTable().GetKeyColumnIds()) { diff --git a/ydb/core/tx/tx_proxy/datareq.cpp b/ydb/core/tx/tx_proxy/datareq.cpp index 41fe51828ed..90444545da3 100644 --- a/ydb/core/tx/tx_proxy/datareq.cpp +++ b/ydb/core/tx/tx_proxy/datareq.cpp @@ -1162,7 +1162,7 @@ void TDataReq::ProcessReadTableResolve(NSchemeCache::TSchemeCacheRequest *cacheR auto &c = *tx.AddColumns(); c.SetId(col.Id); c.SetName(col.Name); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(col.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(col.PType, col.PTypeMod); c.SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *c.MutableTypeInfo() = *columnType.TypeInfo; diff --git a/ydb/core/tx/tx_proxy/describe.cpp b/ydb/core/tx/tx_proxy/describe.cpp index ba4b6305065..4d179a53436 100644 --- a/ydb/core/tx/tx_proxy/describe.cpp +++ b/ydb/core/tx/tx_proxy/describe.cpp @@ -103,9 +103,8 @@ class TDescribeReq : public TActor<TDescribeReq> { for (const auto& [id, column] : entry.Columns) { auto* col = table->AddColumns(); col->SetName(column.Name); - // TODO: support pg types (name) - col->SetType(AppData(ctx)->TypeRegistry->GetTypeName(column.PType.GetTypeId())); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(column.PType); + col->SetType(NScheme::TypeName(column.PType, column.PTypeMod)); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(column.PType, column.PTypeMod); col->SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *col->MutableTypeInfo() = *columnType.TypeInfo; diff --git a/ydb/core/tx/tx_proxy/read_table_impl.cpp b/ydb/core/tx/tx_proxy/read_table_impl.cpp index b19a85de34b..f05e344c7bb 100644 --- a/ydb/core/tx/tx_proxy/read_table_impl.cpp +++ b/ydb/core/tx/tx_proxy/read_table_impl.cpp @@ -1433,7 +1433,7 @@ private: auto &c = *tx.AddColumns(); c.SetId(col.Id); c.SetName(col.Name); - auto columnType = NScheme::ProtoColumnTypeFromTypeInfo(col.PType); + auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(col.PType, col.PTypeMod); c.SetTypeId(columnType.TypeId); if (columnType.TypeInfo) { *c.MutableTypeInfo() = *columnType.TypeInfo; @@ -1731,8 +1731,10 @@ private: meta->set_name(col.Name); if (col.PType.GetTypeId() == NScheme::NTypeIds::Pg) { - auto pgType = meta->mutable_type()->mutable_pg_type(); - pgType->set_oid(NPg::PgTypeIdFromTypeDesc(col.PType.GetTypeDesc())); + auto* typeDesc = col.PType.GetTypeDesc(); + auto* pg = meta->mutable_type()->mutable_pg_type(); + pg->set_type_name(NPg::PgTypeNameFromTypeDesc(typeDesc)); + pg->set_oid(NPg::PgTypeIdFromTypeDesc(typeDesc)); } else { auto id = static_cast<NYql::NProto::TypeIds>(col.PType.GetTypeId()); if (id == NYql::NProto::Decimal) { diff --git a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h index 19260691b0e..a9acdd64e2e 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h +++ b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h @@ -137,6 +137,7 @@ protected: TString ColName; ui32 PositionInStruct; NScheme::TTypeInfo Type; + i32 Typmod; bool NotNull = false; }; TVector<TString> KeyColumnNames; @@ -335,13 +336,15 @@ private: if (!errorMessage.empty()) { return false; } else if (reqColumns.empty()) { - for (auto& [name, type] : SrcColumns) { + for (auto& [name, typeInfo] : SrcColumns) { Ydb::Type ydbType; - if (type.GetTypeId() == NScheme::NTypeIds::Pg) { - Ydb::PgType* pgType = ydbType.mutable_pg_type(); - pgType->set_oid(NPg::PgTypeIdFromTypeDesc(type.GetTypeDesc())); + if (typeInfo.GetTypeId() != NScheme::NTypeIds::Pg) { + ydbType.set_type_id((Ydb::Type::PrimitiveTypeId)typeInfo.GetTypeId()); } else { - ydbType.set_type_id((Ydb::Type::PrimitiveTypeId)type.GetTypeId()); + auto* typeDesc = typeInfo.GetTypeDesc(); + auto* pg = ydbType.mutable_pg_type(); + pg->set_type_name(NPg::PgTypeNameFromTypeDesc(typeDesc)); + pg->set_oid(NPg::PgTypeIdFromTypeDesc(typeDesc)); } reqColumns.emplace_back(name, std::move(ydbType)); } @@ -354,6 +357,7 @@ private: errorMessage = Sprintf("Unknown column: %s", name.c_str()); return false; } + i32 typmod = -1; ui32 colId = *cp; auto& ci = *entry.Columns.FindPtr(colId); @@ -364,8 +368,8 @@ private: bool ok = SameDstType(typeInRequest, ci.PType, GetSourceType() != EUploadSource::ProtoValues); if (!ok) { errorMessage = Sprintf("Type mismatch for column %s: expected %s, got %s", - name.c_str(), NScheme::TypeName(ci.PType), - NScheme::TypeName(typeInRequest)); + name.c_str(), NScheme::TypeName(ci.PType).c_str(), + NScheme::TypeName(typeInRequest).c_str()); return false; } } else if (typeInProto.has_decimal_type() && ci.PType.GetTypeId() == NScheme::NTypeIds::Decimal) { @@ -380,24 +384,34 @@ private: return false; } } else if (typeInProto.has_pg_type()) { - auto pgTypeId = typeInProto.pg_type().oid(); - auto* typeDesc = NPg::TypeDescFromPgTypeId(pgTypeId); + const auto& typeName = typeInProto.pg_type().type_name(); + auto* typeDesc = NPg::TypeDescFromPgTypeName(typeName); if (!typeDesc) { - errorMessage = Sprintf("Unknown pg type for column %s: %u", - name.c_str(), pgTypeId); + errorMessage = Sprintf("Unknown pg type for column %s: %s", + name.c_str(), typeName.c_str()); return false; } auto typeInRequest = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, typeDesc); bool ok = SameDstType(typeInRequest, ci.PType, false); if (!ok) { errorMessage = Sprintf("Type mismatch for column %s: expected %s, got %s", - name.c_str(), NScheme::TypeName(ci.PType), - NScheme::TypeName(typeInRequest)); + name.c_str(), NScheme::TypeName(ci.PType).c_str(), + NScheme::TypeName(typeInRequest).c_str()); return false; } + if (!ci.PTypeMod.empty() && NPg::TypeDescNeedsCoercion(typeDesc)) { + auto result = NPg::BinaryTypeModFromTextTypeMod(ci.PTypeMod, typeDesc); + if (result.Error) { + errorMessage = Sprintf("Invalid typemod for column %s: type %s, error %s", + name.c_str(), NScheme::TypeName(ci.PType, ci.PTypeMod).c_str(), + result.Error->c_str()); + return false; + } + typmod = result.Typmod; + } } else { errorMessage = Sprintf("Unexpected type for column %s: expected %s", - name.c_str(), NScheme::TypeName(ci.PType)); + name.c_str(), NScheme::TypeName(ci.PType).c_str()); return false; } @@ -407,11 +421,11 @@ private: } if (ci.KeyOrder != -1) { - KeyColumnPositions[ci.KeyOrder] = TFieldDescription{ci.Id, ci.Name, (ui32)pos, ci.PType, notNull}; + KeyColumnPositions[ci.KeyOrder] = TFieldDescription{ci.Id, ci.Name, (ui32)pos, ci.PType, typmod, notNull}; keyColumnsLeft.erase(ci.Name); KeyColumnNames[ci.KeyOrder] = ci.Name; } else { - ValueColumnPositions.emplace_back(TFieldDescription{ci.Id, ci.Name, (ui32)pos, ci.PType, notNull}); + ValueColumnPositions.emplace_back(TFieldDescription{ci.Id, ci.Name, (ui32)pos, ci.PType, typmod, notNull}); ValueColumnNames.emplace_back(ci.Name); ValueColumnTypes.emplace_back(ci.PType); } diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp index 435ce40153f..62a4044ec3f 100644 --- a/ydb/core/ydb_convert/table_description.cpp +++ b/ydb/core/ydb_convert/table_description.cpp @@ -35,29 +35,40 @@ static void FillStoragePool(TStoragePoolHolder* out, TAddStoragePoolFunc<TStorag template <typename TColumn> static Ydb::Type* AddColumn(Ydb::Table::ColumnMeta* newColumn, const TColumn& column) { - NYql::NProto::TypeIds protoType; - if (!NYql::NProto::TypeIds_Parse(column.GetType(), &protoType)) { - throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) - << "Got invalid type: " << column.GetType() << " for column: " << column.GetName(); - } - newColumn->set_name(column.GetName()); Ydb::Type* columnType = nullptr; - if (column.GetNotNull() || protoType == NScheme::NTypeIds::Pg) { + auto* typeDesc = NPg::TypeDescFromPgTypeName(column.GetType()); + if (typeDesc) { columnType = newColumn->mutable_type(); + auto* pg = columnType->mutable_pg_type(); + pg->set_type_name(NPg::PgTypeNameFromTypeDesc(typeDesc)); + pg->set_type_modifier(NPg::TypeModFromPgTypeName(column.GetType())); + pg->set_oid(NPg::PgTypeIdFromTypeDesc(typeDesc)); + pg->set_typlen(0); + pg->set_typmod(0); } else { - columnType = newColumn->mutable_type()->mutable_optional_type()->mutable_item(); - } + NYql::NProto::TypeIds protoType; + if (!NYql::NProto::TypeIds_Parse(column.GetType(), &protoType)) { + throw NYql::TErrorException(NKikimrIssues::TIssuesIds::DEFAULT_ERROR) + << "Got invalid type: " << column.GetType() << " for column: " << column.GetName(); + } - Y_ENSURE(columnType); - if (protoType == NYql::NProto::TypeIds::Decimal) { - auto typeParams = columnType->mutable_decimal_type(); - // TODO: Change TEvDescribeSchemeResult to return decimal params - typeParams->set_precision(22); - typeParams->set_scale(9); - } else { - NMiniKQL::ExportPrimitiveTypeToProto(protoType, *columnType); + if (column.GetNotNull()) { + columnType = newColumn->mutable_type(); + } else { + columnType = newColumn->mutable_type()->mutable_optional_type()->mutable_item(); + } + + Y_ENSURE(columnType); + if (protoType == NYql::NProto::TypeIds::Decimal) { + auto typeParams = columnType->mutable_decimal_type(); + // TODO: Change TEvDescribeSchemeResult to return decimal params + typeParams->set_precision(22); + typeParams->set_scale(9); + } else { + NMiniKQL::ExportPrimitiveTypeToProto(protoType, *columnType); + } } return columnType; } @@ -164,7 +175,9 @@ void FillColumnDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSc } } -bool ExtractColumnTypeInfo(NScheme::TTypeInfo& outTypeInfo, const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error) { +bool ExtractColumnTypeInfo(NScheme::TTypeInfo& outTypeInfo, TString& outTypeMod, + const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error) +{ ui32 typeId = 0; auto itemType = inType.has_optional_type() ? inType.optional_type().item() : inType; switch (itemType.type_case()) { @@ -192,14 +205,16 @@ bool ExtractColumnTypeInfo(NScheme::TTypeInfo& outTypeInfo, const Ydb::Type& inT break; } case Ydb::Type::kPgType: { - ui32 pgTypeId = itemType.pg_type().oid(); - auto* desc = NPg::TypeDescFromPgTypeId(pgTypeId); + const auto& pgType = itemType.pg_type(); + const auto& typeName = pgType.type_name(); + auto* desc = NPg::TypeDescFromPgTypeName(typeName); if (!desc) { status = Ydb::StatusIds::BAD_REQUEST; - error = TStringBuilder() << "Invalid PG typeId: " << pgTypeId; + error = TStringBuilder() << "Invalid PG type name: " << typeName; return false; } outTypeInfo = NScheme::TTypeInfo(NScheme::NTypeIds::Pg, desc); + outTypeMod = pgType.type_modifier(); return true; } @@ -239,10 +254,11 @@ bool FillColumnDescription(NKikimrSchemeOp::TTableDescription& out, } NScheme::TTypeInfo typeInfo; - if (!ExtractColumnTypeInfo(typeInfo, column.type(), status, error)) { + TString typeMod; + if (!ExtractColumnTypeInfo(typeInfo, typeMod, column.type(), status, error)) { return false; } - cd->SetType(NScheme::TypeName(typeInfo)); + cd->SetType(NScheme::TypeName(typeInfo, typeMod)); if (!column.family().empty()) { cd->SetFamilyName(column.family()); diff --git a/ydb/core/ydb_convert/table_description.h b/ydb/core/ydb_convert/table_description.h index 86d304f62b0..7a7be50aa57 100644 --- a/ydb/core/ydb_convert/table_description.h +++ b/ydb/core/ydb_convert/table_description.h @@ -18,7 +18,8 @@ void FillColumnDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSc // in bool FillColumnDescription(NKikimrSchemeOp::TTableDescription& out, const google::protobuf::RepeatedPtrField<Ydb::Table::ColumnMeta>& in, Ydb::StatusIds::StatusCode& status, TString& error); -bool ExtractColumnTypeInfo(NScheme::TTypeInfo& outTypeInfo, const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error); +bool ExtractColumnTypeInfo(NScheme::TTypeInfo& outTypeInfo, TString& outTypeMod, + const Ydb::Type& inType, Ydb::StatusIds::StatusCode& status, TString& error); // out void FillTableBoundary(Ydb::Table::DescribeTableResult& out, diff --git a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp index ea6cf3a7d89..8a820b0ff20 100644 --- a/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp +++ b/ydb/core/yq/libs/result_formatter/result_formatter_ut.cpp @@ -526,6 +526,7 @@ Y_UNIT_TEST_SUITE(ResultFormatter) { auto& column = *rs.add_columns(); column.set_name("column0"); auto& pg_type = *column.mutable_type()->mutable_pg_type(); + pg_type.set_type_name("pgbool"); pg_type.set_oid(16); pg_type.set_typlen(234); pg_type.set_typmod(-987); diff --git a/ydb/library/mkql_proto/mkql_proto.cpp b/ydb/library/mkql_proto/mkql_proto.cpp index c1b5abea1d4..38eeba2ba82 100644 --- a/ydb/library/mkql_proto/mkql_proto.cpp +++ b/ydb/library/mkql_proto/mkql_proto.cpp @@ -7,6 +7,7 @@ #include <ydb/library/yql/minikql/mkql_type_ops.h> #include <ydb/library/yql/parser/pg_catalog/catalog.h> #include <ydb/library/yql/parser/pg_wrapper/interface/codec.h> +#include <ydb/library/yql/parser/pg_wrapper/interface/type_desc.h> #include <ydb/library/yql/public/decimal/yql_decimal.h> #include <ydb/library/yql/minikql/dom/json.h> #include <ydb/library/yql/utils/utf8.h> @@ -278,12 +279,17 @@ void ExportTypeToProtoImpl(TType* type, Ydb::Type& res, const TVector<ui32>* col } case TType::EKind::Pg: { - auto pgType = static_cast<TPgType*>(type); - auto t = res.mutable_pg_type(); - t->set_oid(pgType->GetTypeId()); - t->set_typmod(-1); + auto* pgType = static_cast<TPgType*>(type); + auto typeId = pgType->GetTypeId(); + auto* typeDesc = NKikimr::NPg::TypeDescFromPgTypeId(typeId); + MKQL_ENSURE(typeDesc, TStringBuilder() << "Unknown PG type id: " << typeId); + + auto* pg = res.mutable_pg_type(); + pg->set_type_name(NKikimr::NPg::PgTypeNameFromTypeDesc(typeDesc)); + pg->set_oid(typeId); + pg->set_typmod(-1); const i32 typlen = NYql::NPg::LookupType(pgType->GetTypeId()).TypeLen; - t->set_typlen(typlen); + pg->set_typlen(typlen); break; } @@ -1323,8 +1329,12 @@ TType* TProtoImporter::ImportTypeFromProto(const Ydb::Type& input) { return env.GetTypeOfEmptyList(); case Ydb::Type::kEmptyDictType: return env.GetTypeOfEmptyDict(); - case Ydb::Type::kPgType: - return TPgType::Create(input.pg_type().oid(), env); + case Ydb::Type::kPgType: { + const auto& typeName = input.pg_type().type_name(); + auto* typeDesc = NKikimr::NPg::TypeDescFromPgTypeName(typeName); + MKQL_ENSURE(typeDesc, TStringBuilder() << "Unknown PG type name: " << typeName); + return TPgType::Create(NKikimr::NPg::PgTypeIdFromTypeDesc(typeDesc), env); + } case Ydb::Type::kTypeId: { MKQL_ENSURE(NUdf::FindDataSlot(input.type_id()), TStringBuilder() << "unknown type id: " << ui32(input.type_id())); return TDataType::Create(input.type_id(), env); diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index 7e0a7553517..6946d9748be 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -15,6 +15,7 @@ #include <ydb/library/yql/public/udf/udf_value_builder.h> #include <ydb/library/yql/utils/fp_bits.h> #include <library/cpp/yson/detail.h> +#include <util/string/split.h> #define TypeName PG_TypeName #define SortBy PG_SortBy @@ -2931,16 +2932,18 @@ void get_type_io_data(Oid typid, namespace NKikimr::NPg { +constexpr char INTERNAL_TYPE_AND_MOD_SEPARATOR = ':'; + class TPgTypeDescriptor : public NYql::NPg::TTypeDesc { public: explicit TPgTypeDescriptor(const NYql::NPg::TTypeDesc& desc) : NYql::NPg::TTypeDesc(desc) - , YdbTypeName(desc.Name + ".pg") // to distinguish from native ydb types (e.g. "int8") { if (TypeId == ArrayTypeId) { const auto& typeDesc = NYql::NPg::LookupType(ElementTypeId); + YdbTypeName = TString("_pg") + typeDesc.Name; if (typeDesc.CompareProcId) { CompareProcId = NYql::NPg::LookupProc("btarraycmp", { 0, 0 }).ProcId; } @@ -2960,10 +2963,14 @@ public: OutFuncId = NYql::NPg::LookupProc("array_out", { 0 }).ProcId; } } else { + YdbTypeName = TString("pg") + desc.Name; StoredSize = TypeLen < 0 ? 0 : TypeLen; if (TypeId == NAMEOID) { StoredSize = 0; // store 'name' as usual string } + if (NYql::NPg::HasCast(TypeId, TypeId) && TypeModInFuncId != 0) { + NeedsCoercion = true; + } } } @@ -3085,7 +3092,7 @@ public: serialized = (text*)finfo.fn_addr(callInfo); Y_ENSURE(!callInfo->isnull); - return {TString(NMiniKQL::GetVarBuf(serialized)), ""}; + return {TString(NMiniKQL::GetVarBuf(serialized)), {}}; } PG_CATCH(); { @@ -3127,7 +3134,7 @@ public: str = (char*)finfo.fn_addr(callInfo); Y_ENSURE(!callInfo->isnull); - return {TString(str), ""}; + return {TString(str), {}}; } PG_CATCH(); { @@ -3141,6 +3148,198 @@ public: PG_END_TRY(); } + TTypeModResult ReadTypeMod(const TString& str) const { + TVector<TString> params; + ::Split(str, ",", params); + + if (params.size() > 2) { + TStringBuilder errMsg; + errMsg << "Error in 'typemodin' function: " + << NYql::NPg::LookupProc(TypeModInFuncId).Name + << ", reason: too many parameters"; + return {-1, errMsg}; + } + + TVector<Datum> dvalues; + TVector<bool> dnulls; + dnulls.resize(params.size(), false); + dvalues.reserve(params.size()); + + TString textNumberParam; + if (TypeId == INTERVALOID) { + i32 typmod = -1; + auto ok = NYql::ParsePgIntervalModifier(params[0], typmod); + if (!ok) { + TStringBuilder errMsg; + errMsg << "Error in 'typemodin' function: " + << NYql::NPg::LookupProc(TypeModInFuncId).Name + << ", reason: invalid parameter '" << params[0] + << "' for type pginterval"; + return {-1, errMsg}; + } + textNumberParam = Sprintf("%d", typmod); + dvalues.push_back(PointerGetDatum(textNumberParam.data())); + if (params.size() > 1) { + dvalues.push_back(PointerGetDatum(params[1].data())); + } + } else { + for (size_t i = 0; i < params.size(); ++i) { + dvalues.push_back(PointerGetDatum(params[i].data())); + } + } + + NMiniKQL::TScopedAlloc alloc(__LOCATION__); + NMiniKQL::TPAllocScope scope; + ArrayType* paramsArray = nullptr; + Y_DEFER { + if (paramsArray) { + pfree(paramsArray); + } + }; + PG_TRY(); + { + int ndims = 0; + int dims[MAXDIM]; + int lbs[MAXDIM]; + + ndims = 1; + dims[0] = params.size(); + lbs[0] = 1; + + const auto& cstringDesc = NYql::NPg::LookupType(CSTRINGOID); + paramsArray = construct_md_array(dvalues.data(), dnulls.data(), ndims, dims, lbs, + cstringDesc.TypeId, + cstringDesc.TypeLen, + cstringDesc.PassByValue, + cstringDesc.TypeAlign); + + FmgrInfo finfo; + InitFunc(TypeModInFuncId, &finfo, 1, 1); + LOCAL_FCINFO(callInfo, 1); + Zero(*callInfo); + callInfo->flinfo = &finfo; + callInfo->nargs = 1; + callInfo->fncollation = DEFAULT_COLLATION_OID; + callInfo->isnull = false; + callInfo->args[0] = { PointerGetDatum(paramsArray), false }; + + auto result = finfo.fn_addr(callInfo); + Y_ENSURE(!callInfo->isnull); + return {DatumGetInt32(result), {}}; + } + PG_CATCH(); + { + auto error_data = CopyErrorData(); + TStringBuilder errMsg; + errMsg << "Error in 'typemodin' function: " + << NYql::NPg::LookupProc(TypeModInFuncId).Name + << ", reason: " << error_data->message; + FreeErrorData(error_data); + FlushErrorState(); + return {-1, errMsg}; + } + PG_END_TRY(); + } + + TMaybe<TString> Validate(const TStringBuf binary) { + NMiniKQL::TScopedAlloc alloc(__LOCATION__); + NMiniKQL::TPAllocScope scope; + Datum datum = 0; + Y_DEFER { + if (!PassByValue && datum) { + pfree((void*)datum); + } + }; + PG_TRY(); + { + datum = Receive(binary.Data(), binary.Size()); + return {}; + } + PG_CATCH(); + { + auto error_data = CopyErrorData(); + TStringBuilder errMsg; + errMsg << "Error in 'recv' function: " + << NYql::NPg::LookupProc(ReceiveFuncId).Name + << ", reason: " << error_data->message; + FreeErrorData(error_data); + FlushErrorState(); + return errMsg; + } + PG_END_TRY(); + } + + TCoerceResult Coerce(const TStringBuf binary, i32 typmod) { + NMiniKQL::TScopedAlloc alloc(__LOCATION__); + NMiniKQL::TPAllocScope scope; + Datum datum = 0; + Datum datumCasted = 0; + text* serialized = nullptr; + Y_DEFER { + if (!PassByValue) { + if (datum) { + pfree((void*)datum); + } + if (datumCasted && datumCasted != datum) { + pfree((void*)datumCasted); + } + } + if (serialized) { + pfree(serialized); + } + }; + PG_TRY(); + { + datum = Receive(binary.Data(), binary.Size()); + + const auto& cast = NYql::NPg::LookupCast(TypeId, TypeId); + FmgrInfo finfo; + InitFunc(cast.FunctionId, &finfo, 2, 3); + LOCAL_FCINFO(callInfo, 3); + Zero(*callInfo); + callInfo->flinfo = &finfo; + callInfo->nargs = 3; + callInfo->fncollation = DEFAULT_COLLATION_OID; + callInfo->isnull = false; + callInfo->args[0] = { datum, false }; + callInfo->args[1] = { Int32GetDatum(typmod), false }; + callInfo->args[2] = { BoolGetDatum(false), false }; + + datumCasted = finfo.fn_addr(callInfo); + Y_ENSURE(!callInfo->isnull); + + if (datum == datumCasted) { + return {{}, {}}; + } else { + FmgrInfo finfo; + InitFunc(SendFuncId, &finfo, 1, 1); + LOCAL_FCINFO(callInfo, 1); + Zero(*callInfo); + callInfo->flinfo = &finfo; + callInfo->nargs = 1; + callInfo->fncollation = DEFAULT_COLLATION_OID; + callInfo->isnull = false; + callInfo->args[0] = { datumCasted, false }; + + serialized = (text*)finfo.fn_addr(callInfo); + Y_ENSURE(!callInfo->isnull); + return {TString(NMiniKQL::GetVarBuf(serialized)), {}}; + } + } + PG_CATCH(); + { + auto error_data = CopyErrorData(); + TStringBuilder errMsg; + errMsg << "Error in 'cast' function: " + << NYql::NPg::LookupProc(ReceiveFuncId).Name + << ", reason: " << error_data->message; + FreeErrorData(error_data); + FlushErrorState(); + return {{}, errMsg}; + } + PG_END_TRY(); + } + private: Datum Receive(const char* data, size_t size) const { StringInfoData stringInfo; @@ -3175,8 +3374,9 @@ private: } public: - const TString YdbTypeName; + TString YdbTypeName; ui32 StoredSize = 0; // size in local db, 0 for variable size + bool NeedsCoercion = false; }; class TPgTypeDescriptors { @@ -3230,17 +3430,33 @@ void* TypeDescFromPgTypeId(ui32 pgTypeId) { return (void*)TPgTypeDescriptors::Instance().Find(pgTypeId); } -const char* PgTypeNameFromTypeDesc(void* typeDesc) { +TString PgTypeNameFromTypeDesc(void* typeDesc, const TString& typeMod) { if (!typeDesc) { return ""; } - return static_cast<TPgTypeDescriptor*>(typeDesc)->YdbTypeName.data(); + auto* pgTypeDesc = static_cast<TPgTypeDescriptor*>(typeDesc); + if (typeMod.empty()) { + return pgTypeDesc->YdbTypeName; + } + return pgTypeDesc->YdbTypeName + INTERNAL_TYPE_AND_MOD_SEPARATOR + typeMod; } void* TypeDescFromPgTypeName(const TStringBuf name) { + auto space = name.find_first_of(INTERNAL_TYPE_AND_MOD_SEPARATOR); + if (space != TStringBuf::npos) { + return (void*)TPgTypeDescriptors::Instance().Find(name.substr(0, space)); + } return (void*)TPgTypeDescriptors::Instance().Find(name); } +TString TypeModFromPgTypeName(const TStringBuf name) { + auto space = name.find_first_of(INTERNAL_TYPE_AND_MOD_SEPARATOR); + if (space != TStringBuf::npos) { + return TString(name.substr(space + 1)); + } + return {}; +} + bool TypeDescIsComparable(void* typeDesc) { if (!typeDesc) { return false; @@ -3255,6 +3471,13 @@ ui32 TypeDescGetStoredSize(void* typeDesc) { return static_cast<TPgTypeDescriptor*>(typeDesc)->StoredSize; } +bool TypeDescNeedsCoercion(void* typeDesc) { + if (!typeDesc) { + return false; + } + return static_cast<TPgTypeDescriptor*>(typeDesc)->NeedsCoercion; +} + int PgNativeBinaryCompare(const char* dataL, size_t sizeL, const char* dataR, size_t sizeR, void* typeDesc) { return static_cast<TPgTypeDescriptor*>(typeDesc)->Compare(dataL, sizeL, dataR, sizeR); } @@ -3263,6 +3486,27 @@ ui64 PgNativeBinaryHash(const char* data, size_t size, void* typeDesc) { return static_cast<TPgTypeDescriptor*>(typeDesc)->Hash(data, size); } +TTypeModResult BinaryTypeModFromTextTypeMod(const TString& str, void* typeDesc) { + if (!typeDesc) { + return {-1, "invalid type descriptor"}; + } + return static_cast<TPgTypeDescriptor*>(typeDesc)->ReadTypeMod(str); +} + +TMaybe<TString> PgNativeBinaryValidate(const TStringBuf binary, void* typeDesc) { + if (!typeDesc) { + return "invalid type descriptor"; + } + return static_cast<TPgTypeDescriptor*>(typeDesc)->Validate(binary); +} + +TCoerceResult PgNativeBinaryCoerce(const TStringBuf binary, void* typeDesc, i32 typmod) { + if (!typeDesc) { + return {{}, "invalid type descriptor"}; + } + return static_cast<TPgTypeDescriptor*>(typeDesc)->Coerce(binary, typmod); +} + TConvertResult PgNativeBinaryFromNativeText(const TString& str, ui32 pgTypeId) { auto* typeDesc = TypeDescFromPgTypeId(pgTypeId); Y_VERIFY(typeDesc); diff --git a/ydb/library/yql/parser/pg_wrapper/interface/type_desc.h b/ydb/library/yql/parser/pg_wrapper/interface/type_desc.h index 831ffac1440..0a8fb161b88 100644 --- a/ydb/library/yql/parser/pg_wrapper/interface/type_desc.h +++ b/ydb/library/yql/parser/pg_wrapper/interface/type_desc.h @@ -1,25 +1,44 @@ #pragma once #include <util/generic/strbuf.h> +#include <util/generic/maybe.h> namespace NKikimr::NPg { ui32 PgTypeIdFromTypeDesc(void* typeDesc); void* TypeDescFromPgTypeId(ui32 pgTypeId); -const char* PgTypeNameFromTypeDesc(void* typeDesc); +TString PgTypeNameFromTypeDesc(void* typeDesc, const TString& typeMod = {}); void* TypeDescFromPgTypeName(const TStringBuf name); +TString TypeModFromPgTypeName(const TStringBuf name); bool TypeDescIsComparable(void* typeDesc); ui32 TypeDescGetStoredSize(void* typeDesc); +bool TypeDescNeedsCoercion(void* typeDesc); int PgNativeBinaryCompare(const char* dataL, size_t sizeL, const char* dataR, size_t sizeR, void* typeDesc); ui64 PgNativeBinaryHash(const char* data, size_t size, void* typeDesc); +struct TTypeModResult { + i32 Typmod = -1; + TMaybe<TString> Error; +}; + +TTypeModResult BinaryTypeModFromTextTypeMod(const TString& str, void* typeDesc); + +TMaybe<TString> PgNativeBinaryValidate(const TStringBuf binary, void* typeDesc); + +struct TCoerceResult { + TMaybe<TString> NewValue; + TMaybe<TString> Error; +}; + +TCoerceResult PgNativeBinaryCoerce(const TStringBuf binary, void* typeDesc, i32 typmod); + struct TConvertResult { TString Str; - TString Error; + TMaybe<TString> Error; }; TConvertResult PgNativeBinaryFromNativeText(const TString& str, ui32 pgTypeId); diff --git a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp index ac9404e30d9..0366da57efc 100644 --- a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp +++ b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp @@ -285,8 +285,9 @@ void* TypeDescFromPgTypeId(ui32 pgTypeId) { return {}; } -const char* PgTypeNameFromTypeDesc(void* typeDesc) { +TString PgTypeNameFromTypeDesc(void* typeDesc, const TString& typeMod) { Y_UNUSED(typeDesc); + Y_UNUSED(typeMod); return ""; } @@ -295,6 +296,11 @@ void* TypeDescFromPgTypeName(const TStringBuf name) { return {}; } +TString TypeModFromPgTypeName(const TStringBuf name) { + Y_UNUSED(name); + return {}; +} + bool TypeDescIsComparable(void* typeDesc) { Y_UNUSED(typeDesc); throw yexception() << "PG types are not supported"; @@ -305,6 +311,11 @@ ui32 TypeDescGetStoredSize(void* typeDesc) { throw yexception() << "PG types are not supported"; } +bool TypeDescNeedsCoercion(void* typeDesc) { + Y_UNUSED(typeDesc); + throw yexception() << "PG types are not supported"; +} + int PgNativeBinaryCompare(const char* dataL, size_t sizeL, const char* dataR, size_t sizeR, void* typeDesc) { Y_UNUSED(dataL); Y_UNUSED(sizeL); @@ -321,6 +332,25 @@ ui64 PgNativeBinaryHash(const char* data, size_t size, void* typeDesc) { throw yexception() << "PG types are not supported"; } +TTypeModResult BinaryTypeModFromTextTypeMod(const TString& str, void* typeDesc) { + Y_UNUSED(str); + Y_UNUSED(typeDesc); + throw yexception() << "PG types are not supported"; +} + +TMaybe<TString> PgNativeBinaryValidate(const TStringBuf binary, void* typeDesc) { + Y_UNUSED(binary); + Y_UNUSED(typeDesc); + throw yexception() << "PG types are not supported"; +} + +TCoerceResult PgNativeBinaryCoerce(const TStringBuf binary, void* typeDesc, i32 typmod) { + Y_UNUSED(binary); + Y_UNUSED(typeDesc); + Y_UNUSED(typmod); + throw yexception() << "PG types are not supported"; +} + TConvertResult PgNativeBinaryFromNativeText(const TString& str, ui32 pgTypeId) { Y_UNUSED(str); Y_UNUSED(pgTypeId); diff --git a/ydb/public/api/protos/ydb_value.proto b/ydb/public/api/protos/ydb_value.proto index bbee9e5c035..4093a50a271 100644 --- a/ydb/public/api/protos/ydb_value.proto +++ b/ydb/public/api/protos/ydb_value.proto @@ -51,12 +51,15 @@ message TaggedType { } message PgType { + string type_name = 10; + string type_modifier = 11; + // pg object id of the type // full registry could be found here: https://github.com/postgres/postgres/blob/master/src/include/catalog/pg_type.dat - uint32 oid = 1; // required + uint32 oid = 1; // advanced type details useful for pg wire format proxying - int32 typlen = 2; // optional, set to 0 by default - int32 typmod = 3; // optional, set to 0 by default + int32 typlen = 2; + int32 typmod = 3; } message Type { diff --git a/ydb/public/lib/json_value/ydb_json_value.cpp b/ydb/public/lib/json_value/ydb_json_value.cpp index 50f2ea3f5b3..083127b89b3 100644 --- a/ydb/public/lib/json_value/ydb_json_value.cpp +++ b/ydb/public/lib/json_value/ydb_json_value.cpp @@ -658,7 +658,7 @@ namespace { break; case TTypeParser::ETypeKind::Pg: { - TPgType pgType(0, -1, -1); + TPgType pgType(""); // TODO: correct type? if (jsonValue.GetType() == NJson::JSON_STRING) { ValueBuilder.Pg(TPgValue(TPgValue::VK_TEXT, jsonValue.GetString(), pgType)); } else if (jsonValue.GetType() == NJson::JSON_NULL) { diff --git a/ydb/public/lib/json_value/ydb_json_value_ut.cpp b/ydb/public/lib/json_value/ydb_json_value_ut.cpp index 0856192673d..365e821fa3a 100644 --- a/ydb/public/lib/json_value/ydb_json_value_ut.cpp +++ b/ydb/public/lib/json_value/ydb_json_value_ut.cpp @@ -576,7 +576,7 @@ Y_UNIT_TEST_SUITE(JsonValueTest) { } Y_UNIT_TEST(PgValue) { - TPgType pgType(1, 2, 3); + TPgType pgType(""); TPgValue v1(TPgValue::VK_TEXT, "text_value", pgType); TPgValue v2(TPgValue::VK_BINARY, "binary_value", pgType); TPgValue v3(TPgValue::VK_TEXT, "", pgType); diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp index d2eff31232e..637dfde518a 100644 --- a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp +++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp @@ -17,9 +17,8 @@ bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2) { return t1.decimal_type().precision() == t2.decimal_type().precision() && t1.decimal_type().scale() == t2.decimal_type().scale(); case Ydb::Type::kPgType: - return t1.pg_type().oid() == t2.pg_type().oid() - && t1.pg_type().typlen() == t2.pg_type().typlen() - && t1.pg_type().typmod() == t2.pg_type().typmod(); + return t1.pg_type().type_name() == t2.pg_type().type_name() + && t1.pg_type().type_modifier() == t2.pg_type().type_modifier(); case Ydb::Type::kOptionalType: return TypesEqual(t1.optional_type().item(), t2.optional_type().item()); case Ydb::Type::kTaggedType: diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.cpp b/ydb/public/sdk/cpp/client/ydb_value/value.cpp index 71c69959c69..1bb65d5e322 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value.cpp @@ -140,7 +140,11 @@ public: TPgType GetPg() const { CheckKind(ETypeKind::Pg, "GetPg"); const auto& pg = GetProto().pg_type(); - return TPgType(pg.oid(), pg.typlen(), pg.typmod()); + TPgType type(pg.type_name(), pg.type_modifier()); + type.Oid = pg.oid(); + type.Typlen = pg.typlen(); + type.Typmod = pg.typmod(); + return type; } template<ETypeKind kind> @@ -472,7 +476,8 @@ void FormatTypeInternal(TTypeParser& parser, IOutputStream& out) { case TTypeParser::ETypeKind::Pg: { auto pg = parser.GetPg(); - out << "Pg("sv << pg.Oid << ',' << pg.Typlen << ',' << pg.Typmod << ')'; + out << "Pg('"sv << pg.TypeName << "\',\'" << pg.TypeModifier << "\'," + << pg.Oid << "," << pg.Typlen << ',' << pg.Typmod << ')'; break; } @@ -603,9 +608,8 @@ public: void Pg(const TPgType& pgType) { auto& pg = *GetProto().mutable_pg_type(); - pg.set_oid(pgType.Oid); - pg.set_typlen(pgType.Typlen); - pg.set_typmod(pgType.Typmod); + pg.set_type_name(pgType.TypeName); + pg.set_type_modifier(pgType.TypeModifier); } void BeginOptional() { diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.h b/ydb/public/sdk/cpp/client/ydb_value/value.h index 5c7c885daa7..43d516afb9a 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value.h +++ b/ydb/public/sdk/cpp/client/ydb_value/value.h @@ -70,14 +70,16 @@ struct TDecimalType { }; struct TPgType { - ui32 Oid; - i32 Typlen; - i32 Typmod; - - TPgType(ui32 oid, i32 typlen, i32 typmod) - : Oid(oid) - , Typlen(typlen) - , Typmod(typmod) + TString TypeName; + TString TypeModifier; + + ui32 Oid = 0; + i32 Typlen = 0; + i32 Typmod = 0; + + TPgType(const TString& typeName, const TString& typeModifier = {}) + : TypeName(typeName) + , TypeModifier(typeModifier) {} }; diff --git a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp index 78f6ca72a2f..9a75fa76350 100644 --- a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp @@ -142,7 +142,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { .AddElement() .Decimal(TDecimalType(8, 13)) .AddElement() - .Pg(TPgType(1, 2, -3)) + .Pg(TPgType("pgint2")) .AddElement() .BeginOptional() .Primitive(EPrimitiveType::Utf8) @@ -153,7 +153,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { .Build(); UNIT_ASSERT_NO_DIFF(FormatType(type), - R"(Struct<'Member1':List<Uint32?>,'Member2':Dict<Int64,Tuple<Decimal(8,13),Pg(1,2,-3),Utf8?>>>)"); + R"(Struct<'Member1':List<Uint32?>,'Member2':Dict<Int64,Tuple<Decimal(8,13),Pg('pgint2','',0,0,0),Utf8?>>>)"); } Y_UNIT_TEST(BuildTypeReuse) { |