aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorazevaykin <azevaykin@yandex-team.com>2023-11-20 08:46:38 +0300
committerazevaykin <azevaykin@yandex-team.com>2023-11-20 09:01:15 +0300
commitc3c9ba92e055a8db262ef921ec9cdd20a5321e5c (patch)
treefc01c85f63e99deb14f1daa080bc2dd45afc36e5
parente540871359c8a049f7b3228a73081504b6898683 (diff)
downloadydb-c3c9ba92e055a8db262ef921ec9cdd20a5321e5c.tar.gz
Export&Import pg types
-rw-r--r--ydb/core/engine/mkql_proto.cpp2
-rw-r--r--ydb/core/grpc_services/rpc_read_rows.cpp11
-rw-r--r--ydb/core/io_formats/csv_ydb_dump.cpp113
-rw-r--r--ydb/core/kqp/ut/pg/kqp_pg_ut.cpp4
-rw-r--r--ydb/core/tx/datashard/export_common.cpp11
-rw-r--r--ydb/core/tx/datashard/export_common.h1
-rw-r--r--ydb/core/tx/datashard/export_s3_buffer_raw.cpp4
-rw-r--r--ydb/core/tx/datashard/read_table_scan.cpp5
-rw-r--r--ydb/core/tx/schemeshard/ut_helpers/helpers.cpp2
-rw-r--r--ydb/core/tx/schemeshard/ut_restore/CMakeLists.darwin-x86_64.txt3
-rw-r--r--ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux-aarch64.txt3
-rw-r--r--ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux-x86_64.txt3
-rw-r--r--ydb/core/tx/schemeshard/ut_restore/CMakeLists.windows-x86_64.txt3
-rw-r--r--ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp48
-rw-r--r--ydb/core/tx/schemeshard/ut_restore/ya.make3
-rw-r--r--ydb/core/ydb_convert/ydb_convert.cpp3
-rw-r--r--ydb/library/yql/parser/pg_wrapper/comp_factory.cpp26
-rw-r--r--ydb/library/yql/parser/pg_wrapper/interface/type_desc.h4
-rw-r--r--ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp14
19 files changed, 203 insertions, 60 deletions
diff --git a/ydb/core/engine/mkql_proto.cpp b/ydb/core/engine/mkql_proto.cpp
index 0a3867d8760..7ed500cc1a7 100644
--- a/ydb/core/engine/mkql_proto.cpp
+++ b/ydb/core/engine/mkql_proto.cpp
@@ -291,7 +291,7 @@ bool CellToValue(NScheme::TTypeInfo type, const TCell& c, NKikimrMiniKQL::TValue
break;
case NScheme::NTypeIds::Pg: {
- auto convert = NPg::PgNativeTextFromNativeBinary(TString(c.Data(), c.Size()), NPg::PgTypeIdFromTypeDesc(type.GetTypeDesc()));
+ auto convert = NPg::PgNativeTextFromNativeBinary(c.AsBuf(), type.GetTypeDesc());
if (convert.Error) {
errStr = *convert.Error;
return false;
diff --git a/ydb/core/grpc_services/rpc_read_rows.cpp b/ydb/core/grpc_services/rpc_read_rows.cpp
index 46f18a1ca35..11e172ad9b3 100644
--- a/ydb/core/grpc_services/rpc_read_rows.cpp
+++ b/ydb/core/grpc_services/rpc_read_rows.cpp
@@ -563,12 +563,11 @@ public:
vb.AddMember(colMeta.Name);
if (colMeta.Type.GetTypeId() == NScheme::NTypeIds::Pg)
{
- const auto pgTypeId = NPg::PgTypeIdFromTypeDesc(colMeta.Type.GetTypeDesc());
- const NYdb::TPgValue pgValue{
- cell.IsNull() ? NYdb::TPgValue::VK_NULL : NYdb::TPgValue::VK_TEXT,
- NPg::PgNativeTextFromNativeBinary({cell.AsBuf().data(), cell.AsBuf().size()}, pgTypeId).Str,
- getPgTypeFromColMeta(colMeta)
- };
+ const NPg::TConvertResult& pgResult = NPg::PgNativeTextFromNativeBinary(cell.AsBuf(), colMeta.Type.GetTypeDesc());
+ if (pgResult.Error) {
+ LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::RPC_REQUEST, "PgNativeTextFromNativeBinary error " << *pgResult.Error);
+ }
+ const NYdb::TPgValue pgValue{cell.IsNull() ? NYdb::TPgValue::VK_NULL : NYdb::TPgValue::VK_TEXT, pgResult.Str, getPgTypeFromColMeta(colMeta)};
vb.Pg(pgValue);
}
else
diff --git a/ydb/core/io_formats/csv_ydb_dump.cpp b/ydb/core/io_formats/csv_ydb_dump.cpp
index b46158c630c..3c8176132ea 100644
--- a/ydb/core/io_formats/csv_ydb_dump.cpp
+++ b/ydb/core/io_formats/csv_ydb_dump.cpp
@@ -16,6 +16,8 @@
#include <util/datetime/base.h>
#include <util/string/cast.h>
+#include <typeinfo>
+
namespace NKikimr::NFormats {
namespace {
@@ -30,6 +32,11 @@ namespace {
}
template <typename T>
+ TString MakeError() {
+ return TStringBuilder() << "" << typeid(T).name() << " is expected.";
+ }
+
+ template <typename T>
bool TryParse(TStringBuf value, T& result) {
return TryFromString(value, result);
}
@@ -97,6 +104,32 @@ namespace {
return result.Defined();
}
+ template <typename T>
+ bool TryParse(TStringBuf value, T& result, TString& err, void* parseParam) {
+ Y_UNUSED(value);
+ Y_UNUSED(result);
+ Y_UNUSED(err);
+ Y_UNUSED(parseParam);
+ Y_ABORT("TryParse with parseParam is unimplemented");
+ }
+
+ template <>
+ bool TryParse(TStringBuf value, NPg::TConvertResult& result, TString& err, void* typeDesc) {
+ TString unescaped;
+ if (!CheckedUnescape(value, unescaped)) {
+ err = MakeError<NPg::TConvertResult>();
+ return false;
+ }
+
+ result = NPg::PgNativeBinaryFromNativeText(unescaped, typeDesc);
+ if (result.Error) {
+ err = *result.Error;
+ return false;
+ }
+
+ return true;
+ }
+
template <typename T, typename U>
using TConverter = std::function<U(const T&)>;
@@ -131,11 +164,17 @@ namespace {
return TStringBuf(*v);
}
+ TStringBuf PgToStringBuf(const NPg::TConvertResult& v) {
+ Y_ABORT_UNLESS(!v.Error);
+ return v.Str;
+ }
+
template <typename T, typename U = T>
struct TCellMaker {
- static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TConverter<T, U> conv = &Implicit<T, U>) {
+ static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TString& err, TConverter<T, U> conv = &Implicit<T, U>) {
T t;
if (!TryParse<T>(v, t)) {
+ err = MakeError<T>();
return false;
}
@@ -149,9 +188,22 @@ namespace {
template <typename T>
struct TCellMaker<T, TStringBuf> {
- static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TConverter<T, TStringBuf> conv = &Implicit<T, TStringBuf>) {
+ static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TString& err, TConverter<T, TStringBuf> conv = &Implicit<T, TStringBuf>) {
T t;
if (!TryParse<T>(v, t)) {
+ err = MakeError<T>();
+ return false;
+ }
+
+ const auto u = pool.AppendString(conv(t));
+ c = TCell(u.data(), u.size());
+
+ return true;
+ }
+
+ static bool Make(TCell& c, TStringBuf v, TMemoryPool& pool, TString& err, TConverter<T, TStringBuf> conv, void* parseParam) {
+ T t;
+ if (!TryParse<T>(v, t, err, parseParam)) {
return false;
}
@@ -164,58 +216,57 @@ namespace {
} // anonymous
-bool MakeCell(TCell& cell, TStringBuf value, NScheme::TTypeInfo type, TMemoryPool& pool) {
+bool MakeCell(TCell& cell, TStringBuf value, NScheme::TTypeInfo type, TMemoryPool& pool, TString& err) {
if (value == "null") {
return true;
}
switch (type.GetTypeId()) {
case NScheme::NTypeIds::Bool:
- return TCellMaker<bool>::Make(cell, value, pool);
+ return TCellMaker<bool>::Make(cell, value, pool, err);
case NScheme::NTypeIds::Int8:
- return TCellMaker<i8>::Make(cell, value, pool);
+ return TCellMaker<i8>::Make(cell, value, pool, err);
case NScheme::NTypeIds::Uint8:
- return TCellMaker<ui8>::Make(cell, value, pool);
+ return TCellMaker<ui8>::Make(cell, value, pool, err);
case NScheme::NTypeIds::Int16:
- return TCellMaker<i16>::Make(cell, value, pool);
+ return TCellMaker<i16>::Make(cell, value, pool, err);
case NScheme::NTypeIds::Uint16:
- return TCellMaker<ui16>::Make(cell, value, pool);
+ return TCellMaker<ui16>::Make(cell, value, pool, err);
case NScheme::NTypeIds::Int32:
- return TCellMaker<i32>::Make(cell, value, pool);
+ return TCellMaker<i32>::Make(cell, value, pool, err);
case NScheme::NTypeIds::Uint32:
- return TCellMaker<ui32>::Make(cell, value, pool);
+ return TCellMaker<ui32>::Make(cell, value, pool, err);
case NScheme::NTypeIds::Int64:
- return TCellMaker<i64>::Make(cell, value, pool);
+ return TCellMaker<i64>::Make(cell, value, pool, err);
case NScheme::NTypeIds::Uint64:
- return TCellMaker<ui64>::Make(cell, value, pool);
+ return TCellMaker<ui64>::Make(cell, value, pool, err);
case NScheme::NTypeIds::Float:
- return TCellMaker<float>::Make(cell, value, pool);
+ return TCellMaker<float>::Make(cell, value, pool, err);
case NScheme::NTypeIds::Double:
- return TCellMaker<double>::Make(cell, value, pool);
+ return TCellMaker<double>::Make(cell, value, pool, err);
case NScheme::NTypeIds::Date:
- return TCellMaker<TInstant, ui16>::Make(cell, value, pool, &Days);
+ return TCellMaker<TInstant, ui16>::Make(cell, value, pool, err, &Days);
case NScheme::NTypeIds::Datetime:
- return TCellMaker<TInstant, ui32>::Make(cell, value, pool, &Seconds);
+ return TCellMaker<TInstant, ui32>::Make(cell, value, pool, err, &Seconds);
case NScheme::NTypeIds::Timestamp:
- return TCellMaker<TInstant, ui64>::Make(cell, value, pool, &MicroSeconds);
+ return TCellMaker<TInstant, ui64>::Make(cell, value, pool, err, &MicroSeconds);
case NScheme::NTypeIds::Interval:
- return TCellMaker<i64>::Make(cell, value, pool);
+ return TCellMaker<i64>::Make(cell, value, pool, err);
case NScheme::NTypeIds::String:
case NScheme::NTypeIds::String4k:
case NScheme::NTypeIds::String2m:
case NScheme::NTypeIds::Utf8:
case NScheme::NTypeIds::Yson:
case NScheme::NTypeIds::Json:
- return TCellMaker<TString, TStringBuf>::Make(cell, value, pool);
+ return TCellMaker<TString, TStringBuf>::Make(cell, value, pool, err);
case NScheme::NTypeIds::JsonDocument:
- return TCellMaker<TMaybe<NBinaryJson::TBinaryJson>, TStringBuf>::Make(cell, value, pool, &BinaryJsonToStringBuf);
+ return TCellMaker<TMaybe<NBinaryJson::TBinaryJson>, TStringBuf>::Make(cell, value, pool, err, &BinaryJsonToStringBuf);
case NScheme::NTypeIds::DyNumber:
- return TCellMaker<TMaybe<TString>, TStringBuf>::Make(cell, value, pool, &DyNumberToStringBuf);
+ return TCellMaker<TMaybe<TString>, TStringBuf>::Make(cell, value, pool, err, &DyNumberToStringBuf);
case NScheme::NTypeIds::Decimal:
- return TCellMaker<NYql::NDecimal::TInt128, std::pair<ui64, ui64>>::Make(cell, value, pool, &Int128ToPair);
+ return TCellMaker<NYql::NDecimal::TInt128, std::pair<ui64, ui64>>::Make(cell, value, pool, err, &Int128ToPair);
case NScheme::NTypeIds::Pg:
- // TODO: support pg types
- Y_ABORT_UNLESS(false, "pg types are not supported");
+ return TCellMaker<NPg::TConvertResult, TStringBuf>::Make(cell, value, pool, err, &PgToStringBuf, type.GetTypeDesc());
default:
return false;
}
@@ -243,6 +294,7 @@ bool CheckCellValue(const TCell& cell, NScheme::TTypeInfo type) {
case NScheme::NTypeIds::String2m:
case NScheme::NTypeIds::JsonDocument: // checked at parsing time
case NScheme::NTypeIds::DyNumber: // checked at parsing time
+ case NScheme::NTypeIds::Pg: // checked at parsing time
return true;
case NScheme::NTypeIds::Date:
return cell.AsValue<ui16>() < NUdf::MAX_DATE;
@@ -260,9 +312,6 @@ bool CheckCellValue(const TCell& cell, NScheme::TTypeInfo type) {
return NYql::NDom::IsValidJson(cell.AsBuf());
case NScheme::NTypeIds::Decimal:
return !NYql::NDecimal::IsError(cell.AsValue<NYql::NDecimal::TInt128>());
- case NScheme::NTypeIds::Pg:
- // TODO: support pg types
- Y_ABORT_UNLESS(false, "pg types are not supported");
default:
return false;
}
@@ -292,8 +341,14 @@ bool TYdbDump::ParseLine(TStringBuf line, const std::vector<std::pair<i32, NSche
Y_ABORT_UNLESS(cell);
- if (!MakeCell(*cell, value, pType, pool) || !CheckCellValue(*cell, pType)) {
- strError = TStringBuilder() << "Invalid value: '" << value << "'";
+ if (!CheckCellValue(*cell, pType)) {
+ strError = TStringBuilder() << "Value check error: '" << value << "'";
+ return false;
+ }
+
+ TString parseError;
+ if (!MakeCell(*cell, value, pType, pool, parseError)) {
+ strError = TStringBuilder() << "Value parse error: '" << value << "' " << parseError;
return false;
}
diff --git a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp
index 6fd21e3ca42..5661cae7e07 100644
--- a/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp
+++ b/ydb/core/kqp/ut/pg/kqp_pg_ut.cpp
@@ -750,6 +750,7 @@ Y_UNIT_TEST_SUITE(KqpPg) {
rows.BeginList();
for (size_t i = 0; i < rowCount; ++i) {
auto str = isText ? textIn(i) : NPg::PgNativeBinaryFromNativeText(textIn(i), typeId).Str;
+ UNIT_ASSERT(!str.empty());
auto mode = isText ? TPgValue::VK_TEXT : TPgValue::VK_BINARY;
if (isKey) {
rows.AddListItem()
@@ -759,6 +760,7 @@ Y_UNIT_TEST_SUITE(KqpPg) {
.EndStruct();
} else {
auto int2Str = NPg::PgNativeBinaryFromNativeText(Sprintf("%u", i), INT2OID).Str;
+ UNIT_ASSERT(!int2Str.empty());
rows.AddListItem()
.BeginStruct()
.AddMember(colNames[0]).Pg(TPgValue(TPgValue::VK_BINARY, int2Str, TPgType("pgint2")))
@@ -823,6 +825,8 @@ Y_UNIT_TEST_SUITE(KqpPg) {
for (size_t i = 0; i < rowCount; ++i) {
auto str = NPg::PgNativeBinaryFromNativeText(textIn(), typeId).Str;
auto int2Str = NPg::PgNativeBinaryFromNativeText(Sprintf("%u", i), INT2OID).Str;
+ UNIT_ASSERT(!str.empty());
+ UNIT_ASSERT(!int2Str.empty());
rows.AddListItem()
.BeginStruct()
.AddMember("key").Pg(TPgValue(TPgValue::VK_BINARY, int2Str, TPgType("pgint2")))
diff --git a/ydb/core/tx/datashard/export_common.cpp b/ydb/core/tx/datashard/export_common.cpp
index 51d36e83201..bbedc5c02c8 100644
--- a/ydb/core/tx/datashard/export_common.cpp
+++ b/ydb/core/tx/datashard/export_common.cpp
@@ -4,6 +4,7 @@
#include <ydb/core/ydb_convert/table_description.h>
#include <ydb/core/ydb_convert/ydb_convert.h>
#include <ydb/library/dynumber/dynumber.h>
+#include <ydb/library/yql/parser/pg_wrapper/interface/type_desc.h>
#include <ydb/public/lib/scheme_types/scheme_type_id.h>
#include <ydb/library/yql/public/decimal/yql_decimal.h>
@@ -106,5 +107,15 @@ bool DyNumberToStream(TStringBuf data, IOutputStream& out, TString& err) {
return true;
}
+bool PgToStream(TStringBuf data, void* typeDesc, IOutputStream& out, TString& err) {
+ const NPg::TConvertResult& pgResult = NPg::PgNativeTextFromNativeBinary(data, typeDesc);
+ if (pgResult.Error) {
+ err = *pgResult.Error;
+ return false;
+ }
+ out << '"' << CGIEscapeRet(pgResult.Str) << '"';
+ return true;
+}
+
} // NDataShard
} // NKikimr
diff --git a/ydb/core/tx/datashard/export_common.h b/ydb/core/tx/datashard/export_common.h
index 7a7dc19525e..df3434a45cc 100644
--- a/ydb/core/tx/datashard/export_common.h
+++ b/ydb/core/tx/datashard/export_common.h
@@ -38,6 +38,7 @@ TString DecimalToString(const std::pair<ui64, i64>& loHi);
TString DyNumberToString(TStringBuf data);
bool DecimalToStream(const std::pair<ui64, i64>& loHi, IOutputStream& out, TString& err);
bool DyNumberToStream(TStringBuf data, IOutputStream& out, TString& err);
+bool PgToStream(TStringBuf data, void* typeDesc, IOutputStream& out, TString& err);
} // NDataShard
} // NKikimr
diff --git a/ydb/core/tx/datashard/export_s3_buffer_raw.cpp b/ydb/core/tx/datashard/export_s3_buffer_raw.cpp
index 3eac355a9b4..8e4bceadbbf 100644
--- a/ydb/core/tx/datashard/export_s3_buffer_raw.cpp
+++ b/ydb/core/tx/datashard/export_s3_buffer_raw.cpp
@@ -123,8 +123,8 @@ bool TS3BufferRaw::Collect(const NTable::IScan::TRow& row, IOutputStream& out) {
out << '"' << CGIEscapeRet(NBinaryJson::SerializeToJson(cell.AsBuf())) << '"';
break;
case NScheme::NTypeIds::Pg:
- // TODO: support pg types
- Y_ABORT("Unsupported pg type");
+ serialized = PgToStream(cell.AsBuf(), column.Type.GetTypeDesc(), out, ErrorString);
+ break;
default:
Y_ABORT("Unsupported type");
}
diff --git a/ydb/core/tx/datashard/read_table_scan.cpp b/ydb/core/tx/datashard/read_table_scan.cpp
index 818018b3982..2b8b601b54b 100644
--- a/ydb/core/tx/datashard/read_table_scan.cpp
+++ b/ydb/core/tx/datashard/read_table_scan.cpp
@@ -171,10 +171,7 @@ Y_FORCE_INLINE bool AddCell(TOutValue& row, NScheme::TTypeInfo type, const TCell
break;
}
case NScheme::NTypeIds::Pg: {
- auto result = NPg::PgNativeTextFromNativeBinary(
- TString(cell.Data(), cell.Size()),
- NPg::PgTypeIdFromTypeDesc(type.GetTypeDesc())
- );
+ auto result = NPg::PgNativeTextFromNativeBinary(cell.AsBuf(), type.GetTypeDesc());
if (result.Error) {
err = Sprintf("Failed to add cell to Ydb::Value: %s", (*result.Error).c_str());
return false;
diff --git a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp
index 414ee0d9da2..87a18be771d 100644
--- a/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp
+++ b/ydb/core/tx/schemeshard/ut_helpers/helpers.cpp
@@ -1177,7 +1177,7 @@ namespace NSchemeShardUT_Private {
}
if (!found) {
- UNIT_ASSERT_C(found, "Unexpected status: " << Ydb::StatusIds::StatusCode_Name(result));
+ UNIT_ASSERT_C(found, "Unexpected status: " << Ydb::StatusIds::StatusCode_Name(result) << " issues: " << ev->Record.GetResponse().GetEntry().GetIssues());
}
return ev->Record;
diff --git a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.darwin-x86_64.txt
index c96ccfecf77..135c2b299b4 100644
--- a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.darwin-x86_64.txt
+++ b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.darwin-x86_64.txt
@@ -26,7 +26,8 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_restore PUBLIC
tx-schemeshard-ut_helpers
core-wrappers-ut_helpers
ydb-core-ydb_convert
- yql-sql-pg_dummy
+ yql-sql-pg
+ yql-parser-pg_wrapper
)
target_link_options(ydb-core-tx-schemeshard-ut_restore PRIVATE
-Wl,-platform_version,macos,11.0,11.0
diff --git a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux-aarch64.txt
index 321fa77340c..965050bb114 100644
--- a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux-aarch64.txt
@@ -26,7 +26,8 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_restore PUBLIC
tx-schemeshard-ut_helpers
core-wrappers-ut_helpers
ydb-core-ydb_convert
- yql-sql-pg_dummy
+ yql-sql-pg
+ yql-parser-pg_wrapper
)
target_link_options(ydb-core-tx-schemeshard-ut_restore PRIVATE
-ldl
diff --git a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux-x86_64.txt b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux-x86_64.txt
index 184618adc58..41f372da7a6 100644
--- a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux-x86_64.txt
+++ b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.linux-x86_64.txt
@@ -27,7 +27,8 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_restore PUBLIC
tx-schemeshard-ut_helpers
core-wrappers-ut_helpers
ydb-core-ydb_convert
- yql-sql-pg_dummy
+ yql-sql-pg
+ yql-parser-pg_wrapper
)
target_link_options(ydb-core-tx-schemeshard-ut_restore PRIVATE
-ldl
diff --git a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.windows-x86_64.txt b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.windows-x86_64.txt
index ff0f4ff92ce..a4e53b2a985 100644
--- a/ydb/core/tx/schemeshard/ut_restore/CMakeLists.windows-x86_64.txt
+++ b/ydb/core/tx/schemeshard/ut_restore/CMakeLists.windows-x86_64.txt
@@ -26,7 +26,8 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_restore PUBLIC
tx-schemeshard-ut_helpers
core-wrappers-ut_helpers
ydb-core-ydb_convert
- yql-sql-pg_dummy
+ yql-sql-pg
+ yql-parser-pg_wrapper
)
target_sources(ydb-core-tx-schemeshard-ut_restore PRIVATE
${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp
diff --git a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp
index a8d8146f30f..23252f2918f 100644
--- a/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp
+++ b/ydb/core/tx/schemeshard/ut_restore/ut_restore.cpp
@@ -729,6 +729,54 @@ Y_UNIT_TEST_SUITE(TRestoreTests) {
TestGetImport(runtime, txId, "/MyRoot");
}
+ Y_UNIT_TEST(ExportImportPg) {
+ TTestBasicRuntime runtime;
+ TTestEnv env(runtime);
+ ui64 txId = 100;
+
+ TestCreateTable(runtime, ++txId, "/MyRoot", R"(
+ Name: "Table"
+ Columns { Name: "key" Type: "Uint32" }
+ Columns { Name: "value" Type: "pgint4" }
+ KeyColumnNames: ["key"]
+ )");
+ env.TestWaitNotification(runtime, txId);
+
+ UploadRows(runtime, "/MyRoot/Table", 0, {1}, {2}, {55555});
+
+ TPortManager portManager;
+ const ui16 port = portManager.GetPort();
+
+ TS3Mock s3Mock({}, TS3Mock::TSettings(port));
+ UNIT_ASSERT(s3Mock.Start());
+
+ TestExport(runtime, ++txId, "/MyRoot", Sprintf(R"(
+ ExportToS3Settings {
+ endpoint: "localhost:%d"
+ scheme: HTTP
+ items {
+ source_path: "/MyRoot/Table"
+ destination_prefix: "Backup1"
+ }
+ }
+ )", port));
+ env.TestWaitNotification(runtime, txId);
+ TestGetExport(runtime, txId, "/MyRoot");
+
+ TestImport(runtime, txId, "/MyRoot", Sprintf(R"(
+ ImportFromS3Settings {
+ endpoint: "localhost:%d"
+ scheme: HTTP
+ items {
+ source_prefix: "Backup1"
+ destination_path: "/MyRoot/Restored"
+ }
+ }
+ )", port));
+ env.TestWaitNotification(runtime, txId);
+ TestGetImport(runtime, txId, "/MyRoot");
+ }
+
Y_UNIT_TEST_WITH_COMPRESSION(ShouldCountWrittenBytesAndRows) {
TTestBasicRuntime runtime;
TTestEnv env(runtime);
diff --git a/ydb/core/tx/schemeshard/ut_restore/ya.make b/ydb/core/tx/schemeshard/ut_restore/ya.make
index 9dafc9712ac..308dd73133f 100644
--- a/ydb/core/tx/schemeshard/ut_restore/ya.make
+++ b/ydb/core/tx/schemeshard/ut_restore/ya.make
@@ -20,7 +20,8 @@ PEERDIR(
ydb/core/tx/schemeshard/ut_helpers
ydb/core/wrappers/ut_helpers
ydb/core/ydb_convert
- ydb/library/yql/sql/pg_dummy
+ ydb/library/yql/sql/pg
+ ydb/library/yql/parser/pg_wrapper
)
SRCS(
diff --git a/ydb/core/ydb_convert/ydb_convert.cpp b/ydb/core/ydb_convert/ydb_convert.cpp
index 5f915666bb6..5d233987043 100644
--- a/ydb/core/ydb_convert/ydb_convert.cpp
+++ b/ydb/core/ydb_convert/ydb_convert.cpp
@@ -1190,8 +1190,7 @@ bool CellFromProtoVal(NScheme::TTypeInfo type, i32 typmod, const Ydb::Value* vp,
if (!text.empty()) {
isText = true;
auto desc = type.GetTypeDesc();
- auto id = NPg::PgTypeIdFromTypeDesc(desc);
- auto res = NPg::PgNativeBinaryFromNativeText(text, id);
+ auto res = NPg::PgNativeBinaryFromNativeText(text, desc);
if (res.Error) {
err = TStringBuilder() << "Invalid text value for "
<< NPg::PgTypeNameFromTypeDesc(desc) << ": " << *res.Error;
diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp
index fae36570fcf..6eb7599f302 100644
--- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp
+++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp
@@ -3743,7 +3743,7 @@ public:
callInfo->nargs = 3;
callInfo->fncollation = DEFAULT_COLLATION_OID;
callInfo->isnull = false;
- callInfo->args[0] = { (Datum)str.c_str(), false };
+ callInfo->args[0] = { (Datum)str.Data(), false };
callInfo->args[1] = { ObjectIdGetDatum(NMiniKQL::MakeTypeIOParam(*this)), false };
callInfo->args[2] = { Int32GetDatum(-1), false };
@@ -3776,7 +3776,7 @@ public:
PG_END_TRY();
}
- TConvertResult NativeTextFromNativeBinary(const TString& binary) const {
+ TConvertResult NativeTextFromNativeBinary(const TStringBuf binary) const {
NMiniKQL::TScopedAlloc alloc(__LOCATION__);
NMiniKQL::TPAllocScope scope;
Datum datum = 0;
@@ -4257,18 +4257,28 @@ TCoerceResult PgNativeBinaryCoerce(const TStringBuf binary, void* typeDesc, i32
return static_cast<TPgTypeDescriptor*>(typeDesc)->Coerce(binary, typmod);
}
-TConvertResult PgNativeBinaryFromNativeText(const TString& str, ui32 pgTypeId) {
- auto* typeDesc = TypeDescFromPgTypeId(pgTypeId);
- Y_ABORT_UNLESS(typeDesc);
+TConvertResult PgNativeBinaryFromNativeText(const TString& str, void* typeDesc) {
+ if (!typeDesc) {
+ return {{}, "invalid type descriptor"};
+ }
return static_cast<TPgTypeDescriptor*>(typeDesc)->NativeBinaryFromNativeText(str);
}
-TConvertResult PgNativeTextFromNativeBinary(const TString& binary, ui32 pgTypeId) {
- auto* typeDesc = TypeDescFromPgTypeId(pgTypeId);
- Y_ABORT_UNLESS(typeDesc);
+TConvertResult PgNativeBinaryFromNativeText(const TString& str, ui32 pgTypeId) {
+ return PgNativeBinaryFromNativeText(str, TypeDescFromPgTypeId(pgTypeId));
+}
+
+TConvertResult PgNativeTextFromNativeBinary(const TStringBuf binary, void* typeDesc) {
+ if (!typeDesc) {
+ return {{}, "invalid type descriptor"};
+ }
return static_cast<TPgTypeDescriptor*>(typeDesc)->NativeTextFromNativeBinary(binary);
}
+TConvertResult PgNativeTextFromNativeBinary(const TStringBuf binary, ui32 pgTypeId) {
+ return PgNativeTextFromNativeBinary(binary, TypeDescFromPgTypeId(pgTypeId));
+}
+
} // namespace NKikimr::NPg
namespace NYql::NCommon {
diff --git a/ydb/library/yql/parser/pg_wrapper/interface/type_desc.h b/ydb/library/yql/parser/pg_wrapper/interface/type_desc.h
index aeec1732028..8fed8a3f62b 100644
--- a/ydb/library/yql/parser/pg_wrapper/interface/type_desc.h
+++ b/ydb/library/yql/parser/pg_wrapper/interface/type_desc.h
@@ -42,7 +42,9 @@ struct TConvertResult {
TMaybe<TString> Error;
};
+TConvertResult PgNativeBinaryFromNativeText(const TString& str, void* typeDesc);
TConvertResult PgNativeBinaryFromNativeText(const TString& str, ui32 pgTypeId);
-TConvertResult PgNativeTextFromNativeBinary(const TString& binary, ui32 pgTypeId);
+TConvertResult PgNativeTextFromNativeBinary(const TStringBuf binary, void* typeDesc);
+TConvertResult PgNativeTextFromNativeBinary(const TStringBuf binary, ui32 pgTypeId);
} // namespace NKikimr::NPg
diff --git a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp
index b260095db57..de50f8cd711 100644
--- a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp
+++ b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp
@@ -465,13 +465,25 @@ TCoerceResult PgNativeBinaryCoerce(const TStringBuf binary, void* typeDesc, i32
throw yexception() << "PG types are not supported";
}
+TConvertResult PgNativeBinaryFromNativeText(const TString& str, void* typeDesc) {
+ Y_UNUSED(str);
+ Y_UNUSED(typeDesc);
+ throw yexception() << "PG types are not supported";
+}
+
TConvertResult PgNativeBinaryFromNativeText(const TString& str, ui32 pgTypeId) {
Y_UNUSED(str);
Y_UNUSED(pgTypeId);
throw yexception() << "PG types are not supported";
}
-TConvertResult PgNativeTextFromNativeBinary(const TString& binary, ui32 pgTypeId) {
+TConvertResult PgNativeTextFromNativeBinary(const TStringBuf binary, void* typeDesc) {
+ Y_UNUSED(binary);
+ Y_UNUSED(typeDesc);
+ throw yexception() << "PG types are not supported";
+}
+
+TConvertResult PgNativeTextFromNativeBinary(const TStringBuf binary, ui32 pgTypeId) {
Y_UNUSED(binary);
Y_UNUSED(pgTypeId);
throw yexception() << "PG types are not supported";