YQ-1154: represent pg types in json/yson formats

ref:284eb406c38725ab493186fe0321515825f539f9

7 files changed, 267 insertions, 7 deletions

diff --git a/ydb/public/lib/json_value/ydb_json_value.cpp b/ydb/public/lib/json_value/ydb_json_value.cpp
index af126501a73..7842e11dd33 100644
--- a/ydb/public/lib/json_value/ydb_json_value.cpp
+++ b/ydb/public/lib/json_value/ydb_json_value.cpp
@@ -210,6 +210,16 @@ namespace NYdb {
                 Writer.WriteString(Parser.GetDecimal().ToString());
                 break;
 
+            case TTypeParser::ETypeKind::Pg:
+                if (Parser.GetPg().IsText()) {
+                    Writer.WriteString(Parser.GetPg().Content_);
+                } else {
+                    Writer.BeginList();
+                    Writer.UnsafeWriteValue(BinaryStringToJsonString(Parser.GetPg().Content_));
+                    Writer.EndList();
+                }
+                break;
+
             case TTypeParser::ETypeKind::Optional:
                 Parser.OpenOptional();
                 if (Parser.IsNull()) {
@@ -561,6 +571,9 @@ namespace {
             case TTypeParser::ETypeKind::Decimal:
                 typeBuilder.Decimal(TypeParser.GetDecimal());
                 break;
+            case TTypeParser::ETypeKind::Pg:
+                typeBuilder.Pg(TypeParser.GetPg());
+                break;
             case TTypeParser::ETypeKind::Optional:
                 TypeParser.OpenOptional();
                 typeBuilder.BeginOptional();
@@ -629,6 +642,23 @@ namespace {
                 ValueBuilder.Decimal(jsonValue.GetString());
                 break;
 
+            case TTypeParser::ETypeKind::Pg: {
+                    TPgType pgType(0, 0, 0);
+                    if (jsonValue.GetType() == NJson::JSON_STRING) {
+                        ValueBuilder.Pg(TPgValue(true, jsonValue.GetString(), pgType));
+                    } else {
+                        EnsureType(jsonValue, NJson::JSON_ARRAY);
+                        if (jsonValue.GetArray().size() != 1) {
+                            ThrowFatalError(TStringBuilder() << "Pg type should be encoded as array with size 1, but not " << jsonValue.GetArray().size());
+                        }
+                        auto& innerJsonValue = jsonValue.GetArray().at(0);
+                        EnsureType(innerJsonValue, NJson::JSON_STRING);
+                        auto binary = JsonStringToBinaryString(innerJsonValue.GetString());
+                        ValueBuilder.Pg(TPgValue(false, binary, pgType));
+                    }
+                }
+                break;
+
             case TTypeParser::ETypeKind::Optional:
                 TypeParser.OpenOptional();
                 if (jsonValue.IsNull() && TypeParser.GetKind() != TTypeParser::ETypeKind::Optional) {
diff --git a/ydb/public/lib/json_value/ydb_json_value_ut.cpp b/ydb/public/lib/json_value/ydb_json_value_ut.cpp
index d8c1356338f..d6abdee7aff 100644
--- a/ydb/public/lib/json_value/ydb_json_value_ut.cpp
+++ b/ydb/public/lib/json_value/ydb_json_value_ut.cpp
@@ -557,6 +557,46 @@ Y_UNIT_TEST_SUITE(JsonValueTest) {
             TProtoAccessor::GetProto(resultValue).DebugString()
         );
     }
+
+    Y_UNIT_TEST(PgValue) {
+        TPgType pgType(1, 2, 3);
+        TPgValue v1(true, "text_value", pgType);
+        TPgValue v2(false, "binary_value", pgType);
+        TPgValue v3(true, "", pgType);
+        TPgValue v4(false, "", pgType);
+        TValue value = TValueBuilder()
+            .BeginList()
+            .AddListItem()
+                .Pg(v1)
+            .AddListItem()
+                .Pg(v2)
+            .AddListItem()
+                .Pg(v3)
+            .AddListItem()
+                .Pg(v4)
+            .EndList()
+            .Build();
+
+        // unicode
+        const TString jsonString1 = FormatValueJson(value, EBinaryStringEncoding::Unicode);
+        UNIT_ASSERT_NO_DIFF(jsonString1, R"(["text_value",["binary_value"],"",[""]])");
+
+        TValue resultValue1 = JsonToYdbValue(jsonString1, value.GetType(), EBinaryStringEncoding::Unicode);
+        UNIT_ASSERT_NO_DIFF(
+            TProtoAccessor::GetProto(value).DebugString(),
+            TProtoAccessor::GetProto(resultValue1).DebugString()
+        );
+
+        // base64
+        const TString jsonString2 = FormatValueJson(value, EBinaryStringEncoding::Base64);
+        UNIT_ASSERT_NO_DIFF(jsonString2, R"(["text_value",["YmluYXJ5X3ZhbHVl"],"",[""]])");
+
+        TValue resultValue2 = JsonToYdbValue(jsonString2, value.GetType(), EBinaryStringEncoding::Base64);
+        UNIT_ASSERT_NO_DIFF(
+            TProtoAccessor::GetProto(value).DebugString(),
+            TProtoAccessor::GetProto(resultValue2).DebugString()
+        );
+    }
 }
 
 } // namespace NYdb
diff --git a/ydb/public/lib/yson_value/ydb_yson_value.cpp b/ydb/public/lib/yson_value/ydb_yson_value.cpp
index c855acf0a7a..922d14ae2bf 100644
--- a/ydb/public/lib/yson_value/ydb_yson_value.cpp
+++ b/ydb/public/lib/yson_value/ydb_yson_value.cpp
@@ -98,6 +98,17 @@ static void FormatValueYsonInternal(TValueParser& parser, NYson::TYsonWriter& wr
             writer.OnStringScalar(parser.GetDecimal().ToString());
             break;
 
+        case TTypeParser::ETypeKind::Pg:
+            if (parser.GetPg().IsText()) {
+                writer.OnStringScalar(parser.GetPg().Content_);
+            } else {
+                writer.OnBeginList();
+                writer.OnListItem();
+                writer.OnStringScalar(parser.GetPg().Content_);
+                writer.OnEndList();
+            }
+            break;
+
         case TTypeParser::ETypeKind::Optional:
             parser.OpenOptional();
             if (parser.IsNull()) {
diff --git a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp
index 0048cd3ddf1..e87ec8af1e5 100644
--- a/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp
+++ b/ydb/public/sdk/cpp/client/impl/ydb_internal/value_helpers/helpers.cpp
@@ -14,6 +14,10 @@ bool TypesEqual(const Ydb::Type& t1, const Ydb::Type& t2) {
         case Ydb::Type::kDecimalType:
             return t1.decimal_type().precision() == t2.decimal_type().precision()
                 && t1.decimal_type().scale() == t2.decimal_type().scale();
+        case Ydb::Type::kPgType:
+            return t1.pg_type().oid() == t2.pg_type().oid()
+                && t1.pg_type().typlen() == t2.pg_type().typlen()
+                && t1.pg_type().typmod() == t2.pg_type().typmod();
         case Ydb::Type::kOptionalType:
             return TypesEqual(t1.optional_type().item(), t2.optional_type().item());
         case Ydb::Type::kListType:
diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.cpp b/ydb/public/sdk/cpp/client/ydb_value/value.cpp
index 8c18d4d50f8..a4396b7bec7 100644
--- a/ydb/public/sdk/cpp/client/ydb_value/value.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_value/value.cpp
@@ -36,6 +36,8 @@ static TTypeParser::ETypeKind GetKind(const Ydb::Type& type) {
             return ETypeKind::Primitive;
         case Ydb::Type::kDecimalType:
             return ETypeKind::Decimal;
+        case Ydb::Type::kPgType:
+            return ETypeKind::Pg;
         case Ydb::Type::kOptionalType:
             return ETypeKind::Optional;
         case Ydb::Type::kListType:
@@ -134,6 +136,12 @@ public:
             GetProto().decimal_type().scale());
     }
 
+    TPgType GetPg() const {
+        CheckKind(ETypeKind::Pg, "GetPg");
+        const auto& pg = GetProto().pg_type();
+        return TPgType(pg.oid(), pg.typlen(), pg.typmod());
+    }
+
     template<ETypeKind kind>
     void Open() {
         CheckKind(kind, "Open");
@@ -359,6 +367,10 @@ TDecimalType TTypeParser::GetDecimal() const {
     return Impl_->GetDecimal();
 }
 
+TPgType TTypeParser::GetPg() const {
+    return Impl_->GetPg();
+}
+
 void TTypeParser::OpenOptional() {
     Impl_->Open<ETypeKind::Optional>();
 }
@@ -454,7 +466,12 @@ void FormatTypeInternal(TTypeParser& parser, IOutputStream& out) {
         case TTypeParser::ETypeKind::Decimal: {
             auto decimal = parser.GetDecimal();
             out << "Decimal(" << (ui32)decimal.Precision << ',' << (ui32)decimal.Scale << ")";
-            //out << "Decimal";
+            break;
+        }
+
+        case TTypeParser::ETypeKind::Pg: {
+            auto pg = parser.GetPg();
+            out << "Pg("sv << pg.Oid << ',' << pg.Typlen << ',' << pg.Typmod << ')';
             break;
         }
 
@@ -575,6 +592,13 @@ public:
         decimal.set_scale(decimalType.Scale);
     }
 
+    void Pg(const TPgType& pgType) {
+        auto& pg = *GetProto().mutable_pg_type();
+        pg.set_oid(pgType.Oid);
+        pg.set_typlen(pgType.Typlen);
+        pg.set_typmod(pgType.Typmod);
+    }
+
     void BeginOptional() {
         AddPosition(GetProto().mutable_optional_type()->mutable_item());
     }
@@ -761,6 +785,11 @@ TTypeBuilder& TTypeBuilder::Decimal(const TDecimalType& decimalType) {
     return *this;
 }
 
+TTypeBuilder& TTypeBuilder::Pg(const TPgType& pgType) {
+    Impl_->Pg(pgType);
+    return *this;
+}
+
 TTypeBuilder& TTypeBuilder::BeginOptional() {
     Impl_->BeginOptional();
     return *this;
@@ -886,6 +915,32 @@ TString TDecimalValue::ToString() const {
 
 ////////////////////////////////////////////////////////////////////////////////
 
+TPgValue::TPgValue(const Ydb::Value& pgValueProto, const TPgType& pgType)
+    : PgType_(pgType)
+{
+    IsText_ = pgValueProto.has_text_value();
+    if (IsText_) {
+        Content_ = pgValueProto.text_value();
+    }
+
+    if (pgValueProto.has_bytes_value()) {
+        Content_ = pgValueProto.bytes_value();
+    }
+}
+
+TPgValue::TPgValue(bool isText, const TString& content, const TPgType& pgType)
+    : PgType_(pgType)
+    , IsText_(isText)
+    , Content_(content)
+{
+}
+
+bool TPgValue::IsText() const {
+    return IsText_;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
 class TValue::TImpl {
 public:
     TImpl(const TType& type, const Ydb::Value& valueProto)
@@ -1086,6 +1141,11 @@ public:
         return TDecimalValue(GetProto(), TypeParser_.GetDecimal());
     }
 
+    TPgValue GetPg() const {
+        CheckPg();
+        return TPgValue(GetProto(), TypeParser_.GetPg());
+    }
+
     void OpenOptional() {
         TypeParser_.OpenOptional();
 
@@ -1361,6 +1421,10 @@ private:
         CheckTransportKind(Ydb::Value::kLow128);
     }
 
+    void CheckPg() const {
+        CheckKind(ETypeKind::Pg, "Get");
+    }
+
     const Ydb::Value& GetProto() const {
         return *static_cast<const Ydb::Value*>(GetPathBack().Ptr);
     }
@@ -1559,6 +1623,10 @@ TDecimalValue TValueParser::GetDecimal() const {
     return Impl_->GetDecimal();
 }
 
+TPgValue TValueParser::GetPg() const {
+    return Impl_->GetPg();
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 #define RET_OPT_VALUE(Type, Name) \
@@ -1947,6 +2015,15 @@ public:
         GetValue().set_high_128(value.Hi_);
     }
 
+    void Pg(const TPgValue& value) {
+        FillPgType(value.PgType_);
+        if (value.IsText()) {
+            GetValue().set_text_value(value.Content_);
+        } else {
+            GetValue().set_bytes_value(value.Content_);
+        }
+    }
+
     void BeginOptional() {
         SetBuildType(!CheckType(ETypeKind::Optional));
 
@@ -2319,6 +2396,12 @@ private:
         }
     }
 
+    void FillPgType(const TPgType& type) {
+        if (!CheckPgType()) {
+            TypeBuilder_.Pg(type);
+        }
+    }
+
     bool CheckType() {
         if (!GetType().type_case()) {
             return false;
@@ -2372,11 +2455,11 @@ private:
     }
 
     bool CheckDecimalType() {
-        if (!CheckType(ETypeKind::Decimal)) {
-            return false;
-        }
+        return CheckType(ETypeKind::Decimal);
+    }
 
-        return true;
+    bool CheckPgType() {
+        return CheckType(ETypeKind::Pg);
     }
 
     void CheckContainerKind(ETypeKind kind) {
@@ -2632,6 +2715,12 @@ TDerived& TValueBuilderBase<TDerived>::Decimal(const TDecimalValue& value) {
     return static_cast<TDerived&>(*this);
 }
 
+template<typename TDerived>
+TDerived& TValueBuilderBase<TDerived>::Pg(const TPgValue& value) {
+    Impl_->Pg(value);
+    return static_cast<TDerived&>(*this);
+}
+
 #define SET_OPT_VALUE_MAYBE(Name) \
     if (value) { \
         Impl_->BeginOptional(); \
diff --git a/ydb/public/sdk/cpp/client/ydb_value/value.h b/ydb/public/sdk/cpp/client/ydb_value/value.h
index 4a163b92206..363c672285f 100644
--- a/ydb/public/sdk/cpp/client/ydb_value/value.h
+++ b/ydb/public/sdk/cpp/client/ydb_value/value.h
@@ -69,6 +69,18 @@ struct TDecimalType {
         , Scale(scale) {}
 };
 
+struct TPgType {
+    ui32 Oid;
+    i32 Typlen;
+    i32 Typmod;
+
+    TPgType(ui32 oid, i32 typlen, i32 typmod)
+        : Oid(oid)
+        , Typlen(typlen)
+        , Typmod(typmod)
+    {}
+};
+
 //! Types can be complex, so TTypeParser allows to traverse through this hierarchies.
 class TTypeParser : public TMoveOnly {
     friend class TValueParser;
@@ -86,7 +98,8 @@ public:
         Null,
         EmptyList,
         EmptyDict,
-        Tagged
+        Tagged,
+        Pg
     };
 
 public:
@@ -99,6 +112,7 @@ public:
 
     EPrimitiveType GetPrimitive() const;
     TDecimalType GetDecimal() const;
+    TPgType GetPg() const;
 
     // Optional
     void OpenOptional();
@@ -157,6 +171,7 @@ public:
 
     TTypeBuilder& Primitive(const EPrimitiveType& primitiveType);
     TTypeBuilder& Decimal(const TDecimalType& decimalType);
+    TTypeBuilder& Pg(const TPgType& pgType);
 
     // Optional
     TTypeBuilder& BeginOptional();
@@ -205,6 +220,16 @@ struct TDecimalValue {
     i64 Hi_;
 };
 
+struct TPgValue {
+    TPgValue(const Ydb::Value& pgValueProto, const TPgType& pgType);
+    TPgValue(bool isText, const TString& content, const TPgType& pgType);
+    bool IsText() const;
+
+    TPgType PgType_;
+    bool IsText_;
+    TString Content_;
+};
+
 //! Representation of YDB value.
 class TValue {
     friend class TValueParser;
@@ -257,6 +282,7 @@ public:
     const TString& GetYson() const;
     const TString& GetJson() const;
     TDecimalValue GetDecimal() const;
+    TPgValue GetPg() const;
     const TString& GetJsonDocument() const;
     const TString& GetDyNumber() const;
 
@@ -360,6 +386,7 @@ public:
     TDerived& Yson(const TString& value);
     TDerived& Json(const TString& value);
     TDerived& Decimal(const TDecimalValue& value);
+    TDerived& Pg(const TPgValue& value);
     TDerived& JsonDocument(const TString& value);
     TDerived& DyNumber(const TString& value);
 
diff --git a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp
index d7508a5f9b0..82fa017fc59 100644
--- a/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp
+++ b/ydb/public/sdk/cpp/client/ydb_value/value_ut.cpp
@@ -109,6 +109,8 @@ Y_UNIT_TEST_SUITE(YdbValue) {
                     .AddElement()
                         .Decimal(TDecimalType(8, 13))
                     .AddElement()
+                        .Pg(TPgType(1, 2, -3))
+                    .AddElement()
                         .BeginOptional()
                             .Primitive(EPrimitiveType::Utf8)
                         .EndOptional()
@@ -118,7 +120,7 @@ Y_UNIT_TEST_SUITE(YdbValue) {
             .Build();
 
         UNIT_ASSERT_NO_DIFF(FormatType(type),
-            R"(Struct<'Member1':List<Uint32?>,'Member2':Dict<Int64,Tuple<Decimal(8,13),Utf8?>>>)");
+            R"(Struct<'Member1':List<Uint32?>,'Member2':Dict<Int64,Tuple<Decimal(8,13),Pg(1,2,-3),Utf8?>>>)");
     }
 
     Y_UNIT_TEST(BuildTypeReuse) {
@@ -319,6 +321,63 @@ Y_UNIT_TEST_SUITE(YdbValue) {
             R"([[10,{"Member1":"1972-09-27"}],[20,{"Member1":"1975-06-24"}]])");
     }
 
+    Y_UNIT_TEST(ParseValuePg) {
+        auto protoTypeStr = R"(
+            struct_type {
+                members {
+                    name: "A"
+                    type {
+                        pg_type {
+                            oid: 123
+                            typlen: 345
+                            typmod: -321
+                        }
+                    }
+                }
+                members {
+                    name: "B"
+                    type {
+                        pg_type {
+                            oid: 123
+                            typlen: -345
+                            typmod: 321
+                        }
+                    }
+                }
+            }
+            )";
+
+        auto protoValueStr = R"(
+            items {
+                text_value: "my_text_value"
+            }
+            items {
+                bytes_value: "my_binary_value"
+            }
+            items {
+                text_value: ""
+            }
+            items {
+                bytes_value: ""
+            }
+        )";
+
+        Ydb::Type protoType;
+        NProtoBuf::TextFormat::ParseFromString(protoTypeStr, &protoType);
+
+        Ydb::Value protoValue;
+        NProtoBuf::TextFormat::ParseFromString(protoValueStr, &protoValue);
+
+        TValue value(TType(protoType), protoValue);
+
+        UNIT_ASSERT_NO_DIFF(FormatValueYson(value),
+            R"(["my_text_value";["my_binary_value"]])");
+        UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Unicode),
+            R"({"A":"my_text_value","B":["my_binary_value"]})");
+        UNIT_ASSERT_NO_DIFF(FormatValueJson(value, EBinaryStringEncoding::Base64),
+            R"({"A":"my_text_value","B":["bXlfYmluYXJ5X3ZhbHVl"]})");
+    }
+
     Y_UNIT_TEST(ParseValueMaybe) {
         auto protoTypeStr = R"(
             tuple_type {