diff options
| author | imunkin <[email protected]> | 2024-11-08 10:00:23 +0300 |
|---|---|---|
| committer | imunkin <[email protected]> | 2024-11-08 10:12:13 +0300 |
| commit | a784a2f943d6e15caa6241e2e96d80aac6dbf375 (patch) | |
| tree | 05f1e5366c916b988a8afb75bdab8ddeee0f6e6d /yql/essentials/udfs/common/json2 | |
| parent | d70137a7b530ccaa52834274913bbb5a3d1ca06e (diff) | |
Move yql/udfs/common/ to /yql/essentials YQL-19206
Except the following directories:
* clickhouse/client
* datetime
* knn
* roaring
commit_hash:c7da95636144d28db109d6b17ddc762e9bacb59f
Diffstat (limited to 'yql/essentials/udfs/common/json2')
30 files changed, 3873 insertions, 0 deletions
diff --git a/yql/essentials/udfs/common/json2/as_json_node.h b/yql/essentials/udfs/common/json2/as_json_node.h new file mode 100644 index 00000000000..c7463fffa66 --- /dev/null +++ b/yql/essentials/udfs/common/json2/as_json_node.h @@ -0,0 +1,115 @@ +#pragma once + +#include "resource.h" + +#include <yql/essentials/public/udf/udf_value.h> +#include <yql/essentials/public/udf/udf_helpers.h> +#include <yql/essentials/minikql/dom/node.h> +#include <yql/essentials/minikql/dom/json.h> + +namespace NJson2Udf { + using namespace NKikimr; + using namespace NUdf; + using namespace NYql; + using namespace NDom; + + template <typename TSource> + class TAsJsonNode: public TBoxedValue { + public: + TAsJsonNode(TSourcePosition pos) + : Pos_(pos) + { + } + + static TStringRef Name(); + + static bool DeclareSignature( + const TStringRef& name, + TType* userType, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) { + Y_UNUSED(userType); + if (name != Name()) { + return false; + } + + auto optionalSourceType = builder.Optional()->Item<TSource>().Build(); + auto resourceType = builder.Resource(JSON_NODE_RESOURCE_NAME); + builder.Args() + ->Add(optionalSourceType) + .Done() + .Returns(resourceType); + + if (!typesOnly) { + builder.Implementation(new TAsJsonNode<TSource>(builder.GetSourcePosition())); + } + + builder.IsStrict(); + return true; + } + + private: + const size_t MaxParseErrors = 10; + + static TUnboxedValue Interpret(const TUnboxedValue& sourceValue, const IValueBuilder* valueBuilder); + + TUnboxedValue Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const final { + Y_UNUSED(valueBuilder); + try { + if (!args[0].HasValue()) { + return MakeEntity(); + } + return Interpret(args[0], valueBuilder); + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } + } + + TSourcePosition Pos_; + }; + + template <> + TStringRef TAsJsonNode<TUtf8>::Name() { + return TStringRef::Of("Utf8AsJsonNode"); + } + + template <> + TUnboxedValue TAsJsonNode<TUtf8>::Interpret(const TUnboxedValue& sourceValue, const IValueBuilder* valueBuilder) { + return MakeString(sourceValue.AsStringRef(), valueBuilder); + } + + template <> + TStringRef TAsJsonNode<double>::Name() { + return TStringRef::Of("DoubleAsJsonNode"); + } + + template <> + TUnboxedValue TAsJsonNode<double>::Interpret(const TUnboxedValue& sourceValue, const IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + return MakeDouble(sourceValue.Get<double>()); + } + + template <> + TStringRef TAsJsonNode<bool>::Name() { + return TStringRef::Of("BoolAsJsonNode"); + } + + template <> + TUnboxedValue TAsJsonNode<bool>::Interpret(const TUnboxedValue& sourceValue, const IValueBuilder* valueBuilder) { + Y_UNUSED(valueBuilder); + return MakeBool(sourceValue.Get<bool>()); + } + + template <> + TStringRef TAsJsonNode<TJson>::Name() { + return TStringRef::Of("JsonAsJsonNode"); + } + + template <> + TUnboxedValue TAsJsonNode<TJson>::Interpret(const TUnboxedValue& sourceValue, const IValueBuilder* valueBuilder) { + return TryParseJsonDom(sourceValue.AsStringRef(), valueBuilder); + } +} + diff --git a/yql/essentials/udfs/common/json2/compile_path.h b/yql/essentials/udfs/common/json2/compile_path.h new file mode 100644 index 00000000000..8239cfc1eee --- /dev/null +++ b/yql/essentials/udfs/common/json2/compile_path.h @@ -0,0 +1,70 @@ +#pragma once + +#include "resource.h" + +#include <yql/essentials/public/udf/udf_value.h> +#include <yql/essentials/public/udf/udf_helpers.h> + +namespace NJson2Udf { + using namespace NKikimr; + using namespace NUdf; + using namespace NYql; + + class TCompilePath: public TBoxedValue { + public: + TCompilePath(TSourcePosition pos) + : Pos_(pos) + { + } + + static const TStringRef& Name() { + static auto name = TStringRef::Of("CompilePath"); + return name; + } + + static bool DeclareSignature( + const TStringRef& name, + TType* userType, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) { + Y_UNUSED(userType); + if (name != Name()) { + return false; + } + + auto resourceType = builder.Resource(JSONPATH_RESOURCE_NAME); + builder.Args() + ->Add<NUdf::TUtf8>() + .Done() + .Returns(resourceType); + + if (!typesOnly) { + builder.Implementation(new TCompilePath(builder.GetSourcePosition())); + } + return true; + } + + private: + const size_t MaxParseErrors = 10; + + TUnboxedValue Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const final { + Y_UNUSED(valueBuilder); + try { + TIssues issues; + const auto jsonPath = NJsonPath::ParseJsonPath(args[0].AsStringRef(), issues, MaxParseErrors); + if (!issues.Empty()) { + ythrow yexception() << "Error parsing jsonpath:" << Endl << issues.ToString(); + } + + return TUnboxedValuePod(new TJsonPathResource(jsonPath)); + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } + } + + TSourcePosition Pos_; + }; +} + diff --git a/yql/essentials/udfs/common/json2/json2_udf.cpp b/yql/essentials/udfs/common/json2/json2_udf.cpp new file mode 100644 index 00000000000..96ef6ccf00b --- /dev/null +++ b/yql/essentials/udfs/common/json2/json2_udf.cpp @@ -0,0 +1,43 @@ +#include "as_json_node.h" +#include "compile_path.h" +#include "parse.h" +#include "serialize.h" +#include "sql_exists.h" +#include "sql_query.h" +#include "sql_value.h" + +#include <yql/essentials/public/udf/udf_helpers.h> + +namespace NJson2Udf { + SIMPLE_MODULE(TJson2Module, + TParse, + TSerialize<EDataSlot::Json>, + TSerialize<EDataSlot::JsonDocument>, + TCompilePath, + TSqlValue<EDataSlot::Json, TUtf8>, + TSqlValue<EDataSlot::Json, TUtf8, true>, + TSqlValue<EDataSlot::Json, i64>, + TSqlValue<EDataSlot::Json, double>, + TSqlValue<EDataSlot::Json, bool>, + TSqlValue<EDataSlot::JsonDocument, TUtf8>, + TSqlValue<EDataSlot::JsonDocument, TUtf8, true>, + TSqlValue<EDataSlot::JsonDocument, i64>, + TSqlValue<EDataSlot::JsonDocument, double>, + TSqlValue<EDataSlot::JsonDocument, bool>, + TSqlExists<EDataSlot::Json, false>, + TSqlExists<EDataSlot::Json, true>, + TSqlExists<EDataSlot::JsonDocument, false>, + TSqlExists<EDataSlot::JsonDocument, true>, + TSqlQuery<EDataSlot::Json, EJsonQueryWrap::NoWrap>, + TSqlQuery<EDataSlot::Json, EJsonQueryWrap::Wrap>, + TSqlQuery<EDataSlot::Json, EJsonQueryWrap::ConditionalWrap>, + TSqlQuery<EDataSlot::JsonDocument, EJsonQueryWrap::NoWrap>, + TSqlQuery<EDataSlot::JsonDocument, EJsonQueryWrap::Wrap>, + TSqlQuery<EDataSlot::JsonDocument, EJsonQueryWrap::ConditionalWrap>, + TAsJsonNode<TUtf8>, + TAsJsonNode<double>, + TAsJsonNode<bool>, + TAsJsonNode<TJson>) +} + +REGISTER_MODULES(NJson2Udf::TJson2Module) diff --git a/yql/essentials/udfs/common/json2/parse.h b/yql/essentials/udfs/common/json2/parse.h new file mode 100644 index 00000000000..0020c164c2b --- /dev/null +++ b/yql/essentials/udfs/common/json2/parse.h @@ -0,0 +1,66 @@ +#pragma once + +#include "resource.h" + +#include <yql/essentials/public/udf/udf_value.h> +#include <yql/essentials/public/udf/udf_helpers.h> +#include <yql/essentials/minikql/dom/json.h> + +#include <library/cpp/json/json_reader.h> + +namespace NJson2Udf { + using namespace NKikimr; + using namespace NUdf; + using namespace NYql; + using namespace NDom; + + class TParse: public TBoxedValue { + public: + TParse(TSourcePosition pos) + : Pos_(pos) + { + } + + static const TStringRef& Name() { + static auto name = TStringRef::Of("Parse"); + return name; + } + + static bool DeclareSignature( + const TStringRef& name, + TType* userType, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) { + Y_UNUSED(userType); + if (name != Name()) { + return false; + } + + builder.Args() + ->Add<TAutoMap<TJson>>() + .Done() + .Returns<TJsonNodeResource>(); + + if (!typesOnly) { + builder.Implementation(new TParse(builder.GetSourcePosition())); + } + return true; + } + + private: + TUnboxedValue Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const final { + Y_UNUSED(valueBuilder); + try { + const auto json = args[0].AsStringRef(); + return TryParseJsonDom(json, valueBuilder); + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } + } + + TSourcePosition Pos_; + }; +} + diff --git a/yql/essentials/udfs/common/json2/resource.h b/yql/essentials/udfs/common/json2/resource.h new file mode 100644 index 00000000000..aa65b14818d --- /dev/null +++ b/yql/essentials/udfs/common/json2/resource.h @@ -0,0 +1,17 @@ +#pragma once + +#include <yql/essentials/public/udf/udf_value.h> +#include <yql/essentials/minikql/jsonpath/jsonpath.h> + +namespace NJson2Udf { + using namespace NKikimr; + using namespace NUdf; + using namespace NYql; + + extern const char JSONPATH_RESOURCE_NAME[] = "JsonPath"; + using TJsonPathResource = TBoxedResource<NJsonPath::TJsonPathPtr, JSONPATH_RESOURCE_NAME>; + + extern const char JSON_NODE_RESOURCE_NAME[] = "JsonNode"; + using TJsonNodeResource = TResource<JSON_NODE_RESOURCE_NAME>; +} + diff --git a/yql/essentials/udfs/common/json2/serialize.h b/yql/essentials/udfs/common/json2/serialize.h new file mode 100644 index 00000000000..a7077cb6e6d --- /dev/null +++ b/yql/essentials/udfs/common/json2/serialize.h @@ -0,0 +1,89 @@ +#pragma once + +#include "resource.h" + +#include <yql/essentials/public/udf/udf_value.h> +#include <yql/essentials/public/udf/udf_helpers.h> +#include <yql/essentials/minikql/dom/json.h> + +#include <yql/essentials/types/binary_json/write.h> + +namespace NJson2Udf { + using namespace NKikimr; + using namespace NUdf; + using namespace NYql; + using namespace NDom; + using namespace NBinaryJson; + + template <EDataSlot ResultType> + class TSerialize : public TBoxedValue { + public: + TSerialize(TSourcePosition pos) + : Pos_(pos) + { + } + + static const TStringRef& Name(); + + static bool DeclareSignature( + const TStringRef& name, + TType* userType, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) { + Y_UNUSED(userType); + if (name != Name()) { + return false; + } + + TType* resultType = nullptr; + if constexpr (ResultType == EDataSlot::Json) { + resultType = builder.SimpleType<TJson>(); + } else { + resultType = builder.SimpleType<TJsonDocument>(); + } + + builder.Args() + ->Add<TAutoMap<TJsonNodeResource>>() + .Done() + .Returns(resultType); + + if (!typesOnly) { + builder.Implementation(new TSerialize(builder.GetSourcePosition())); + } + return true; + } + + private: + TUnboxedValue Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const final { + try { + const TUnboxedValue& jsonDom = args[0]; + + if constexpr (ResultType == EDataSlot::Json) { + return valueBuilder->NewString(SerializeJsonDom(jsonDom)); + } else { + const auto binaryJson = SerializeToBinaryJson(jsonDom); + return valueBuilder->NewString(TStringBuf(binaryJson.Data(), binaryJson.Size())); + } + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } + } + + TSourcePosition Pos_; + }; + + template <> + const TStringRef& TSerialize<EDataSlot::Json>::Name() { + static auto name = TStringRef::Of("Serialize"); + return name; + } + + template <> + const TStringRef& TSerialize<EDataSlot::JsonDocument>::Name() { + static auto name = TStringRef::Of("SerializeToJsonDocument"); + return name; + } +} + diff --git a/yql/essentials/udfs/common/json2/sql_exists.h b/yql/essentials/udfs/common/json2/sql_exists.h new file mode 100644 index 00000000000..8a049b49d42 --- /dev/null +++ b/yql/essentials/udfs/common/json2/sql_exists.h @@ -0,0 +1,135 @@ +#pragma once + +#include "resource.h" +#include "compile_path.h" + +#include <yql/essentials/public/udf/udf_type_builder.h> +#include <yql/essentials/public/udf/udf_value.h> +#include <yql/essentials/public/udf/udf_helpers.h> + +#include <util/generic/yexception.h> + +namespace NJson2Udf { + using namespace NKikimr; + using namespace NUdf; + using namespace NYql; + using namespace NJsonPath; + + template <EDataSlot InputType, bool ThrowException> + class TSqlExists: public TBoxedValue { + public: + explicit TSqlExists(TSourcePosition pos) + : Pos_(pos) + { + } + + static TStringRef Name(); + + static bool DeclareSignature( + const TStringRef& name, + TType* userType, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) { + Y_UNUSED(userType); + if (name != Name()) { + return false; + } + + auto jsonType = builder.Resource(JSON_NODE_RESOURCE_NAME); + TType* inputType = nullptr; + if constexpr (InputType == EDataSlot::JsonDocument) { + inputType = builder.SimpleType<TJsonDocument>(); + } else { + inputType = jsonType; + } + auto inputOptionalType = builder.Optional()->Item(inputType).Build(); + auto jsonPathType = builder.Resource(JSONPATH_RESOURCE_NAME); + auto dictType = builder.Dict()->Key<TUtf8>().Value(jsonType).Build(); + auto optionalBoolType = builder.Optional()->Item<bool>().Build(); + + if constexpr (ThrowException) { + builder.Args() + ->Add(inputOptionalType) + .Add(jsonPathType) + .Add(dictType) + .Done() + .Returns(optionalBoolType); + } else { + builder.Args() + ->Add(inputOptionalType) + .Add(jsonPathType) + .Add(dictType) + .Add(optionalBoolType) + .Done() + .Returns(optionalBoolType); + } + + if (!typesOnly) { + builder.Implementation(new TSqlExists(builder.GetSourcePosition())); + } + if constexpr (!ThrowException) { + builder.IsStrict(); + } + return true; + } + + private: + TUnboxedValue Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const final { + Y_UNUSED(valueBuilder); + try { + if (!args[0].HasValue()) { + return TUnboxedValuePod(); + } + + TValue jsonDom; + if constexpr (InputType == EDataSlot::JsonDocument) { + jsonDom = TValue(NBinaryJson::TBinaryJsonReader::Make(args[0].AsStringRef())->GetRootCursor()); + } else { + jsonDom = TValue(args[0]); + } + + auto* jsonPathResource = static_cast<TJsonPathResource*>(args[1].AsBoxed().Get()); + const auto& jsonPath = *jsonPathResource->Get(); + const auto variables = DictToVariables(args[2]); + + const auto result = ExecuteJsonPath(jsonPath, jsonDom, variables, valueBuilder); + if (result.IsError()) { + if constexpr (ThrowException) { + ythrow yexception() << "Error executing jsonpath:" << Endl << result.GetError() << Endl; + } else { + return args[3]; + } + } + + return TUnboxedValuePod(!result.GetNodes().empty()); + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } + } + + TSourcePosition Pos_; + }; + + template <> + TStringRef TSqlExists<EDataSlot::Json, false>::Name() { + return "SqlExists"; + } + + template <> + TStringRef TSqlExists<EDataSlot::Json, true>::Name() { + return "SqlTryExists"; + } + + template <> + TStringRef TSqlExists<EDataSlot::JsonDocument, false>::Name() { + return "JsonDocumentSqlExists"; + } + + template <> + TStringRef TSqlExists<EDataSlot::JsonDocument, true>::Name() { + return "JsonDocumentSqlTryExists"; + } +} + diff --git a/yql/essentials/udfs/common/json2/sql_query.h b/yql/essentials/udfs/common/json2/sql_query.h new file mode 100644 index 00000000000..cb3bafd3b0b --- /dev/null +++ b/yql/essentials/udfs/common/json2/sql_query.h @@ -0,0 +1,184 @@ +#pragma once + +#include "resource.h" +#include "compile_path.h" + +#include <yql/essentials/core/sql_types/yql_atom_enums.h> +#include <yql/essentials/public/udf/udf_type_builder.h> +#include <yql/essentials/public/udf/udf_value.h> +#include <yql/essentials/public/udf/udf_helpers.h> +#include <yql/essentials/minikql/dom/node.h> + +#include <util/generic/yexception.h> + +namespace NJson2Udf { + using namespace NKikimr; + using namespace NUdf; + using namespace NYql; + using namespace NDom; + using namespace NJsonPath; + + template <EDataSlot InputType, EJsonQueryWrap Mode> + class TSqlQuery: public TBoxedValue { + public: + explicit TSqlQuery(TSourcePosition pos) + : Pos_(pos) + { + } + + static TStringRef Name(); + + static bool DeclareSignature( + const TStringRef& name, + TType* userType, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) { + Y_UNUSED(userType); + if (name != Name()) { + return false; + } + + auto jsonType = builder.Resource(JSON_NODE_RESOURCE_NAME); + auto optionalJsonType = builder.Optional()->Item(jsonType).Build(); + TType* inputType = nullptr; + if constexpr (InputType == EDataSlot::JsonDocument) { + inputType = builder.SimpleType<TJsonDocument>(); + } else { + inputType = jsonType; + } + auto inputOptionalType = builder.Optional()->Item(inputType).Build(); + auto jsonPathType = builder.Resource(JSONPATH_RESOURCE_NAME); + auto dictType = builder.Dict()->Key<TUtf8>().Value(jsonType).Build(); + + /* + Arguments: + 0. Resource<JsonNode>? or JsonDocument?. Input json + 1. Resource<JsonPath>. Jsonpath to execute on json + 2. Dict<TUtf8, Resource<JsonNode>>. Variables to pass into jsonpath + 3. Bool. True - throw on empty result, false otherwise + 4. Resource<JsonNode>?. Default value to return on empty result. Ignored if 2d argument is true + 5. Bool. True - throw on error, false - otherwise + 6. Resource<JsonNode>?. Default value to return on error. Ignored if 4th argument is true + */ + // we can't mark TSqlQuery as strict due to runtime throw policy setting + // TODO: optimizer can mark SqlQuery as strict if 3th/5th arguments are literal booleans + builder.Args() + ->Add(inputOptionalType) + .Add(jsonPathType) + .Add(dictType) + .Add<bool>() + .Add(optionalJsonType) + .Add<bool>() + .Add(optionalJsonType) + .Done() + .Returns(optionalJsonType); + + if (!typesOnly) { + builder.Implementation(new TSqlQuery(builder.GetSourcePosition())); + } + return true; + } + + private: + TUnboxedValue Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const final { + Y_UNUSED(valueBuilder); + try { + if (!args[0].HasValue()) { + return TUnboxedValuePod(); + } + + TValue jsonDom; + if constexpr (InputType == EDataSlot::JsonDocument) { + jsonDom = TValue(NBinaryJson::TBinaryJsonReader::Make(args[0].AsStringRef())->GetRootCursor()); + } else { + jsonDom = TValue(args[0]); + } + + auto* jsonPathResource = static_cast<TJsonPathResource*>(args[1].AsBoxed().Get()); + const auto& jsonPath = *jsonPathResource->Get(); + + const bool throwOnEmpty = args[3].Get<bool>(); + const auto emptyDefault = args[4]; + const bool throwOnError = args[5].Get<bool>(); + const auto errorDefault = args[6]; + const auto variables = DictToVariables(args[2]); + + auto result = ExecuteJsonPath(jsonPath, jsonDom, variables, valueBuilder); + + const auto handleCase = [](TStringBuf message, bool throws, const TUnboxedValuePod& caseDefault) { + if (throws) { + ythrow yexception() << message; + } + return caseDefault; + }; + + if (result.IsError()) { + return handleCase(TStringBuilder() << "Error executing jsonpath:" << Endl << result.GetError() << Endl, throwOnError, errorDefault); + } + + auto& nodes = result.GetNodes(); + const bool isSingleStruct = nodes.size() == 1 && (nodes[0].Is(EValueType::Array) || nodes[0].Is(EValueType::Object)); + if (Mode == EJsonQueryWrap::Wrap || (Mode == EJsonQueryWrap::ConditionalWrap && !isSingleStruct)) { + TVector<TUnboxedValue> converted; + converted.reserve(nodes.size()); + for (auto& node : nodes) { + converted.push_back(node.ConvertToUnboxedValue(valueBuilder)); + } + return MakeList(converted.data(), converted.size(), valueBuilder); + } + + if (nodes.empty()) { + return handleCase("Empty result", throwOnEmpty, emptyDefault); + } + + // No wrapping is applicable and result is not empty. Result must be a single object or array + if (nodes.size() > 1) { + return handleCase("Result consists of multiple items", throwOnError, errorDefault); + } + + if (!nodes[0].Is(EValueType::Array) && !nodes[0].Is(EValueType::Object)) { + return handleCase("Result is neither object nor array", throwOnError, errorDefault); + } + + return nodes[0].ConvertToUnboxedValue(valueBuilder); + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } + } + + TSourcePosition Pos_; + }; + + template <> + TStringRef TSqlQuery<EDataSlot::Json, EJsonQueryWrap::NoWrap>::Name() { + return "SqlQuery"; + } + + template <> + TStringRef TSqlQuery<EDataSlot::Json, EJsonQueryWrap::Wrap>::Name() { + return "SqlQueryWrap"; + } + + template <> + TStringRef TSqlQuery<EDataSlot::Json, EJsonQueryWrap::ConditionalWrap>::Name() { + return "SqlQueryConditionalWrap"; + } + + template <> + TStringRef TSqlQuery<EDataSlot::JsonDocument, EJsonQueryWrap::NoWrap>::Name() { + return "JsonDocumentSqlQuery"; + } + + template <> + TStringRef TSqlQuery<EDataSlot::JsonDocument, EJsonQueryWrap::Wrap>::Name() { + return "JsonDocumentSqlQueryWrap"; + } + + template <> + TStringRef TSqlQuery<EDataSlot::JsonDocument, EJsonQueryWrap::ConditionalWrap>::Name() { + return "JsonDocumentSqlQueryConditionalWrap"; + } +} + diff --git a/yql/essentials/udfs/common/json2/sql_value.h b/yql/essentials/udfs/common/json2/sql_value.h new file mode 100644 index 00000000000..8d3318a8c54 --- /dev/null +++ b/yql/essentials/udfs/common/json2/sql_value.h @@ -0,0 +1,296 @@ +#pragma once + +#include "resource.h" +#include "compile_path.h" + +#include <yql/essentials/public/udf/udf_type_builder.h> +#include <yql/essentials/public/udf/udf_value.h> +#include <yql/essentials/public/udf/udf_helpers.h> +#include <yql/essentials/minikql/dom/node.h> + +#include <yql/essentials/types/binary_json/read.h> + +#include <util/generic/yexception.h> +#include <util/generic/ylimits.h> +#include <util/string/cast.h> + +namespace NJson2Udf { + using namespace NKikimr; + using namespace NUdf; + using namespace NYql; + using namespace NDom; + using namespace NJsonPath; + + namespace { + template <class TValueType, bool ForceConvert = false> + TUnboxedValue TryConvertJson(const IValueBuilder* valueBuilder, const TUnboxedValue& source) { + Y_UNUSED(valueBuilder); + Y_UNUSED(source); + Y_ABORT("Unsupported type"); + } + + template <> + TUnboxedValue TryConvertJson<TUtf8>(const IValueBuilder* valueBuilder, const TUnboxedValue& source) { + Y_UNUSED(valueBuilder); + if (IsNodeType(source, ENodeType::String)) { + return source; + } + return {}; + } + + template <> + TUnboxedValue TryConvertJson<TUtf8, true>(const IValueBuilder* valueBuilder, const TUnboxedValue& source) { + switch (GetNodeType(source)) { + case ENodeType::String: + return source; + case ENodeType::Uint64: + return valueBuilder->NewString(ToString(source.Get<ui64>())).Release(); + case ENodeType::Int64: + return valueBuilder->NewString(ToString(source.Get<i64>())).Release(); + case ENodeType::Bool: + return source.Get<bool>() ? TUnboxedValuePod::Embedded("true") : TUnboxedValuePod::Embedded("false"); + case ENodeType::Double: + return valueBuilder->NewString(ToString(source.Get<double>())).Release(); + case ENodeType::Entity: + return TUnboxedValuePod::Embedded("null"); + case ENodeType::List: + case ENodeType::Dict: + case ENodeType::Attr: + return {}; + } + } + + template <> + TUnboxedValue TryConvertJson<i64>(const IValueBuilder* valueBuilder, const TUnboxedValue& source) { + Y_UNUSED(valueBuilder); + if (!source.IsEmbedded()) { + return {}; + } + + if (IsNodeType(source, ENodeType::Int64)) { + return TUnboxedValuePod(source.Get<i64>()); + } else if (IsNodeType(source, ENodeType::Uint64) && source.Get<ui64>() < Max<i64>()) { + return TUnboxedValuePod(static_cast<i64>(source.Get<ui64>())); + } else if (IsNodeType(source, ENodeType::Double) && static_cast<i64>(source.Get<double>()) == source.Get<double>()) { + return TUnboxedValuePod(static_cast<i64>(source.Get<double>())); + } + + return {}; + } + + template <> + TUnboxedValue TryConvertJson<double>(const IValueBuilder* valueBuilder, const TUnboxedValue& source) { + Y_UNUSED(valueBuilder); + if (!source.IsEmbedded()) { + return {}; + } + + if (IsNodeType(source, ENodeType::Double)) { + return TUnboxedValuePod(source.Get<double>()); + } else if (IsNodeType(source, ENodeType::Int64)) { + return TUnboxedValuePod(static_cast<double>(source.Get<i64>())); + } else if (IsNodeType(source, ENodeType::Uint64)) { + return TUnboxedValuePod(static_cast<double>(source.Get<ui64>())); + } + + return {}; + } + + template <> + TUnboxedValue TryConvertJson<bool>(const IValueBuilder* valueBuilder, const TUnboxedValue& source) { + Y_UNUSED(valueBuilder); + if (!source.IsEmbedded() || !IsNodeType(source, ENodeType::Bool)) { + return {}; + } + return {TUnboxedValuePod(source.Get<bool>())}; + } + } + + template <EDataSlot InputType, class TValueType, bool ForceConvert = false> + class TSqlValue: public TBoxedValue { + public: + enum class TErrorCode : ui8 { + Empty = 0, + Error = 1 + }; + + TSqlValue(TSourcePosition pos) + : Pos_(pos) + { + } + + static TStringRef Name(); + + static bool DeclareSignature( + const TStringRef& name, + TType* userType, + IFunctionTypeInfoBuilder& builder, + bool typesOnly) { + Y_UNUSED(userType); + if (name != Name()) { + return false; + } + + auto optionalValueType = builder.Optional()->Item<TValueType>().Build(); + auto errorTupleType = builder.Tuple(2)->Add<ui8>().Add<char*>().Build(); + auto returnTypeTuple = builder.Tuple(2) + ->Add(errorTupleType) + .Add(optionalValueType) + .Build(); + auto returnType = builder.Variant()->Over(returnTypeTuple).Build(); + + TType* jsonType = nullptr; + if constexpr (InputType == EDataSlot::Json) { + jsonType = builder.Resource(JSON_NODE_RESOURCE_NAME); + } else { + jsonType = builder.SimpleType<TJsonDocument>(); + } + auto optionalJsonType = builder.Optional()->Item(jsonType).Build(); + auto jsonPathType = builder.Resource(JSONPATH_RESOURCE_NAME); + auto dictType = builder.Dict()->Key<TUtf8>().Value(builder.Resource(JSON_NODE_RESOURCE_NAME)).Build(); + + builder.Args() + ->Add(optionalJsonType) + .Add(jsonPathType) + .Add(dictType) + .Done() + .Returns(returnType); + + builder.IsStrict(); + + if (!typesOnly) { + builder.Implementation(new TSqlValue(builder.GetSourcePosition())); + } + return true; + } + + private: + TUnboxedValue BuildErrorResult(const IValueBuilder* valueBuilder, TErrorCode code, const TStringBuf message) const { + TUnboxedValue* items = nullptr; + auto errorTuple = valueBuilder->NewArray(2, items); + items[0] = TUnboxedValuePod(static_cast<ui8>(code)); + items[1] = valueBuilder->NewString(message); + return valueBuilder->NewVariant(0, std::move(errorTuple)); + } + + TUnboxedValue BuildSuccessfulResult(const IValueBuilder* valueBuilder, TUnboxedValue&& value) const { + return valueBuilder->NewVariant(1, std::move(value)); + } + + TUnboxedValue Run( + const IValueBuilder* valueBuilder, + const TUnboxedValuePod* args) const final { + try { + if (!args[0].HasValue()) { + return BuildSuccessfulResult(valueBuilder, TUnboxedValuePod()); + } + + TValue jsonDom; + if constexpr (InputType == EDataSlot::JsonDocument) { + jsonDom = TValue(NBinaryJson::TBinaryJsonReader::Make(args[0].AsStringRef())->GetRootCursor()); + } else { + jsonDom = TValue(args[0]); + } + + auto* jsonPathResource = static_cast<TJsonPathResource*>(args[1].AsBoxed().Get()); + const auto& jsonPath = *jsonPathResource->Get(); + const auto variables = DictToVariables(args[2]); + + const auto result = ExecuteJsonPath(jsonPath, jsonDom, variables, valueBuilder); + + if (result.IsError()) { + return BuildErrorResult(valueBuilder, TErrorCode::Error, TStringBuilder() << "Error executing jsonpath:" << Endl << result.GetError() << Endl); + } + + const auto& nodes = result.GetNodes(); + if (nodes.empty()) { + return BuildErrorResult(valueBuilder, TErrorCode::Empty, "Result is empty"); + } + + if (nodes.size() > 1) { + return BuildErrorResult(valueBuilder, TErrorCode::Error, "Result consists of multiple items"); + } + + const auto& value = nodes[0]; + if (value.Is(EValueType::Array) || value.Is(EValueType::Object)) { + // SqlValue can return only scalar values + return BuildErrorResult(valueBuilder, TErrorCode::Error, "Extracted JSON value is either object or array"); + } + + if (value.Is(EValueType::Null)) { + // JSON nulls must be converted to SQL nulls + return BuildSuccessfulResult(valueBuilder, TUnboxedValuePod()); + } + + const auto source = value.ConvertToUnboxedValue(valueBuilder); + TUnboxedValue convertedValue = TryConvertJson<TValueType, ForceConvert>(valueBuilder, source); + if (!convertedValue) { + // error while converting JSON value type to TValueType + return BuildErrorResult(valueBuilder, TErrorCode::Error, "Cannot convert extracted JSON value to target type"); + } + + return BuildSuccessfulResult(valueBuilder, std::move(convertedValue)); + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } + } + + TSourcePosition Pos_; + }; + + template <EDataSlot InputType, class TValueType, bool ForceConvert> + TStringRef TSqlValue<InputType, TValueType, ForceConvert>::Name() { + Y_ABORT("Unknown name"); + } + + template<> + TStringRef TSqlValue<EDataSlot::Json, TUtf8, true>::Name() { + return TStringRef::Of("SqlValueConvertToUtf8"); + } + + template <> + TStringRef TSqlValue<EDataSlot::Json, TUtf8>::Name() { + return TStringRef::Of("SqlValueUtf8"); + } + + template <> + TStringRef TSqlValue<EDataSlot::Json, i64>::Name() { + return TStringRef::Of("SqlValueInt64"); + } + + template <> + TStringRef TSqlValue<EDataSlot::Json, double>::Name() { + return TStringRef::Of("SqlValueNumber"); + } + + template <> + TStringRef TSqlValue<EDataSlot::Json, bool>::Name() { + return TStringRef::Of("SqlValueBool"); + } + + template<> + TStringRef TSqlValue<EDataSlot::JsonDocument, TUtf8, true>::Name() { + return TStringRef::Of("JsonDocumentSqlValueConvertToUtf8"); + } + + template <> + TStringRef TSqlValue<EDataSlot::JsonDocument, TUtf8>::Name() { + return TStringRef::Of("JsonDocumentSqlValueUtf8"); + } + + template <> + TStringRef TSqlValue<EDataSlot::JsonDocument, i64>::Name() { + return TStringRef::Of("JsonDocumentSqlValueInt64"); + } + + template <> + TStringRef TSqlValue<EDataSlot::JsonDocument, double>::Name() { + return TStringRef::Of("JsonDocumentSqlValueNumber"); + } + + template <> + TStringRef TSqlValue<EDataSlot::JsonDocument, bool>::Name() { + return TStringRef::Of("JsonDocumentSqlValueBool"); + } + +} diff --git a/yql/essentials/udfs/common/json2/test/canondata/result.json b/yql/essentials/udfs/common/json2/test/canondata/result.json new file mode 100644 index 00000000000..086f5e77ead --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/canondata/result.json @@ -0,0 +1,42 @@ +{ + "test.test[AsJsonNode]": [ + { + "uri": "file://test.test_AsJsonNode_/results.txt" + } + ], + "test.test[SerializeParse]": [ + { + "uri": "file://test.test_SerializeParse_/results.txt" + } + ], + "test.test[SqlExists]": [ + { + "uri": "file://test.test_SqlExists_/results.txt" + } + ], + "test.test[SqlQueryError]": [ + { + "uri": "file://test.test_SqlQueryError_/extracted" + } + ], + "test.test[SqlQuery]": [ + { + "uri": "file://test.test_SqlQuery_/results.txt" + } + ], + "test.test[SqlTryExistsError]": [ + { + "uri": "file://test.test_SqlTryExistsError_/extracted" + } + ], + "test.test[SqlTryExists]": [ + { + "uri": "file://test.test_SqlTryExists_/results.txt" + } + ], + "test.test[SqlValue]": [ + { + "uri": "file://test.test_SqlValue_/results.txt" + } + ] +} diff --git a/yql/essentials/udfs/common/json2/test/canondata/test.test_AsJsonNode_/results.txt b/yql/essentials/udfs/common/json2/test/canondata/test.test_AsJsonNode_/results.txt new file mode 100644 index 00000000000..fd6bba35bca --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/canondata/test.test_AsJsonNode_/results.txt @@ -0,0 +1,84 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "DataType"; + "Json" + ] + ]; + [ + "column1"; + [ + "DataType"; + "Json" + ] + ]; + [ + "column2"; + [ + "DataType"; + "Json" + ] + ]; + [ + "column3"; + [ + "DataType"; + "Json" + ] + ]; + [ + "column4"; + [ + "DataType"; + "Json" + ] + ]; + [ + "column5"; + [ + "DataType"; + "Json" + ] + ]; + [ + "column6"; + [ + "DataType"; + "Json" + ] + ]; + [ + "column7"; + [ + "DataType"; + "Json" + ] + ] + ] + ] + ]; + "Data" = [ + [ + "\"string\""; + "null"; + "1.2345"; + "null"; + "true"; + "null"; + "{\"key\":28}"; + "null" + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/canondata/test.test_SerializeParse_/results.txt b/yql/essentials/udfs/common/json2/test/canondata/test.test_SerializeParse_/results.txt new file mode 100644 index 00000000000..58a867b34e4 --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/canondata/test.test_SerializeParse_/results.txt @@ -0,0 +1,102 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "DataType"; + "Json" + ] + ]; + [ + "column1"; + [ + "DataType"; + "Json" + ] + ]; + [ + "column2"; + [ + "DataType"; + "Json" + ] + ]; + [ + "column3"; + [ + "DataType"; + "Json" + ] + ] + ] + ] + ]; + "Data" = [ + [ + "[]"; + "{}"; + "[1,3,4,5,6]"; + "{\"x\":1234}" + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "DataType"; + "JsonDocument" + ] + ]; + [ + "column1"; + [ + "DataType"; + "JsonDocument" + ] + ]; + [ + "column2"; + [ + "DataType"; + "JsonDocument" + ] + ]; + [ + "column3"; + [ + "DataType"; + "JsonDocument" + ] + ] + ] + ] + ]; + "Data" = [ + [ + "[]"; + "{}"; + "[1,3,4,5,6]"; + "{\"x\":1234}" + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlExists_/results.txt b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlExists_/results.txt new file mode 100644 index 00000000000..1b74c43a71d --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlExists_/results.txt @@ -0,0 +1,195 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column2"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column3"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + %true + ]; + [ + %true + ]; + [ + %true + ]; + [ + %true + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column2"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column3"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + #; + [ + %false + ]; + [ + %false + ]; + [ + %false + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column2"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + %false + ]; + [ + %true + ]; + # + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlQueryError_/extracted b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlQueryError_/extracted new file mode 100644 index 00000000000..12f3e1927de --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlQueryError_/extracted @@ -0,0 +1,10 @@ +<tmp_path>/program.sql:<main>: Fatal: Execution + + <tmp_path>/program.sql:<main>:12:1: Fatal: Execution of node: Result + SELECT + ^ + <tmp_path>/program.sql:<main>:14:12: Fatal: yql/essentials/udfs/common/json2/sql_query.h:xxx: Error executing jsonpath: +jsonpath:1:8: Error: Member not found, code: 4702 + + Json2::SqlQuery($jsonpath_error, $path, AsDict(), false, NULL, true, NULL); + ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlQuery_/results.txt b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlQuery_/results.txt new file mode 100644 index 00000000000..0773abf2be8 --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlQuery_/results.txt @@ -0,0 +1,400 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "{\"y\":123}" + ]; + [ + "[123,456]" + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + # + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column2"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + #; + [ + "{}" + ]; + [ + "[]" + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column2"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + #; + [ + "{}" + ]; + [ + "[]" + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "[123]" + ]; + [ + "[123]" + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column2"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "{\"y\":123}" + ]; + [ + "[{\"y\":123}]" + ]; + [ + "{\"y\":123}" + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column2"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "[123,456]" + ]; + [ + "[[123,456]]" + ]; + [ + "[123,456]" + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ]; + [ + "column2"; + [ + "OptionalType"; + [ + "DataType"; + "Json" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "{}" + ]; + [ + "[]" + ]; + [ + "[]" + ] + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlTryExistsError_/extracted b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlTryExistsError_/extracted new file mode 100644 index 00000000000..7761bc9a04e --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlTryExistsError_/extracted @@ -0,0 +1,10 @@ +<tmp_path>/program.sql:<main>: Fatal: Execution + + <tmp_path>/program.sql:<main>:12:1: Fatal: Execution of node: Result + SELECT + ^ + <tmp_path>/program.sql:<main>:14:12: Fatal: yql/essentials/udfs/common/json2/sql_exists.h:xxx: Error executing jsonpath: +jsonpath:1:8: Error: Expected object, code: 4701 + + Json2::SqlTryExists($json, $path, AsDict()); + ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlTryExists_/results.txt b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlTryExists_/results.txt new file mode 100644 index 00000000000..4a5f62bc86b --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlTryExists_/results.txt @@ -0,0 +1,83 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column2"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column3"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ]; + [ + "column4"; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + #; + [ + %true + ]; + [ + %true + ]; + [ + %true + ]; + [ + %true + ] + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlValue_/results.txt b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlValue_/results.txt new file mode 100644 index 00000000000..b5aeb82c9ab --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/canondata/test.test_SqlValue_/results.txt @@ -0,0 +1,1663 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + [ + "column1"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + [ + "column2"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + [ + "column3"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + [ + "column4"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "1"; + [ + "some string value" + ] + ]; + [ + "0"; + [ + "1"; + "Error executing jsonpath:\njsonpath:1:8: Error: Member not found, code: 4702\n" + ] + ]; + [ + "0"; + [ + "1"; + "Error executing jsonpath:\njsonpath:1:8: Error: Expected object, code: 4701\n" + ] + ]; + [ + "1"; + # + ]; + [ + "1"; + # + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ] + ] + ] + ]; + [ + "column1"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ] + ] + ] + ]; + [ + "column2"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ] + ] + ] + ]; + [ + "column3"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ] + ] + ] + ]; + [ + "column4"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "1"; + [ + "2856" + ] + ]; + [ + "0"; + [ + "1"; + "Error executing jsonpath:\njsonpath:1:8: Error: Member not found, code: 4702\n" + ] + ]; + [ + "0"; + [ + "1"; + "Error executing jsonpath:\njsonpath:1:8: Error: Expected object, code: 4701\n" + ] + ]; + [ + "1"; + # + ]; + [ + "1"; + # + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Double" + ] + ] + ] + ] + ] + ]; + [ + "column1"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Double" + ] + ] + ] + ] + ] + ]; + [ + "column2"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Double" + ] + ] + ] + ] + ] + ]; + [ + "column3"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Double" + ] + ] + ] + ] + ] + ]; + [ + "column4"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Double" + ] + ] + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "1"; + [ + "2.71828" + ] + ]; + [ + "0"; + [ + "1"; + "Error executing jsonpath:\njsonpath:1:8: Error: Member not found, code: 4702\n" + ] + ]; + [ + "0"; + [ + "1"; + "Error executing jsonpath:\njsonpath:1:8: Error: Expected object, code: 4701\n" + ] + ]; + [ + "1"; + # + ]; + [ + "1"; + # + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ]; + [ + "column1"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ]; + [ + "column2"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ]; + [ + "column3"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ]; + [ + "column4"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "1"; + [ + %true + ] + ]; + [ + "0"; + [ + "1"; + "Error executing jsonpath:\njsonpath:1:8: Error: Member not found, code: 4702\n" + ] + ]; + [ + "0"; + [ + "1"; + "Error executing jsonpath:\njsonpath:1:8: Error: Expected object, code: 4701\n" + ] + ]; + [ + "1"; + # + ]; + [ + "1"; + # + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + [ + "column1"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + [ + "column2"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + [ + "column3"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + [ + "column4"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + [ + "column5"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "1"; + [ + "some string value" + ] + ]; + [ + "1"; + [ + "2856" + ] + ]; + [ + "1"; + [ + "2.71828" + ] + ]; + [ + "1"; + [ + "true" + ] + ]; + [ + "1"; + # + ]; + [ + "1"; + # + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + [ + "column1"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ]; + [ + "column2"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Utf8" + ] + ] + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "0"; + [ + "1"; + "Cannot convert extracted JSON value to target type" + ] + ]; + [ + "0"; + [ + "1"; + "Cannot convert extracted JSON value to target type" + ] + ]; + [ + "0"; + [ + "1"; + "Cannot convert extracted JSON value to target type" + ] + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ] + ] + ] + ]; + [ + "column1"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ] + ] + ] + ]; + [ + "column2"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Int64" + ] + ] + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "0"; + [ + "1"; + "Cannot convert extracted JSON value to target type" + ] + ]; + [ + "0"; + [ + "1"; + "Cannot convert extracted JSON value to target type" + ] + ]; + [ + "0"; + [ + "1"; + "Cannot convert extracted JSON value to target type" + ] + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Double" + ] + ] + ] + ] + ] + ]; + [ + "column1"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Double" + ] + ] + ] + ] + ] + ]; + [ + "column2"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Double" + ] + ] + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "0"; + [ + "1"; + "Cannot convert extracted JSON value to target type" + ] + ]; + [ + "1"; + [ + "2856" + ] + ]; + [ + "0"; + [ + "1"; + "Cannot convert extracted JSON value to target type" + ] + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ]; + [ + "column1"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ]; + [ + "column2"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "0"; + [ + "1"; + "Cannot convert extracted JSON value to target type" + ] + ]; + [ + "0"; + [ + "1"; + "Cannot convert extracted JSON value to target type" + ] + ]; + [ + "0"; + [ + "1"; + "Cannot convert extracted JSON value to target type" + ] + ] + ] + ] + } + ] + }; + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ]; + [ + "column1"; + [ + "VariantType"; + [ + "TupleType"; + [ + [ + "TupleType"; + [ + [ + "DataType"; + "Uint8" + ]; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "OptionalType"; + [ + "DataType"; + "Bool" + ] + ] + ] + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "0"; + [ + "1"; + "Extracted JSON value is either object or array" + ] + ]; + [ + "0"; + [ + "1"; + "Extracted JSON value is either object or array" + ] + ] + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/cases/AsJsonNode.sql b/yql/essentials/udfs/common/json2/test/cases/AsJsonNode.sql new file mode 100644 index 00000000000..2d85d5576ce --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/cases/AsJsonNode.sql @@ -0,0 +1,9 @@ +SELECT + Json2::Utf8AsJsonNode(CAST("string" as Utf8)), + Json2::Utf8AsJsonNode(NULL), + Json2::DoubleAsJsonNode(1.2345), + Json2::DoubleAsJsonNode(NULL), + Json2::BoolAsJsonNode(true), + Json2::BoolAsJsonNode(NULL), + Json2::JsonAsJsonNode(CAST(@@{"key": 28}@@ as Json)), + Json2::JsonAsJsonNode(NULL); diff --git a/yql/essentials/udfs/common/json2/test/cases/SerializeParse.sql b/yql/essentials/udfs/common/json2/test/cases/SerializeParse.sql new file mode 100644 index 00000000000..1d5eb42d0cf --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/cases/SerializeParse.sql @@ -0,0 +1,15 @@ +$id = ($json) -> { RETURN Json2::Serialize(Json2::Parse($json)); }; + +SELECT + $id("[]"), + $id("{}"), + $id("[1, 3, 4, 5, 6]"), + $id(@@{"x": 1234}@@); + +$id_jd = ($json) -> { RETURN Json2::SerializeToJsonDocument(Json2::Parse($json)); }; + +SELECT + $id_jd("[]"), + $id_jd("{}"), + $id_jd("[1, 3, 4, 5, 6]"), + $id_jd(@@{"x": 1234}@@); diff --git a/yql/essentials/udfs/common/json2/test/cases/SqlExists.sql b/yql/essentials/udfs/common/json2/test/cases/SqlExists.sql new file mode 100644 index 00000000000..34f475fe5a8 --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/cases/SqlExists.sql @@ -0,0 +1,25 @@ +/* syntax version 1 */ + +$path = Json2::CompilePath("strict $.x"); + +-- Key exists +SELECT + Json2::SqlExists(CAST(@@{"x": 123}@@ as Json), $path, AsDict(), false), + Json2::SqlExists(CAST(@@{"x": {"key": "value"}}@@ as Json), $path, AsDict(), false), + Json2::SqlExists(CAST(@@{"x": [1, 2, 3]}@@ as Json), $path, AsDict(), false), + Json2::SqlExists(CAST(@@{"x": null}@@ as Json), $path, AsDict(), false); + +-- Key is missing +SELECT + Json2::SqlExists(NULL, $path, AsDict(), false), + Json2::SqlExists(CAST(@@{"not_x": 123}@@ as Json), $path, AsDict(), false), + Json2::SqlExists(CAST("{}" as Json), $path, AsDict(), false), + Json2::SqlExists(CAST("[]" as Json), $path, AsDict(), false); + +-- Error handling +$json = CAST("[]" as Json); + +SELECT + Json2::SqlExists($json, $path, AsDict(), false), + Json2::SqlExists($json, $path, AsDict(), true), + Json2::SqlExists($json, $path, AsDict(), NULL);
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/cases/SqlQuery.sql b/yql/essentials/udfs/common/json2/test/cases/SqlQuery.sql new file mode 100644 index 00000000000..38750aec512 --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/cases/SqlQuery.sql @@ -0,0 +1,52 @@ +/* syntax version 1 */ + +$path = Json2::CompilePath("strict $.x"); +$array = CAST("[]" as Json); +$object = CAST("{}" as Json); + +-- Valid cases +$nested_object = CAST(@@{"x": {"y": 123}}@@ as Json); +$nested_array = CAST(@@{"x": [123, 456]}@@ as Json); +SELECT + Json2::SqlQuery($nested_object, $path, AsDict(), false, $array, false, $object), + Json2::SqlQuery($nested_array, $path, AsDict(), false, $array, false, $object); + +-- Null handling +SELECT + Json2::SqlQuery(NULL, $path, AsDict(), false, $array, false, $object); + +-- Errors +$jsonpath_error = CAST(@@{"y": []}@@ as Json); +SELECT + Json2::SqlQuery($jsonpath_error, $path, AsDict(), false, $array, false, NULL), + Json2::SqlQuery($jsonpath_error, $path, AsDict(), false, $array, false, $object), + Json2::SqlQuery($jsonpath_error, $path, AsDict(), false, $object, false, $array); + +$mismatch_error = CAST(@@{"x": 123}@@ as Json); +SELECT + Json2::SqlQuery($mismatch_error, $path, AsDict(), false, $array, false, NULL), + Json2::SqlQuery($mismatch_error, $path, AsDict(), false, $array, false, $object), + Json2::SqlQuery($mismatch_error, $path, AsDict(), false, $object, false, $array); + +-- Wrap +$nested_value = CAST(@@{"x": 123}@@ as Json); +SELECT + Json2::SqlQueryWrap($nested_value, $path, AsDict(), false, $object, false, $array), + Json2::SqlQueryConditionalWrap($nested_value, $path, AsDict(), false, $object, false, $array); + +SELECT + Json2::SqlQuery($nested_object, $path, AsDict(), false, $object, false, $array), + Json2::SqlQueryWrap($nested_object, $path, AsDict(), false, $object, false, $array), + Json2::SqlQueryConditionalWrap($nested_object, $path, AsDict(), false, $object, false, $array); + +SELECT + Json2::SqlQuery($nested_array, $path, AsDict(), false, $object, false, $array), + Json2::SqlQueryWrap($nested_array, $path, AsDict(), false, $object, false, $array), + Json2::SqlQueryConditionalWrap($nested_array, $path, AsDict(), false, $object, false, $array); + +-- Wrap empty result +$path_lax = Json2::CompilePath("lax $.x"); +SELECT + Json2::SqlQuery($object, $path_lax, AsDict(), false, $object, false, $object), + Json2::SqlQueryWrap($object, $path_lax, AsDict(), false, $object, false, $object), + Json2::SqlQueryConditionalWrap($object, $path_lax, AsDict(), false, $object, false, $object);
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/cases/SqlQueryError.cfg b/yql/essentials/udfs/common/json2/test/cases/SqlQueryError.cfg new file mode 100644 index 00000000000..eb2e5315d1e --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/cases/SqlQueryError.cfg @@ -0,0 +1 @@ +xfail
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/cases/SqlQueryError.sql b/yql/essentials/udfs/common/json2/test/cases/SqlQueryError.sql new file mode 100644 index 00000000000..4aaa329fc06 --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/cases/SqlQueryError.sql @@ -0,0 +1,7 @@ +/* syntax version 1 */ + +$path = Json2::CompilePath("strict $.x"); + +$jsonpath_error = CAST(@@{"y": []}@@ as Json); +SELECT + Json2::SqlQuery($jsonpath_error, $path, AsDict(), false, NULL, true, NULL);
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/cases/SqlTryExists.sql b/yql/essentials/udfs/common/json2/test/cases/SqlTryExists.sql new file mode 100644 index 00000000000..f42bd5628db --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/cases/SqlTryExists.sql @@ -0,0 +1,11 @@ +/* syntax version 1 */ + +$path = Json2::CompilePath("strict $.x"); + +-- Key exists +SELECT + Json2::SqlTryExists(NULL, $path, AsDict()), + Json2::SqlTryExists(CAST(@@{"x": 123}@@ as Json), $path, AsDict()), + Json2::SqlTryExists(CAST(@@{"x": {"key": "value"}}@@ as Json), $path, AsDict()), + Json2::SqlTryExists(CAST(@@{"x": [1, 2, 3]}@@ as Json), $path, AsDict()), + Json2::SqlTryExists(CAST(@@{"x": null}@@ as Json), $path, AsDict()); diff --git a/yql/essentials/udfs/common/json2/test/cases/SqlTryExistsError.cfg b/yql/essentials/udfs/common/json2/test/cases/SqlTryExistsError.cfg new file mode 100644 index 00000000000..eb2e5315d1e --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/cases/SqlTryExistsError.cfg @@ -0,0 +1 @@ +xfail
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/cases/SqlTryExistsError.sql b/yql/essentials/udfs/common/json2/test/cases/SqlTryExistsError.sql new file mode 100644 index 00000000000..3d0440b3cad --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/cases/SqlTryExistsError.sql @@ -0,0 +1,7 @@ +/* syntax version 1 */ + +$path = Json2::CompilePath("strict $.x"); +$json = CAST("[]" as Json); + +SELECT + Json2::SqlTryExists($json, $path, AsDict());
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/cases/SqlValue.sql b/yql/essentials/udfs/common/json2/test/cases/SqlValue.sql new file mode 100644 index 00000000000..8f86edee795 --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/cases/SqlValue.sql @@ -0,0 +1,95 @@ +/* syntax version 1 */ + +-- Plain cases +$path = Json2::CompilePath("strict $.x"); +$empty_object = CAST("{}" as Json); +$empty_array = CAST("[]" as Json); +$null_key = CAST(@@{ + "x": null +}@@ as Json); + +$string_json = CAST(@@{ + "x": "some string value" +}@@ as Json); +SELECT + Json2::SqlValueUtf8($string_json, $path, AsDict()), + Json2::SqlValueUtf8($empty_object, $path, AsDict()), + Json2::SqlValueUtf8($empty_array, $path, AsDict()), + Json2::SqlValueUtf8($null_key, $path, AsDict()), + Json2::SqlValueUtf8(NULL, $path, AsDict()); + +$int64_json = CAST(@@{ + "x": 2856 +}@@ as Json); +SELECT + Json2::SqlValueInt64($int64_json, $path, AsDict()), + Json2::SqlValueInt64($empty_object, $path, AsDict()), + Json2::SqlValueInt64($empty_array, $path, AsDict()), + Json2::SqlValueInt64($null_key, $path, AsDict()), + Json2::SqlValueInt64(NULL, $path, AsDict()); + +$double_json = CAST(@@{ + "x": 2.71828 +}@@ as Json); +SELECT + Json2::SqlValueNumber($double_json, $path, AsDict()), + Json2::SqlValueNumber($empty_object, $path, AsDict()), + Json2::SqlValueNumber($empty_array, $path, AsDict()), + Json2::SqlValueNumber($null_key, $path, AsDict()), + Json2::SqlValueNumber(NULL, $path, AsDict()); + +$bool_json = CAST(@@{ + "x": true +}@@ as Json); +SELECT + Json2::SqlValueBool($bool_json, $path, AsDict()), + Json2::SqlValueBool($empty_object, $path, AsDict()), + Json2::SqlValueBool($empty_array, $path, AsDict()), + Json2::SqlValueBool($null_key, $path, AsDict()), + Json2::SqlValueBool(NULL, $path, AsDict()); + +-- Convert cases +SELECT + Json2::SqlValueConvertToUtf8($string_json, $path, AsDict()), + Json2::SqlValueConvertToUtf8($int64_json, $path, AsDict()), + Json2::SqlValueConvertToUtf8($double_json, $path, AsDict()), + Json2::SqlValueConvertToUtf8($bool_json, $path, AsDict()), + -- NOTE: Here SQL null must be returned, not "null" string + Json2::SqlValueConvertToUtf8($null_key, $path, AsDict()), + Json2::SqlValueConvertToUtf8(NULL, $path, AsDict()); + +-- Error cases +SELECT + Json2::SqlValueUtf8($int64_json, $path, AsDict()), + Json2::SqlValueUtf8($double_json, $path, AsDict()), + Json2::SqlValueUtf8($bool_json, $path, AsDict()); + +SELECT + Json2::SqlValueInt64($string_json, $path, AsDict()), + Json2::SqlValueInt64($double_json, $path, AsDict()), + Json2::SqlValueInt64($bool_json, $path, AsDict()); + +SELECT + Json2::SqlValueNumber($string_json, $path, AsDict()), + -- NOTE: Here int64 is automatically converted to double as it is possible without precision loss + Json2::SqlValueNumber($int64_json, $path, AsDict()), + Json2::SqlValueNumber($bool_json, $path, AsDict()); + +SELECT + Json2::SqlValueBool($string_json, $path, AsDict()), + Json2::SqlValueBool($int64_json, $path, AsDict()), + Json2::SqlValueBool($double_json, $path, AsDict()); + +$nested_object_json = CAST(@@{ + "x": { + "a": 1 + } +}@@ as Json); + +$nested_array_json = CAST(@@{ + "x": [29, 32, "some string"] +}@@ as Json); + +SELECT + Json2::SqlValueBool($nested_object_json, $path, AsDict()), + Json2::SqlValueBool($nested_array_json, $path, AsDict());
\ No newline at end of file diff --git a/yql/essentials/udfs/common/json2/test/ya.make b/yql/essentials/udfs/common/json2/test/ya.make new file mode 100644 index 00000000000..c7079c1bd86 --- /dev/null +++ b/yql/essentials/udfs/common/json2/test/ya.make @@ -0,0 +1,13 @@ +YQL_UDF_TEST_CONTRIB() + +DEPENDS(yql/essentials/udfs/common/json2) + +TIMEOUT(300) + +SIZE(MEDIUM) + +IF (SANITIZER_TYPE == "memory") + TAG(ya:not_autocheck) # YQL-15385 +ENDIF() + +END() diff --git a/yql/essentials/udfs/common/json2/ya.make b/yql/essentials/udfs/common/json2/ya.make new file mode 100644 index 00000000000..202b4aee9ca --- /dev/null +++ b/yql/essentials/udfs/common/json2/ya.make @@ -0,0 +1,33 @@ +IF (YQL_PACKAGED) + PACKAGE() + FROM_SANDBOX(FILE 7319901430 OUT_NOAUTO libjson2_udf.so + ) + END() +ELSE () +YQL_UDF_CONTRIB(json2_udf) + + YQL_ABI_VERSION( + 2 + 28 + 0 + ) + + SRCS( + json2_udf.cpp + ) + + PEERDIR( + yql/essentials/core/sql_types + yql/essentials/types/binary_json + yql/essentials/minikql/dom + yql/essentials/minikql/jsonpath + ) + + END() +ENDIF () + + +RECURSE_FOR_TESTS( + test +) + |