diff options
author | lambda-delta <[email protected]> | 2025-08-04 19:40:46 +0300 |
---|---|---|
committer | lambda-delta <[email protected]> | 2025-08-04 20:05:53 +0300 |
commit | b5c31e9554edee086eed97d94e7908d6d6ef3ec2 (patch) | |
tree | 06f2ac35cc4d8aa62f3c64f023e56a0e7256352c | |
parent | c0bc4e708e27a153b958eed0f42d2a67f245b288 (diff) |
YQL-19784: Implement Ipv4FromUint32 and Ipv4ToUint32 functions in IP UDF
Implement Ipv4FromUint32 and Ipv4ToUint32 functions in IP UDF
commit_hash:ac21513d59acaf686ad0ab1482753863eb2d9a12
14 files changed, 247 insertions, 13 deletions
diff --git a/yql/essentials/docs/en/changelog/2025.03.md b/yql/essentials/docs/en/changelog/2025.03.md index 361474ae616..19b7d486be8 100644 --- a/yql/essentials/docs/en/changelog/2025.03.md +++ b/yql/essentials/docs/en/changelog/2025.03.md @@ -15,6 +15,10 @@ * Added `Math::SwapBytes` function +## Changes in IP module {#ip-module} + +* Added `Ip::Ipv4FromUint32` and `Ip::Ipv4ToUint32` functions. + ## Changes in SELECT syntax * Added the ability to explicitly specify `DISTINCT` for `UNION`; diff --git a/yql/essentials/docs/en/udf/list/ip.md b/yql/essentials/docs/en/udf/list/ip.md index c635d5009e1..e214aecaa5f 100644 --- a/yql/essentials/docs/en/udf/list/ip.md +++ b/yql/essentials/docs/en/udf/list/ip.md @@ -5,14 +5,16 @@ The `Ip` module supports both the IPv4 and IPv6 addresses. By default, they are ## List of functions * `Ip::FromString(String{Flags:AutoMap}) -> String?` - From a human-readable representation to a binary representation. +* `Ip::Ipv4FromUint32(Uint32{Flags:AutoMap}) -> String` - From a Uint32 to a binary representation. Integer `A << 24 | B << 16 | C << 8 | D` corresponds to address `A.B.C.D`. * `Ip::SubnetFromString(String{Flags:AutoMap}) -> String?` - From a human-readable representation of subnet to a binary representation. * `Ip::ToString(String{Flags:AutoMap}) -> String?` - From a binary representation to a human-readable representation. +* `Ip::Ipv4ToUint32(String{Flags:AutoMap}) -> Uint32?` - From a binary representation of IPv4 to a Uint32. Address `A.B.C.D` corresponds to integer `A << 24 | B << 16 | C << 8 | D`. IPv6 is not supported. * `Ip::SubnetToString(String{Flags:AutoMap}) -> String?` - From a binary representation of subnet to a human-readable representation. * `Ip::IsIPv4(String?) -> Bool` * `Ip::IsIPv6(String?) -> Bool` * `Ip::IsEmbeddedIPv4(String?) -> Bool` -* `Ip::ConvertToIPv6(String{Flags:AutoMap}) -> String`: IPv6 remains unchanged, and IPv4 becomes embedded in IPv6 -* `Ip::GetSubnet(String{Flags:AutoMap}, [Uint8?]) -> String`: The second argument is the subnet size, by default it's 24 for IPv4 and 64 for IPv6 +* `Ip::ConvertToIPv6(String{Flags:AutoMap}) -> String`: IPv6 remains unchanged, and IPv4 becomes embedded in IPv6. +* `Ip::GetSubnet(String{Flags:AutoMap}, [Uint8?]) -> String`: The second argument is the subnet size, by default it's 24 for IPv4 and 64 for IPv6. * `Ip::GetSubnetByMask(String{Flags:AutoMap}, String{Flags:AutoMap}) -> String`: The first argument is the base address, the second argument is the bit mask of a desired subnet. * `Ip::SubnetMatch(String{Flags:AutoMap}, String{Flags:AutoMap}) -> Bool`: The first argument is a subnet, the second argument is a subnet or an address. @@ -24,6 +26,14 @@ SELECT Ip::IsEmbeddedIPv4( Ip::FromString("::ffff:77.75.155.3") ); -- true +SELECT Ip::ToString( + Ip::Ipv4FromUint32(1) +); -- "0.0.0.1" + +SELECT Ip::Ipv4ToUint32( + Ip::FromString("0.0.0.1") +); -- 1 + SELECT Ip::ToString( Ip::GetSubnet( diff --git a/yql/essentials/docs/ru/changelog/2025.03.md b/yql/essentials/docs/ru/changelog/2025.03.md index f50f09d8a0e..4ec16ac99f7 100644 --- a/yql/essentials/docs/ru/changelog/2025.03.md +++ b/yql/essentials/docs/ru/changelog/2025.03.md @@ -15,6 +15,10 @@ * Добавлена функция `Math::SwapBytes` +## Изменения в модуле IP {#ip-module} + +* Добавлены функции `Ip::Ipv4FromUint32` и `Ip::Ipv4ToUint32`. + ## Изменения в синтаксисе SELECT * Добавлена возможность явного указания `DISTINCT` для `UNION`; diff --git a/yql/essentials/docs/ru/udf/list/ip.md b/yql/essentials/docs/ru/udf/list/ip.md index 8ec01015c9e..f114e36ad09 100644 --- a/yql/essentials/docs/ru/udf/list/ip.md +++ b/yql/essentials/docs/ru/udf/list/ip.md @@ -4,17 +4,19 @@ ## Список функций -* `Ip::FromString(String{Flags:AutoMap}) -> String?` - из человекочитаемого представления в бинарное -* `Ip::SubnetFromString(String{Flags:AutoMap}) -> String?` - из человекочитаемого представления подсети в бинарное -* `Ip::ToString(String{Flags:AutoMap}) -> String?` - из бинарного представления в человекочитаемое -* `Ip::ToString(String{Flags:AutoMap}) -> String?` - из бинарного представления подсети в человекочитаемое +* `Ip::FromString(String{Flags:AutoMap}) -> String?` - из человекочитаемого представления в бинарное. +* `Ip::Ipv4FromUint32(Uint32{Flags:AutoMap}) -> String` - из Uint32 в бинарное представление. Целое число `A << 24 | B << 16 | C << 8 | D` соответствует адресу `A.B.C.D`. +* `Ip::SubnetFromString(String{Flags:AutoMap}) -> String?` - из человекочитаемого представления подсети в бинарное. +* `Ip::ToString(String{Flags:AutoMap}) -> String?` - из бинарного представления в человекочитаемое. +* `Ip::Ipv4ToUint32(String{Flags:AutoMap}) -> Uint32?` - из бинарного представления IPv4 в Uint32. Адрес `A.B.C.D` соответствует целому числу `A << 24 | B << 16 | C << 8 | D`. IPv6 не поддерживается. +* `Ip::SubnetToString(String{Flags:AutoMap}) -> String?` - из бинарного представления подсети в человекочитаемое. * `Ip::IsIPv4(String?) -> Bool` * `Ip::IsIPv6(String?) -> Bool` * `Ip::IsEmbeddedIPv4(String?) -> Bool` -* `Ip::ConvertToIPv6(String{Flags:AutoMap}) -> String` - IPv6 остается без изменений, а IPv4 становится embedded в IPv6 -* `Ip::GetSubnet(String{Flags:AutoMap}, [Uint8?]) -> String` - во втором аргументе размер подсети, по умолчанию 24 для IPv4 и 64 для IPv6 -* `Ip::GetSubnetByMask(String{Flags:AutoMap}, String{Flags:AutoMap}) -> String` - во втором аргументе битовая маска подсети -* `Ip::SubnetMatch(String{Flags:AutoMap}, String{Flags:AutoMap}) -> Bool` - в первом аргументе подсеть, во втором аргументе подсеть или адрес +* `Ip::ConvertToIPv6(String{Flags:AutoMap}) -> String` - IPv6 остается без изменений, а IPv4 становится embedded в IPv6. +* `Ip::GetSubnet(String{Flags:AutoMap}, [Uint8?]) -> String` - во втором аргументе размер подсети, по умолчанию 24 для IPv4 и 64 для IPv6. +* `Ip::GetSubnetByMask(String{Flags:AutoMap}, String{Flags:AutoMap}) -> String` - во втором аргументе битовая маска подсети. +* `Ip::SubnetMatch(String{Flags:AutoMap}, String{Flags:AutoMap}) -> Bool` - в первом аргументе подсеть, во втором аргументе подсеть или адрес. ## Примеры @@ -23,6 +25,14 @@ SELECT Ip::IsEmbeddedIPv4( Ip::FromString("::ffff:77.75.155.3") ); -- true +SELECT Ip::ToString( + Ip::Ipv4FromUint32(1) +); -- "0.0.0.1" + +SELECT Ip::Ipv4ToUint32( + Ip::FromString("0.0.0.1") +); -- 1 + SELECT Ip::ToString( Ip::GetSubnet( diff --git a/yql/essentials/udfs/common/ip_base/lib/ip_base_udf.h b/yql/essentials/udfs/common/ip_base/lib/ip_base_udf.h index 3644207f8b3..2bb4f987190 100644 --- a/yql/essentials/udfs/common/ip_base/lib/ip_base_udf.h +++ b/yql/essentials/udfs/common/ip_base/lib/ip_base_udf.h @@ -1,6 +1,7 @@ #pragma once #include <yql/essentials/public/udf/udf_helpers.h> +#include <yql/essentials/public/langver/yql_langver.h> #include <library/cpp/ipv6_address/ipv6_address.h> #include <library/cpp/ipmath/ipmath.h> @@ -8,7 +9,9 @@ namespace { using TAutoMapString = NKikimr::NUdf::TAutoMap<char*>; + using TAutoMapUint32 = NKikimr::NUdf::TAutoMap<ui32>; using TOptionalString = NKikimr::NUdf::TOptional<char*>; + using TOptionalUint32 = NKikimr::NUdf::TOptional<ui32>; using TOptionalByte = NKikimr::NUdf::TOptional<ui8>; using TStringRef = NKikimr::NUdf::TStringRef; using TUnboxedValue = NKikimr::NUdf::TUnboxedValue; @@ -229,6 +232,13 @@ namespace { return valueBuilder->NewString(SerializeAddress(addr)); } + SIMPLE_STRICT_UDF_OPTIONS(TIpv4FromUint32, char*(TAutoMapUint32), builder.SetMinLangVer(NYql::MakeLangVersion(2025, 3))) { + // in_addr expects bytes in network byte order. + in_addr addr; + addr.s_addr = htonl(args[0].Get<ui32>()); + return valueBuilder->NewString(SerializeAddress(TIpv6Address{addr})); + } + SIMPLE_STRICT_UDF(TSubnetFromString, TOptionalString(TAutoMapString)) { TIpAddressRange range = TIpAddressRange::FromCompactString(args[0].AsStringRef()); auto res = SerializeSubnet(range); @@ -239,6 +249,19 @@ namespace { return valueBuilder->NewString(DeserializeAddress(args[0].AsStringRef()).ToString(false)); } + SIMPLE_UDF_OPTIONS(TIpv4ToUint32, TOptionalUint32(TAutoMapString), builder.SetMinLangVer(NYql::MakeLangVersion(2025, 3))) { + Y_UNUSED(valueBuilder); + TIpv6Address addr = DeserializeAddress(args[0].AsStringRef()); + if (addr.Type() != TIpv6Address::Ipv4) { + return TUnboxedValue(); + } + + in_addr tmp; + addr.ToInAddr(tmp); + ui32 ret = ntohl(tmp.s_addr); + return TUnboxedValuePod(ret); + } + SIMPLE_UDF(TSubnetToString, char*(TAutoMapString)) { TStringBuilder result; auto range = DeserializeSubnet(args[0].AsStringRef()); @@ -345,8 +368,10 @@ namespace { #define EXPORTED_IP_BASE_UDF \ TFromString, \ + TIpv4FromUint32, \ TSubnetFromString, \ TToString, \ + TIpv4ToUint32, \ TSubnetToString, \ TIsIPv4, \ TIsIPv6, \ diff --git a/yql/essentials/udfs/common/ip_base/lib/ya.make b/yql/essentials/udfs/common/ip_base/lib/ya.make index 72633514771..ab9b2bce8e2 100644 --- a/yql/essentials/udfs/common/ip_base/lib/ya.make +++ b/yql/essentials/udfs/common/ip_base/lib/ya.make @@ -2,7 +2,7 @@ LIBRARY() YQL_ABI_VERSION( 2 - 28 + 43 0 ) diff --git a/yql/essentials/udfs/common/ip_base/test/canondata/result.json b/yql/essentials/udfs/common/ip_base/test/canondata/result.json index a9602f6bf0c..59a83495eac 100644 --- a/yql/essentials/udfs/common/ip_base/test/canondata/result.json +++ b/yql/essentials/udfs/common/ip_base/test/canondata/result.json @@ -4,6 +4,16 @@ "uri": "file://test.test_Basic_/results.txt" } ], + "test.test[Ipv4Uint32]": [ + { + "uri": "file://test.test_Ipv4Uint32_/results.txt" + } + ], + "test.test[Ipv4Uint32_2025_02]": [ + { + "uri": "file://test.test_Ipv4Uint32_2025_02_/extracted" + } + ], "test.test[Subnets]": [ { "uri": "file://test.test_Subnets_/results.txt" diff --git a/yql/essentials/udfs/common/ip_base/test/canondata/test.test_Ipv4Uint32_/results.txt b/yql/essentials/udfs/common/ip_base/test/canondata/test.test_Ipv4Uint32_/results.txt new file mode 100644 index 00000000000..c572880165f --- /dev/null +++ b/yql/essentials/udfs/common/ip_base/test/canondata/test.test_Ipv4Uint32_/results.txt @@ -0,0 +1,133 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "internal_representation"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "uint32_repr"; + [ + "OptionalType"; + [ + "DataType"; + "Uint32" + ] + ] + ]; + [ + "internal_repr_uint32"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "\x7F\0\0\1" + ]; + [ + "2130706433" + ]; + [ + "\x7F\0\0\1" + ] + ]; + [ + [ + "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1" + ]; + #; + # + ]; + [ + [ + [ + "1bTBAw==" + ] + ]; + [ + "3585392899" + ]; + [ + [ + "1bTBAw==" + ] + ] + ]; + [ + [ + [ + "KgIGuAAAAAAAAAAAAAAAAw==" + ] + ]; + #; + # + ]; + [ + [ + [ + "JADLACBIAAEAAAAAaBwbZQ==" + ] + ]; + #; + # + ]; + [ + [ + [ + "/oAAAAAAAAACFbL//qlnzg==" + ] + ]; + #; + # + ]; + [ + [ + [ + "AAAAAAAAAAAAAP//TUubAw==" + ] + ]; + #; + # + ]; + [ + #; + #; + # + ]; + [ + [ + "\0\0\0\0" + ]; + [ + "0" + ]; + [ + "\0\0\0\0" + ] + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/ip_base/test/canondata/test.test_Ipv4Uint32_2025_02_/extracted b/yql/essentials/udfs/common/ip_base/test/canondata/test.test_Ipv4Uint32_2025_02_/extracted new file mode 100644 index 00000000000..16a3af0eebe --- /dev/null +++ b/yql/essentials/udfs/common/ip_base/test/canondata/test.test_Ipv4Uint32_2025_02_/extracted @@ -0,0 +1,23 @@ +<tmp_path>/program.sql:<main>: Error: Type annotation + + <tmp_path>/program.sql:<main>:2:1: Error: At function: RemovePrefixMembers, At function: Unordered, At function: PersistableRepr, At function: OrderedSqlProject, At tuple + SELECT + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:4:9: Error: At function: Apply, At function: Udf + Ip::Ipv4ToUint32(Ip::FromString("127.0.0.1")) AS uint32_repr, + ^ + <tmp_path>/program.sql:<main>:4:9: Error: UDF 'Ip.Ipv4ToUint32' is not available before version 2025.03 + Ip::Ipv4ToUint32(Ip::FromString("127.0.0.1")) AS uint32_repr, + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:6:9: Error: At function: Apply, At function: Udf + Ip::Ipv4FromUint32(0x7F000001U) AS internal_repr_uint32; + ^ + <tmp_path>/program.sql:<main>:6:9: Error: UDF 'Ip.Ipv4FromUint32' is not available before version 2025.03 + Ip::Ipv4FromUint32(0x7F000001U) AS internal_repr_uint32; + ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32.cfg b/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32.cfg new file mode 100644 index 00000000000..a0eb39c67fd --- /dev/null +++ b/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32.cfg @@ -0,0 +1,2 @@ +langver 2025.03 +in plato.Input Basic.in diff --git a/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32.sql b/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32.sql new file mode 100644 index 00000000000..a11aa895173 --- /dev/null +++ b/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32.sql @@ -0,0 +1,8 @@ +/* syntax version 1 */ +SELECT + internal_representation AS internal_representation, + Ip::Ipv4ToUint32(internal_representation) AS uint32_repr, + Ip::Ipv4FromUint32(Ip::Ipv4ToUint32(internal_representation)) AS internal_repr_uint32, +FROM ( + SELECT Ip::FromString(key) AS internal_representation FROM Input +); diff --git a/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32_2025_02.cfg b/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32_2025_02.cfg new file mode 100644 index 00000000000..989226cf3d4 --- /dev/null +++ b/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32_2025_02.cfg @@ -0,0 +1,2 @@ +xfail +langver 2025.02 diff --git a/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32_2025_02.sql b/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32_2025_02.sql new file mode 100644 index 00000000000..66681990156 --- /dev/null +++ b/yql/essentials/udfs/common/ip_base/test/cases/Ipv4Uint32_2025_02.sql @@ -0,0 +1,3 @@ +SELECT + Ip::Ipv4ToUint32(Ip::FromString("127.0.0.1")) AS uint32_repr, + Ip::Ipv4FromUint32(0x7F000001U) AS internal_repr_uint32; diff --git a/yql/essentials/udfs/common/ip_base/ya.make b/yql/essentials/udfs/common/ip_base/ya.make index 0e5537607a2..0a2859c0af2 100644 --- a/yql/essentials/udfs/common/ip_base/ya.make +++ b/yql/essentials/udfs/common/ip_base/ya.make @@ -2,7 +2,7 @@ YQL_UDF_CONTRIB(ip_udf) YQL_ABI_VERSION( 2 - 28 + 43 0 ) @@ -18,4 +18,4 @@ YQL_UDF_CONTRIB(ip_udf) RECURSE_FOR_TESTS( test -)
\ No newline at end of file +) |