YQL-18303: Introduce From* converters overloads for Timestamp64

commit_hash:9e4f59aa752831d906745606e53e3f36a370a0a4

12 files changed, 391 insertions, 36 deletions

diff --git a/yql/essentials/udfs/common/datetime2/datetime_udf.cpp b/yql/essentials/udfs/common/datetime2/datetime_udf.cpp
index be41a0b8ed..3798557b91 100644
--- a/yql/essentials/udfs/common/datetime2/datetime_udf.cpp
+++ b/yql/essentials/udfs/common/datetime2/datetime_udf.cpp
@@ -548,14 +548,19 @@ TValue DoAddYears(const TValue& date, i64 years, const NUdf::IDateBuilder& build
         return false;
     }
 
-    inline bool ValidateDatetime(ui32 datetime) {
-        return datetime < MAX_DATETIME;
-    }
+    template<typename TType>
+    inline bool Validate(typename TDataType<TType>::TLayout arg);
 
-    inline bool ValidateTimestamp(ui64 timestamp) {
+    template<>
+    inline bool Validate<TTimestamp>(ui64 timestamp) {
         return timestamp < MAX_TIMESTAMP;
     }
 
+    template<>
+    inline bool Validate<TTimestamp64>(i64 timestamp) {
+        return timestamp >= MIN_TIMESTAMP64 && timestamp <= MAX_TIMESTAMP64;
+    }
+
     inline bool ValidateInterval(i64 interval) {
         return interval > -i64(MAX_TIMESTAMP) && interval < i64(MAX_TIMESTAMP);
     }
@@ -1268,44 +1273,39 @@ TValue DoAddYears(const TValue& date, i64 years, const NUdf::IDateBuilder& build
 
     // From*
 
-    BEGIN_SIMPLE_STRICT_ARROW_UDF(TFromSeconds, TOptional<TTimestamp>(TAutoMap<ui32>)) {
-        Y_UNUSED(valueBuilder);
-        auto res = args[0].Get<ui32>();
-        if (!ValidateDatetime(res)) {
-            return TUnboxedValuePod();
-        }
-        return TUnboxedValuePod((ui64)(res * 1000000ull));
+    template<typename TInput, typename TOutput, i64 UsecMultiplier>
+    inline TUnboxedValuePod TFromConverter(TInput arg) {
+        using TLayout = TDataType<TOutput>::TLayout;
+        const TLayout usec = TLayout(arg) * UsecMultiplier;
+        return Validate<TOutput>(usec) ? TUnboxedValuePod(usec) : TUnboxedValuePod();
     }
 
-    using TFromSecondsKernel = TUnaryUnsafeFixedSizeFilterKernel<ui32, ui64,
-        [] (ui32 seconds) { return std::make_pair(ui64(seconds * 1000000ull), ValidateDatetime(seconds)); }>;
-    END_SIMPLE_ARROW_UDF(TFromSeconds, TFromSecondsKernel::Do);
 
-    BEGIN_SIMPLE_STRICT_ARROW_UDF(TFromMilliseconds, TOptional<TTimestamp>(TAutoMap<ui64>)) {
-        Y_UNUSED(valueBuilder);
-        auto res = args[0].Get<ui64>();
-        if (res >= MAX_TIMESTAMP / 1000u) {
-            return TUnboxedValuePod();
-        }
-        return TUnboxedValuePod(res * 1000u);
-    }
+    template<typename TInput, typename TOutput, i64 UsecMultiplier>
+    using TFromConverterKernel = TUnaryUnsafeFixedSizeFilterKernel<TInput,
+        typename TDataType<TOutput>::TLayout, [] (TInput arg) {
+            using TLayout = TDataType<TOutput>::TLayout;
+            const TLayout usec = TLayout(arg) * UsecMultiplier;
+            return std::make_pair(usec, Validate<TOutput>(usec));
+        }>;
 
-    using TFromMillisecondsKernel = TUnaryUnsafeFixedSizeFilterKernel<ui64, ui64,
-        [] (ui64 milliseconds) { return std::make_pair(ui64(milliseconds * 1000u), milliseconds < MAX_TIMESTAMP / 1000u); }>;
-    END_SIMPLE_ARROW_UDF(TFromMilliseconds, TFromMillisecondsKernel::Do);
 
-    BEGIN_SIMPLE_STRICT_ARROW_UDF(TFromMicroseconds, TOptional<TTimestamp>(TAutoMap<ui64>)) {
-        Y_UNUSED(valueBuilder);
-        auto res = args[0].Get<ui64>();
-        if (!ValidateTimestamp(res)) {
-            return TUnboxedValuePod();
-        }
-        return TUnboxedValuePod(res);
-    }
+#define DATETIME_FROM_CONVERTER_UDF(name, retType, argType, usecMultiplier)              \
+    BEGIN_SIMPLE_STRICT_ARROW_UDF(T##name, TOptional<retType>(TAutoMap<argType>)) {      \
+        Y_UNUSED(valueBuilder);                                                          \
+        return TFromConverter<argType, retType, usecMultiplier>(args[0].Get<argType>()); \
+    }                                                                                    \
+                                                                                         \
+    END_SIMPLE_ARROW_UDF(T##name, (TFromConverterKernel<argType, retType, usecMultiplier>::Do))
+
+    DATETIME_FROM_CONVERTER_UDF(FromSeconds, TTimestamp, ui32, UsecondsInSecond);
+    DATETIME_FROM_CONVERTER_UDF(FromMilliseconds, TTimestamp, ui64, UsecondsInMilliseconds);
+    DATETIME_FROM_CONVERTER_UDF(FromMicroseconds, TTimestamp, ui64, 1);
+
+    DATETIME_FROM_CONVERTER_UDF(FromSeconds64, TTimestamp64, i64, UsecondsInSecond);
+    DATETIME_FROM_CONVERTER_UDF(FromMilliseconds64, TTimestamp64, i64, UsecondsInMilliseconds);
+    DATETIME_FROM_CONVERTER_UDF(FromMicroseconds64, TTimestamp64, i64, 1);
 
-    using TFromMicrosecondsKernel = TUnaryUnsafeFixedSizeFilterKernel<ui64, ui64,
-        [] (ui64 timestamp) { return std::make_pair(timestamp, ValidateTimestamp(timestamp)); }>;
-    END_SIMPLE_ARROW_UDF(TFromMicroseconds, TFromMicrosecondsKernel::Do);
 
     template <typename TInput, i64 Multiplier>
     using TIntervalFromKernel = TUnaryUnsafeFixedSizeFilterKernel<TInput, i64,
@@ -2426,6 +2426,10 @@ TValue DoAddYears(const TValue& date, i64 years, const NUdf::IDateBuilder& build
         TFromMilliseconds,
         TFromMicroseconds,
 
+        TFromSeconds64,
+        TFromMilliseconds64,
+        TFromMicroseconds64,
+
         TIntervalFromDays,
         TIntervalFromHours,
         TIntervalFromMinutes,
diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/result.json b/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/result.json
index f83bc10798..c804444f63 100644
--- a/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/result.json
+++ b/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/result.json
@@ -1,4 +1,14 @@
 {
+    "test.test[BlockFrom]": [
+        {
+            "uri": "file://test.test_BlockFrom_/results.txt"
+        }
+    ],
+    "test.test[From]": [
+        {
+            "uri": "file://test.test_From_/results.txt"
+        }
+    ],
     "test.test[SplitMake]": [
         {
             "uri": "file://test.test_SplitMake_/results.txt"
diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/test.test_BlockFrom_/results.txt b/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/test.test_BlockFrom_/results.txt
new file mode 100644
index 0000000000..2e8e14b3b2
--- /dev/null
+++ b/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/test.test_BlockFrom_/results.txt
@@ -0,0 +1,102 @@
+[
+    {
+        "Write" = [
+            {
+                "Type" = [
+                    "ListType";
+                    [
+                        "StructType";
+                        [
+                            [
+                                "ts64_sec";
+                                [
+                                    "OptionalType";
+                                    [
+                                        "DataType";
+                                        "Timestamp64"
+                                    ]
+                                ]
+                            ];
+                            [
+                                "ts64_msec";
+                                [
+                                    "OptionalType";
+                                    [
+                                        "DataType";
+                                        "Timestamp64"
+                                    ]
+                                ]
+                            ];
+                            [
+                                "ts64_usec";
+                                [
+                                    "OptionalType";
+                                    [
+                                        "DataType";
+                                        "Timestamp64"
+                                    ]
+                                ]
+                            ]
+                        ]
+                    ]
+                ];
+                "Data" = [
+                    [
+                        [
+                            "3875345000000"
+                        ];
+                        [
+                            "3875345000000"
+                        ];
+                        [
+                            "3875345000000"
+                        ]
+                    ];
+                    [
+                        [
+                            "4291747200000000"
+                        ];
+                        [
+                            "4291747200000000"
+                        ];
+                        [
+                            "4291747200000000"
+                        ]
+                    ];
+                    [
+                        [
+                            "-4611669897600000000"
+                        ];
+                        [
+                            "-4611669897600000000"
+                        ];
+                        [
+                            "-4611669897600000000"
+                        ]
+                    ];
+                    [
+                        #;
+                        #;
+                        #
+                    ];
+                    [
+                        [
+                            "4611669811199000000"
+                        ];
+                        [
+                            "4611669811199999000"
+                        ];
+                        [
+                            "4611669811199999999"
+                        ]
+                    ];
+                    [
+                        #;
+                        #;
+                        #
+                    ]
+                ]
+            }
+        ]
+    }
+]
\ No newline at end of file
diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/test.test_From_/results.txt b/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/test.test_From_/results.txt
new file mode 100644
index 0000000000..2e8e14b3b2
--- /dev/null
+++ b/yql/essentials/udfs/common/datetime2/test_bigdates/canondata/test.test_From_/results.txt
@@ -0,0 +1,102 @@
+[
+    {
+        "Write" = [
+            {
+                "Type" = [
+                    "ListType";
+                    [
+                        "StructType";
+                        [
+                            [
+                                "ts64_sec";
+                                [
+                                    "OptionalType";
+                                    [
+                                        "DataType";
+                                        "Timestamp64"
+                                    ]
+                                ]
+                            ];
+                            [
+                                "ts64_msec";
+                                [
+                                    "OptionalType";
+                                    [
+                                        "DataType";
+                                        "Timestamp64"
+                                    ]
+                                ]
+                            ];
+                            [
+                                "ts64_usec";
+                                [
+                                    "OptionalType";
+                                    [
+                                        "DataType";
+                                        "Timestamp64"
+                                    ]
+                                ]
+                            ]
+                        ]
+                    ]
+                ];
+                "Data" = [
+                    [
+                        [
+                            "3875345000000"
+                        ];
+                        [
+                            "3875345000000"
+                        ];
+                        [
+                            "3875345000000"
+                        ]
+                    ];
+                    [
+                        [
+                            "4291747200000000"
+                        ];
+                        [
+                            "4291747200000000"
+                        ];
+                        [
+                            "4291747200000000"
+                        ]
+                    ];
+                    [
+                        [
+                            "-4611669897600000000"
+                        ];
+                        [
+                            "-4611669897600000000"
+                        ];
+                        [
+                            "-4611669897600000000"
+                        ]
+                    ];
+                    [
+                        #;
+                        #;
+                        #
+                    ];
+                    [
+                        [
+                            "4611669811199000000"
+                        ];
+                        [
+                            "4611669811199999000"
+                        ];
+                        [
+                            "4611669811199999999"
+                        ]
+                    ];
+                    [
+                        #;
+                        #;
+                        #
+                    ]
+                ]
+            }
+        ]
+    }
+]
\ No newline at end of file
diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.cfg b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.cfg
new file mode 100644
index 0000000000..018e43ffd8
--- /dev/null
+++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.cfg
@@ -0,0 +1 @@
+in plato.Input BlockFrom.in
diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.in b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.in
new file mode 100644
index 0000000000..66cc696756
--- /dev/null
+++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.in
@@ -0,0 +1,30 @@
+{
+    "fts64_sec" =  3875345;
+    "fts64_msec" = 3875345000;
+    "fts64_usec" = 3875345000000;
+};
+{
+    "fts64_sec"  = 4291747200;
+    "fts64_msec" = 4291747200000;
+    "fts64_usec" = 4291747200000000;
+};
+{
+    "fts64_sec"  = -4611669897600;
+    "fts64_msec" = -4611669897600000;
+    "fts64_usec" = -4611669897600000000;
+};
+{
+    "fts64_sec"  = -4611669897601;
+    "fts64_msec" = -4611669897600001;
+    "fts64_usec" = -4611669897600000001;
+};
+{
+    "fts64_sec"  = 4611669811199;
+    "fts64_msec" = 4611669811199999;
+    "fts64_usec" = 4611669811199999999;
+};
+{
+    "fts64_sec"  = 4611669811200;
+    "fts64_msec" = 4611669811200000;
+    "fts64_usec" = 4611669811200000000;
+};
diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.in.attr b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.in.attr
new file mode 100644
index 0000000000..ae712ef6cb
--- /dev/null
+++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.in.attr
@@ -0,0 +1,30 @@
+{
+    "_yql_row_spec" = {
+        "Type" = [
+            "StructType";
+            [
+                [
+                    "fts64_sec";
+                    [
+                        "DataType";
+                        "Int64"
+                    ]
+                ];
+                [
+                    "fts64_msec";
+                    [
+                        "DataType";
+                        "Int64"
+                    ]
+                ];
+                [
+                    "fts64_usec";
+                    [
+                        "DataType";
+                        "Int64"
+                    ]
+                ];
+            ]
+        ]
+    }
+}
diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.sql b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.sql
new file mode 100644
index 0000000000..052a6da8ab
--- /dev/null
+++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/BlockFrom.sql
@@ -0,0 +1,8 @@
+/* syntax version 1 */
+pragma UseBlocks;
+
+select
+    DateTime::FromSeconds64(fts64_sec) as ts64_sec,
+    DateTime::FromMilliseconds64(fts64_msec) as ts64_msec,
+    DateTime::FromMicroseconds64(fts64_usec) as ts64_usec,
+from Input
diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.cfg b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.cfg
new file mode 100644
index 0000000000..8fc79a8b6b
--- /dev/null
+++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.cfg
@@ -0,0 +1 @@
+in plato.Input From.in
diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.in b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.in
new file mode 100644
index 0000000000..66cc696756
--- /dev/null
+++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.in
@@ -0,0 +1,30 @@
+{
+    "fts64_sec" =  3875345;
+    "fts64_msec" = 3875345000;
+    "fts64_usec" = 3875345000000;
+};
+{
+    "fts64_sec"  = 4291747200;
+    "fts64_msec" = 4291747200000;
+    "fts64_usec" = 4291747200000000;
+};
+{
+    "fts64_sec"  = -4611669897600;
+    "fts64_msec" = -4611669897600000;
+    "fts64_usec" = -4611669897600000000;
+};
+{
+    "fts64_sec"  = -4611669897601;
+    "fts64_msec" = -4611669897600001;
+    "fts64_usec" = -4611669897600000001;
+};
+{
+    "fts64_sec"  = 4611669811199;
+    "fts64_msec" = 4611669811199999;
+    "fts64_usec" = 4611669811199999999;
+};
+{
+    "fts64_sec"  = 4611669811200;
+    "fts64_msec" = 4611669811200000;
+    "fts64_usec" = 4611669811200000000;
+};
diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.in.attr b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.in.attr
new file mode 100644
index 0000000000..2e2c2d1067
--- /dev/null
+++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.in.attr
@@ -0,0 +1,31 @@
+{
+    "_yql_row_spec" = {
+        "Type" = [
+            "StructType";
+            [
+                [
+                    "fts64_sec";
+                    [
+                        "DataType";
+                        "Int64"
+                    ]
+                ];
+                [
+                    "fts64_msec";
+                    [
+                        "DataType";
+                        "Int64"
+                    ]
+                ];
+                [
+                    "fts64_usec";
+                    [
+                        "DataType";
+                        "Int64"
+                    ]
+                ];
+            ]
+        ]
+    }
+}
+
diff --git a/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.sql b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.sql
new file mode 100644
index 0000000000..f1b89fc028
--- /dev/null
+++ b/yql/essentials/udfs/common/datetime2/test_bigdates/cases/From.sql
@@ -0,0 +1,6 @@
+/* syntax version 1 */
+select
+    DateTime::FromSeconds64(fts64_sec) as ts64_sec,
+    DateTime::FromMilliseconds64(fts64_msec) as ts64_msec,
+    DateTime::FromMicroseconds64(fts64_usec) as ts64_usec,
+from Input