diff options
author | avevad <avevad@yandex-team.com> | 2024-11-13 02:42:42 +0300 |
---|---|---|
committer | avevad <avevad@yandex-team.com> | 2024-11-13 02:52:02 +0300 |
commit | 1017fdddd5035f4b9e8a49328dd793625dd99f97 (patch) | |
tree | 26794fed86aeab8e9871e8e4419aee093f3261b7 | |
parent | de501ac78131d9de78c5baf13288273a97a73b11 (diff) | |
download | ydb-1017fdddd5035f4b9e8a49328dd793625dd99f97.tar.gz |
YQL-17471 Add EndOf... functions in Datetime UDF
Add EndOf... functions in Datetime UDF
commit_hash:6ac57411cfaabbf9abb6bed20b0e52e0a88918d0
9 files changed, 293 insertions, 109 deletions
diff --git a/yql/essentials/udfs/common/datetime2/datetime_udf.cpp b/yql/essentials/udfs/common/datetime2/datetime_udf.cpp index 76ef7014d7..143eaefbcd 100644 --- a/yql/essentials/udfs/common/datetime2/datetime_udf.cpp +++ b/yql/essentials/udfs/common/datetime2/datetime_udf.cpp @@ -1395,6 +1395,17 @@ TValue DoAddYears(const TValue& date, i64 years, const NUdf::IDateBuilder& build } }; + template<auto Core> + TUnboxedValue SimpleDatetimeToDatetimeUdf(const IValueBuilder* valueBuilder, const TUnboxedValuePod* args) { + auto result = args[0]; + auto& storage = Reference(result); + if (auto res = Core(storage, *valueBuilder)) { + storage = res.GetRef(); + return result; + } + return TUnboxedValuePod{}; + } + TMaybe<TTMStorage> StartOfYear(TTMStorage storage, const IValueBuilder& valueBuilder) { storage.Month = 1; storage.Day = 1; @@ -1408,16 +1419,31 @@ TValue DoAddYears(const TValue& date, i64 years, const NUdf::IDateBuilder& build return storage; } BEGIN_SIMPLE_STRICT_ARROW_UDF(TStartOfYear, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { - auto result = args[0]; - auto& storage = Reference(result); - if (auto res = StartOfYear(storage, *valueBuilder)) { - storage = res.GetRef(); - return result; - } - return TUnboxedValuePod{}; + return SimpleDatetimeToDatetimeUdf<StartOfYear>(valueBuilder, args); } END_SIMPLE_ARROW_UDF(TStartOfYear, TStartOfKernelExec<StartOfYear>::Do); + void SetEndOfDay(TTMStorage& storage) { + storage.Hour = 23; + storage.Minute = 59; + storage.Second = 59; + storage.Microsecond = 999999; + } + + TMaybe<TTMStorage> EndOfYear(TTMStorage storage, const IValueBuilder& valueBuilder) { + storage.Month = 12; + storage.Day = 31; + SetEndOfDay(storage); + if (!storage.Validate(valueBuilder.GetDateBuilder())) { + return {}; + } + return storage; + } + BEGIN_SIMPLE_STRICT_ARROW_UDF(TEndOfYear, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { + return SimpleDatetimeToDatetimeUdf<EndOfYear>(valueBuilder, args); + } + END_SIMPLE_ARROW_UDF(TEndOfYear, TStartOfKernelExec<EndOfYear>::Do); + TMaybe<TTMStorage> StartOfQuarter(TTMStorage storage, const IValueBuilder& valueBuilder) { storage.Month = (storage.Month - 1) / 3 * 3 + 1; storage.Day = 1; @@ -1431,16 +1457,24 @@ TValue DoAddYears(const TValue& date, i64 years, const NUdf::IDateBuilder& build return storage; } BEGIN_SIMPLE_STRICT_ARROW_UDF(TStartOfQuarter, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { - auto result = args[0]; - auto& storage = Reference(result); - if (auto res = StartOfQuarter(storage, *valueBuilder)) { - storage = res.GetRef(); - return result; - } - return TUnboxedValuePod{}; + return SimpleDatetimeToDatetimeUdf<StartOfQuarter>(valueBuilder, args); } END_SIMPLE_ARROW_UDF(TStartOfQuarter, TStartOfKernelExec<StartOfQuarter>::Do); + TMaybe<TTMStorage> EndOfQuarter(TTMStorage storage, const IValueBuilder& valueBuilder) { + storage.Month = ((storage.Month - 1) / 3 + 1) * 3; + storage.Day = NMiniKQL::GetMonthLength(storage.Month, NMiniKQL::IsLeapYear(storage.Year)); + SetEndOfDay(storage); + if (!storage.Validate(valueBuilder.GetDateBuilder())) { + return {}; + } + return storage; + } + BEGIN_SIMPLE_STRICT_ARROW_UDF(TEndOfQuarter, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { + return SimpleDatetimeToDatetimeUdf<EndOfQuarter>(valueBuilder, args); + } + END_SIMPLE_ARROW_UDF(TEndOfQuarter, TStartOfKernelExec<EndOfQuarter>::Do); + TMaybe<TTMStorage> StartOfMonth(TTMStorage storage, const IValueBuilder& valueBuilder) { storage.Day = 1; storage.Hour = 0; @@ -1453,23 +1487,13 @@ TValue DoAddYears(const TValue& date, i64 years, const NUdf::IDateBuilder& build return storage; } BEGIN_SIMPLE_STRICT_ARROW_UDF(TStartOfMonth, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { - auto result = args[0]; - auto& storage = Reference(result); - if (auto res = StartOfMonth(storage, *valueBuilder)) { - storage = res.GetRef(); - return result; - } - return TUnboxedValuePod{}; + return SimpleDatetimeToDatetimeUdf<StartOfMonth>(valueBuilder, args); } END_SIMPLE_ARROW_UDF(TStartOfMonth, TStartOfKernelExec<StartOfMonth>::Do); TMaybe<TTMStorage> EndOfMonth(TTMStorage storage, const IValueBuilder& valueBuilder) { storage.Day = NMiniKQL::GetMonthLength(storage.Month, NMiniKQL::IsLeapYear(storage.Year)); - storage.Hour = 0; - storage.Minute = 0; - storage.Second = 0; - storage.Microsecond = 0; - + SetEndOfDay(storage); if (!storage.Validate(valueBuilder.GetDateBuilder())) { return {}; } @@ -1477,13 +1501,7 @@ TValue DoAddYears(const TValue& date, i64 years, const NUdf::IDateBuilder& build } BEGIN_SIMPLE_STRICT_ARROW_UDF(TEndOfMonth, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { - auto result = args[0]; - auto& storage = Reference(result); - if (auto res = EndOfMonth(storage, *valueBuilder)) { - storage = res.GetRef(); - return result; - } - return TUnboxedValuePod{}; + return SimpleDatetimeToDatetimeUdf<EndOfMonth>(valueBuilder, args); } END_SIMPLE_ARROW_UDF(TEndOfMonth, TStartOfKernelExec<EndOfMonth>::Do); @@ -1497,20 +1515,36 @@ TValue DoAddYears(const TValue& date, i64 years, const NUdf::IDateBuilder& build storage.Minute = 0; storage.Second = 0; storage.Microsecond = 0; + if (!storage.Validate(valueBuilder.GetDateBuilder())) { + return {}; + } return storage; } BEGIN_SIMPLE_STRICT_ARROW_UDF(TStartOfWeek, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { - auto result = args[0]; - auto& storage = Reference(result); - if (auto res = StartOfWeek(storage, *valueBuilder)) { - storage = res.GetRef(); - return result; - } - return TUnboxedValuePod{}; + return SimpleDatetimeToDatetimeUdf<StartOfWeek>(valueBuilder, args); } END_SIMPLE_ARROW_UDF(TStartOfWeek, TStartOfKernelExec<StartOfWeek>::Do); + TMaybe<TTMStorage> EndOfWeek(TTMStorage storage, const IValueBuilder& valueBuilder) { + const ui32 shift = 86400u * (7u - storage.DayOfWeek); + auto dt = storage.ToDatetime(valueBuilder.GetDateBuilder()); + if (NUdf::MAX_DATETIME - shift <= dt) { + return {}; + } + storage.FromDatetime(valueBuilder.GetDateBuilder(), dt + shift, storage.TimezoneId); + SetEndOfDay(storage); + if (!storage.Validate(valueBuilder.GetDateBuilder())) { + return {}; + } + return storage; + } + + BEGIN_SIMPLE_STRICT_ARROW_UDF(TEndOfWeek, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { + return SimpleDatetimeToDatetimeUdf<EndOfWeek>(valueBuilder, args); + } + END_SIMPLE_ARROW_UDF(TEndOfWeek, TStartOfKernelExec<EndOfWeek>::Do); + TMaybe<TTMStorage> StartOfDay(TTMStorage storage, const IValueBuilder& valueBuilder) { storage.Hour = 0; storage.Minute = 0; @@ -1524,16 +1558,24 @@ TValue DoAddYears(const TValue& date, i64 years, const NUdf::IDateBuilder& build } BEGIN_SIMPLE_STRICT_ARROW_UDF(TStartOfDay, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { - auto result = args[0]; - auto& storage = Reference(result); - if (auto res = StartOfDay(storage, *valueBuilder)) { - storage = res.GetRef(); - return result; - } - return TUnboxedValuePod{}; + return SimpleDatetimeToDatetimeUdf<StartOfDay>(valueBuilder, args); } END_SIMPLE_ARROW_UDF(TStartOfDay, TStartOfKernelExec<StartOfDay>::Do); + TMaybe<TTMStorage> EndOfDay(TTMStorage storage, const IValueBuilder& valueBuilder) { + SetEndOfDay(storage); + auto& builder = valueBuilder.GetDateBuilder(); + if (!storage.Validate(builder)) { + return {}; + } + return storage; + } + + BEGIN_SIMPLE_STRICT_ARROW_UDF(TEndOfDay, TOptional<TResource<TMResourceName>>(TAutoMap<TResource<TMResourceName>>)) { + return SimpleDatetimeToDatetimeUdf<EndOfDay>(valueBuilder, args); + } + END_SIMPLE_ARROW_UDF(TEndOfDay, TStartOfKernelExec<EndOfDay>::Do); + TMaybe<TTMStorage> StartOf(TTMStorage storage, ui64 interval, const IValueBuilder& valueBuilder) { if (interval >= 86400000000ull) { // treat as StartOfDay @@ -2375,7 +2417,11 @@ TValue DoAddYears(const TValue& date, i64 years, const NUdf::IDateBuilder& build TShiftQuarters, TShiftMonths, + TEndOfYear, + TEndOfQuarter, TEndOfMonth, + TEndOfWeek, + TEndOfDay, TToUnits<ToSecondsName, ui32, 1>, TToUnits<ToMillisecondsName, ui64, 1000>, diff --git a/yql/essentials/udfs/common/datetime2/test/canondata/test.test_BlockStartOf_/results.txt b/yql/essentials/udfs/common/datetime2/test/canondata/test.test_BlockStartOf_/results.txt index 2089000383..59d2c26df8 100644 --- a/yql/essentials/udfs/common/datetime2/test/canondata/test.test_BlockStartOf_/results.txt +++ b/yql/essentials/udfs/common/datetime2/test/canondata/test.test_BlockStartOf_/results.txt @@ -116,16 +116,6 @@ "Interval" ] ] - ]; - [ - "column11"; - [ - "OptionalType"; - [ - "DataType"; - "TzTimestamp" - ] - ] ] ] ] @@ -152,9 +142,6 @@ ]; [ "18000000000" - ]; - [ - "1970-01-31T00:00:00,Europe/Moscow" ] ]; [ @@ -190,9 +177,6 @@ ]; [ "3723456789" - ]; - [ - "2018-12-31T00:00:00,Europe/Moscow" ] ]; [ @@ -228,9 +212,6 @@ ]; [ "59025000000" - ]; - [ - "2105-12-31T00:00:00,GMT" ] ]; [ @@ -266,8 +247,7 @@ ]; [ "3600000000" - ]; - # + ] ]; [ [ @@ -302,9 +282,6 @@ ]; [ "43200000000" - ]; - [ - "2019-07-31T00:00:00,Europe/Moscow" ] ] ] diff --git a/yql/essentials/udfs/common/datetime2/test/canondata/test.test_EndOf_/results.txt b/yql/essentials/udfs/common/datetime2/test/canondata/test.test_EndOf_/results.txt index 508cd4438e..8dafce2473 100644 --- a/yql/essentials/udfs/common/datetime2/test/canondata/test.test_EndOf_/results.txt +++ b/yql/essentials/udfs/common/datetime2/test/canondata/test.test_EndOf_/results.txt @@ -64,19 +64,19 @@ "Data" = [ [ [ - "2023-07-31 00:00:00 Europe/Moscow" + "2023-07-31 23:59:59.999999 Europe/Moscow" ]; [ - "2023-08-31 00:00:00 GMT" + "2023-08-31 23:59:59.999999 GMT" ]; [ - "2023-09-30 00:00:00 GMT" + "2023-09-30 23:59:59.999999 GMT" ]; [ - "2023-02-28 00:00:00 GMT" + "2023-02-28 23:59:59.999999 GMT" ]; [ - "2024-02-29 00:00:00 GMT" + "2024-02-29 23:59:59.999999 GMT" ] ] ] @@ -131,7 +131,7 @@ "1970-01-01 00:00:00 GMT" ]; [ - "1970-01-31 00:00:00 GMT" + "1970-01-31 23:59:59.999999 GMT" ]; # ] @@ -197,10 +197,10 @@ "2105-12-31 23:59:59.999999 GMT" ]; [ - "2105-12-31 00:00:00 GMT" + "2105-12-31 23:59:59.999999 GMT" ]; [ - "2105-12-31 00:00:00 GMT" + "2105-12-31 23:59:59.999999 GMT" ]; # ] @@ -245,7 +245,9 @@ [ "1969-12-31 23:59:59.999999 Atlantic/Azores" ]; - # + [ + "1969-12-31 23:59:59.999999 Atlantic/Azores" + ] ] ] } @@ -291,5 +293,145 @@ ] } ] + }; + { + "Label" = "Other cases"; + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "column1"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "column2"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "column3"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "column4"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [ + "1970-12-31 23:59:59.999999 Europe/Moscow" + ]; + [ + "1970-03-31 23:59:59.999999 Europe/Moscow" + ]; + [ + "1970-01-31 23:59:59.999999 Europe/Moscow" + ]; + [ + "1970-01-04 23:59:59.999999 Europe/Moscow" + ]; + [ + "1970-01-01 23:59:59.999999 Europe/Moscow" + ] + ]; + [ + [ + "2018-12-31 23:59:59.999999 Europe/Moscow" + ]; + [ + "2018-12-31 23:59:59.999999 Europe/Moscow" + ]; + [ + "2018-12-31 23:59:59.999999 Europe/Moscow" + ]; + [ + "2018-12-16 23:59:59.999999 Europe/Moscow" + ]; + [ + "2018-12-15 23:59:59.999999 Europe/Moscow" + ] + ]; + [ + [ + "2105-12-31 23:59:59.999999 GMT" + ]; + [ + "2105-12-31 23:59:59.999999 GMT" + ]; + [ + "2105-12-31 23:59:59.999999 GMT" + ]; + #; + [ + "2105-12-31 23:59:59.999999 GMT" + ] + ]; + [ + #; + #; + #; + #; + # + ]; + [ + [ + "2019-12-31 23:59:59.999999 Europe/Moscow" + ]; + [ + "2019-09-30 23:59:59.999999 Europe/Moscow" + ]; + [ + "2019-07-31 23:59:59.999999 Europe/Moscow" + ]; + [ + "2019-07-28 23:59:59.999999 Europe/Moscow" + ]; + [ + "2019-07-24 23:59:59.999999 Europe/Moscow" + ] + ] + ] + } + ] } ]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/datetime2/test/canondata/test.test_StartOf_/results.txt b/yql/essentials/udfs/common/datetime2/test/canondata/test.test_StartOf_/results.txt index b548b00a7c..622ad731d9 100644 --- a/yql/essentials/udfs/common/datetime2/test/canondata/test.test_StartOf_/results.txt +++ b/yql/essentials/udfs/common/datetime2/test/canondata/test.test_StartOf_/results.txt @@ -116,16 +116,6 @@ "Interval" ] ] - ]; - [ - "column11"; - [ - "OptionalType"; - [ - "DataType"; - "String" - ] - ] ] ] ] @@ -152,9 +142,6 @@ ]; [ "18000000000" - ]; - [ - "1970-01-31 00:00:00 Europe/Moscow" ] ]; [ @@ -190,9 +177,6 @@ ]; [ "3723456789" - ]; - [ - "2018-12-31 00:00:00 Europe/Moscow" ] ]; [ @@ -228,9 +212,6 @@ ]; [ "59025000000" - ]; - [ - "2105-12-31 00:00:00 GMT" ] ]; [ @@ -266,8 +247,7 @@ ]; [ "3600000000" - ]; - # + ] ]; [ [ @@ -302,9 +282,6 @@ ]; [ "43200000000" - ]; - [ - "2019-07-31 00:00:00 Europe/Moscow" ] ] ] diff --git a/yql/essentials/udfs/common/datetime2/test/cases/BlockStartOf.sql b/yql/essentials/udfs/common/datetime2/test/cases/BlockStartOf.sql index e531d6f1c8..915f8da347 100644 --- a/yql/essentials/udfs/common/datetime2/test/cases/BlockStartOf.sql +++ b/yql/essentials/udfs/common/datetime2/test/cases/BlockStartOf.sql @@ -24,7 +24,5 @@ select DateTime::StartOf(`tztimestamp`, Interval("PT15M")), DateTime::StartOf(`tztimestamp`, Interval("PT20S")), DateTime::StartOf(`tztimestamp`, Interval("PT7S")), - DateTime::TimeOfDay(`tztimestamp`), - - DateTime::EndOfMonth(`tztimestamp`), + DateTime::TimeOfDay(`tztimestamp`) from @t; diff --git a/yql/essentials/udfs/common/datetime2/test/cases/EndOf.in b/yql/essentials/udfs/common/datetime2/test/cases/EndOf.in new file mode 100644 index 0000000000..f482585e72 --- /dev/null +++ b/yql/essentials/udfs/common/datetime2/test/cases/EndOf.in @@ -0,0 +1,15 @@ +{ + "ftztimestamp"="1970-01-01T05:00:00.000000,Europe/Moscow" +}; +{ + "ftztimestamp"="2018-12-15T01:02:03.456789,Europe/Moscow" +}; +{ + "ftztimestamp"="2105-12-31T16:23:45.000000,GMT" +}; +{ + "ftztimestamp"="2106-01-01T01:00:00.000000,Europe/Moscow" +}; +{ + "ftztimestamp"="2019-07-24T12:00:00,Europe/Moscow" +}; diff --git a/yql/essentials/udfs/common/datetime2/test/cases/EndOf.in.attr b/yql/essentials/udfs/common/datetime2/test/cases/EndOf.in.attr new file mode 100644 index 0000000000..2cc4f8c0d6 --- /dev/null +++ b/yql/essentials/udfs/common/datetime2/test/cases/EndOf.in.attr @@ -0,0 +1,17 @@ +{ + "_yql_row_spec" = { + "Type" = [ + "StructType"; + [ + [ + "ftztimestamp"; + [ + "DataType"; + "String" + ] + ] + ] + ] + } +} + diff --git a/yql/essentials/udfs/common/datetime2/test/cases/EndOf.sql b/yql/essentials/udfs/common/datetime2/test/cases/EndOf.sql index 61b4a29e53..ee0eb55e6a 100644 --- a/yql/essentials/udfs/common/datetime2/test/cases/EndOf.sql +++ b/yql/essentials/udfs/common/datetime2/test/cases/EndOf.sql @@ -32,3 +32,16 @@ into result `Timestamp below minimum`; select $format(cast($tsAbove || ',Atlantic/Madeira' as TzTimestamp)) , $format(DateTime::EndOfMonth(cast($tsAbove || ',Atlantic/Madeira' as TzTimestamp))) into result `Timestamp above maximum`; + +select + $format(DateTime::EndOfYear(`tztimestamp`)), + $format(DateTime::EndOfQuarter(`tztimestamp`)), + $format(DateTime::EndOfMonth(`tztimestamp`)), + $format(DateTime::EndOfWeek(`tztimestamp`)), + $format(DateTime::EndOfDay(`tztimestamp`)), +from ( + select + cast(ftztimestamp as TzTimestamp) as `tztimestamp` + from Input +) +into result `Other cases`; diff --git a/yql/essentials/udfs/common/datetime2/test/cases/StartOf.sql b/yql/essentials/udfs/common/datetime2/test/cases/StartOf.sql index 201db38230..81fad12632 100644 --- a/yql/essentials/udfs/common/datetime2/test/cases/StartOf.sql +++ b/yql/essentials/udfs/common/datetime2/test/cases/StartOf.sql @@ -12,8 +12,7 @@ select $format(DateTime::StartOf(`tztimestamp`, Interval("PT15M"))), $format(DateTime::StartOf(`tztimestamp`, Interval("PT20S"))), $format(DateTime::StartOf(`tztimestamp`, Interval("PT7S"))), - DateTime::TimeOfDay(`tztimestamp`), - $format(DateTime::EndOfMonth(`tztimestamp`)), + DateTime::TimeOfDay(`tztimestamp`) from ( select cast(ftztimestamp as TzTimestamp) as `tztimestamp` |