Support for big datetime in table_flat (#5829)

3 files changed, 184 insertions, 7 deletions

diff --git a/ydb/core/tablet_flat/datetime_ut.cpp b/ydb/core/tablet_flat/datetime_ut.cpp
index 6311a4b9d73..1e4883aad4d 100644
--- a/ydb/core/tablet_flat/datetime_ut.cpp
+++ b/ydb/core/tablet_flat/datetime_ut.cpp
@@ -21,9 +21,13 @@ Y_UNIT_TEST_SUITE(TFlatTableDatetime) {
                 struct Datetime : Column<3, NScheme::NTypeIds::Datetime> {};
                 struct Timestamp : Column<4, NScheme::NTypeIds::Timestamp> {};
                 struct Interval : Column<5, NScheme::NTypeIds::Interval> {};
+                struct Date32 : Column<6, NScheme::NTypeIds::Date32> {};
+                struct Datetime64 : Column<7, NScheme::NTypeIds::Datetime64> {};
+                struct Timestamp64 : Column<8, NScheme::NTypeIds::Timestamp64> {};
+                struct Interval64 : Column<9, NScheme::NTypeIds::Interval64> {};
 
                 using TKey = TableKey<Key>;
-                using TColumns = TableColumns<Key, Date, Datetime, Timestamp, Interval>;
+                using TColumns = TableColumns<Key, Date, Datetime, Timestamp, Interval, Date32, Datetime64, Timestamp64, Interval64>;
             };
 
             struct DateKey : Table<2> {
@@ -58,8 +62,39 @@ Y_UNIT_TEST_SUITE(TFlatTableDatetime) {
                 using TColumns = TableColumns<Key, Value>;
             };
 
+            struct Date32Key : Table<6> {
+                struct Key : Column<1, NScheme::NTypeIds::Date32> {};
+                struct Value : Column<2, NScheme::NTypeIds::Uint32> {};
+
+                using TKey = TableKey<Key>;
+                using TColumns = TableColumns<Key, Value>;
+            };
+
+            struct Datetime64Key : Table<7> {
+                struct Key : Column<1, NScheme::NTypeIds::Datetime64> {};
+                struct Value : Column<2, NScheme::NTypeIds::Uint32> {};
+
+                using TKey = TableKey<Key>;
+                using TColumns = TableColumns<Key, Value>;
+            };
+
+            struct Timestamp64Key : Table<8> {
+                struct Key : Column<1, NScheme::NTypeIds::Timestamp64> {};
+                struct Value : Column<2, NScheme::NTypeIds::Uint32> {};
+
+                using TKey = TableKey<Key>;
+                using TColumns = TableColumns<Key, Value>;
+            };
+
+            struct Interval64Key : Table<9> {
+                struct Key : Column<1, NScheme::NTypeIds::Interval64> {};
+                struct Value : Column<2, NScheme::NTypeIds::Uint32> {};
+
+                using TKey = TableKey<Key>;
+                using TColumns = TableColumns<Key, Value>;
+            };
 
-            using TTables = SchemaTables<DateValue, DateKey, DatetimeKey, TimestampKey, IntervalKey>;
+            using TTables = SchemaTables<DateValue, DateKey, DatetimeKey, TimestampKey, IntervalKey, Date32Key, Datetime64Key, Timestamp64Key, Interval64Key>;
             using TSettings = SchemaSettings<ExecutorLogBatching<true>,
                                              ExecutorLogFlushPeriod<TDuration::MicroSeconds(512).GetValue()>>;
         };
@@ -105,17 +140,30 @@ Y_UNIT_TEST_SUITE(TFlatTableDatetime) {
                     .Update<Schema::DateValue::Date>(Min<ui16>())
                     .Update<Schema::DateValue::Datetime>(Min<ui32>())
                     .Update<Schema::DateValue::Timestamp>(Min<ui64>())
-                    .Update<Schema::DateValue::Interval>(Min<i64>());
+                    .Update<Schema::DateValue::Interval>(Min<i64>())
+                    .Update<Schema::DateValue::Date32>(Min<i32>())
+                    .Update<Schema::DateValue::Datetime64>(Min<i64>())
+                    .Update<Schema::DateValue::Timestamp64>(Min<i64>())
+                    .Update<Schema::DateValue::Interval64>(Min<i64>());
                 db.Table<Schema::DateValue>().Key(2)
                     .Update<Schema::DateValue::Date>((ui16)100)
                     .Update<Schema::DateValue::Datetime>((ui32)100)
                     .Update<Schema::DateValue::Timestamp>((ui64)100)
-                    .Update<Schema::DateValue::Interval>((i64)100);
+                    .Update<Schema::DateValue::Interval>((i64)100)
+                    .Update<Schema::DateValue::Date32>((i32)-100)
+                    .Update<Schema::DateValue::Datetime64>((i64)-100)
+                    .Update<Schema::DateValue::Timestamp64>((i64)-100)
+                    .Update<Schema::DateValue::Interval64>((i64)-100);
+                    
                 db.Table<Schema::DateValue>().Key(3)
                     .Update<Schema::DateValue::Date>(Max<ui16>())
                     .Update<Schema::DateValue::Datetime>(Max<ui32>())
                     .Update<Schema::DateValue::Timestamp>(Max<ui64>())
-                    .Update<Schema::DateValue::Interval>(Max<i64>());
+                    .Update<Schema::DateValue::Interval>(Max<i64>())
+                    .Update<Schema::DateValue::Date32>(Max<i32>())
+                    .Update<Schema::DateValue::Datetime64>(Max<i64>())
+                    .Update<Schema::DateValue::Timestamp64>(Max<i64>())
+                    .Update<Schema::DateValue::Interval64>(Max<i64>());
 
                 // DateKey
                 for (ui16 i = 1; i <= 10; ++i) {
@@ -127,6 +175,14 @@ Y_UNIT_TEST_SUITE(TFlatTableDatetime) {
                         .Update<Schema::DateKey::Value>(i);
                     db.Table<Schema::IntervalKey>().Key((i64)i - 5)
                         .Update<Schema::DateKey::Value>(i);
+                    db.Table<Schema::Date32Key>().Key(i)
+                        .Update<Schema::Date32Key::Value>(i);
+                    db.Table<Schema::Datetime64Key>().Key((ui32)i)
+                        .Update<Schema::DateKey::Value>(i);
+                    db.Table<Schema::Timestamp64Key>().Key((ui64)i)
+                        .Update<Schema::DateKey::Value>(i);
+                    db.Table<Schema::Interval64Key>().Key((i64)i - 5)
+                        .Update<Schema::DateKey::Value>(i);
                 }
 
                 return true;
@@ -158,14 +214,26 @@ Y_UNIT_TEST_SUITE(TFlatTableDatetime) {
                     UNIT_ASSERT_VALUES_EQUAL(row1.GetValue<Schema::DateValue::Datetime>(), Min<ui32>());
                     UNIT_ASSERT_VALUES_EQUAL(row1.GetValue<Schema::DateValue::Timestamp>(), Min<ui64>());
                     UNIT_ASSERT_VALUES_EQUAL(row1.GetValue<Schema::DateValue::Interval>(), Min<i64>());
+                    UNIT_ASSERT_VALUES_EQUAL(row1.GetValue<Schema::DateValue::Date32>(), Min<i32>());
+                    UNIT_ASSERT_VALUES_EQUAL(row1.GetValue<Schema::DateValue::Datetime64>(), Min<i64>());
+                    UNIT_ASSERT_VALUES_EQUAL(row1.GetValue<Schema::DateValue::Timestamp64>(), Min<i64>());
+                    UNIT_ASSERT_VALUES_EQUAL(row1.GetValue<Schema::DateValue::Interval64>(), Min<i64>());
                     UNIT_ASSERT_VALUES_EQUAL(row2.GetValue<Schema::DateValue::Date>(), (ui16)100);
                     UNIT_ASSERT_VALUES_EQUAL(row2.GetValue<Schema::DateValue::Datetime>(), (ui32)100);
                     UNIT_ASSERT_VALUES_EQUAL(row2.GetValue<Schema::DateValue::Timestamp>(), (ui64)100);
                     UNIT_ASSERT_VALUES_EQUAL(row2.GetValue<Schema::DateValue::Interval>(), (i64)100);
+                    UNIT_ASSERT_VALUES_EQUAL(row2.GetValue<Schema::DateValue::Date32>(), (i32)-100);
+                    UNIT_ASSERT_VALUES_EQUAL(row2.GetValue<Schema::DateValue::Datetime64>(), (i64)-100);
+                    UNIT_ASSERT_VALUES_EQUAL(row2.GetValue<Schema::DateValue::Timestamp64>(), (i64)-100);
+                    UNIT_ASSERT_VALUES_EQUAL(row2.GetValue<Schema::DateValue::Interval64>(), (i64)-100);
                     UNIT_ASSERT_VALUES_EQUAL(row3.GetValue<Schema::DateValue::Date>(), Max<ui16>());
                     UNIT_ASSERT_VALUES_EQUAL(row3.GetValue<Schema::DateValue::Datetime>(), Max<ui32>());
                     UNIT_ASSERT_VALUES_EQUAL(row3.GetValue<Schema::DateValue::Timestamp>(), Max<ui64>());
                     UNIT_ASSERT_VALUES_EQUAL(row3.GetValue<Schema::DateValue::Interval>(), Max<i64>());
+                    UNIT_ASSERT_VALUES_EQUAL(row3.GetValue<Schema::DateValue::Date32>(), Max<i32>());
+                    UNIT_ASSERT_VALUES_EQUAL(row3.GetValue<Schema::DateValue::Datetime64>(), Max<i64>());
+                    UNIT_ASSERT_VALUES_EQUAL(row3.GetValue<Schema::DateValue::Timestamp64>(), Max<i64>());
+                    UNIT_ASSERT_VALUES_EQUAL(row3.GetValue<Schema::DateValue::Interval64>(), Max<i64>());
                 }
 
                 // DateKey
@@ -200,6 +268,38 @@ Y_UNIT_TEST_SUITE(TFlatTableDatetime) {
                     UNIT_ASSERT_VALUES_EQUAL(row.GetValue<Schema::IntervalKey::Value>(), i);
                 }
 
+                // Date32Key
+                for (ui16 i = 1; i <= 10; ++i) {
+                    auto row = db.Table<Schema::Date32Key>().Key(i).Select();
+                    if (!IsReady(row))
+                        return false;
+                    UNIT_ASSERT_VALUES_EQUAL(row.GetValue<Schema::Date32Key::Value>(), (ui16)i);
+                }
+
+                // Datetime64Key
+                for (ui32 i = 1; i <= 10; ++i) {
+                    auto row = db.Table<Schema::Datetime64Key>().Key(i).Select();
+                    if (!IsReady(row))
+                        return false;
+                    UNIT_ASSERT_VALUES_EQUAL(row.GetValue<Schema::Datetime64Key::Value>(), i);
+                }
+
+                // Timestamp64Key
+                for (ui64 i = 1; i <= 10; ++i) {
+                    auto row = db.Table<Schema::Timestamp64Key>().Key(i).Select();
+                    if (!IsReady(row))
+                        return false;
+                    UNIT_ASSERT_VALUES_EQUAL(row.GetValue<Schema::Timestamp64Key::Value>(), i);
+                }
+
+                // Interval64Key
+                for (i16 i = 1; i <= 10; ++i) {
+                    auto row = db.Table<Schema::Interval64Key>().Key(i - 5).Select();
+                    if (!IsReady(row))
+                        return false;
+                    UNIT_ASSERT_VALUES_EQUAL(row.GetValue<Schema::Interval64Key::Value>(), i);
+                }                
+
                 return true;
             }
 
@@ -276,6 +376,66 @@ Y_UNIT_TEST_SUITE(TFlatTableDatetime) {
                     UNIT_ASSERT(rowset.EndOfSet());
                 }
 
+                // Date32Key
+                {
+                    auto rowset = db.Table<Schema::Date32Key>().GreaterOrEqual(5).Select();
+                    if (!rowset.IsReady())
+                        return false;
+                    for (ui16 i = 5; i <= 10; ++i) {
+                        UNIT_ASSERT(!rowset.EndOfSet());
+                        UNIT_ASSERT_VALUES_EQUAL(rowset.GetKey(), i);
+                        UNIT_ASSERT_VALUES_EQUAL(rowset.GetValue<Schema::Date32Key::Value>(), i);
+                        if (!rowset.Next())
+                            return false;
+                    }
+                    UNIT_ASSERT(rowset.EndOfSet());
+                }
+
+                // Datetime64Key
+                {
+                    auto rowset = db.Table<Schema::Datetime64Key>().GreaterOrEqual(5).Select();
+                    if (!rowset.IsReady())
+                        return false;
+                    for (ui32 i = 5; i <= 10; ++i) {
+                        UNIT_ASSERT(!rowset.EndOfSet());
+                        UNIT_ASSERT_VALUES_EQUAL(rowset.GetKey(), i);
+                        UNIT_ASSERT_VALUES_EQUAL(rowset.GetValue<Schema::Datetime64Key::Value>(), i);
+                        if (!rowset.Next())
+                            return false;
+                    }
+                    UNIT_ASSERT(rowset.EndOfSet());
+                }
+
+                // Timestamp64Key
+                {
+                    auto rowset = db.Table<Schema::Timestamp64Key>().GreaterOrEqual(5).Select();
+                    if (!rowset.IsReady())
+                        return false;
+                    for (ui64 i = 5; i <= 10; ++i) {
+                        UNIT_ASSERT(!rowset.EndOfSet());
+                        UNIT_ASSERT_VALUES_EQUAL(rowset.GetKey(), i);
+                        UNIT_ASSERT_VALUES_EQUAL(rowset.GetValue<Schema::Timestamp64Key::Value>(), i);
+                        if (!rowset.Next())
+                            return false;
+                    }
+                    UNIT_ASSERT(rowset.EndOfSet());
+                }
+
+                // Interval64Key
+                {
+                    auto rowset = db.Table<Schema::Interval64Key>().GreaterOrEqual(-2).Select();
+                    if (!rowset.IsReady())
+                        return false;
+                    for (i64 i = 3; i <= 10; ++i) {
+                        UNIT_ASSERT(!rowset.EndOfSet());
+                        UNIT_ASSERT_VALUES_EQUAL(rowset.GetKey(), i - 5);
+                        UNIT_ASSERT_VALUES_EQUAL(rowset.GetValue<Schema::Interval64Key::Value>(), i);
+                        if (!rowset.Next())
+                            return false;
+                    }
+                    UNIT_ASSERT(rowset.EndOfSet());
+                }                
+
                 return true;
             }
 
diff --git a/ydb/core/tablet_flat/flat_cxx_database.h b/ydb/core/tablet_flat/flat_cxx_database.h
index 06acbcfd69d..aa4ed80e554 100644
--- a/ydb/core/tablet_flat/flat_cxx_database.h
+++ b/ydb/core/tablet_flat/flat_cxx_database.h
@@ -110,7 +110,10 @@ public:
 
     operator i64() const {
         Y_ABORT_UNLESS((Type() == NScheme::NTypeIds::Int64
-                  || Type() == NScheme::NTypeIds::Interval)
+                  || Type() == NScheme::NTypeIds::Interval
+                  || Type() == NScheme::NTypeIds::Datetime64
+                  || Type() == NScheme::NTypeIds::Timestamp64
+                  || Type() == NScheme::NTypeIds::Interval64)
                  && Size() == sizeof(i64), "Data=%" PRIxPTR ", Type=%" PRIi64 ", Size=%" PRIi64, (ui64)Data(), (i64)Type(), (i64)Size());
         return ReadUnaligned<i64>(reinterpret_cast<const i64*>(Data()));
     }
@@ -124,7 +127,9 @@ public:
     }
 
     operator i32() const {
-        Y_ABORT_UNLESS(Type() == NScheme::NTypeIds::Int32 && Size() == sizeof(i32), "Data=%" PRIxPTR ", Type=%" PRIi64 ", Size=%" PRIi64, (ui64)Data(), (i64)Type(), (i64)Size());
+        Y_ABORT_UNLESS((Type() == NScheme::NTypeIds::Int32 
+                  || Type() == NScheme::NTypeIds::Date32)
+                 && Size() == sizeof(i32), "Data=%" PRIxPTR ", Type=%" PRIi64 ", Size=%" PRIi64, (ui64)Data(), (i64)Type(), (i64)Size());
         i32 value = ReadUnaligned<i32>(reinterpret_cast<const i32*>(Data()));
         return value;
     }
@@ -230,6 +235,10 @@ template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Date> { typedef ui16 Typ
 template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Datetime> { typedef ui32 Type; };
 template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Timestamp> { typedef ui64 Type; };
 template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Interval> { typedef i64 Type; };
+template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Date32> { typedef i32 Type; };
+template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Datetime64> { typedef i64 Type; };
+template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Timestamp64> { typedef i64 Type; };
+template <> struct NSchemeTypeMapper<NScheme::NTypeIds::Interval64> { typedef i64 Type; };
 
 /// only for compatibility with old code
 template <NScheme::TTypeId ValType>
diff --git a/ydb/core/tablet_flat/flat_executor_db_mon.cpp b/ydb/core/tablet_flat/flat_executor_db_mon.cpp
index 4f779591d44..469d74564ef 100644
--- a/ydb/core/tablet_flat/flat_executor_db_mon.cpp
+++ b/ydb/core/tablet_flat/flat_executor_db_mon.cpp
@@ -196,6 +196,14 @@ public:
                                         case NScheme::NTypeIds::Interval:
                                             str << *(i64*)data;
                                             break;
+                                        case NScheme::NTypeIds::Date32:
+                                            str << *(i32*)data;
+                                            break;
+                                        case NScheme::NTypeIds::Datetime64:
+                                        case NScheme::NTypeIds::Timestamp64:
+                                        case NScheme::NTypeIds::Interval64:
+                                            str << *(i64*)data;
+                                            break;
                                         case NScheme::NTypeIds::PairUi64Ui64:
                                             str << "(" << ((std::pair<ui64,ui64>*)data)->first << "," << ((std::pair<ui64,ui64>*)data)->second << ")";
                                             break;