aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorhcpp <hcpp@ydb.tech>2022-09-16 17:15:27 +0300
committerhcpp <hcpp@ydb.tech>2022-09-16 17:15:27 +0300
commit3bae125f8c3520947187a60ce193b839064aaf1b (patch)
treedef228611a124aaa46f402817de2a1d68f8d1a8f
parent2df8226558ba61b949a051ef03f59e2a60443cd4 (diff)
downloadydb-3bae125f8c3520947187a60ce193b839064aaf1b.tar.gz
date for projection has been fixed
-rw-r--r--ydb/library/yql/providers/s3/path_generator/ut/CMakeLists.darwin.txt2
-rw-r--r--ydb/library/yql/providers/s3/path_generator/ut/CMakeLists.linux.txt2
-rw-r--r--ydb/library/yql/providers/s3/path_generator/ut/yql_generate_partitioning_rules_ut.cpp26
-rw-r--r--ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp3
4 files changed, 31 insertions, 2 deletions
diff --git a/ydb/library/yql/providers/s3/path_generator/ut/CMakeLists.darwin.txt b/ydb/library/yql/providers/s3/path_generator/ut/CMakeLists.darwin.txt
index 9788a322ecf..3d2faa56a52 100644
--- a/ydb/library/yql/providers/s3/path_generator/ut/CMakeLists.darwin.txt
+++ b/ydb/library/yql/providers/s3/path_generator/ut/CMakeLists.darwin.txt
@@ -20,6 +20,8 @@ target_link_libraries(ydb-library-yql-providers-s3-path_generator-ut PUBLIC
library-cpp-cpuid_check
cpp-testing-unittest_main
providers-s3-path_generator
+ library-yql-minikql
+ udf-service-stub
)
target_link_options(ydb-library-yql-providers-s3-path_generator-ut PRIVATE
-Wl,-no_deduplicate
diff --git a/ydb/library/yql/providers/s3/path_generator/ut/CMakeLists.linux.txt b/ydb/library/yql/providers/s3/path_generator/ut/CMakeLists.linux.txt
index 7fda3164bb7..4209b249dde 100644
--- a/ydb/library/yql/providers/s3/path_generator/ut/CMakeLists.linux.txt
+++ b/ydb/library/yql/providers/s3/path_generator/ut/CMakeLists.linux.txt
@@ -22,6 +22,8 @@ target_link_libraries(ydb-library-yql-providers-s3-path_generator-ut PUBLIC
library-cpp-cpuid_check
cpp-testing-unittest_main
providers-s3-path_generator
+ library-yql-minikql
+ udf-service-stub
)
target_link_options(ydb-library-yql-providers-s3-path_generator-ut PRIVATE
-ldl
diff --git a/ydb/library/yql/providers/s3/path_generator/ut/yql_generate_partitioning_rules_ut.cpp b/ydb/library/yql/providers/s3/path_generator/ut/yql_generate_partitioning_rules_ut.cpp
index 413110a5909..0ab1ff286eb 100644
--- a/ydb/library/yql/providers/s3/path_generator/ut/yql_generate_partitioning_rules_ut.cpp
+++ b/ydb/library/yql/providers/s3/path_generator/ut/yql_generate_partitioning_rules_ut.cpp
@@ -2,6 +2,8 @@
#include <library/cpp/testing/unittest/registar.h>
+#include <ydb/library/yql/minikql/mkql_type_ops.h>
+
namespace NYql::NPathGenerator {
Y_UNIT_TEST_SUITE(TGenerateTests) {
@@ -239,6 +241,30 @@ Y_UNIT_TEST_SUITE(TGenerateTests) {
UNIT_ASSERT_VALUES_EQUAL(rules[1].Path, "yellow_tripdata_asdf%0 asdf2013%0 444-01.csv");
UNIT_ASSERT_VALUES_EQUAL(rules[1].ColumnValues.size(), 1);
}
+
+ Y_UNIT_TEST(TimestampFormatCheck) {
+ auto generator = CreatePathGenerator(R"(
+ {
+ "projection.enabled" : true,
+ "projection.dt.type" : "date",
+ "projection.dt.min" : "2012-01-01",
+ "projection.dt.max" : "2012-02-01",
+ "projection.dt.interval" : "1",
+ "projection.dt.format" : "asdf asdf 444",
+ "projection.dt.unit" : "YEARS",
+ "storage.location.template" : "yellow_tripdata_${dt}-01.csv"
+ }
+ )", {"dt"});
+
+ auto rules = generator->GetRules();
+ UNIT_ASSERT_VALUES_EQUAL(rules.size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(rules[0].Path, "yellow_tripdata_asdf asdf 444-01.csv");
+ UNIT_ASSERT_VALUES_EQUAL(rules[0].ColumnValues.size(), 1);
+ const auto& columnValue = rules[0].ColumnValues[0];
+ auto result = NKikimr::NMiniKQL::ValueFromString(columnValue.Type, TStringBuf{columnValue.Value});
+ UNIT_ASSERT(result.HasValue());
+ UNIT_ASSERT_VALUES_EQUAL(result.Get<ui32>(), 15340);
+ }
}
}
diff --git a/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp b/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp
index 2a30fca5ac1..3b180e84719 100644
--- a/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp
+++ b/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp
@@ -540,10 +540,9 @@ private:
TString copyLocationTemplate = locationTemplate;
const TString time = Strftime(rule.Format.c_str(), current);
ReplaceAll(copyLocationTemplate, "${" + rule.Name + "}", time);
- columnsWithValue.push_back(TColumnWithValue{.Name=rule.Name, .Type=NUdf::EDataSlot::String, .Value=time});
+ columnsWithValue.push_back(TColumnWithValue{.Name=rule.Name, .Type=NUdf::EDataSlot::Date, .Value=Strftime("%F", current)});
DoGenerate(rules, copyLocationTemplate, columnsWithValue, result, pathsLimit, now, p + 1);
columnsWithValue.pop_back();
-
if (IsOverflow(current.GetValue(), interval.GetValue())) {
return; // correct overflow handling
}