summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorOleg Doronin <[email protected]>2022-06-24 16:30:52 +0300
committerOleg Doronin <[email protected]>2022-06-24 16:30:52 +0300
commit0796a2f91010abe8890cee7570eb2f61efab27c7 (patch)
treee959082bf994f049a121988bd614c4f1a728aa43
parentc9301d00fe2cd9d315f2298e1dc5101c04d252f0 (diff)
string parsers has been added YQ-1129
ref:089cee53d3991768b7f11917c513bf0ad6cf9c7b
-rw-r--r--ydb/library/yql/providers/s3/path_generator/ut/yql_parse_partitioning_rules_ut.cpp42
-rw-r--r--ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp40
2 files changed, 71 insertions, 11 deletions
diff --git a/ydb/library/yql/providers/s3/path_generator/ut/yql_parse_partitioning_rules_ut.cpp b/ydb/library/yql/providers/s3/path_generator/ut/yql_parse_partitioning_rules_ut.cpp
index d94358dfb2d..2cfbb77e010 100644
--- a/ydb/library/yql/providers/s3/path_generator/ut/yql_parse_partitioning_rules_ut.cpp
+++ b/ydb/library/yql/providers/s3/path_generator/ut/yql_parse_partitioning_rules_ut.cpp
@@ -94,12 +94,52 @@ Y_UNIT_TEST_SUITE(TParseTests) {
{
"projection.enabled" : true,
"projection.city.type" : "enum",
- "projection.city.values" : 23,
+ "projection.city.values" : null,
"storage.location.template" : "/${city}/"
}
)", {"city"}), yexception, "The values must be a string");
}
+ Y_UNIT_TEST(SuccessParseInteger) {
+ auto result = ParsePartitioningRules(R"(
+ {
+ "projection.enabled" : true,
+ "projection.id.type" : "integer",
+ "projection.id.min" : "5",
+ "projection.id.max" : "6",
+ "storage.location.template" : "/${id}/"
+ }
+ )", {"id"});
+ UNIT_ASSERT_VALUES_EQUAL(result.Enabled, true);
+ UNIT_ASSERT_VALUES_EQUAL(result.LocationTemplate, "/${id}/");
+ UNIT_ASSERT_VALUES_EQUAL(result.Rules.size(), 1);
+ const auto& rule = result.Rules.front();
+ UNIT_ASSERT_VALUES_EQUAL(rule.Type, EType::INTEGER);
+ UNIT_ASSERT_VALUES_EQUAL(rule.Name, "id");
+ UNIT_ASSERT_VALUES_EQUAL(rule.Min, 5);
+ UNIT_ASSERT_VALUES_EQUAL(rule.Max, 6);
+ }
+
+ Y_UNIT_TEST(SuccessDisableProjection) {
+ auto result = ParsePartitioningRules(R"(
+ {
+ "projection.enabled" : false,
+ "projection.id.type" : "integer",
+ "projection.id.min" : "5",
+ "projection.id.max" : "6",
+ "storage.location.template" : "/${id}/"
+ }
+ )", {"id"});
+ UNIT_ASSERT_VALUES_EQUAL(result.Enabled, false);
+ UNIT_ASSERT_VALUES_EQUAL(result.LocationTemplate, "/${id}/");
+ UNIT_ASSERT_VALUES_EQUAL(result.Rules.size(), 1);
+ const auto& rule = result.Rules.front();
+ UNIT_ASSERT_VALUES_EQUAL(rule.Type, EType::INTEGER);
+ UNIT_ASSERT_VALUES_EQUAL(rule.Name, "id");
+ UNIT_ASSERT_VALUES_EQUAL(rule.Min, 5);
+ UNIT_ASSERT_VALUES_EQUAL(rule.Max, 6);
+ }
+
}
}
diff --git a/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp b/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp
index dc0de907ff3..98d5c5834a1 100644
--- a/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp
+++ b/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp
@@ -23,24 +23,46 @@ TVector<TString> GetPath(const TStringBuf& key) {
}
i64 GetIntOrThrow(const NSc::TValue& json, const TString& error) {
- if (json.IsIntNumber()) {
+ if (json.IsNumber()) {
return json.GetIntNumber();
}
+
+ if (TStringBuf str = json.GetString(TStringBuf())) {
+ {
+ i64 result = 0;
+ if (TryFromString<i64>(str, result)) {
+ return result;
+ }
+ }
+ {
+ ui64 result = 0;
+ if (TryFromString<ui64>(str, result)) {
+ return result;
+ }
+ }
+ {
+ double result = 0;
+ if (TryFromString<double>(str, result)) {
+ return result;
+ }
+ }
+ }
+
ythrow yexception() << error;
}
TString GetStringOrThrow(const NSc::TValue& json, const TString& error) {
- if (json.IsString()) {
+ if (json.IsString() || json.IsIntNumber() || json.IsNumber()) {
return json.ForceString();
}
ythrow yexception() << error;
}
i64 GetBoolOrThrow(const NSc::TValue& json, const TString& error) {
- if (json.IsBool()) {
- return json.GetBool();
+ if (json.IsNull()) {
+ ythrow yexception() << error;
}
- ythrow yexception() << error;
+ return json.IsTrue();
}
EIntervalUnit ToIntervalUnit(const TString& unit) {
@@ -432,7 +454,6 @@ ExplicitPartitioningConfig ParsePartitioningRules(const TString& config, const s
}
if (!config) {
- result.Enabled = true;
for (const auto& columnName: partitionBy) {
result.LocationTemplate += "/" + columnName + "=${" + columnName + "}";
}
@@ -466,10 +487,9 @@ ExplicitPartitioningConfig ParsePartitioningRules(const TString& config, const s
std::vector<ExpandedPartitioningRule> ExpandPartitioningRules(const ExplicitPartitioningConfig& config, int pathsLimit) {
if (!config.Enabled) {
- return {};
- }
- if (config.Rules.empty()) {
- return {ExpandedPartitioningRule{.Path=config.LocationTemplate}};
+ return config.LocationTemplate
+ ? std::vector{ExpandedPartitioningRule{.Path=config.LocationTemplate}}
+ : std::vector<ExpandedPartitioningRule>{};
}
auto now = TInstant::Now();
TMap<TString, std::vector<ColumnWithValue>> result;