diff options
author | Oleg Doronin <[email protected]> | 2022-06-24 16:30:52 +0300 |
---|---|---|
committer | Oleg Doronin <[email protected]> | 2022-06-24 16:30:52 +0300 |
commit | 0796a2f91010abe8890cee7570eb2f61efab27c7 (patch) | |
tree | e959082bf994f049a121988bd614c4f1a728aa43 | |
parent | c9301d00fe2cd9d315f2298e1dc5101c04d252f0 (diff) |
string parsers has been added YQ-1129
ref:089cee53d3991768b7f11917c513bf0ad6cf9c7b
-rw-r--r-- | ydb/library/yql/providers/s3/path_generator/ut/yql_parse_partitioning_rules_ut.cpp | 42 | ||||
-rw-r--r-- | ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp | 40 |
2 files changed, 71 insertions, 11 deletions
diff --git a/ydb/library/yql/providers/s3/path_generator/ut/yql_parse_partitioning_rules_ut.cpp b/ydb/library/yql/providers/s3/path_generator/ut/yql_parse_partitioning_rules_ut.cpp index d94358dfb2d..2cfbb77e010 100644 --- a/ydb/library/yql/providers/s3/path_generator/ut/yql_parse_partitioning_rules_ut.cpp +++ b/ydb/library/yql/providers/s3/path_generator/ut/yql_parse_partitioning_rules_ut.cpp @@ -94,12 +94,52 @@ Y_UNIT_TEST_SUITE(TParseTests) { { "projection.enabled" : true, "projection.city.type" : "enum", - "projection.city.values" : 23, + "projection.city.values" : null, "storage.location.template" : "/${city}/" } )", {"city"}), yexception, "The values must be a string"); } + Y_UNIT_TEST(SuccessParseInteger) { + auto result = ParsePartitioningRules(R"( + { + "projection.enabled" : true, + "projection.id.type" : "integer", + "projection.id.min" : "5", + "projection.id.max" : "6", + "storage.location.template" : "/${id}/" + } + )", {"id"}); + UNIT_ASSERT_VALUES_EQUAL(result.Enabled, true); + UNIT_ASSERT_VALUES_EQUAL(result.LocationTemplate, "/${id}/"); + UNIT_ASSERT_VALUES_EQUAL(result.Rules.size(), 1); + const auto& rule = result.Rules.front(); + UNIT_ASSERT_VALUES_EQUAL(rule.Type, EType::INTEGER); + UNIT_ASSERT_VALUES_EQUAL(rule.Name, "id"); + UNIT_ASSERT_VALUES_EQUAL(rule.Min, 5); + UNIT_ASSERT_VALUES_EQUAL(rule.Max, 6); + } + + Y_UNIT_TEST(SuccessDisableProjection) { + auto result = ParsePartitioningRules(R"( + { + "projection.enabled" : false, + "projection.id.type" : "integer", + "projection.id.min" : "5", + "projection.id.max" : "6", + "storage.location.template" : "/${id}/" + } + )", {"id"}); + UNIT_ASSERT_VALUES_EQUAL(result.Enabled, false); + UNIT_ASSERT_VALUES_EQUAL(result.LocationTemplate, "/${id}/"); + UNIT_ASSERT_VALUES_EQUAL(result.Rules.size(), 1); + const auto& rule = result.Rules.front(); + UNIT_ASSERT_VALUES_EQUAL(rule.Type, EType::INTEGER); + UNIT_ASSERT_VALUES_EQUAL(rule.Name, "id"); + UNIT_ASSERT_VALUES_EQUAL(rule.Min, 5); + UNIT_ASSERT_VALUES_EQUAL(rule.Max, 6); + } + } } diff --git a/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp b/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp index dc0de907ff3..98d5c5834a1 100644 --- a/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp +++ b/ydb/library/yql/providers/s3/path_generator/yql_s3_path_generator.cpp @@ -23,24 +23,46 @@ TVector<TString> GetPath(const TStringBuf& key) { } i64 GetIntOrThrow(const NSc::TValue& json, const TString& error) { - if (json.IsIntNumber()) { + if (json.IsNumber()) { return json.GetIntNumber(); } + + if (TStringBuf str = json.GetString(TStringBuf())) { + { + i64 result = 0; + if (TryFromString<i64>(str, result)) { + return result; + } + } + { + ui64 result = 0; + if (TryFromString<ui64>(str, result)) { + return result; + } + } + { + double result = 0; + if (TryFromString<double>(str, result)) { + return result; + } + } + } + ythrow yexception() << error; } TString GetStringOrThrow(const NSc::TValue& json, const TString& error) { - if (json.IsString()) { + if (json.IsString() || json.IsIntNumber() || json.IsNumber()) { return json.ForceString(); } ythrow yexception() << error; } i64 GetBoolOrThrow(const NSc::TValue& json, const TString& error) { - if (json.IsBool()) { - return json.GetBool(); + if (json.IsNull()) { + ythrow yexception() << error; } - ythrow yexception() << error; + return json.IsTrue(); } EIntervalUnit ToIntervalUnit(const TString& unit) { @@ -432,7 +454,6 @@ ExplicitPartitioningConfig ParsePartitioningRules(const TString& config, const s } if (!config) { - result.Enabled = true; for (const auto& columnName: partitionBy) { result.LocationTemplate += "/" + columnName + "=${" + columnName + "}"; } @@ -466,10 +487,9 @@ ExplicitPartitioningConfig ParsePartitioningRules(const TString& config, const s std::vector<ExpandedPartitioningRule> ExpandPartitioningRules(const ExplicitPartitioningConfig& config, int pathsLimit) { if (!config.Enabled) { - return {}; - } - if (config.Rules.empty()) { - return {ExpandedPartitioningRule{.Path=config.LocationTemplate}}; + return config.LocationTemplate + ? std::vector{ExpandedPartitioningRule{.Path=config.LocationTemplate}} + : std::vector<ExpandedPartitioningRule>{}; } auto now = TInstant::Now(); TMap<TString, std::vector<ColumnWithValue>> result; |