summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorhcpp <[email protected]>2023-11-02 18:23:01 +0300
committerhcpp <[email protected]>2023-11-02 19:05:59 +0300
commit44ce0972717174bb8c38e0387067be6fce736ce2 (patch)
treec3d114957526b63e3e0fca9eccba722d1da05903
parentc336c58dfb573341b17f27c7b128e80416d87422 (diff)
location without slash has been fixed
-rw-r--r--ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp62
-rw-r--r--ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp3
2 files changed, 64 insertions, 1 deletions
diff --git a/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp b/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp
index bc48c8e10d1..1907857bfbf 100644
--- a/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp
+++ b/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp
@@ -1250,7 +1250,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) {
FROM `/Root/external_data_source`.`/`
WITH (
FORMAT="raw",
-
SCHEMA=(
`data` String NOT NULL
)
@@ -1381,6 +1380,67 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) {
UNIT_ASSERT_STRING_CONTAINS(resultFuture.GetValueSync().GetIssues().ToString(), "Callable not expected in effects tx: Unwrap");
}
}
+
+ Y_UNIT_TEST(ExecuteScriptWithLocationWithoutSlashAtTheEnd) {
+ using namespace fmt::literals;
+ const TString externalDataSourceName = "/Root/external_data_source";
+ const TString externalTableName = "/Root/test_binding_resolve";
+ const TString bucket = "test_bucket_with_location_without_slash_at_the_end";
+ const TString object = "year=1/month=2/test_object";
+ const TString content = "data,year,month\ntest,1,2";
+
+ CreateBucketWithObject(bucket, object, content);
+
+ auto kikimr = MakeKikimrRunner(NYql::IHTTPGateway::Make());
+
+ auto tc = kikimr->GetTableClient();
+ auto session = tc.CreateSession().GetValueSync().GetSession();
+ const TString query = fmt::format(R"(
+ CREATE EXTERNAL DATA SOURCE `{external_source}` WITH (
+ SOURCE_TYPE="ObjectStorage",
+ LOCATION="{location}",
+ AUTH_METHOD="NONE"
+ );
+ CREATE EXTERNAL TABLE `{external_table}` (
+ data STRING NOT NULL,
+ year UINT32 NOT NULL,
+ month UINT32 NOT NULL
+ ) WITH (
+ DATA_SOURCE="{external_source}",
+ LOCATION="/",
+ FORMAT="csv_with_names",
+ PARTITIONED_BY="[year, month]"
+ );)",
+ "external_source"_a = externalDataSourceName,
+ "external_table"_a = externalTableName,
+ "location"_a = TStringBuilder() << GetEnv("S3_ENDPOINT") << '/' << bucket
+ );
+
+ auto result = session.ExecuteSchemeQuery(query).GetValueSync();
+ UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString());
+
+ auto db = kikimr->GetQueryClient();
+ const TString sql = fmt::format(R"(
+ SELECT * FROM `{external_table}`
+ )", "external_table"_a = externalTableName);
+
+ auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString());
+
+ NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver());
+ UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Completed);
+ TFetchScriptResultsResult results = db.FetchScriptResults(scriptExecutionOperation.Id(), 0).ExtractValueSync();
+ UNIT_ASSERT_C(results.IsSuccess(), results.GetIssues().ToString());
+
+ TResultSetParser resultSet(results.ExtractResultSet());
+ UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnsCount(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(resultSet.RowsCount(), 1);
+
+ UNIT_ASSERT(resultSet.TryNextRow());
+ UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser("data").GetString(), "test");
+ UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser("year").GetUint32(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser("month").GetUint32(), 2);
+ }
}
} // namespace NKqp
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp
index 25257c554c6..32dad4aec9a 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp
@@ -33,6 +33,9 @@ public:
State_->Configuration->AddValidCluster(name);
auto& settings = State_->Configuration->Clusters[name];
settings.Url = properties.Value("location", "");
+ if (!settings.Url.EndsWith("/")) {
+ settings.Url += "/";
+ }
auto signReference = properties.Value("serviceAccountIdSignatureReference", "");
if (signReference) {
State_->Configuration->Tokens[name] = ComposeStructuredTokenJsonForServiceAccountWithSecret(properties.Value("serviceAccountId", ""), signReference, properties.Value("serviceAccountIdSignature", ""));