diff options
author | hcpp <[email protected]> | 2023-11-02 18:23:01 +0300 |
---|---|---|
committer | hcpp <[email protected]> | 2023-11-02 19:05:59 +0300 |
commit | 44ce0972717174bb8c38e0387067be6fce736ce2 (patch) | |
tree | c3d114957526b63e3e0fca9eccba722d1da05903 | |
parent | c336c58dfb573341b17f27c7b128e80416d87422 (diff) |
location without slash has been fixed
-rw-r--r-- | ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp | 62 | ||||
-rw-r--r-- | ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp | 3 |
2 files changed, 64 insertions, 1 deletions
diff --git a/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp b/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp index bc48c8e10d1..1907857bfbf 100644 --- a/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/s3/kqp_federated_query_ut.cpp @@ -1250,7 +1250,6 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { FROM `/Root/external_data_source`.`/` WITH ( FORMAT="raw", - SCHEMA=( `data` String NOT NULL ) @@ -1381,6 +1380,67 @@ Y_UNIT_TEST_SUITE(KqpFederatedQuery) { UNIT_ASSERT_STRING_CONTAINS(resultFuture.GetValueSync().GetIssues().ToString(), "Callable not expected in effects tx: Unwrap"); } } + + Y_UNIT_TEST(ExecuteScriptWithLocationWithoutSlashAtTheEnd) { + using namespace fmt::literals; + const TString externalDataSourceName = "/Root/external_data_source"; + const TString externalTableName = "/Root/test_binding_resolve"; + const TString bucket = "test_bucket_with_location_without_slash_at_the_end"; + const TString object = "year=1/month=2/test_object"; + const TString content = "data,year,month\ntest,1,2"; + + CreateBucketWithObject(bucket, object, content); + + auto kikimr = MakeKikimrRunner(NYql::IHTTPGateway::Make()); + + auto tc = kikimr->GetTableClient(); + auto session = tc.CreateSession().GetValueSync().GetSession(); + const TString query = fmt::format(R"( + CREATE EXTERNAL DATA SOURCE `{external_source}` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="{location}", + AUTH_METHOD="NONE" + ); + CREATE EXTERNAL TABLE `{external_table}` ( + data STRING NOT NULL, + year UINT32 NOT NULL, + month UINT32 NOT NULL + ) WITH ( + DATA_SOURCE="{external_source}", + LOCATION="/", + FORMAT="csv_with_names", + PARTITIONED_BY="[year, month]" + );)", + "external_source"_a = externalDataSourceName, + "external_table"_a = externalTableName, + "location"_a = TStringBuilder() << GetEnv("S3_ENDPOINT") << '/' << bucket + ); + + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + + auto db = kikimr->GetQueryClient(); + const TString sql = fmt::format(R"( + SELECT * FROM `{external_table}` + )", "external_table"_a = externalTableName); + + auto scriptExecutionOperation = db.ExecuteScript(sql).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); + + NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); + UNIT_ASSERT_EQUAL(readyOp.Metadata().ExecStatus, EExecStatus::Completed); + TFetchScriptResultsResult results = db.FetchScriptResults(scriptExecutionOperation.Id(), 0).ExtractValueSync(); + UNIT_ASSERT_C(results.IsSuccess(), results.GetIssues().ToString()); + + TResultSetParser resultSet(results.ExtractResultSet()); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnsCount(), 3); + UNIT_ASSERT_VALUES_EQUAL(resultSet.RowsCount(), 1); + + UNIT_ASSERT(resultSet.TryNextRow()); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser("data").GetString(), "test"); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser("year").GetUint32(), 1); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser("month").GetUint32(), 2); + } } } // namespace NKqp diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp index 25257c554c6..32dad4aec9a 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_datasource.cpp @@ -33,6 +33,9 @@ public: State_->Configuration->AddValidCluster(name); auto& settings = State_->Configuration->Clusters[name]; settings.Url = properties.Value("location", ""); + if (!settings.Url.EndsWith("/")) { + settings.Url += "/"; + } auto signReference = properties.Value("serviceAccountIdSignatureReference", ""); if (signReference) { State_->Configuration->Tokens[name] = ComposeStructuredTokenJsonForServiceAccountWithSecret(properties.Value("serviceAccountId", ""), signReference, properties.Value("serviceAccountIdSignature", "")); |