aws credentials have been fixed (#12262)

4 files changed, 75 insertions, 8 deletions

diff --git a/ydb/core/external_sources/s3/ut/s3_aws_credentials_ut.cpp b/ydb/core/external_sources/s3/ut/s3_aws_credentials_ut.cpp
index 30e5f14deb..bf2b1d0f2b 100644
--- a/ydb/core/external_sources/s3/ut/s3_aws_credentials_ut.cpp
+++ b/ydb/core/external_sources/s3/ut/s3_aws_credentials_ut.cpp
@@ -44,6 +44,30 @@ TString GetExternalPort(const TString& service, const TString& port) {
     return result ? Strip(result.back()) : TString{};
 }
 
+void WaitBucket(std::shared_ptr<TKikimrRunner> kikimr, const TString& externalDataSourceName) {
+    auto db = kikimr->GetQueryClient();
+    for (size_t i = 0; i < 100; i++) {
+        auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"(
+            SELECT * FROM `{external_source}`.`/a/` WITH (
+                format="json_each_row",
+                schema(
+                    key Utf8 NOT NULL,
+                    value Utf8 NOT NULL
+                )
+            )
+        )", "external_source"_a = externalDataSourceName)).ExtractValueSync();
+        UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString());
+        UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId);
+
+        NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver());
+        if (readyOp.Metadata().ExecStatus == EExecStatus::Completed) {
+            return;
+        }
+        Sleep(TDuration::Seconds(1));
+    }
+    UNIT_FAIL("Bucket isn't ready");
+}
+
 Y_UNIT_TEST_SUITE(S3AwsCredentials) {
     Y_UNIT_TEST(ExecuteScriptWithEqSymbol) {
         const TString externalDataSourceName = "/Root/external_data_source";
@@ -69,6 +93,7 @@ Y_UNIT_TEST_SUITE(S3AwsCredentials) {
             );
         auto result = session.ExecuteSchemeQuery(query).GetValueSync();
         UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString());
+        WaitBucket(kikimr, externalDataSourceName);
         auto db = kikimr->GetQueryClient();
         {
             auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"(
@@ -183,6 +208,46 @@ Y_UNIT_TEST_SUITE(S3AwsCredentials) {
                 UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "2");
                 UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "hello world");
             }
+            {
+                auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"(
+                    SELECT * FROM `{external_source}`.`/` WITH (
+                        format="json_each_row",
+                        schema(
+                            key Utf8 NOT NULL,
+                            value Utf8 NOT NULL
+                        )
+                    )
+                )", "external_source"_a = externalDataSourceName)).ExtractValueSync();
+                UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString());
+                UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId);
+
+                NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver());
+                UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString());
+                TFetchScriptResultsResult results = db.FetchScriptResults(scriptExecutionOperation.Id(), 0).ExtractValueSync();
+                UNIT_ASSERT_C(results.IsSuccess(), results.GetIssues().ToString());
+
+                TResultSetParser resultSet(results.ExtractResultSet());
+                UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnsCount(), 2);
+                UNIT_ASSERT_VALUES_EQUAL(resultSet.RowsCount(), 4);
+                UNIT_ASSERT(resultSet.TryNextRow());
+                UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "1");
+                UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "trololo");
+                UNIT_ASSERT(resultSet.TryNextRow());
+                UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "2");
+                UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "hello world");
+            }
+
+            {
+                auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"(
+                    INSERT INTO `{external_source}`.`exp_folder/` WITH (FORMAT = "csv_with_names")
+                    SELECT "Hello, world!" AS Data
+                )", "external_source"_a = externalDataSourceName)).ExtractValueSync();
+                UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString());
+                UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId);
+
+                NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver());
+                UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString());             
+            }
         }
         
     }
diff --git a/ydb/core/external_sources/ya.make b/ydb/core/external_sources/ya.make
index aa7d7dc912..8465ec8960 100644
--- a/ydb/core/external_sources/ya.make
+++ b/ydb/core/external_sources/ya.make
@@ -35,4 +35,5 @@ RECURSE_FOR_TESTS(
 RECURSE(
     hive_metastore
     object_storage
+    s3
 )
diff --git a/ydb/library/yql/providers/common/http_gateway/yql_aws_signature.cpp b/ydb/library/yql/providers/common/http_gateway/yql_aws_signature.cpp
index f7623091fb..699d43f4f1 100644
--- a/ydb/library/yql/providers/common/http_gateway/yql_aws_signature.cpp
+++ b/ydb/library/yql/providers/common/http_gateway/yql_aws_signature.cpp
@@ -63,7 +63,7 @@ TString TAwsSignature::GetAuthorization() const {
 }
 
 TString TAwsSignature::GetXAmzContentSha256() const {
-    return HashSHA256(TStringBuilder{} << "\"" << Payload << "\"");
+    return HashSHA256(TStringBuilder{} << Payload);
 }
 
 TString TAwsSignature::GetAmzDate() const {
@@ -158,7 +158,8 @@ TString TAwsSignature::UriEncode(const TStringBuf input, bool encodeSlash) {
 }
 
 void TAwsSignature::PrepareCgiParameters() {
-    TCgiParameters cgi(Cgi);
+    TCgiParameters cgi;
+    cgi.ScanAddAll(Cgi);
     TMap<TString, TVector<TString>> sortedCgi;
 
     for (const auto& [key, value] : cgi) {
diff --git a/ydb/library/yql/providers/common/http_gateway/yql_aws_signature_ut.cpp b/ydb/library/yql/providers/common/http_gateway/yql_aws_signature_ut.cpp
index 7a4489ffce..b3f053bf21 100644
--- a/ydb/library/yql/providers/common/http_gateway/yql_aws_signature_ut.cpp
+++ b/ydb/library/yql/providers/common/http_gateway/yql_aws_signature_ut.cpp
@@ -9,7 +9,7 @@ Y_UNIT_TEST_SUITE(TAwsSignature) {
     Y_UNIT_TEST(Sign) {
         NYql::TAwsSignature signature("GET", "http://os.com/my-bucket/year=2024/day=03/", "application/json", {}, "key", "pwd");
         UNIT_ASSERT_VALUES_EQUAL(signature.GetContentType(), "application/json");
-        UNIT_ASSERT_VALUES_EQUAL(signature.GetXAmzContentSha256(), "12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126");
+        UNIT_ASSERT_VALUES_EQUAL(signature.GetXAmzContentSha256(), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
         UNIT_ASSERT_STRING_CONTAINS(signature.GetAuthorization(), "SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=");
         UNIT_ASSERT_STRING_CONTAINS(signature.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/");
         UNIT_ASSERT_STRING_CONTAINS(signature.GetAuthorization(), "///aws4_request");
@@ -40,7 +40,7 @@ Y_UNIT_TEST_SUITE(TAwsSignature) {
         NYql::TAwsSignature signature3("GET", "http://os.com/my-bucket/year=2024/day=03/", "application/json", "", "key2", "pwd", time);
         NYql::TAwsSignature signature4("POST", "http://os.com/my-bucket/year=2024/day=03/", "application/json", "", "key2", "pwd", time);
         static const TString CONTENT_TYPE = "application/json";
-        static const TString X_AMZ_CONTENT_SHA_256 = "12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126";
+        static const TString X_AMZ_CONTENT_SHA_256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
         static const TString X_AMX_DATE = "19700101T000000Z";
         UNIT_ASSERT_VALUES_EQUAL(signature1.GetContentType(), CONTENT_TYPE);
         UNIT_ASSERT_VALUES_EQUAL(signature2.GetContentType(), CONTENT_TYPE);
@@ -54,10 +54,10 @@ Y_UNIT_TEST_SUITE(TAwsSignature) {
         UNIT_ASSERT_VALUES_EQUAL(signature2.GetAmzDate(), X_AMX_DATE);
         UNIT_ASSERT_VALUES_EQUAL(signature3.GetAmzDate(), X_AMX_DATE);
         UNIT_ASSERT_VALUES_EQUAL(signature4.GetAmzDate(), X_AMX_DATE);
-        UNIT_ASSERT_VALUES_EQUAL(signature1.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=fba1374eb117fe9d00fc04bb1b709a3ea6f152232ac1f7dc49117a505f7e9f3f");
-        UNIT_ASSERT_VALUES_EQUAL(signature2.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=39b47595c16b5d7b8256fd00e3d17e2157c5050c293306c995ae6980f11c689f");
-        UNIT_ASSERT_VALUES_EQUAL(signature3.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=fba1374eb117fe9d00fc04bb1b709a3ea6f152232ac1f7dc49117a505f7e9f3f");
-        UNIT_ASSERT_VALUES_EQUAL(signature4.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=a495d0ada1156d7278a56fb754a728d3b9470725208ad422bc299d6d6f793a8b");
+        UNIT_ASSERT_VALUES_EQUAL(signature1.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=32e344ef8df96a217ac1780393acc92caf87e73238e444d290cc713b9e81f7e8");
+        UNIT_ASSERT_VALUES_EQUAL(signature2.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=90e3a59639f80cff944aae40f3b4dba3435b3cf39b56e6395722469160c21f23");
+        UNIT_ASSERT_VALUES_EQUAL(signature3.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=32e344ef8df96a217ac1780393acc92caf87e73238e444d290cc713b9e81f7e8");
+        UNIT_ASSERT_VALUES_EQUAL(signature4.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=29af4a2f6e5be24030218e83c8a516a0511449951b078a9e4792a57e983d111a");
     }
 
     Y_UNIT_TEST(SignWithCanonization) {