Intermediate changes

commit_hash:8af0bdb0e64115388dd042cdf03368fd7611a883

7 files changed, 154 insertions, 43 deletions

diff --git a/contrib/python/s3transfer/py3/.dist-info/METADATA b/contrib/python/s3transfer/py3/.dist-info/METADATA
index 629c988c78a..cd598e2d609 100644
--- a/contrib/python/s3transfer/py3/.dist-info/METADATA
+++ b/contrib/python/s3transfer/py3/.dist-info/METADATA
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: s3transfer
-Version: 0.11.0
+Version: 0.11.3
 Summary: An Amazon S3 Transfer Manager
 Home-page: https://github.com/boto/s3transfer
 Author: Amazon Web Services
@@ -22,9 +22,9 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >= 3.8
 License-File: LICENSE.txt
 License-File: NOTICE.txt
-Requires-Dist: botocore (<2.0a.0,>=1.33.2)
+Requires-Dist: botocore (<2.0a.0,>=1.36.0)
 Provides-Extra: crt
-Requires-Dist: botocore[crt] (<2.0a.0,>=1.33.2) ; extra == 'crt'
+Requires-Dist: botocore[crt] (<2.0a.0,>=1.36.0) ; extra == 'crt'
 
 =====================================================
 s3transfer - An Amazon S3 Transfer Manager for Python
diff --git a/contrib/python/s3transfer/py3/s3transfer/__init__.py b/contrib/python/s3transfer/py3/s3transfer/__init__.py
index de62a74eba5..a15bac0cb9f 100644
--- a/contrib/python/s3transfer/py3/s3transfer/__init__.py
+++ b/contrib/python/s3transfer/py3/s3transfer/__init__.py
@@ -145,7 +145,7 @@ import s3transfer.compat
 from s3transfer.exceptions import RetriesExceededError, S3UploadFailedError
 
 __author__ = 'Amazon Web Services'
-__version__ = '0.11.0'
+__version__ = '0.11.3'
 
 
 class NullHandler(logging.Handler):
diff --git a/contrib/python/s3transfer/py3/s3transfer/crt.py b/contrib/python/s3transfer/py3/s3transfer/crt.py
index 37b041d8a96..b11c368261c 100644
--- a/contrib/python/s3transfer/py3/s3transfer/crt.py
+++ b/contrib/python/s3transfer/py3/s3transfer/crt.py
@@ -895,14 +895,22 @@ class S3ClientArgsCreator:
         ) and accesspoint_arn_details['region'] == "":
             # Configure our region to `*` to propogate in `x-amz-region-set`
             # for multi-region support in MRAP accesspoints.
+            # use_double_uri_encode and should_normalize_uri_path are defaulted to be True
+            # But SDK already encoded the URI, and it's for S3, so set both to False
             make_request_args['signing_config'] = AwsSigningConfig(
                 algorithm=AwsSigningAlgorithm.V4_ASYMMETRIC,
                 region="*",
+                use_double_uri_encode=False,
+                should_normalize_uri_path=False,
             )
             call_args.bucket = accesspoint_arn_details['resource_name']
         elif is_s3express_bucket(call_args.bucket):
+            # use_double_uri_encode and should_normalize_uri_path are defaulted to be True
+            # But SDK already encoded the URI, and it's for S3, so set both to False
             make_request_args['signing_config'] = AwsSigningConfig(
-                algorithm=AwsSigningAlgorithm.V4_S3EXPRESS
+                algorithm=AwsSigningAlgorithm.V4_S3EXPRESS,
+                use_double_uri_encode=False,
+                should_normalize_uri_path=False,
             )
         return make_request_args
 
diff --git a/contrib/python/s3transfer/py3/s3transfer/manager.py b/contrib/python/s3transfer/py3/s3transfer/manager.py
index 23b6f305ef0..8e1cdf068bb 100644
--- a/contrib/python/s3transfer/py3/s3transfer/manager.py
+++ b/contrib/python/s3transfer/py3/s3transfer/manager.py
@@ -519,7 +519,11 @@ class TransferManager:
                 )
 
     def _add_operation_defaults(self, extra_args):
-        set_default_checksum_algorithm(extra_args)
+        if (
+            self.client.meta.config.request_checksum_calculation
+            == "when_supported"
+        ):
+            set_default_checksum_algorithm(extra_args)
 
     def _submit_transfer(
         self, call_args, submission_task_cls, extra_main_kwargs=None
diff --git a/contrib/python/s3transfer/py3/tests/functional/test_crt.py b/contrib/python/s3transfer/py3/tests/functional/test_crt.py
index 5ef25144e85..98e3fffbe7a 100644
--- a/contrib/python/s3transfer/py3/tests/functional/test_crt.py
+++ b/contrib/python/s3transfer/py3/tests/functional/test_crt.py
@@ -163,6 +163,12 @@ class TestCRTTransferManager(unittest.TestCase):
             make_request_kwargs['signing_config'].algorithm,
             awscrt.auth.AwsSigningAlgorithm.V4_S3EXPRESS,
         )
+        self.assertFalse(
+            make_request_kwargs['signing_config'].use_double_uri_encode,
+        )
+        self.assertFalse(
+            make_request_kwargs['signing_config'].should_normalize_uri_path,
+        )
 
     def _assert_expected_mrap_request(
         self, make_request_kwargs, expected_http_method='GET'
@@ -179,6 +185,12 @@ class TestCRTTransferManager(unittest.TestCase):
             awscrt.auth.AwsSigningAlgorithm.V4_ASYMMETRIC,
         )
         self.assertEqual(make_request_kwargs['signing_config'].region, "*")
+        self.assertFalse(
+            make_request_kwargs['signing_config'].use_double_uri_encode,
+        )
+        self.assertFalse(
+            make_request_kwargs['signing_config'].should_normalize_uri_path,
+        )
 
     def _assert_subscribers_called(self, expected_future=None):
         self.assertTrue(self.record_subscriber.on_queued_called)
diff --git a/contrib/python/s3transfer/py3/tests/functional/test_upload.py b/contrib/python/s3transfer/py3/tests/functional/test_upload.py
index 00ac284f62f..efbc55af06c 100644
--- a/contrib/python/s3transfer/py3/tests/functional/test_upload.py
+++ b/contrib/python/s3transfer/py3/tests/functional/test_upload.py
@@ -19,6 +19,7 @@ from io import BytesIO
 from botocore.awsrequest import AWSRequest
 from botocore.client import Config
 from botocore.exceptions import ClientError
+from botocore.httpchecksum import DEFAULT_CHECKSUM_ALGORITHM
 from botocore.stub import ANY
 
 from s3transfer.manager import TransferConfig, TransferManager
@@ -143,7 +144,7 @@ class TestNonMultipartUpload(BaseUploadTest):
     __test__ = True
 
     def add_put_object_response_with_default_expected_params(
-        self, extra_expected_params=None, bucket=None
+        self, extra_expected_params=None, bucket=None, include_checksum=True
     ):
         if bucket is None:
             bucket = self.bucket
@@ -152,8 +153,9 @@ class TestNonMultipartUpload(BaseUploadTest):
             'Body': ANY,
             'Bucket': bucket,
             'Key': self.key,
-            'ChecksumAlgorithm': 'CRC32',
         }
+        if include_checksum:
+            expected_params["ChecksumAlgorithm"] = DEFAULT_CHECKSUM_ALGORITHM
         if extra_expected_params:
             expected_params.update(extra_expected_params)
         upload_response = self.create_stubbed_responses()[0]
@@ -187,6 +189,46 @@ class TestNonMultipartUpload(BaseUploadTest):
         self.assert_expected_client_calls_were_correct()
         self.assert_put_object_body_was_correct()
 
+    def test_upload_with_default_checksum_when_supported(self):
+        # Reset client to configure `request_checksum_calculation` to "when_supported".
+        self.reset_stubber_with_new_client(
+            {'config': Config(request_checksum_calculation="when_supported")}
+        )
+        self.client.meta.events.register(
+            'before-parameter-build.s3.*', self.collect_body
+        )
+        self._manager = TransferManager(self.client, self.config)
+
+        self.add_put_object_response_with_default_expected_params(
+            include_checksum=True
+        )
+        future = self.manager.upload(
+            self.filename, self.bucket, self.key, self.extra_args
+        )
+        future.result()
+        self.assert_expected_client_calls_were_correct()
+        self.assert_put_object_body_was_correct()
+
+    def test_upload_with_default_checksum_when_required(self):
+        # Reset client to configure `request_checksum_calculation` to "when_required".
+        self.reset_stubber_with_new_client(
+            {'config': Config(request_checksum_calculation="when_required")}
+        )
+        self.client.meta.events.register(
+            'before-parameter-build.s3.*', self.collect_body
+        )
+        self._manager = TransferManager(self.client, self.config)
+
+        self.add_put_object_response_with_default_expected_params(
+            include_checksum=False
+        )
+        future = self.manager.upload(
+            self.filename, self.bucket, self.key, self.extra_args
+        )
+        future.result()
+        self.assert_expected_client_calls_were_correct()
+        self.assert_put_object_body_was_correct()
+
     def test_upload_with_s3express_default_checksum(self):
         s3express_bucket = "mytestbucket--usw2-az6--x-s3"
         self.assertFalse("ChecksumAlgorithm" in self.extra_args)
@@ -370,9 +412,7 @@ class TestMultipartUpload(BaseUploadTest):
         self.assertEqual(self.sent_bodies, expected_contents)
 
     def add_create_multipart_response_with_default_expected_params(
-        self,
-        extra_expected_params=None,
-        bucket=None,
+        self, extra_expected_params=None, bucket=None, include_checksum=True
     ):
         if bucket is None:
             bucket = self.bucket
@@ -380,8 +420,9 @@ class TestMultipartUpload(BaseUploadTest):
         expected_params = {
             'Bucket': bucket,
             'Key': self.key,
-            'ChecksumAlgorithm': 'CRC32',
         }
+        if include_checksum:
+            expected_params["ChecksumAlgorithm"] = DEFAULT_CHECKSUM_ALGORITHM
         if extra_expected_params:
             expected_params.update(extra_expected_params)
         response = self.create_stubbed_responses()[0]
@@ -389,9 +430,7 @@ class TestMultipartUpload(BaseUploadTest):
         self.stubber.add_response(**response)
 
     def add_upload_part_responses_with_default_expected_params(
-        self,
-        extra_expected_params=None,
-        bucket=None,
+        self, extra_expected_params=None, bucket=None, include_checksum=True
     ):
         if bucket is None:
             bucket = self.bucket
@@ -406,50 +445,48 @@ class TestMultipartUpload(BaseUploadTest):
                 'UploadId': self.multipart_id,
                 'Body': ANY,
                 'PartNumber': i + 1,
-                'ChecksumAlgorithm': 'CRC32',
             }
+            if include_checksum:
+                expected_params["ChecksumAlgorithm"] = (
+                    DEFAULT_CHECKSUM_ALGORITHM
+                )
             if extra_expected_params:
                 expected_params.update(extra_expected_params)
 
-            name = expected_params['ChecksumAlgorithm']
-            checksum_member = f'Checksum{name.upper()}'
-            response = upload_part_response['service_response']
-            response[checksum_member] = f'sum{i+1}=='
+            # If ChecksumAlgorithm is in expected parameters, add checksum to the response
+            checksum_algorithm = expected_params.get('ChecksumAlgorithm')
+            if checksum_algorithm:
+                checksum_member = f'Checksum{checksum_algorithm.upper()}'
+                response = upload_part_response['service_response']
+                response[checksum_member] = f'sum{i+1}=='
 
             upload_part_response['expected_params'] = expected_params
             self.stubber.add_response(**upload_part_response)
 
     def add_complete_multipart_response_with_default_expected_params(
-        self,
-        extra_expected_params=None,
-        bucket=None,
+        self, extra_expected_params=None, bucket=None, include_checksum=True
     ):
         if bucket is None:
             bucket = self.bucket
 
+        num_parts = 3
+        parts = []
+        for part_num in range(1, num_parts + 1):
+            part = {
+                'ETag': f'etag-{part_num}',
+                'PartNumber': part_num,
+            }
+            if include_checksum:
+                part[f"Checksum{DEFAULT_CHECKSUM_ALGORITHM}"] = (
+                    f"sum{part_num}=="
+                )
+            parts.append(part)
+
         expected_params = {
             'Bucket': bucket,
             'Key': self.key,
             'UploadId': self.multipart_id,
-            'MultipartUpload': {
-                'Parts': [
-                    {
-                        'ETag': 'etag-1',
-                        'PartNumber': 1,
-                        'ChecksumCRC32': 'sum1==',
-                    },
-                    {
-                        'ETag': 'etag-2',
-                        'PartNumber': 2,
-                        'ChecksumCRC32': 'sum2==',
-                    },
-                    {
-                        'ETag': 'etag-3',
-                        'PartNumber': 3,
-                        'ChecksumCRC32': 'sum3==',
-                    },
-                ]
-            },
+            'MultipartUpload': {'Parts': parts},
         }
         if extra_expected_params:
             expected_params.update(extra_expected_params)
@@ -761,3 +798,53 @@ class TestMultipartUpload(BaseUploadTest):
         )
         future.result()
         self.assert_expected_client_calls_were_correct()
+
+    def test_multipart_upload_with_default_checksum_when_supported(self):
+        # Reset client to configure `request_checksum_calculation` to "when_supported".
+        self.reset_stubber_with_new_client(
+            {'config': Config(request_checksum_calculation="when_supported")}
+        )
+        self.client.meta.events.register(
+            'before-parameter-build.s3.*', self.collect_body
+        )
+        self._manager = TransferManager(self.client, self.config)
+
+        self.add_create_multipart_response_with_default_expected_params(
+            include_checksum=True
+        )
+        self.add_upload_part_responses_with_default_expected_params(
+            include_checksum=True
+        )
+        self.add_complete_multipart_response_with_default_expected_params()
+
+        future = self.manager.upload(
+            self.filename, self.bucket, self.key, self.extra_args
+        )
+        future.result()
+        self.assert_expected_client_calls_were_correct()
+
+    def test_multipart_upload_with_default_checksum_when_required(self):
+        # Reset client to configure `request_checksum_calculation` to "when_required".
+        self.reset_stubber_with_new_client(
+            {'config': Config(request_checksum_calculation="when_required")}
+        )
+        self.client.meta.events.register(
+            'before-parameter-build.s3.*', self.collect_body
+        )
+        self._manager = TransferManager(self.client, self.config)
+
+        self.add_create_multipart_response_with_default_expected_params(
+            include_checksum=False
+        )
+        self.add_upload_part_responses_with_default_expected_params(
+            include_checksum=False
+        )
+        self.add_complete_multipart_response_with_default_expected_params(
+            include_checksum=False
+        )
+
+        future = self.manager.upload(
+            self.filename, self.bucket, self.key, self.extra_args
+        )
+        future.result()
+        self.assert_expected_client_calls_were_correct()
diff --git a/contrib/python/s3transfer/py3/ya.make b/contrib/python/s3transfer/py3/ya.make
index 57776c6f7ce..7c6a4458dcf 100644
--- a/contrib/python/s3transfer/py3/ya.make
+++ b/contrib/python/s3transfer/py3/ya.make
@@ -2,7 +2,7 @@
 
 PY3_LIBRARY()
 
-VERSION(0.11.0)
+VERSION(0.11.3)
 
 LICENSE(Apache-2.0)