diff options
author | vitalyisaev <vitalyisaev@ydb.tech> | 2023-12-12 21:55:07 +0300 |
---|---|---|
committer | vitalyisaev <vitalyisaev@ydb.tech> | 2023-12-12 22:25:10 +0300 |
commit | 4967f99474a4040ba150eb04995de06342252718 (patch) | |
tree | c9c118836513a8fab6e9fcfb25be5d404338bca7 /vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal | |
parent | 2ce9cccb9b0bdd4cd7a3491dc5cbf8687cda51de (diff) | |
download | ydb-4967f99474a4040ba150eb04995de06342252718.tar.gz |
YQ Connector: prepare code base for S3 integration
1. Кодовая база Коннектора переписана с помощью Go дженериков так, чтобы добавление нового источника данных (в частности S3 + csv) максимально переиспользовало имеющийся код (чтобы сохранялась логика нарезания на блоки данных, учёт трафика и пр.)
2. API Connector расширено для работы с S3, но ещё пока не протестировано.
Diffstat (limited to 'vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal')
25 files changed, 5248 insertions, 0 deletions
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/arn/arn_parser.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/arn/arn_parser.go new file mode 100644 index 0000000000..97b5771bb1 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/arn/arn_parser.go @@ -0,0 +1,106 @@ +package arn + +import ( + "fmt" + "strings" + + awsarn "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/service/internal/s3shared/arn" +) + +const ( + s3Namespace = "s3" + s3ObjectsLambdaNamespace = "s3-object-lambda" + s3OutpostsNamespace = "s3-outposts" +) + +// ParseEndpointARN parses a given generic aws ARN into a s3 arn resource. +func ParseEndpointARN(v awsarn.ARN) (arn.Resource, error) { + return arn.ParseResource(v, accessPointResourceParser) +} + +func accessPointResourceParser(a awsarn.ARN) (arn.Resource, error) { + resParts := arn.SplitResource(a.Resource) + + switch resParts[0] { + case "accesspoint": + switch a.Service { + case s3Namespace: + return arn.ParseAccessPointResource(a, resParts[1:]) + case s3ObjectsLambdaNamespace: + return parseS3ObjectLambdaAccessPointResource(a, resParts) + default: + return arn.AccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: fmt.Sprintf("service is not %s or %s", s3Namespace, s3ObjectsLambdaNamespace)} + } + case "outpost": + if a.Service != s3OutpostsNamespace { + return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "service is not %s"} + } + return parseOutpostAccessPointResource(a, resParts[1:]) + default: + return nil, arn.InvalidARNError{ARN: a, Reason: "unknown resource type"} + } +} + +func parseOutpostAccessPointResource(a awsarn.ARN, resParts []string) (arn.OutpostAccessPointARN, error) { + // outpost accesspoint arn is only valid if service is s3-outposts + if a.Service != "s3-outposts" { + return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "service is not s3-outposts"} + } + + if len(resParts) == 0 { + return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "outpost resource-id not set"} + } + + if len(resParts) < 3 { + return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ + ARN: a, Reason: "access-point resource not set in Outpost ARN", + } + } + + resID := strings.TrimSpace(resParts[0]) + if len(resID) == 0 { + return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "outpost resource-id not set"} + } + + var outpostAccessPointARN = arn.OutpostAccessPointARN{} + switch resParts[1] { + case "accesspoint": + // Do not allow region-less outpost access-point arns. + if len(a.Region) == 0 { + return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "region is not set"} + } + + accessPointARN, err := arn.ParseAccessPointResource(a, resParts[2:]) + if err != nil { + return arn.OutpostAccessPointARN{}, err + } + // set access-point arn + outpostAccessPointARN.AccessPointARN = accessPointARN + default: + return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "access-point resource not set in Outpost ARN"} + } + + // set outpost id + outpostAccessPointARN.OutpostID = resID + return outpostAccessPointARN, nil +} + +func parseS3ObjectLambdaAccessPointResource(a awsarn.ARN, resParts []string) (arn.S3ObjectLambdaAccessPointARN, error) { + if a.Service != s3ObjectsLambdaNamespace { + return arn.S3ObjectLambdaAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: fmt.Sprintf("service is not %s", s3ObjectsLambdaNamespace)} + } + + if len(a.Region) == 0 { + return arn.S3ObjectLambdaAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: fmt.Sprintf("%s region not set", s3ObjectsLambdaNamespace)} + } + + accessPointARN, err := arn.ParseAccessPointResource(a, resParts[1:]) + if err != nil { + return arn.S3ObjectLambdaAccessPointARN{}, err + } + + return arn.S3ObjectLambdaAccessPointARN{ + AccessPointARN: accessPointARN, + }, nil +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/arn/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/arn/ya.make new file mode 100644 index 0000000000..8befaddf00 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/arn/ya.make @@ -0,0 +1,9 @@ +GO_LIBRARY() + +LICENSE(Apache-2.0) + +SRCS( + arn_parser.go +) + +END() diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/doc.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/doc.go new file mode 100644 index 0000000000..e1d1cbefa4 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/doc.go @@ -0,0 +1,79 @@ +/* +Package customizations provides customizations for the Amazon S3 API client. + +This package provides support for following S3 customizations + + ProcessARN Middleware: processes an ARN if provided as input and updates the endpoint as per the arn type + + UpdateEndpoint Middleware: resolves a custom endpoint as per s3 config options + + RemoveBucket Middleware: removes a serialized bucket name from request url path + + processResponseWith200Error Middleware: Deserializing response error with 200 status code + +# Virtual Host style url addressing + +Since serializers serialize by default as path style url, we use customization +to modify the endpoint url when `UsePathStyle` option on S3Client is unset or +false. This flag will be ignored if `UseAccelerate` option is set to true. + +If UseAccelerate is not enabled, and the bucket name is not a valid hostname +label, they SDK will fallback to forcing the request to be made as if +UsePathStyle was enabled. This behavior is also used if UseDualStackEndpoint is enabled. + +https://docs.aws.amazon.com/AmazonS3/latest/dev/dual-stack-endpoints.html#dual-stack-endpoints-description + +# Transfer acceleration + +By default S3 Transfer acceleration support is disabled. By enabling `UseAccelerate` +option on S3Client, one can enable s3 transfer acceleration support. Transfer +acceleration only works with Virtual Host style addressing, and thus `UsePathStyle` +option if set is ignored. Transfer acceleration is not supported for S3 operations +DeleteBucket, ListBuckets, and CreateBucket. + +# Dualstack support + +By default dualstack support for s3 client is disabled. By enabling `UseDualstack` +option on s3 client, you can enable dualstack endpoint support. + +# Endpoint customizations + +Customizations to lookup ARN, process ARN needs to happen before request serialization. +UpdateEndpoint middleware which mutates resources based on Options such as +UseDualstack, UseAccelerate for modifying resolved endpoint are executed after +request serialization. Remove bucket middleware is executed after +an request is serialized, and removes the serialized bucket name from request path + + Middleware layering: + + Initialize : HTTP Request -> ARN Lookup -> Input-Validation -> Serialize step + + Serialize : HTTP Request -> Process ARN -> operation serializer -> Update-Endpoint customization -> Remove-Bucket -> next middleware + +Customization options: + + UseARNRegion (Disabled by Default) + + UsePathStyle (Disabled by Default) + + UseAccelerate (Disabled by Default) + + UseDualstack (Disabled by Default) + +# Handle Error response with 200 status code + +S3 operations: CopyObject, CompleteMultipartUpload, UploadPartCopy can have an +error Response with status code 2xx. The processResponseWith200Error middleware +customizations enables SDK to check for an error within response body prior to +deserialization. + +As the check for 2xx response containing an error needs to be performed earlier +than response deserialization. Since the behavior of Deserialization is in +reverse order to the other stack steps its easier to consider that "after" means +"before". + + Middleware layering: + + HTTP Response -> handle 200 error customization -> deserialize +*/ +package customizations diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/gotest/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/gotest/ya.make new file mode 100644 index 0000000000..d05b863028 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/gotest/ya.make @@ -0,0 +1,5 @@ +GO_TEST_FOR(vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations) + +LICENSE(Apache-2.0) + +END() diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/handle_200_error.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/handle_200_error.go new file mode 100644 index 0000000000..2b11b1fa27 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/handle_200_error.go @@ -0,0 +1,74 @@ +package customizations + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "strings" + + "github.com/aws/smithy-go" + smithyxml "github.com/aws/smithy-go/encoding/xml" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +// HandleResponseErrorWith200Status check for S3 200 error response. +// If an s3 200 error is found, status code for the response is modified temporarily to +// 5xx response status code. +func HandleResponseErrorWith200Status(stack *middleware.Stack) error { + return stack.Deserialize.Insert(&processResponseFor200ErrorMiddleware{}, "OperationDeserializer", middleware.After) +} + +// middleware to process raw response and look for error response with 200 status code +type processResponseFor200ErrorMiddleware struct{} + +// ID returns the middleware ID. +func (*processResponseFor200ErrorMiddleware) ID() string { + return "S3:ProcessResponseFor200Error" +} + +func (m *processResponseFor200ErrorMiddleware) HandleDeserialize( + ctx context.Context, in middleware.DeserializeInput, next middleware.DeserializeHandler) ( + out middleware.DeserializeOutput, metadata middleware.Metadata, err error, +) { + out, metadata, err = next.HandleDeserialize(ctx, in) + if err != nil { + return out, metadata, err + } + + response, ok := out.RawResponse.(*smithyhttp.Response) + if !ok { + return out, metadata, &smithy.DeserializationError{Err: fmt.Errorf("unknown transport type %T", out.RawResponse)} + } + + // check if response status code is 2xx. + if response.StatusCode < 200 || response.StatusCode >= 300 { + return + } + + var readBuff bytes.Buffer + body := io.TeeReader(response.Body, &readBuff) + + rootDecoder := xml.NewDecoder(body) + t, err := smithyxml.FetchRootElement(rootDecoder) + if err == io.EOF { + return out, metadata, &smithy.DeserializationError{ + Err: fmt.Errorf("received empty response payload"), + } + } + + // rewind response body + response.Body = ioutil.NopCloser(io.MultiReader(&readBuff, response.Body)) + + // if start tag is "Error", the response is consider error response. + if strings.EqualFold(t.Name.Local, "Error") { + // according to https://aws.amazon.com/premiumsupport/knowledge-center/s3-resolve-200-internalerror/ + // 200 error responses are similar to 5xx errors. + response.StatusCode = 500 + } + + return out, metadata, err +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/handle_200_error_test.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/handle_200_error_test.go new file mode 100644 index 0000000000..57c062f838 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/handle_200_error_test.go @@ -0,0 +1,128 @@ +package customizations_test + +import ( + "context" + "io" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/internal/awstesting/unit" + + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type EndpointResolverFunc func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) + +func (fn EndpointResolverFunc) ResolveEndpoint(region string, options s3.EndpointResolverOptions) (endpoint aws.Endpoint, err error) { + return fn(region, options) +} + +type mockHTTPClient struct { + r *http.Response +} + +func (m *mockHTTPClient) Do(*http.Request) (*http.Response, error) { + return m.r, nil +} + +var _ s3.HTTPClient = &mockHTTPClient{} + +func asReadCloser(s string) io.ReadCloser { + return ioutil.NopCloser(strings.NewReader(s)) +} + +func TestErrorResponseWith200StatusCode(t *testing.T) { + cases := map[string]struct { + response *http.Response + expectedError string + expectedBucket string + }{ + "200ErrorBody": { + response: &http.Response{ + StatusCode: 200, + Body: asReadCloser( + `<Error> + <Type>Sender</Type> + <Code>InvalidGreeting</Code> + <Message>Hi</Message> + <AnotherSetting>setting</AnotherSetting> + <RequestId>foo-id</RequestId> + </Error>`, + ), + }, + expectedError: "InvalidGreeting", + }, + "200NoResponse": { + response: &http.Response{ + StatusCode: 200, + Body: asReadCloser(""), + }, + expectedError: "received empty response payload", + }, + "200InvalidResponse": { + response: &http.Response{ + StatusCode: 200, + Body: asReadCloser( + `<Error> + <Type>Sender</Type> + <Code>InvalidGreeting</Code> + <Message>Hi</Message> + <AnotherSetting>setting</AnotherSetting> + <RequestId>foo-id`, + ), + }, + expectedError: "unexpected EOF", + }, + "200SuccessResponse": { + response: &http.Response{ + StatusCode: 200, + Body: asReadCloser( + `<CompleteMultipartUploadResult> + <Bucket>bucket</Bucket> + </CompleteMultipartUploadResult>`, + ), + }, + expectedError: "", + expectedBucket: "bucket", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + options := s3.Options{ + Credentials: unit.StubCredentialsProvider{}, + Retryer: aws.NopRetryer{}, + Region: "mock-region", + UsePathStyle: true, + HTTPClient: &mockHTTPClient{c.response}, + } + + svc := s3.New(options) + resp, err := svc.CompleteMultipartUpload(context.Background(), &s3.CompleteMultipartUploadInput{ + UploadId: aws.String("mockID"), + RequestPayer: "requester", + Bucket: aws.String("bucket"), + Key: aws.String("mockKey"), + }) + + if len(c.expectedError) != 0 { + if err == nil { + t.Fatalf("expected error, got none") + } + + if e, a := c.expectedError, err.Error(); !strings.Contains(a, e) { + t.Fatalf("expected %v, got %v", e, a) + } + } + + if len(c.expectedBucket) != 0 { + if e, a := c.expectedBucket, *resp.Bucket; !strings.EqualFold(e, a) { + t.Fatalf("expected bucket name to be %v, got %v", e, a) + } + } + }) + } +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/host.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/host.go new file mode 100644 index 0000000000..87f7a22327 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/host.go @@ -0,0 +1,22 @@ +package customizations + +import ( + "github.com/aws/smithy-go/transport/http" + "strings" +) + +func updateS3HostForS3AccessPoint(req *http.Request) { + updateHostPrefix(req, "s3", s3AccessPoint) +} + +func updateS3HostForS3ObjectLambda(req *http.Request) { + updateHostPrefix(req, "s3", s3ObjectLambda) +} + +func updateHostPrefix(req *http.Request, oldEndpointPrefix, newEndpointPrefix string) { + host := req.URL.Host + if strings.HasPrefix(host, oldEndpointPrefix) { + // For example if oldEndpointPrefix=s3 would replace to newEndpointPrefix + req.URL.Host = newEndpointPrefix + host[len(oldEndpointPrefix):] + } +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/presign_test.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/presign_test.go new file mode 100644 index 0000000000..c2e23f5aea --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/presign_test.go @@ -0,0 +1,472 @@ +package customizations_test + +import ( + "bytes" + "context" + "net/http" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/internal/awstesting/unit" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +func TestPutObject_PresignURL(t *testing.T) { + cases := map[string]struct { + input s3.PutObjectInput + options []func(*s3.PresignOptions) + expectPresignedURLHost string + expectRequestURIQuery []string + expectSignedHeader http.Header + expectMethod string + expectError string + }{ + "standard case": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: strings.NewReader("hello-world"), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Content-Length": []string{"11"}, + "Content-Type": []string{"application/octet-stream"}, + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "seekable payload": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: bytes.NewReader([]byte("hello-world")), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Content-Length": []string{"11"}, + "Content-Type": []string{"application/octet-stream"}, + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "unseekable payload": { + // unseekable payload succeeds as we disable content sha256 computation for streaming input + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: bytes.NewBuffer([]byte(`hello-world`)), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Content-Length": []string{"11"}, + "Content-Type": []string{"application/octet-stream"}, + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "empty body": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: bytes.NewReader([]byte(``)), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "nil body": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "nil body with content-length": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + ContentLength: 100, + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + "Content-Length": []string{"100"}, + }, + }, + "mrap presigned": { + input: s3.PutObjectInput{ + Bucket: aws.String("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"), + Key: aws.String("mockkey"), + Body: strings.NewReader("hello-world"), + }, + expectPresignedURLHost: "https://mfzwi23gnjvgw.mrap.accesspoint.s3-global.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + "X-Amz-Region-Set", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Content-Length": []string{"11"}, + "Content-Type": []string{"application/octet-stream"}, + "Host": []string{"mfzwi23gnjvgw.mrap.accesspoint.s3-global.amazonaws.com"}, + }, + }, + "mrap presigned with mrap disabled": { + input: s3.PutObjectInput{ + Bucket: aws.String("arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap"), + Key: aws.String("mockkey"), + Body: strings.NewReader("hello-world"), + }, + options: []func(option *s3.PresignOptions){ + func(option *s3.PresignOptions) { + option.ClientOptions = []func(o *s3.Options){ + func(o *s3.Options) { + o.DisableMultiRegionAccessPoints = true + }, + } + }, + }, + expectError: "Invalid configuration: Multi-Region Access Point ARNs are disabled.", + }, + "standard case with checksum preset checksum": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: strings.NewReader("hello world"), + ChecksumAlgorithm: s3types.ChecksumAlgorithmCrc32c, + ChecksumCRC32: aws.String("DUoRhQ=="), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + "X-Amz-Checksum-Crc32", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Content-Length": []string{"11"}, + "Content-Type": []string{"application/octet-stream"}, + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "standard case with checksum empty body": { + input: s3.PutObjectInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + Body: strings.NewReader(""), + ChecksumAlgorithm: s3types.ChecksumAlgorithmCrc32c, + ChecksumCRC32: aws.String("AAAAAA=="), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "x-id=PutObject", + "X-Amz-Signature", + "X-Amz-Checksum-Crc32", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + cfg := aws.Config{ + Region: "us-west-2", + Credentials: unit.StubCredentialsProvider{}, + Retryer: func() aws.Retryer { + return aws.NopRetryer{} + }, + } + presignClient := s3.NewPresignClient(s3.NewFromConfig(cfg), c.options...) + + req, err := presignClient.PresignPutObject(ctx, &c.input) + if err != nil { + if len(c.expectError) == 0 { + t.Fatalf("expected no error, got %v", err) + } + // if expect error, match error and skip rest + if e, a := c.expectError, err.Error(); !strings.Contains(a, e) { + t.Fatalf("expected error to be %s, got %s", e, a) + } + return + } else { + if len(c.expectError) != 0 { + t.Fatalf("expected error to be %v, got none", c.expectError) + } + } + + if e, a := c.expectPresignedURLHost, req.URL; !strings.Contains(a, e) { + t.Fatalf("expected presigned url to contain host %s, got %s", e, a) + } + + if len(c.expectRequestURIQuery) != 0 { + for _, label := range c.expectRequestURIQuery { + if e, a := label, req.URL; !strings.Contains(a, e) { + t.Fatalf("expected presigned url to contain %v label in url: %v", label, req.URL) + } + } + } + + if e, a := c.expectSignedHeader, req.SignedHeader; len(cmp.Diff(e, a)) != 0 { + t.Fatalf("expected signed header to be %s, got %s, \n diff : %s", e, a, cmp.Diff(e, a)) + } + + if e, a := c.expectMethod, req.Method; !strings.EqualFold(e, a) { + t.Fatalf("expected presigning Method to be %s, got %s", e, a) + } + + }) + } +} + +func TestUploadPart_PresignURL(t *testing.T) { + cases := map[string]struct { + input s3.UploadPartInput + options s3.PresignOptions + expectPresignedURLHost string + expectRequestURIQuery []string + expectSignedHeader http.Header + expectMethod string + expectError string + }{ + "standard case": { + input: s3.UploadPartInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + PartNumber: 1, + UploadId: aws.String("123456"), + Body: strings.NewReader("hello-world"), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "partNumber=1", + "uploadId=123456", + "x-id=UploadPart", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Content-Length": []string{"11"}, + "Content-Type": []string{"application/octet-stream"}, + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "seekable payload": { + input: s3.UploadPartInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + PartNumber: 1, + UploadId: aws.String("123456"), + Body: bytes.NewReader([]byte("hello-world")), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "partNumber=1", + "uploadId=123456", + "x-id=UploadPart", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Content-Length": []string{"11"}, + "Content-Type": []string{"application/octet-stream"}, + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "unseekable payload": { + // unseekable payload succeeds as we disable content sha256 computation for streaming input + input: s3.UploadPartInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + PartNumber: 1, + UploadId: aws.String("123456"), + Body: bytes.NewBuffer([]byte(`hello-world`)), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "partNumber=1", + "uploadId=123456", + "x-id=UploadPart", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Content-Length": []string{"11"}, + "Content-Type": []string{"application/octet-stream"}, + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "empty body": { + input: s3.UploadPartInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + PartNumber: 1, + UploadId: aws.String("123456"), + Body: bytes.NewReader([]byte(``)), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "partNumber=1", + "uploadId=123456", + "x-id=UploadPart", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + "nil body": { + input: s3.UploadPartInput{ + Bucket: aws.String("mock-bucket"), + Key: aws.String("mockkey"), + PartNumber: 1, + UploadId: aws.String("123456"), + }, + expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?", + expectRequestURIQuery: []string{ + "X-Amz-Expires=900", + "X-Amz-Credential", + "X-Amz-Date", + "partNumber=1", + "uploadId=123456", + "x-id=UploadPart", + "X-Amz-Signature", + }, + expectMethod: "PUT", + expectSignedHeader: http.Header{ + "Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"}, + }, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + cfg := aws.Config{ + Region: "us-west-2", + Credentials: unit.StubCredentialsProvider{}, + Retryer: func() aws.Retryer { + return aws.NopRetryer{} + }, + } + presignClient := s3.NewPresignClient(s3.NewFromConfig(cfg), func(options *s3.PresignOptions) { + options = &c.options + }) + + req, err := presignClient.PresignUploadPart(ctx, &c.input) + if err != nil { + if len(c.expectError) == 0 { + t.Fatalf("expected no error, got %v", err) + } + // if expect error, match error and skip rest + if e, a := c.expectError, err.Error(); !strings.Contains(a, e) { + t.Fatalf("expected error to be %s, got %s", e, a) + } + } else { + if len(c.expectError) != 0 { + t.Fatalf("expected error to be %v, got none", c.expectError) + } + } + + if e, a := c.expectPresignedURLHost, req.URL; !strings.Contains(a, e) { + t.Fatalf("expected presigned url to contain host %s, got %s", e, a) + } + + if len(c.expectRequestURIQuery) != 0 { + for _, label := range c.expectRequestURIQuery { + if e, a := label, req.URL; !strings.Contains(a, e) { + t.Fatalf("expected presigned url to contain %v label in url: %v", label, req.URL) + } + } + } + + if e, a := c.expectSignedHeader, req.SignedHeader; len(cmp.Diff(e, a)) != 0 { + t.Fatalf("expected signed header to be %s, got %s, \n diff : %s", e, a, cmp.Diff(e, a)) + } + + if e, a := c.expectMethod, req.Method; !strings.EqualFold(e, a) { + t.Fatalf("expected presigning Method to be %s, got %s", e, a) + } + + }) + } +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/presigned_expires.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/presigned_expires.go new file mode 100644 index 0000000000..f4bbb4b6de --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/presigned_expires.go @@ -0,0 +1,49 @@ +package customizations + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +// AddExpiresOnPresignedURL represents a build middleware used to assign +// expiration on a presigned URL. +type AddExpiresOnPresignedURL struct { + + // Expires is time.Duration within which presigned url should be expired. + // This should be the duration in seconds the presigned URL should be considered valid for. + // By default the S3 presigned url expires in 15 minutes ie. 900 seconds. + Expires time.Duration +} + +// ID representing the middleware +func (*AddExpiresOnPresignedURL) ID() string { + return "S3:AddExpiresOnPresignedURL" +} + +// HandleBuild handles the build step middleware behavior +func (m *AddExpiresOnPresignedURL) HandleBuild(ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error, +) { + // if expiration is unset skip this middleware + if m.Expires == 0 { + // default to 15 * time.Minutes + m.Expires = 15 * time.Minute + } + + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown transport type %T", req) + } + + // set S3 X-AMZ-Expires header + query := req.URL.Query() + query.Set("X-Amz-Expires", strconv.FormatInt(int64(m.Expires/time.Second), 10)) + req.URL.RawQuery = query.Encode() + + return next.HandleBuild(ctx, in) +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/process_arn_resource.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/process_arn_resource.go new file mode 100644 index 0000000000..bbc971f2a2 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/process_arn_resource.go @@ -0,0 +1,568 @@ +package customizations + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/aws/smithy-go/middleware" + "github.com/aws/smithy-go/transport/http" + + "github.com/aws/aws-sdk-go-v2/aws" + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/aws-sdk-go-v2/internal/v4a" + "github.com/aws/aws-sdk-go-v2/service/internal/s3shared" + "github.com/aws/aws-sdk-go-v2/service/internal/s3shared/arn" + s3arn "github.com/aws/aws-sdk-go-v2/service/s3/internal/arn" + "github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints" +) + +const ( + s3AccessPoint = "s3-accesspoint" + s3ObjectLambda = "s3-object-lambda" +) + +// processARNResource is used to process an ARN resource. +type processARNResource struct { + + // UseARNRegion indicates if region parsed from an ARN should be used. + UseARNRegion bool + + // UseAccelerate indicates if s3 transfer acceleration is enabled + UseAccelerate bool + + // EndpointResolver used to resolve endpoints. This may be a custom endpoint resolver + EndpointResolver EndpointResolver + + // EndpointResolverOptions used by endpoint resolver + EndpointResolverOptions EndpointResolverOptions + + // DisableMultiRegionAccessPoints indicates multi-region access point support is disabled + DisableMultiRegionAccessPoints bool +} + +// ID returns the middleware ID. +func (*processARNResource) ID() string { return "S3:ProcessARNResource" } + +func (m *processARNResource) HandleSerialize( + ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler, +) ( + out middleware.SerializeOutput, metadata middleware.Metadata, err error, +) { + if !awsmiddleware.GetRequiresLegacyEndpoints(ctx) { + return next.HandleSerialize(ctx, in) + } + + // check if arn was provided, if not skip this middleware + arnValue, ok := s3shared.GetARNResourceFromContext(ctx) + if !ok { + return next.HandleSerialize(ctx, in) + } + + req, ok := in.Request.(*http.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown request type %T", req) + } + + // parse arn into an endpoint arn wrt to service + resource, err := s3arn.ParseEndpointARN(arnValue) + if err != nil { + return out, metadata, err + } + + // build a resource request struct + resourceRequest := s3shared.ResourceRequest{ + Resource: resource, + UseARNRegion: m.UseARNRegion, + UseFIPS: m.EndpointResolverOptions.UseFIPSEndpoint == aws.FIPSEndpointStateEnabled, + RequestRegion: awsmiddleware.GetRegion(ctx), + SigningRegion: awsmiddleware.GetSigningRegion(ctx), + PartitionID: awsmiddleware.GetPartitionID(ctx), + } + + // switch to correct endpoint updater + switch tv := resource.(type) { + case arn.AccessPointARN: + // multi-region arns do not need to validate for cross partition request + if len(tv.Region) != 0 { + // validate resource request + if err := validateRegionForResourceRequest(resourceRequest); err != nil { + return out, metadata, err + } + } + + // Special handling for region-less ap-arns. + if len(tv.Region) == 0 { + // check if multi-region arn support is disabled + if m.DisableMultiRegionAccessPoints { + return out, metadata, fmt.Errorf("Invalid configuration, Multi-Region access point ARNs are disabled") + } + + // Do not allow dual-stack configuration with multi-region arns. + if m.EndpointResolverOptions.UseDualStackEndpoint == aws.DualStackEndpointStateEnabled { + return out, metadata, s3shared.NewClientConfiguredForDualStackError(tv, + resourceRequest.PartitionID, resourceRequest.RequestRegion, nil) + } + } + + // check if accelerate + if m.UseAccelerate { + return out, metadata, s3shared.NewClientConfiguredForAccelerateError(tv, + resourceRequest.PartitionID, resourceRequest.RequestRegion, nil) + } + + // fetch arn region to resolve request + resolveRegion := tv.Region + // check if request region is FIPS + if resourceRequest.UseFIPS && len(resolveRegion) == 0 { + // Do not allow Fips support within multi-region arns. + return out, metadata, s3shared.NewClientConfiguredForFIPSError( + tv, resourceRequest.PartitionID, resourceRequest.RequestRegion, nil) + } + + var requestBuilder func(context.Context, accesspointOptions) (context.Context, error) + if len(resolveRegion) == 0 { + requestBuilder = buildMultiRegionAccessPointsRequest + } else { + requestBuilder = buildAccessPointRequest + } + + // build request as per accesspoint builder + ctx, err = requestBuilder(ctx, accesspointOptions{ + processARNResource: *m, + request: req, + resource: tv, + resolveRegion: resolveRegion, + partitionID: resourceRequest.PartitionID, + requestRegion: resourceRequest.RequestRegion, + }) + if err != nil { + return out, metadata, err + } + + case arn.S3ObjectLambdaAccessPointARN: + // validate region for resource request + if err := validateRegionForResourceRequest(resourceRequest); err != nil { + return out, metadata, err + } + + // check if accelerate + if m.UseAccelerate { + return out, metadata, s3shared.NewClientConfiguredForAccelerateError(tv, + resourceRequest.PartitionID, resourceRequest.RequestRegion, nil) + } + + // check if dualstack + if m.EndpointResolverOptions.UseDualStackEndpoint == aws.DualStackEndpointStateEnabled { + return out, metadata, s3shared.NewClientConfiguredForDualStackError(tv, + resourceRequest.PartitionID, resourceRequest.RequestRegion, nil) + } + + // fetch arn region to resolve request + resolveRegion := tv.Region + + // build access point request + ctx, err = buildS3ObjectLambdaAccessPointRequest(ctx, accesspointOptions{ + processARNResource: *m, + request: req, + resource: tv.AccessPointARN, + resolveRegion: resolveRegion, + partitionID: resourceRequest.PartitionID, + requestRegion: resourceRequest.RequestRegion, + }) + if err != nil { + return out, metadata, err + } + + // process outpost accesspoint ARN + case arn.OutpostAccessPointARN: + // validate region for resource request + if err := validateRegionForResourceRequest(resourceRequest); err != nil { + return out, metadata, err + } + + // check if accelerate + if m.UseAccelerate { + return out, metadata, s3shared.NewClientConfiguredForAccelerateError(tv, + resourceRequest.PartitionID, resourceRequest.RequestRegion, nil) + } + + // check if dual stack + if m.EndpointResolverOptions.UseDualStackEndpoint == aws.DualStackEndpointStateEnabled { + return out, metadata, s3shared.NewClientConfiguredForDualStackError(tv, + resourceRequest.PartitionID, resourceRequest.RequestRegion, nil) + } + + // check if request region is FIPS + if resourceRequest.UseFIPS { + return out, metadata, s3shared.NewFIPSConfigurationError(tv, resourceRequest.PartitionID, + resourceRequest.RequestRegion, nil) + } + + // build outpost access point request + ctx, err = buildOutpostAccessPointRequest(ctx, outpostAccessPointOptions{ + processARNResource: *m, + resource: tv, + request: req, + partitionID: resourceRequest.PartitionID, + requestRegion: resourceRequest.RequestRegion, + }) + if err != nil { + return out, metadata, err + } + + default: + return out, metadata, s3shared.NewInvalidARNError(resource, nil) + } + + return next.HandleSerialize(ctx, in) +} + +// validate if s3 resource and request region config is compatible. +func validateRegionForResourceRequest(resourceRequest s3shared.ResourceRequest) error { + // check if resourceRequest leads to a cross partition error + v, err := resourceRequest.IsCrossPartition() + if err != nil { + return err + } + if v { + // if cross partition + return s3shared.NewClientPartitionMismatchError(resourceRequest.Resource, + resourceRequest.PartitionID, resourceRequest.RequestRegion, nil) + } + + // check if resourceRequest leads to a cross region error + if !resourceRequest.AllowCrossRegion() && resourceRequest.IsCrossRegion() { + // if cross region, but not use ARN region is not enabled + return s3shared.NewClientRegionMismatchError(resourceRequest.Resource, + resourceRequest.PartitionID, resourceRequest.RequestRegion, nil) + } + + return nil +} + +// === Accesspoint ========== + +type accesspointOptions struct { + processARNResource + request *http.Request + resource arn.AccessPointARN + resolveRegion string + partitionID string + requestRegion string +} + +func buildAccessPointRequest(ctx context.Context, options accesspointOptions) (context.Context, error) { + tv := options.resource + req := options.request + resolveRegion := options.resolveRegion + + resolveService := tv.Service + + ero := options.EndpointResolverOptions + ero.Logger = middleware.GetLogger(ctx) + ero.ResolvedRegion = "" // clear endpoint option's resolved region so that we resolve using the passed in region + + // resolve endpoint + endpoint, err := options.EndpointResolver.ResolveEndpoint(resolveRegion, ero) + if err != nil { + return ctx, s3shared.NewFailedToResolveEndpointError( + tv, + options.partitionID, + options.requestRegion, + err, + ) + } + + // assign resolved endpoint url to request url + req.URL, err = url.Parse(endpoint.URL) + if err != nil { + return ctx, fmt.Errorf("failed to parse endpoint URL: %w", err) + } + + if len(endpoint.SigningName) != 0 && endpoint.Source == aws.EndpointSourceCustom { + ctx = awsmiddleware.SetSigningName(ctx, endpoint.SigningName) + } else { + // Must sign with s3-object-lambda + ctx = awsmiddleware.SetSigningName(ctx, resolveService) + } + + if len(endpoint.SigningRegion) != 0 { + ctx = awsmiddleware.SetSigningRegion(ctx, endpoint.SigningRegion) + } else { + ctx = awsmiddleware.SetSigningRegion(ctx, resolveRegion) + } + + // update serviceID to "s3-accesspoint" + ctx = awsmiddleware.SetServiceID(ctx, s3AccessPoint) + + // disable host prefix behavior + ctx = http.DisableEndpointHostPrefix(ctx, true) + + // remove the serialized arn in place of /{Bucket} + ctx = setBucketToRemoveOnContext(ctx, tv.String()) + + // skip arn processing, if arn region resolves to a immutable endpoint + if endpoint.HostnameImmutable { + return ctx, nil + } + + updateS3HostForS3AccessPoint(req) + + ctx, err = buildAccessPointHostPrefix(ctx, req, tv) + if err != nil { + return ctx, err + } + + return ctx, nil +} + +func buildS3ObjectLambdaAccessPointRequest(ctx context.Context, options accesspointOptions) (context.Context, error) { + tv := options.resource + req := options.request + resolveRegion := options.resolveRegion + + resolveService := tv.Service + + ero := options.EndpointResolverOptions + ero.Logger = middleware.GetLogger(ctx) + ero.ResolvedRegion = "" // clear endpoint options resolved region so we resolve the passed in region + + // resolve endpoint + endpoint, err := options.EndpointResolver.ResolveEndpoint(resolveRegion, ero) + if err != nil { + return ctx, s3shared.NewFailedToResolveEndpointError( + tv, + options.partitionID, + options.requestRegion, + err, + ) + } + + // assign resolved endpoint url to request url + req.URL, err = url.Parse(endpoint.URL) + if err != nil { + return ctx, fmt.Errorf("failed to parse endpoint URL: %w", err) + } + + if len(endpoint.SigningName) != 0 && endpoint.Source == aws.EndpointSourceCustom { + ctx = awsmiddleware.SetSigningName(ctx, endpoint.SigningName) + } else { + // Must sign with s3-object-lambda + ctx = awsmiddleware.SetSigningName(ctx, resolveService) + } + + if len(endpoint.SigningRegion) != 0 { + ctx = awsmiddleware.SetSigningRegion(ctx, endpoint.SigningRegion) + } else { + ctx = awsmiddleware.SetSigningRegion(ctx, resolveRegion) + } + + // update serviceID to "s3-object-lambda" + ctx = awsmiddleware.SetServiceID(ctx, s3ObjectLambda) + + // disable host prefix behavior + ctx = http.DisableEndpointHostPrefix(ctx, true) + + // remove the serialized arn in place of /{Bucket} + ctx = setBucketToRemoveOnContext(ctx, tv.String()) + + // skip arn processing, if arn region resolves to a immutable endpoint + if endpoint.HostnameImmutable { + return ctx, nil + } + + if endpoint.Source == aws.EndpointSourceServiceMetadata { + updateS3HostForS3ObjectLambda(req) + } + + ctx, err = buildAccessPointHostPrefix(ctx, req, tv) + if err != nil { + return ctx, err + } + + return ctx, nil +} + +func buildMultiRegionAccessPointsRequest(ctx context.Context, options accesspointOptions) (context.Context, error) { + const s3GlobalLabel = "s3-global." + const accesspointLabel = "accesspoint." + + tv := options.resource + req := options.request + resolveService := tv.Service + resolveRegion := options.requestRegion + arnPartition := tv.Partition + + // resolve endpoint + ero := options.EndpointResolverOptions + ero.Logger = middleware.GetLogger(ctx) + + endpoint, err := options.EndpointResolver.ResolveEndpoint(resolveRegion, ero) + if err != nil { + return ctx, s3shared.NewFailedToResolveEndpointError( + tv, + options.partitionID, + options.requestRegion, + err, + ) + } + + // set signing region and version for MRAP + endpoint.SigningRegion = "*" + ctx = awsmiddleware.SetSigningRegion(ctx, endpoint.SigningRegion) + ctx = SetSignerVersion(ctx, v4a.Version) + + if len(endpoint.SigningName) != 0 { + ctx = awsmiddleware.SetSigningName(ctx, endpoint.SigningName) + } else { + ctx = awsmiddleware.SetSigningName(ctx, resolveService) + } + + // skip arn processing, if arn region resolves to a immutable endpoint + if endpoint.HostnameImmutable { + return ctx, nil + } + + // modify endpoint host to use s3-global host prefix + scheme := strings.SplitN(endpoint.URL, "://", 2) + dnsSuffix, err := endpoints.GetDNSSuffix(arnPartition, ero) + if err != nil { + return ctx, fmt.Errorf("Error determining dns suffix from arn partition, %w", err) + } + // set url as per partition + endpoint.URL = scheme[0] + "://" + s3GlobalLabel + dnsSuffix + + // assign resolved endpoint url to request url + req.URL, err = url.Parse(endpoint.URL) + if err != nil { + return ctx, fmt.Errorf("failed to parse endpoint URL: %w", err) + } + + // build access point host prefix + accessPointHostPrefix := tv.AccessPointName + "." + accesspointLabel + + // add host prefix to url + req.URL.Host = accessPointHostPrefix + req.URL.Host + if len(req.Host) > 0 { + req.Host = accessPointHostPrefix + req.Host + } + + // validate the endpoint host + if err := http.ValidateEndpointHost(req.URL.Host); err != nil { + return ctx, fmt.Errorf("endpoint validation error: %w, when using arn %v", err, tv) + } + + // disable host prefix behavior + ctx = http.DisableEndpointHostPrefix(ctx, true) + + // remove the serialized arn in place of /{Bucket} + ctx = setBucketToRemoveOnContext(ctx, tv.String()) + + return ctx, nil +} + +func buildAccessPointHostPrefix(ctx context.Context, req *http.Request, tv arn.AccessPointARN) (context.Context, error) { + // add host prefix for access point + accessPointHostPrefix := tv.AccessPointName + "-" + tv.AccountID + "." + req.URL.Host = accessPointHostPrefix + req.URL.Host + if len(req.Host) > 0 { + req.Host = accessPointHostPrefix + req.Host + } + + // validate the endpoint host + if err := http.ValidateEndpointHost(req.URL.Host); err != nil { + return ctx, s3shared.NewInvalidARNError(tv, err) + } + + return ctx, nil +} + +// ====== Outpost Accesspoint ======== + +type outpostAccessPointOptions struct { + processARNResource + request *http.Request + resource arn.OutpostAccessPointARN + partitionID string + requestRegion string +} + +func buildOutpostAccessPointRequest(ctx context.Context, options outpostAccessPointOptions) (context.Context, error) { + tv := options.resource + req := options.request + + resolveRegion := tv.Region + resolveService := tv.Service + endpointsID := resolveService + if strings.EqualFold(resolveService, "s3-outposts") { + // assign endpoints ID as "S3" + endpointsID = "s3" + } + + ero := options.EndpointResolverOptions + ero.Logger = middleware.GetLogger(ctx) + ero.ResolvedRegion = "" + + // resolve regional endpoint for resolved region. + endpoint, err := options.EndpointResolver.ResolveEndpoint(resolveRegion, ero) + if err != nil { + return ctx, s3shared.NewFailedToResolveEndpointError( + tv, + options.partitionID, + options.requestRegion, + err, + ) + } + + // assign resolved endpoint url to request url + req.URL, err = url.Parse(endpoint.URL) + if err != nil { + return ctx, fmt.Errorf("failed to parse endpoint URL: %w", err) + } + + // assign resolved service from arn as signing name + if len(endpoint.SigningName) != 0 && endpoint.Source == aws.EndpointSourceCustom { + ctx = awsmiddleware.SetSigningName(ctx, endpoint.SigningName) + } else { + ctx = awsmiddleware.SetSigningName(ctx, resolveService) + } + + if len(endpoint.SigningRegion) != 0 { + // redirect signer to use resolved endpoint signing name and region + ctx = awsmiddleware.SetSigningRegion(ctx, endpoint.SigningRegion) + } else { + ctx = awsmiddleware.SetSigningRegion(ctx, resolveRegion) + } + + // update serviceID to resolved service id + ctx = awsmiddleware.SetServiceID(ctx, resolveService) + + // disable host prefix behavior + ctx = http.DisableEndpointHostPrefix(ctx, true) + + // remove the serialized arn in place of /{Bucket} + ctx = setBucketToRemoveOnContext(ctx, tv.String()) + + // skip further customizations, if arn region resolves to a immutable endpoint + if endpoint.HostnameImmutable { + return ctx, nil + } + + updateHostPrefix(req, endpointsID, resolveService) + + // add host prefix for s3-outposts + outpostAPHostPrefix := tv.AccessPointName + "-" + tv.AccountID + "." + tv.OutpostID + "." + req.URL.Host = outpostAPHostPrefix + req.URL.Host + if len(req.Host) > 0 { + req.Host = outpostAPHostPrefix + req.Host + } + + // validate the endpoint host + if err := http.ValidateEndpointHost(req.URL.Host); err != nil { + return ctx, s3shared.NewInvalidARNError(tv, err) + } + + return ctx, nil +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/remove_bucket_middleware.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/remove_bucket_middleware.go new file mode 100644 index 0000000000..cf3f4dc8b6 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/remove_bucket_middleware.go @@ -0,0 +1,63 @@ +package customizations + +import ( + "context" + "fmt" + + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/smithy-go/middleware" + "github.com/aws/smithy-go/transport/http" +) + +// removeBucketFromPathMiddleware needs to be executed after serialize step is performed +type removeBucketFromPathMiddleware struct { +} + +func (m *removeBucketFromPathMiddleware) ID() string { + return "S3:RemoveBucketFromPathMiddleware" +} + +func (m *removeBucketFromPathMiddleware) HandleSerialize( + ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler, +) ( + out middleware.SerializeOutput, metadata middleware.Metadata, err error, +) { + if !awsmiddleware.GetRequiresLegacyEndpoints(ctx) { + return next.HandleSerialize(ctx, in) + } + + // check if a bucket removal from HTTP path is required + bucket, ok := getRemoveBucketFromPath(ctx) + if !ok { + return next.HandleSerialize(ctx, in) + } + + req, ok := in.Request.(*http.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown request type %T", req) + } + + removeBucketFromPath(req.URL, bucket) + return next.HandleSerialize(ctx, in) +} + +type removeBucketKey struct { + bucket string +} + +// setBucketToRemoveOnContext sets the bucket name to be removed. +// +// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues +// to clear all stack values. +func setBucketToRemoveOnContext(ctx context.Context, bucket string) context.Context { + return middleware.WithStackValue(ctx, removeBucketKey{}, bucket) +} + +// getRemoveBucketFromPath returns the bucket name to remove from the path. +// +// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues +// to clear all stack values. +func getRemoveBucketFromPath(ctx context.Context) (string, bool) { + v, ok := middleware.GetStackValue(ctx, removeBucketKey{}).(string) + return v, ok +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/s3_object_lambda.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/s3_object_lambda.go new file mode 100644 index 0000000000..6e1d447243 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/s3_object_lambda.go @@ -0,0 +1,88 @@ +package customizations + +import ( + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/smithy-go/middleware" + "github.com/aws/smithy-go/transport/http" + "net/url" +) + +type s3ObjectLambdaEndpoint struct { + // whether the operation should use the s3-object-lambda endpoint + UseEndpoint bool + + // use transfer acceleration + UseAccelerate bool + + EndpointResolver EndpointResolver + EndpointResolverOptions EndpointResolverOptions +} + +func (t *s3ObjectLambdaEndpoint) ID() string { + return "S3:ObjectLambdaEndpoint" +} + +func (t *s3ObjectLambdaEndpoint) HandleSerialize( + ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler, +) ( + out middleware.SerializeOutput, metadata middleware.Metadata, err error, +) { + if !awsmiddleware.GetRequiresLegacyEndpoints(ctx) { + return next.HandleSerialize(ctx, in) + } + + if !t.UseEndpoint { + return next.HandleSerialize(ctx, in) + } + + req, ok := in.Request.(*http.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown transport type: %T", in.Request) + } + + if t.EndpointResolverOptions.UseDualStackEndpoint == aws.DualStackEndpointStateEnabled { + return out, metadata, fmt.Errorf("client configured for dualstack but not supported for operation") + } + + if t.UseAccelerate { + return out, metadata, fmt.Errorf("client configured for accelerate but not supported for operation") + } + + region := awsmiddleware.GetRegion(ctx) + + ero := t.EndpointResolverOptions + + endpoint, err := t.EndpointResolver.ResolveEndpoint(region, ero) + if err != nil { + return out, metadata, err + } + + // Set the ServiceID and SigningName + ctx = awsmiddleware.SetServiceID(ctx, s3ObjectLambda) + + if len(endpoint.SigningName) > 0 && endpoint.Source == aws.EndpointSourceCustom { + ctx = awsmiddleware.SetSigningName(ctx, endpoint.SigningName) + } else { + ctx = awsmiddleware.SetSigningName(ctx, s3ObjectLambda) + } + + req.URL, err = url.Parse(endpoint.URL) + if err != nil { + return out, metadata, err + } + + if len(endpoint.SigningRegion) > 0 { + ctx = awsmiddleware.SetSigningRegion(ctx, endpoint.SigningRegion) + } else { + ctx = awsmiddleware.SetSigningRegion(ctx, region) + } + + if endpoint.Source == aws.EndpointSourceServiceMetadata || !endpoint.HostnameImmutable { + updateS3HostForS3ObjectLambda(req) + } + + return next.HandleSerialize(ctx, in) +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/signer_wrapper.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/signer_wrapper.go new file mode 100644 index 0000000000..4408f3e8a2 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/signer_wrapper.go @@ -0,0 +1,213 @@ +package customizations + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/internal/v4a" + "github.com/aws/smithy-go/middleware" +) + +type signerVersionKey struct{} + +// GetSignerVersion retrieves the signer version to use for signing +// +// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues +// to clear all stack values. +func GetSignerVersion(ctx context.Context) (v string) { + v, _ = middleware.GetStackValue(ctx, signerVersionKey{}).(string) + return v +} + +// SetSignerVersion sets the signer version to be used for signing the request +// +// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues +// to clear all stack values. +func SetSignerVersion(ctx context.Context, version string) context.Context { + return middleware.WithStackValue(ctx, signerVersionKey{}, version) +} + +// SignHTTPRequestMiddlewareOptions is the configuration options for the SignHTTPRequestMiddleware middleware. +type SignHTTPRequestMiddlewareOptions struct { + + // credential provider + CredentialsProvider aws.CredentialsProvider + + // log signing + LogSigning bool + + // v4 signer + V4Signer v4.HTTPSigner + + //v4a signer + V4aSigner v4a.HTTPSigner +} + +// NewSignHTTPRequestMiddleware constructs a SignHTTPRequestMiddleware using the given Signer for signing requests +func NewSignHTTPRequestMiddleware(options SignHTTPRequestMiddlewareOptions) *SignHTTPRequestMiddleware { + return &SignHTTPRequestMiddleware{ + credentialsProvider: options.CredentialsProvider, + v4Signer: options.V4Signer, + v4aSigner: options.V4aSigner, + logSigning: options.LogSigning, + } +} + +// SignHTTPRequestMiddleware is a `FinalizeMiddleware` implementation to select HTTP Signing method +type SignHTTPRequestMiddleware struct { + + // credential provider + credentialsProvider aws.CredentialsProvider + + // log signing + logSigning bool + + // v4 signer + v4Signer v4.HTTPSigner + + //v4a signer + v4aSigner v4a.HTTPSigner +} + +// ID is the SignHTTPRequestMiddleware identifier +func (s *SignHTTPRequestMiddleware) ID() string { + return "Signing" +} + +// HandleFinalize will take the provided input and sign the request using the SigV4 authentication scheme +func (s *SignHTTPRequestMiddleware) HandleFinalize(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) ( + out middleware.FinalizeOutput, metadata middleware.Metadata, err error, +) { + // fetch signer type from context + signerVersion := GetSignerVersion(ctx) + + // SigV4a + if strings.EqualFold(signerVersion, v4a.Version) { + v4aCredentialProvider, ok := s.credentialsProvider.(v4a.CredentialsProvider) + if !ok { + return out, metadata, fmt.Errorf("invalid credential-provider provided for sigV4a Signer") + } + + mw := v4a.NewSignHTTPRequestMiddleware(v4a.SignHTTPRequestMiddlewareOptions{ + Credentials: v4aCredentialProvider, + Signer: s.v4aSigner, + LogSigning: s.logSigning, + }) + return mw.HandleFinalize(ctx, in, next) + } + // SigV4 + mw := v4.NewSignHTTPRequestMiddleware(v4.SignHTTPRequestMiddlewareOptions{ + CredentialsProvider: s.credentialsProvider, + Signer: s.v4Signer, + LogSigning: s.logSigning, + }) + return mw.HandleFinalize(ctx, in, next) +} + +// RegisterSigningMiddleware registers the wrapper signing middleware to the stack. If a signing middleware is already +// present, this provided middleware will be swapped. Otherwise the middleware will be added at the tail of the +// finalize step. +func RegisterSigningMiddleware(stack *middleware.Stack, signingMiddleware *SignHTTPRequestMiddleware) (err error) { + const signedID = "Signing" + _, present := stack.Finalize.Get(signedID) + if present { + _, err = stack.Finalize.Swap(signedID, signingMiddleware) + } else { + err = stack.Finalize.Add(signingMiddleware, middleware.After) + } + return err +} + +// PresignHTTPRequestMiddlewareOptions is the options for the PresignHTTPRequestMiddleware middleware. +type PresignHTTPRequestMiddlewareOptions struct { + CredentialsProvider aws.CredentialsProvider + V4Presigner v4.HTTPPresigner + V4aPresigner v4a.HTTPPresigner + LogSigning bool +} + +// PresignHTTPRequestMiddleware provides the Finalize middleware for creating a +// presigned URL for an HTTP request. +// +// Will short circuit the middleware stack and not forward onto the next +// Finalize handler. +type PresignHTTPRequestMiddleware struct { + + // cred provider and signer for sigv4 + credentialsProvider aws.CredentialsProvider + + // sigV4 signer + v4Signer v4.HTTPPresigner + + // sigV4a signer + v4aSigner v4a.HTTPPresigner + + // log signing + logSigning bool +} + +// NewPresignHTTPRequestMiddleware constructs a PresignHTTPRequestMiddleware using the given Signer for signing requests +func NewPresignHTTPRequestMiddleware(options PresignHTTPRequestMiddlewareOptions) *PresignHTTPRequestMiddleware { + return &PresignHTTPRequestMiddleware{ + credentialsProvider: options.CredentialsProvider, + v4Signer: options.V4Presigner, + v4aSigner: options.V4aPresigner, + logSigning: options.LogSigning, + } +} + +// ID provides the middleware ID. +func (*PresignHTTPRequestMiddleware) ID() string { return "PresignHTTPRequest" } + +// HandleFinalize will take the provided input and create a presigned url for +// the http request using the SigV4 or SigV4a presign authentication scheme. +// +// Since the signed request is not a valid HTTP request +func (p *PresignHTTPRequestMiddleware) HandleFinalize( + ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler, +) ( + out middleware.FinalizeOutput, metadata middleware.Metadata, err error, +) { + // fetch signer type from context + signerVersion := GetSignerVersion(ctx) + + switch signerVersion { + case v4a.Version: + v4aCredentialProvider, ok := p.credentialsProvider.(v4a.CredentialsProvider) + if !ok { + return out, metadata, fmt.Errorf("invalid credential-provider provided for sigV4a Signer") + } + + mw := v4a.NewPresignHTTPRequestMiddleware(v4a.PresignHTTPRequestMiddlewareOptions{ + CredentialsProvider: v4aCredentialProvider, + Presigner: p.v4aSigner, + LogSigning: p.logSigning, + }) + return mw.HandleFinalize(ctx, in, next) + + default: + mw := v4.NewPresignHTTPRequestMiddleware(v4.PresignHTTPRequestMiddlewareOptions{ + CredentialsProvider: p.credentialsProvider, + Presigner: p.v4Signer, + LogSigning: p.logSigning, + }) + return mw.HandleFinalize(ctx, in, next) + } +} + +// RegisterPreSigningMiddleware registers the wrapper pre-signing middleware to the stack. If a pre-signing middleware is already +// present, this provided middleware will be swapped. Otherwise the middleware will be added at the tail of the +// finalize step. +func RegisterPreSigningMiddleware(stack *middleware.Stack, signingMiddleware *PresignHTTPRequestMiddleware) (err error) { + const signedID = "PresignHTTPRequest" + _, present := stack.Finalize.Get(signedID) + if present { + _, err = stack.Finalize.Swap(signedID, signingMiddleware) + } else { + err = stack.Finalize.Add(signingMiddleware, middleware.After) + } + return err +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/unit_test.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/unit_test.go new file mode 100644 index 0000000000..5ef688c6b6 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/unit_test.go @@ -0,0 +1,167 @@ +package customizations_test + +import ( + "context" + "errors" + "net/http" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/aws/smithy-go" +) + +func Test_EmptyResponse(t *testing.T) { + cases := map[string]struct { + response *http.Response + expectError bool + }{ + "success case with no response body": { + response: &http.Response{ + StatusCode: 200, + Body: asReadCloser( + ``, + ), + }, + }, + "error case with no response body": { + response: &http.Response{ + StatusCode: 400, + Body: asReadCloser( + ``, + ), + }, + expectError: true, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + + ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelFn() + + cfg := aws.Config{ + Region: "mock-region", + Retryer: func() aws.Retryer { + return aws.NopRetryer{} + }, + } + + client := s3.NewFromConfig(cfg, + func(options *s3.Options) { + options.UsePathStyle = true + options.HTTPClient = &mockHTTPClient{c.response} + }, + ) + + params := &s3.HeadBucketInput{Bucket: aws.String("aws-sdk-go-data")} + _, err := client.HeadBucket(ctx, params) + if c.expectError { + var apiErr smithy.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("expect error to be API error, was not, %v", err) + } + if len(apiErr.ErrorCode()) == 0 { + t.Errorf("expect non-empty error code") + } + if len(apiErr.ErrorMessage()) == 0 { + t.Errorf("expect non-empty error message") + } + } else { + if err != nil { + t.Errorf("expected no error, got %v", err.Error()) + } + } + }) + } +} + +func TestBucketLocationPopulation(t *testing.T) { + cases := map[string]struct { + response *http.Response + expectLocation string + expectError string + }{ + "empty location": { + response: &http.Response{ + StatusCode: 200, + Body: asReadCloser( + `<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`, + ), + }, + expectLocation: "", + }, + "EU location": { + response: &http.Response{ + StatusCode: 200, + Body: asReadCloser( + `<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">EU</LocationConstraint>`, + ), + }, + expectLocation: "EU", + }, + "AfSouth1 location": { + response: &http.Response{ + StatusCode: 200, + Body: asReadCloser( + `<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">af-south-1</LocationConstraint>`, + ), + }, + expectLocation: "af-south-1", + }, + "IncompleteResponse": { + response: &http.Response{ + Body: asReadCloser( + `<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`, + ), + }, + expectError: "unexpected EOF", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + + ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelFn() + + cfg := aws.Config{ + Region: "us-east-1", + Retryer: func() aws.Retryer { return aws.NopRetryer{} }, + } + + client := s3.NewFromConfig(cfg, func(options *s3.Options) { + options.UsePathStyle = true + options.HTTPClient = &mockHTTPClient{c.response} + }) + + params := &s3.GetBucketLocationInput{ + Bucket: aws.String("aws-sdk-go-data"), + } + resp, err := client.GetBucketLocation(ctx, params) + if len(c.expectError) != 0 && err == nil { + t.Fatal("expect error, got none") + } + + if err != nil && len(c.expectError) == 0 { + t.Fatalf("expect no error, got %v", err) + } else { + if err != nil { + if !strings.Contains(err.Error(), c.expectError) { + t.Fatalf("expect error to be %v, got %v", err.Error(), c.expectError) + } + return + } + } + + if e, a := c.expectLocation, resp.LocationConstraint; !strings.EqualFold(e, string(a)) { + t.Fatalf("expected location constraint to be deserialized as %v, got %v", e, a) + } + }) + } + +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/update_endpoint.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/update_endpoint.go new file mode 100644 index 0000000000..eedfc7eefa --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/update_endpoint.go @@ -0,0 +1,310 @@ +package customizations + +import ( + "context" + "fmt" + "log" + "net/url" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/aws-sdk-go-v2/service/internal/s3shared" + internalendpoints "github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints" + "github.com/aws/smithy-go/encoding/httpbinding" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +// EndpointResolver interface for resolving service endpoints. +type EndpointResolver interface { + ResolveEndpoint(region string, options EndpointResolverOptions) (aws.Endpoint, error) +} + +// EndpointResolverOptions is the service endpoint resolver options +type EndpointResolverOptions = internalendpoints.Options + +// UpdateEndpointParameterAccessor represents accessor functions used by the middleware +type UpdateEndpointParameterAccessor struct { + // functional pointer to fetch bucket name from provided input. + // The function is intended to take an input value, and + // return a string pointer to value of string, and bool if + // input has no bucket member. + GetBucketFromInput func(interface{}) (*string, bool) +} + +// UpdateEndpointOptions provides the options for the UpdateEndpoint middleware setup. +type UpdateEndpointOptions struct { + // Accessor are parameter accessors used by the middleware + Accessor UpdateEndpointParameterAccessor + + // use path style + UsePathStyle bool + + // use transfer acceleration + UseAccelerate bool + + // indicates if an operation supports s3 transfer acceleration. + SupportsAccelerate bool + + // use ARN region + UseARNRegion bool + + // Indicates that the operation should target the s3-object-lambda endpoint. + // Used to direct operations that do not route based on an input ARN. + TargetS3ObjectLambda bool + + // EndpointResolver used to resolve endpoints. This may be a custom endpoint resolver + EndpointResolver EndpointResolver + + // EndpointResolverOptions used by endpoint resolver + EndpointResolverOptions EndpointResolverOptions + + // DisableMultiRegionAccessPoints indicates multi-region access point support is disabled + DisableMultiRegionAccessPoints bool +} + +// UpdateEndpoint adds the middleware to the middleware stack based on the UpdateEndpointOptions. +func UpdateEndpoint(stack *middleware.Stack, options UpdateEndpointOptions) (err error) { + const serializerID = "OperationSerializer" + + // initial arn look up middleware + err = stack.Initialize.Insert(&s3shared.ARNLookup{ + GetARNValue: options.Accessor.GetBucketFromInput, + }, "legacyEndpointContextSetter", middleware.After) + if err != nil { + return err + } + + // process arn + err = stack.Serialize.Insert(&processARNResource{ + UseARNRegion: options.UseARNRegion, + UseAccelerate: options.UseAccelerate, + EndpointResolver: options.EndpointResolver, + EndpointResolverOptions: options.EndpointResolverOptions, + DisableMultiRegionAccessPoints: options.DisableMultiRegionAccessPoints, + }, serializerID, middleware.Before) + if err != nil { + return err + } + + // process whether the operation requires the s3-object-lambda endpoint + // Occurs before operation serializer so that hostPrefix mutations + // can be handled correctly. + err = stack.Serialize.Insert(&s3ObjectLambdaEndpoint{ + UseEndpoint: options.TargetS3ObjectLambda, + UseAccelerate: options.UseAccelerate, + EndpointResolver: options.EndpointResolver, + EndpointResolverOptions: options.EndpointResolverOptions, + }, serializerID, middleware.Before) + if err != nil { + return err + } + + // remove bucket arn middleware + err = stack.Serialize.Insert(&removeBucketFromPathMiddleware{}, serializerID, middleware.After) + if err != nil { + return err + } + + // update endpoint to use options for path style and accelerate + err = stack.Serialize.Insert(&updateEndpoint{ + usePathStyle: options.UsePathStyle, + getBucketFromInput: options.Accessor.GetBucketFromInput, + useAccelerate: options.UseAccelerate, + supportsAccelerate: options.SupportsAccelerate, + }, serializerID, middleware.After) + if err != nil { + return err + } + + return err +} + +type updateEndpoint struct { + // path style options + usePathStyle bool + getBucketFromInput func(interface{}) (*string, bool) + + // accelerate options + useAccelerate bool + supportsAccelerate bool +} + +// ID returns the middleware ID. +func (*updateEndpoint) ID() string { + return "S3:UpdateEndpoint" +} + +func (u *updateEndpoint) HandleSerialize( + ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler, +) ( + out middleware.SerializeOutput, metadata middleware.Metadata, err error, +) { + if !awsmiddleware.GetRequiresLegacyEndpoints(ctx) { + return next.HandleSerialize(ctx, in) + } + + // if arn was processed, skip this middleware + if _, ok := s3shared.GetARNResourceFromContext(ctx); ok { + return next.HandleSerialize(ctx, in) + } + + // skip this customization if host name is set as immutable + if smithyhttp.GetHostnameImmutable(ctx) { + return next.HandleSerialize(ctx, in) + } + + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown request type %T", req) + } + + // check if accelerate is supported + if u.useAccelerate && !u.supportsAccelerate { + // accelerate is not supported, thus will be ignored + log.Println("Transfer acceleration is not supported for the operation, ignoring UseAccelerate.") + u.useAccelerate = false + } + + // transfer acceleration is not supported with path style urls + if u.useAccelerate && u.usePathStyle { + log.Println("UseAccelerate is not compatible with UsePathStyle, ignoring UsePathStyle.") + u.usePathStyle = false + } + + if u.getBucketFromInput != nil { + // Below customization only apply if bucket name is provided + bucket, ok := u.getBucketFromInput(in.Parameters) + if ok && bucket != nil { + region := awsmiddleware.GetRegion(ctx) + if err := u.updateEndpointFromConfig(req, *bucket, region); err != nil { + return out, metadata, err + } + } + } + + return next.HandleSerialize(ctx, in) +} + +func (u updateEndpoint) updateEndpointFromConfig(req *smithyhttp.Request, bucket string, region string) error { + // do nothing if path style is enforced + if u.usePathStyle { + return nil + } + + if !hostCompatibleBucketName(req.URL, bucket) { + // bucket name must be valid to put into the host for accelerate operations. + // For non-accelerate operations the bucket name can stay in the path if + // not valid hostname. + var err error + if u.useAccelerate { + err = fmt.Errorf("bucket name %s is not compatible with S3", bucket) + } + + // No-Op if not using accelerate. + return err + } + + // accelerate is only supported if use path style is disabled + if u.useAccelerate { + parts := strings.Split(req.URL.Host, ".") + if len(parts) < 3 { + return fmt.Errorf("unable to update endpoint host for S3 accelerate, hostname invalid, %s", req.URL.Host) + } + + if parts[0] == "s3" || strings.HasPrefix(parts[0], "s3-") { + parts[0] = "s3-accelerate" + } + + for i := 1; i+1 < len(parts); i++ { + if strings.EqualFold(parts[i], region) { + parts = append(parts[:i], parts[i+1:]...) + break + } + } + + // construct the url host + req.URL.Host = strings.Join(parts, ".") + } + + // move bucket to follow virtual host style + moveBucketNameToHost(req.URL, bucket) + return nil +} + +// updates endpoint to use virtual host styling +func moveBucketNameToHost(u *url.URL, bucket string) { + u.Host = bucket + "." + u.Host + removeBucketFromPath(u, bucket) +} + +// remove bucket from url +func removeBucketFromPath(u *url.URL, bucket string) { + if strings.HasPrefix(u.Path, "/"+bucket) { + // modify url path + u.Path = strings.Replace(u.Path, "/"+bucket, "", 1) + + // modify url raw path + u.RawPath = strings.Replace(u.RawPath, "/"+httpbinding.EscapePath(bucket, true), "", 1) + } + + if u.Path == "" { + u.Path = "/" + } + + if u.RawPath == "" { + u.RawPath = "/" + } +} + +// hostCompatibleBucketName returns true if the request should +// put the bucket in the host. This is false if the bucket is not +// DNS compatible or the EndpointResolver resolves an aws.Endpoint with +// HostnameImmutable member set to true. +// +// https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/aws#Endpoint.HostnameImmutable +func hostCompatibleBucketName(u *url.URL, bucket string) bool { + // Bucket might be DNS compatible but dots in the hostname will fail + // certificate validation, so do not use host-style. + if u.Scheme == "https" && strings.Contains(bucket, ".") { + return false + } + + // if the bucket is DNS compatible + return dnsCompatibleBucketName(bucket) +} + +// dnsCompatibleBucketName returns true if the bucket name is DNS compatible. +// Buckets created outside of the classic region MUST be DNS compatible. +func dnsCompatibleBucketName(bucket string) bool { + if strings.Contains(bucket, "..") { + return false + } + + // checks for `^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$` domain mapping + if !((bucket[0] > 96 && bucket[0] < 123) || (bucket[0] > 47 && bucket[0] < 58)) { + return false + } + + for _, c := range bucket[1:] { + if !((c > 96 && c < 123) || (c > 47 && c < 58) || c == 46 || c == 45) { + return false + } + } + + // checks for `^(\d+\.){3}\d+$` IPaddressing + v := strings.SplitN(bucket, ".", -1) + if len(v) == 4 { + for _, c := range bucket { + if !((c > 47 && c < 58) || c == 46) { + // we confirm that this is not a IP address + return true + } + } + // this is a IP address + return false + } + + return true +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/update_endpoint_test.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/update_endpoint_test.go new file mode 100644 index 0000000000..c1f4f14f18 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/update_endpoint_test.go @@ -0,0 +1,1609 @@ +package customizations_test + +import ( + "context" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" + "github.com/aws/aws-sdk-go-v2/internal/awstesting/unit" + "github.com/aws/aws-sdk-go-v2/internal/v4a" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints" + "github.com/aws/smithy-go/middleware" + "github.com/aws/smithy-go/ptr" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +type s3BucketTest struct { + bucket string + key string + url string + err string +} + +func Test_UpdateEndpointBuild(t *testing.T) { + cases := map[string]map[string]struct { + tests []s3BucketTest + useAccelerate bool + useDualstack bool + usePathStyle bool + disableHTTPS bool + customEndpoint *aws.Endpoint + }{ + "default endpoint": { + "PathStyleBucket": { + usePathStyle: true, + tests: []s3BucketTest{ + {"abc", "key", "https://s3.mock-region.amazonaws.com/abc/key?x-id=GetObject", ""}, + {"a$b$c", "key", "https://s3.mock-region.amazonaws.com/a%24b%24c/key?x-id=GetObject", ""}, + {"a.b.c", "key", "https://s3.mock-region.amazonaws.com/a.b.c/key?x-id=GetObject", ""}, + {"a..bc", "key", "https://s3.mock-region.amazonaws.com/a..bc/key?x-id=GetObject", ""}, + {"abc", "k:e,y", "https://s3.mock-region.amazonaws.com/abc/k%3Ae%2Cy?x-id=GetObject", ""}, + }, + }, + "VirtualHostStyleBucket": { + tests: []s3BucketTest{ + {"abc", "key", "https://abc.s3.mock-region.amazonaws.com/key?x-id=GetObject", ""}, + {"a$b$c", "key", "https://s3.mock-region.amazonaws.com/a%24b%24c/key?x-id=GetObject", ""}, + {"a.b.c", "key", "https://s3.mock-region.amazonaws.com/a.b.c/key?x-id=GetObject", ""}, + {"a..bc", "key", "https://s3.mock-region.amazonaws.com/a..bc/key?x-id=GetObject", ""}, + {"abc", "k:e,y", "https://abc.s3.mock-region.amazonaws.com/k%3Ae%2Cy?x-id=GetObject", ""}, + }, + }, + "Accelerate": { + useAccelerate: true, + tests: []s3BucketTest{ + {"abc", "key", "https://abc.s3-accelerate.amazonaws.com/key?x-id=GetObject", ""}, + {"a.b.c", "key", "https://s3.mock-region.amazonaws.com/a.b.c/key?x-id=GetObject", "cannot be used with"}, + {"a$b$c", "key", "https://s3.mock-region.amazonaws.com/a%24b%24c/key?x-id=GetObject", "cannot be used with"}, + }, + }, + "AccelerateNoSSLTests": { + useAccelerate: true, + disableHTTPS: true, + tests: []s3BucketTest{ + {"abc", "key", "http://abc.s3-accelerate.amazonaws.com/key?x-id=GetObject", ""}, + {"a$b$c", "key", "http://s3.mock-region.amazonaws.com/a%24b%24c/key?x-id=GetObject", "cannot be used with"}, + }, + }, + "DualStack": { + useDualstack: true, + tests: []s3BucketTest{ + {"abc", "key", "https://abc.s3.dualstack.mock-region.amazonaws.com/key?x-id=GetObject", ""}, + {"a.b.c", "key", "https://s3.dualstack.mock-region.amazonaws.com/a.b.c/key?x-id=GetObject", ""}, + {"a$b$c", "key", "https://s3.dualstack.mock-region.amazonaws.com/a%24b%24c/key?x-id=GetObject", ""}, + }, + }, + "DualStackWithPathStyle": { + useDualstack: true, + usePathStyle: true, + tests: []s3BucketTest{ + {"abc", "key", "https://s3.dualstack.mock-region.amazonaws.com/abc/key?x-id=GetObject", ""}, + {"a.b.c", "key", "https://s3.dualstack.mock-region.amazonaws.com/a.b.c/key?x-id=GetObject", ""}, + {"a$b$c", "key", "https://s3.dualstack.mock-region.amazonaws.com/a%24b%24c/key?x-id=GetObject", ""}, + }, + }, + "AccelerateWithDualStack": { + useAccelerate: true, + useDualstack: true, + tests: []s3BucketTest{ + {"abc", "key", "https://abc.s3-accelerate.dualstack.amazonaws.com/key?x-id=GetObject", ""}, + {"a.b.c", "key", "https://s3.mock-region.dualstack.amazonaws.com/a.b.c/key?x-id=GetObject", "cannot be used with"}, + {"a$b$c", "key", "https://s3.mock-region.dualstack.amazonaws.com/a%24b%24c/key?x-id=GetObject", "cannot be used with"}, + }, + }, + }, + + "immutable endpoint": { + "PathStyleBucket": { + usePathStyle: true, + customEndpoint: &aws.Endpoint{ + URL: "https://example.region.amazonaws.com", + HostnameImmutable: true, + }, + tests: []s3BucketTest{ + {"abc", "key", "https://example.region.amazonaws.com/abc/key?x-id=GetObject", ""}, + {"a$b$c", "key", "https://example.region.amazonaws.com/a%24b%24c/key?x-id=GetObject", ""}, + {"a.b.c", "key", "https://example.region.amazonaws.com/a.b.c/key?x-id=GetObject", ""}, + {"a..bc", "key", "https://example.region.amazonaws.com/a..bc/key?x-id=GetObject", ""}, + {"abc", "k:e,y", "https://example.region.amazonaws.com/abc/k%3Ae%2Cy?x-id=GetObject", ""}, + }, + }, + "VirtualHostStyleBucket": { + customEndpoint: &aws.Endpoint{ + URL: "https://example.region.amazonaws.com", + HostnameImmutable: true, + }, + tests: []s3BucketTest{ + {"abc", "key", "https://example.region.amazonaws.com/abc/key?x-id=GetObject", ""}, + {"a$b$c", "key", "https://example.region.amazonaws.com/a%24b%24c/key?x-id=GetObject", ""}, + {"a.b.c", "key", "https://example.region.amazonaws.com/a.b.c/key?x-id=GetObject", ""}, + {"a..bc", "key", "https://example.region.amazonaws.com/a..bc/key?x-id=GetObject", ""}, + {"abc", "k:e,y", "https://example.region.amazonaws.com/abc/k%3Ae%2Cy?x-id=GetObject", ""}, + }, + }, + "Accelerate": { + useAccelerate: true, + customEndpoint: &aws.Endpoint{ + URL: "https://example.region.amazonaws.com", + HostnameImmutable: true, + }, + tests: []s3BucketTest{ + {"abc", "key", "https://example.region.amazonaws.com/abc/key?x-id=GetObject", ""}, + {"a$b$c", "key", "https://example.region.amazonaws.com/a%24b%24c/key?x-id=GetObject", ""}, + {"a.b.c", "key", "https://example.region.amazonaws.com/a.b.c/key?x-id=GetObject", ""}, + {"a..bc", "key", "https://example.region.amazonaws.com/a..bc/key?x-id=GetObject", ""}, + }, + }, + "AccelerateNoSSLTests": { + useAccelerate: true, + disableHTTPS: true, + customEndpoint: &aws.Endpoint{ + URL: "https://example.region.amazonaws.com", + HostnameImmutable: true, + }, + tests: []s3BucketTest{ + {"abc", "key", "https://example.region.amazonaws.com/abc/key?x-id=GetObject", ""}, + {"a.b.c", "key", "https://example.region.amazonaws.com/a.b.c/key?x-id=GetObject", ""}, + {"a$b$c", "key", "https://example.region.amazonaws.com/a%24b%24c/key?x-id=GetObject", ""}, + }, + }, + }, + } + + for suitName, cs := range cases { + t.Run(suitName, func(t *testing.T) { + for unitName, c := range cs { + t.Run(unitName, func(t *testing.T) { + options := s3.Options{ + Credentials: unit.StubCredentialsProvider{}, + Retryer: aws.NopRetryer{}, + Region: "mock-region", + + HTTPClient: smithyhttp.NopClient{}, + + EndpointOptions: endpoints.Options{ + DisableHTTPS: c.disableHTTPS, + }, + + UsePathStyle: c.usePathStyle, + UseAccelerate: c.useAccelerate, + UseDualstack: c.useDualstack, + } + + if c.customEndpoint != nil { + options.EndpointResolver = s3.EndpointResolverFunc( + func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return *c.customEndpoint, nil + }) + } + + svc := s3.New(options) + for i, test := range c.tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + fm := requestRetrieverMiddleware{} + _, err := svc.GetObject(context.Background(), + &s3.GetObjectInput{Bucket: &test.bucket, Key: &test.key}, + func(options *s3.Options) { + options.APIOptions = append(options.APIOptions, + func(stack *middleware.Stack) error { + stack.Serialize.Insert(&fm, + "OperationSerializer", middleware.After) + return nil + }) + }, + ) + + if test.err != "" { + if err == nil { + t.Fatalf("test %d: expected error, got none", i) + } + if a, e := err.Error(), test.err; !strings.Contains(a, e) { + t.Fatalf("expect error code to contain %q, got %q", e, a) + } + return + } + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + + req := fm.request.Build(context.Background()) + if e, a := test.url, req.URL.String(); e != a { + t.Fatalf("expect url %s, got %s", e, a) + } + }) + } + }) + } + }) + } +} + +// test case struct used to test endpoint customizations +type testCaseForEndpointCustomization struct { + options s3.Options + bucket string + operation func(ctx context.Context, svc *s3.Client, fm *requestRetriever) (interface{}, error) + expectedErr string + expectedReqURL string + expectedSigningName string + expectedSigningRegion string + expectedHeader map[string]string +} + +func TestEndpointWithARN(t *testing.T) { + // test cases + cases := map[string]testCaseForEndpointCustomization{ + "Object Lambda with no UseARNRegion flag set": { + bucket: "arn:aws:s3-object-lambda:us-west-2:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "us-west-2", + }, + expectedReqURL: "https://myap-123456789012.s3-object-lambda.us-west-2.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3-object-lambda", + expectedSigningRegion: "us-west-2", + }, + "Object Lambda with UseARNRegion flag set": { + bucket: "arn:aws:s3-object-lambda:us-east-1:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedReqURL: "https://myap-123456789012.s3-object-lambda.us-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3-object-lambda", + expectedSigningRegion: "us-east-1", + }, + "Object Lambda with Cross-Region error": { + bucket: "arn:aws:s3-object-lambda:us-east-1:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "us-west-2", + }, + expectedErr: "region from ARN `us-east-1` does not match client region `us-west-2` and UseArnRegion is `false`", + }, + "Object Lambda Pseudo-Region with UseARNRegion flag set": { + bucket: "arn:aws:s3-object-lambda:us-east-1:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "aws-global", + UseARNRegion: true, + }, + expectedReqURL: "https://myap-123456789012.s3-object-lambda.us-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningRegion: "us-east-1", + expectedSigningName: "s3-object-lambda", + }, + "Object Lambda Cross-Region DualStack error": { + bucket: "arn:aws:s3-object-lambda:us-east-1:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "us-west-2", + UseDualstack: true, + UseARNRegion: true, + }, + expectedErr: "S3 Object Lambda does not support Dual-stack", + }, + "Object Lambda Cross-Partition error": { + bucket: "arn:aws-cn:s3-object-lambda:cn-north-1:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedErr: "Client was configured for partition `aws` but ARN (`arn:aws-cn:s3-object-lambda:cn-north-1:123456789012:accesspoint/myap`) has `aws-cn`", + }, + "Object Lambda FIPS": { + bucket: "arn:aws-us-gov:s3-object-lambda:us-gov-west-1:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "us-gov-west-1", + EndpointOptions: endpoints.Options{ + UseFIPSEndpoint: aws.FIPSEndpointStateEnabled, + }, + }, + expectedReqURL: "https://myap-123456789012.s3-object-lambda-fips.us-gov-west-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningRegion: "us-gov-west-1", + expectedSigningName: "s3-object-lambda", + }, + "Object Lambda FIPS (ResolvedRegion)": { + bucket: "arn:aws-us-gov:s3-object-lambda:us-gov-west-1:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "fips-us-gov-west-1", + }, + expectedReqURL: "https://myap-123456789012.s3-object-lambda-fips.us-gov-west-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningRegion: "us-gov-west-1", + expectedSigningName: "s3-object-lambda", + }, + "Object Lambda FIPS with UseARNRegion flag set": { + bucket: "arn:aws-us-gov:s3-object-lambda:us-gov-west-1:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "us-gov-west-1", + UseARNRegion: true, + EndpointOptions: s3.EndpointResolverOptions{ + UseFIPSEndpoint: aws.FIPSEndpointStateEnabled, + }, + }, + expectedReqURL: "https://myap-123456789012.s3-object-lambda-fips.us-gov-west-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningRegion: "us-gov-west-1", + expectedSigningName: "s3-object-lambda", + }, + "Object Lambda FIPS (ResolvedRegion) with UseARNRegion flag set": { + bucket: "arn:aws-us-gov:s3-object-lambda:us-gov-west-1:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "fips-us-gov-west-1", + UseARNRegion: true, + }, + expectedReqURL: "https://myap-123456789012.s3-object-lambda-fips.us-gov-west-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningRegion: "us-gov-west-1", + expectedSigningName: "s3-object-lambda", + }, + "Object Lambda with Accelerate": { + bucket: "arn:aws:s3-object-lambda:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + UseAccelerate: true, + }, + expectedErr: "S3 Object Lambda does not support S3 Accelerate", + }, + "Object Lambda with Custom Endpoint Source": { + bucket: "arn:aws:s3-object-lambda:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://my-domain.com", + Source: aws.EndpointSourceCustom, + SigningName: "custom-sign-name", + SigningRegion: region, + }, nil + }), + }, + expectedReqURL: "https://myendpoint-123456789012.my-domain.com/testkey?x-id=GetObject", + expectedSigningName: "custom-sign-name", + expectedSigningRegion: "us-west-2", + }, + "Object Lambda with Custom Endpoint Source Immutable": { + bucket: "arn:aws:s3-object-lambda:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://myendpoint-123456789012.my-domain.com", + Source: aws.EndpointSourceCustom, + SigningName: "custom-sign-name", + SigningRegion: region, + HostnameImmutable: true, + }, nil + }), + }, + expectedReqURL: "https://myendpoint-123456789012.my-domain.com/testkey?x-id=GetObject", + expectedSigningName: "custom-sign-name", + expectedSigningRegion: "us-west-2", + }, + "Outpost AccessPoint with no S3UseARNRegion flag set": { + bucket: "arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-west-2", + }, + expectedReqURL: "https://myaccesspoint-123456789012.op-01234567890123456.s3-outposts.us-west-2.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "us-west-2", + }, + "Outpost AccessPoint Cross-Region Enabled": { + bucket: "arn:aws:s3-outposts:us-east-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedReqURL: "https://myaccesspoint-123456789012.op-01234567890123456.s3-outposts.us-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "us-east-1", + }, + "Outpost AccessPoint Cross-Region Disabled": { + bucket: "arn:aws:s3-outposts:us-east-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-west-2", + }, + expectedErr: "region from ARN `us-east-1` does not match client region `us-west-2` and UseArnRegion is `false`", + }, + "Outpost AccessPoint other partition": { + bucket: "arn:aws-cn:s3-outposts:cn-north-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedErr: "Client was configured for partition `aws` but ARN (`arn:aws-cn:s3-outposts:cn-north-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint`) has `aws-cn`", + }, + "Outpost AccessPoint cn partition": { + bucket: "arn:aws-cn:s3-outposts:cn-north-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "cn-north-1", + }, + expectedReqURL: "https://myaccesspoint-123456789012.op-01234567890123456.s3-outposts.cn-north-1.amazonaws.com.cn/testkey?x-id=GetObject", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "cn-north-1", + }, + "Outpost AccessPoint Custom Endpoint Source": { + bucket: "arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://my-domain.com", + Source: aws.EndpointSourceCustom, + SigningName: "custom-sign-name", + SigningRegion: region, + }, nil + }), + }, + expectedReqURL: "https://myaccesspoint-123456789012.op-01234567890123456.my-domain.com/testkey?x-id=GetObject", + expectedSigningName: "custom-sign-name", + expectedSigningRegion: "us-west-2", + }, + "Outpost AccessPoint Custom Endpoint Source Immutable": { + bucket: "arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://myaccesspoint-123456789012.op-01234567890123456.my-domain.com", + Source: aws.EndpointSourceCustom, + SigningName: "custom-sign-name", + SigningRegion: region, + HostnameImmutable: true, + }, nil + }), + }, + expectedReqURL: "https://myaccesspoint-123456789012.op-01234567890123456.my-domain.com/testkey?x-id=GetObject", + expectedSigningName: "custom-sign-name", + expectedSigningRegion: "us-west-2", + }, + "Outpost AccessPoint us-gov region": { + bucket: "arn:aws-us-gov:s3-outposts:us-gov-east-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-gov-east-1", + UseARNRegion: true, + }, + expectedReqURL: "https://myaccesspoint-123456789012.op-01234567890123456.s3-outposts.us-gov-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "us-gov-east-1", + }, + "Outpost AccessPoint FIPS cross-region": { + bucket: "arn:aws-us-gov:s3-outposts:us-gov-east-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "fips-us-gov-west-1", + }, + expectedErr: "S3 Outposts does not support FIPS", + }, + "Outpost AccessPoint with FIPS cross-region": { + bucket: "arn:aws-us-gov:s3-outposts:us-gov-east-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-gov-west-1", + UseARNRegion: true, + EndpointOptions: s3.EndpointResolverOptions{ + UseFIPSEndpoint: aws.FIPSEndpointStateEnabled, + }, + }, + expectedErr: "S3 Outposts does not support FIPS", + }, + "Outpost AccessPoint with FIPS (ResolvedRegion) cross-region": { + bucket: "arn:aws-us-gov:s3-outposts:us-gov-east-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "fips-us-gov-west-1", + UseARNRegion: true, + }, + expectedErr: "S3 Outposts does not support FIPS", + }, + "Outpost AccessPoint with FIPS matching region": { + bucket: "arn:aws-us-gov:s3-outposts:us-gov-west-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-gov-west-1", + EndpointOptions: s3.EndpointResolverOptions{ + UseFIPSEndpoint: aws.FIPSEndpointStateEnabled, + }, + UseARNRegion: true, + }, + expectedErr: "S3 Outposts does not support FIPS", + }, + "Outpost AccessPoint with FIPS (ResolvedRegion) matching region": { + bucket: "arn:aws-us-gov:s3-outposts:us-gov-west-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "fips-us-gov-west-1", + UseARNRegion: true, + }, + expectedErr: "S3 Outposts does not support FIPS", + }, + "Outpost AccessPoint with Immutable Endpoint": { + bucket: "arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://myaccesspoint-123456789012.op-01234567890123456.my-domain.com", + SigningRegion: region, + HostnameImmutable: true, + }, nil + }), + }, + expectedReqURL: "https://myaccesspoint-123456789012.op-01234567890123456.my-domain.com/testkey?x-id=GetObject", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "us-west-2", + }, + "Outpost AccessPoint with DualStack": { + bucket: "arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-west-2", + UseDualstack: true, + }, + expectedErr: "S3 Outposts does not support Dual-stack", + }, + "Outpost AccessPoint with Accelerate": { + bucket: "arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-west-2", + UseAccelerate: true, + }, + expectedErr: "S3 Outposts does not support S3 Accelerate", + }, + "AccessPoint": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.us-west-2.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "AccessPoint slash delimiter": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint/myendpoint", + options: s3.Options{ + Region: "us-west-2", + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.us-west-2.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "AccessPoint other partition": { + bucket: "arn:aws-cn:s3:cn-north-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "cn-north-1", + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.cn-north-1.amazonaws.com.cn/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "cn-north-1", + }, + "AccessPoint Cross-Region Disabled": { + bucket: "arn:aws:s3:ap-south-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + }, + expectedErr: "region from ARN `ap-south-1` does not match client region `us-west-2` and UseArnRegion is `false`", + }, + "AccessPoint Cross-Region Enabled": { + bucket: "arn:aws:s3:ap-south-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.ap-south-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "ap-south-1", + }, + "AccessPoint us-east-1": { + bucket: "arn:aws:s3:us-east-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-east-1", + UseARNRegion: true, + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.us-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-east-1", + }, + "AccessPoint us-east-1 cross region": { + bucket: "arn:aws:s3:us-east-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.us-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-east-1", + }, + "AccessPoint Cross-Partition not supported": { + bucket: "arn:aws-cn:s3:cn-north-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + UseDualstack: true, + UseARNRegion: true, + }, + expectedErr: "Client was configured for partition `aws` but ARN (`arn:aws-cn:s3:cn-north-1:123456789012:accesspoint:myendpoint`) has `aws-cn`", + }, + "AccessPoint DualStack": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + UseDualstack: true, + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.dualstack.us-west-2.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "AccessPoint FIPS same region with cross region disabled": { + bucket: "arn:aws-us-gov:s3:us-gov-west-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-gov-west-1", + EndpointOptions: s3.EndpointResolverOptions{ + UseFIPSEndpoint: aws.FIPSEndpointStateEnabled, + }, + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint-fips.us-gov-west-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-gov-west-1", + }, + "AccessPoint FIPS (ResolvedRegion) same region with cross region disabled": { + bucket: "arn:aws-us-gov:s3:us-gov-west-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "fips-us-gov-west-1", + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint-fips.us-gov-west-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-gov-west-1", + }, + "AccessPoint FIPS same region with cross region enabled": { + bucket: "arn:aws-us-gov:s3:us-gov-west-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-gov-west-1", + EndpointOptions: s3.EndpointResolverOptions{ + UseFIPSEndpoint: aws.FIPSEndpointStateEnabled, + }, + UseARNRegion: true, + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint-fips.us-gov-west-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-gov-west-1", + }, + "AccessPoint FIPS (ResolvedRegion) same region with cross region enabled": { + bucket: "arn:aws-us-gov:s3:us-gov-west-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "fips-us-gov-west-1", + UseARNRegion: true, + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint-fips.us-gov-west-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-gov-west-1", + }, + "AccessPoint Immutable Endpoint": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://myendpoint-123456789012.s3-accesspoint.us-east-1.amazonaws.com", + SigningRegion: region, + HostnameImmutable: true, + }, nil + }), + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.us-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "AccessPoint Custom Endpoint Source": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://myendpoint-123456789012.my-domain.com", + Source: aws.EndpointSourceCustom, + SigningName: "custom-sign-name", + SigningRegion: region, + HostnameImmutable: true, + }, nil + }), + }, + expectedReqURL: "https://myendpoint-123456789012.my-domain.com/testkey?x-id=GetObject", + expectedSigningName: "custom-sign-name", + expectedSigningRegion: "us-west-2", + }, + "AccessPoint FIPS cross region": { + bucket: "arn:aws-us-gov:s3:us-gov-east-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-gov-west-1", + UseARNRegion: true, + EndpointOptions: s3.EndpointResolverOptions{ + UseFIPSEndpoint: aws.FIPSEndpointStateEnabled, + }, + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint-fips.us-gov-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-gov-east-1", + }, + "AccessPoint FIPS (ResolvedRegion) cross region": { + bucket: "arn:aws-us-gov:s3:us-gov-east-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "fips-us-gov-west-1", + UseARNRegion: true, + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint-fips.us-gov-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-gov-east-1", + }, + "AccessPoint Accelerate not supported": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + UseAccelerate: true, + }, + expectedErr: "Access Points do not support S3 Accelerate", + }, + "Custom Resolver Without PartitionID in ClientInfo": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + switch region { + case "us-west-2": + return aws.Endpoint{ + URL: "https://s3.us-west-2.amazonaws.com", + SigningRegion: "us-west-2", + SigningName: "s3", + SigningMethod: "s3v4", + }, nil + } + return aws.Endpoint{}, nil + }), + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.us-west-2.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Custom Resolver Without PartitionID in Cross-Region Target": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-east-1", + UseARNRegion: true, + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + switch region { + case "us-west-2": + return aws.Endpoint{ + URL: "https://s3.us-west-2.amazonaws.com", + PartitionID: "aws", + SigningRegion: "us-west-2", + SigningName: "s3", + SigningMethod: "s3v4", + }, nil + case "us-east-1": + return aws.Endpoint{ + URL: "https://s3.us-east-1.amazonaws.com", + SigningRegion: "us-east-1", + SigningName: "s3", + SigningMethod: "s3v4", + }, nil + } + return aws.Endpoint{}, nil + }), + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.us-west-2.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "bucket host-style": { + bucket: "mock-bucket", + options: s3.Options{ + Region: "us-west-2", + }, + expectedReqURL: "https://mock-bucket.s3.us-west-2.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "bucket path-style": { + bucket: "mock-bucket", + options: s3.Options{ + Region: "us-west-2", + UsePathStyle: true, + }, + expectedReqURL: "https://s3.us-west-2.amazonaws.com/mock-bucket/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "bucket host-style endpoint with default port": { + bucket: "mock-bucket", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://s3.us-west-2.amazonaws.com:443", + SigningRegion: "us-west-2", + }, nil + }), + }, + expectedReqURL: "https://mock-bucket.s3.us-west-2.amazonaws.com:443/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "bucket host-style endpoint with non-default port": { + bucket: "mock-bucket", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://s3.us-west-2.amazonaws.com:8443", + SigningRegion: "us-west-2", + }, nil + }), + }, + expectedReqURL: "https://mock-bucket.s3.us-west-2.amazonaws.com:8443/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "bucket path-style endpoint with default port": { + bucket: "mock-bucket", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://s3.us-west-2.amazonaws.com:443", + SigningRegion: "us-west-2", + }, nil + }), + UsePathStyle: true, + }, + expectedReqURL: "https://s3.us-west-2.amazonaws.com:443/mock-bucket/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "bucket path-style endpoint with non-default port": { + bucket: "mock-bucket", + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://s3.us-west-2.amazonaws.com:8443", + SigningRegion: "us-west-2", + }, nil + }), + UsePathStyle: true, + }, + expectedReqURL: "https://s3.us-west-2.amazonaws.com:8443/mock-bucket/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Invalid AccessPoint ARN with FIPS pseudo-region (prefix)": { + bucket: "arn:aws:s3:fips-us-east-1:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.fips-us-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "fips-us-east-1", + }, + "Invalid AccessPoint ARN with FIPS pseudo-region (suffix)": { + bucket: "arn:aws:s3:us-east-1-fips:123456789012:accesspoint:myendpoint", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedReqURL: "https://myendpoint-123456789012.s3-accesspoint.us-east-1-fips.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3", + expectedSigningRegion: "us-east-1-fips", + }, + "Invalid Outpost AccessPoint ARN with FIPS pseudo-region (prefix)": { + bucket: "arn:aws:s3-outposts:fips-us-east-1:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedReqURL: "https://myaccesspoint-123456789012.op-01234567890123456.s3-outposts.fips-us-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "fips-us-east-1", + }, + "Invalid Outpost AccessPoint ARN with FIPS pseudo-region (suffix)": { + bucket: "arn:aws:s3-outposts:us-east-1-fips:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedReqURL: "https://myaccesspoint-123456789012.op-01234567890123456.s3-outposts.us-east-1-fips.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "us-east-1-fips", + }, + "Invalid Object Lambda ARN with FIPS pseudo-region (prefix)": { + bucket: "arn:aws:s3-object-lambda:fips-us-east-1:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedReqURL: "https://myap-123456789012.s3-object-lambda.fips-us-east-1.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3-object-lambda", + expectedSigningRegion: "fips-us-east-1", + }, + "Invalid Object Lambda ARN with FIPS pseudo-region (suffix)": { + bucket: "arn:aws:s3-object-lambda:us-east-1-fips:123456789012:accesspoint/myap", + options: s3.Options{ + Region: "us-west-2", + UseARNRegion: true, + }, + expectedReqURL: "https://myap-123456789012.s3-object-lambda.us-east-1-fips.amazonaws.com/testkey?x-id=GetObject", + expectedSigningName: "s3-object-lambda", + expectedSigningRegion: "us-east-1-fips"}, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + runValidations(t, c, func(ctx context.Context, svc *s3.Client, fetcher *requestRetriever) (interface{}, error) { + if c.operation != nil { + return c.operation(ctx, svc, fetcher) + } + return svc.GetObject(ctx, &s3.GetObjectInput{ + Bucket: ptr.String(c.bucket), + Key: ptr.String("testkey"), + }, addRequestRetriever(fetcher)) + }) + }) + } +} + +func TestVPC_CustomEndpoint(t *testing.T) { + cases := map[string]testCaseForEndpointCustomization{ + "standard custom endpoint url": { + bucket: "bucketname", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + }, + expectedReqURL: "https://bucketname.beta.example.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "custom resolver to v2 fallback": { + bucket: "bucketname", + options: s3.Options{ + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{}, &aws.EndpointNotFoundError{} + }), + Region: "us-west-2", + }, + expectedReqURL: "https://bucketname.s3.us-west-2.amazonaws.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "AccessPoint with custom endpoint url": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + }, + expectedReqURL: "https://myendpoint-123456789012.beta.example.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Outpost AccessPoint with custom endpoint url": { + bucket: "arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + }, + expectedReqURL: "https://myaccesspoint-123456789012.op-01234567890123456.beta.example.com/", + expectedSigningName: "s3-outposts", + expectedSigningRegion: "us-west-2", + }, + "ListBucket with custom endpoint url": { + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + }, + operation: func(ctx context.Context, svc *s3.Client, fm *requestRetriever) (interface{}, error) { + return svc.ListBuckets(ctx, &s3.ListBucketsInput{}, addRequestRetriever(fm)) + }, + expectedReqURL: "https://bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Path-style addressing with custom endpoint url": { + bucket: "bucketname", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + UsePathStyle: true, + }, + expectedReqURL: "https://bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com/bucketname", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Virtual host addressing with custom endpoint url": { + bucket: "bucketname", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + }, + expectedReqURL: "https://bucketname.bucket.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Access-point with custom endpoint url and use_arn_region set": { + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:myendpoint", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://accesspoint.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "eu-west-1", + UseARNRegion: true, + }, + expectedReqURL: "https://myendpoint-123456789012.accesspoint.vpce-123-abc.s3.us-west-2.vpce.amazonaws.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Custom endpoint url with Dualstack": { + bucket: "bucketname", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + UseDualstack: true, + }, + expectedReqURL: "https://bucketname.beta.example.com/", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + "Outpost with custom endpoint url and Dualstack": { + bucket: "arn:aws:s3-outposts:us-west-2:123456789012:outpost:op-01234567890123456:accesspoint:myaccesspoint", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + Region: "us-west-2", + UseDualstack: true, + }, + expectedErr: "client configured for S3 Dual-stack but is not supported with resource", + }, + "Standard custom endpoint url with Immutable Host": { + bucket: "bucketname", + options: s3.Options{ + EndpointResolver: s3.EndpointResolverFromURL("https://beta.example.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + endpoint.HostnameImmutable = true + }), + Region: "us-west-2", + }, + expectedReqURL: "https://beta.example.com/bucketname", + expectedSigningName: "s3", + expectedSigningRegion: "us-west-2", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + runValidations(t, c, func(ctx context.Context, svc *s3.Client, fm *requestRetriever) (interface{}, error) { + if c.operation != nil { + return c.operation(ctx, svc, fm) + } + + return svc.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: ptr.String(c.bucket), + }, addRequestRetriever(fm)) + }) + }) + } +} + +func TestWriteGetObjectResponse_UpdateEndpoint(t *testing.T) { + cases := map[string]testCaseForEndpointCustomization{ + "standard endpoint": { + options: s3.Options{ + Region: "us-west-2", + }, + expectedReqURL: "https://test-route.s3-object-lambda.us-west-2.amazonaws.com/WriteGetObjectResponse?x-id=WriteGetObjectResponse", + expectedSigningRegion: "us-west-2", + expectedSigningName: "s3-object-lambda", + }, + "fips endpoint": { + options: s3.Options{ + Region: "us-gov-west-1", + EndpointOptions: s3.EndpointResolverOptions{ + UseFIPSEndpoint: aws.FIPSEndpointStateEnabled, + }, + }, + expectedReqURL: "https://test-route.s3-object-lambda-fips.us-gov-west-1.amazonaws.com/WriteGetObjectResponse?x-id=WriteGetObjectResponse", + expectedSigningRegion: "us-gov-west-1", + expectedSigningName: "s3-object-lambda", + }, + "fips endpoint (ResolvedRegion)": { + options: s3.Options{ + Region: "fips-us-gov-west-1", + }, + expectedReqURL: "https://test-route.s3-object-lambda-fips.us-gov-west-1.amazonaws.com/WriteGetObjectResponse?x-id=WriteGetObjectResponse", + expectedSigningRegion: "us-gov-west-1", + expectedSigningName: "s3-object-lambda", + }, + "dualstack endpoint": { + options: s3.Options{ + Region: "us-west-2", + UseDualstack: true, + }, + expectedErr: "S3 Object Lambda does not support Dual-stack", + }, + "accelerate endpoint": { + options: s3.Options{ + Region: "us-west-2", + UseAccelerate: true, + }, + expectedErr: "S3 Object Lambda does not support S3 Accelerate", + }, + "custom endpoint": { + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://my-domain.com", + SigningRegion: region, + SigningName: "s3", // incorrect signing name gets overwritten + }, nil + }), + }, + expectedReqURL: "https://test-route.my-domain.com/WriteGetObjectResponse?x-id=WriteGetObjectResponse", + expectedSigningRegion: "us-west-2", + expectedSigningName: "s3-object-lambda", + }, + "custom endpoint immutable": { + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: "https://test-route.my-domain.com", + SigningRegion: region, + SigningName: "s3", // incorrect signing name gets overwritten + HostnameImmutable: true, + }, nil + }), + }, + expectedReqURL: "https://test-route.my-domain.com/WriteGetObjectResponse?x-id=WriteGetObjectResponse", + expectedSigningRegion: "us-west-2", + expectedSigningName: "s3-object-lambda", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + runValidations(t, c, func(ctx context.Context, client *s3.Client, retrieverMiddleware *requestRetriever) (interface{}, error) { + return client.WriteGetObjectResponse(context.Background(), + &s3.WriteGetObjectResponseInput{ + RequestRoute: aws.String("test-route"), + RequestToken: aws.String("test-token"), + }, addRequestRetriever(retrieverMiddleware)) + }) + }) + } +} + +func TestUseDualStackClientBehavior(t *testing.T) { + cases := map[string]testCaseForEndpointCustomization{ + "client options dual-stack false, endpoint resolver dual-stack unset": { + options: s3.Options{ + Region: "us-west-2", + UseDualstack: false, + }, + expectedReqURL: "https://test-bucket.s3.us-west-2.amazonaws.com/test-key?x-id=GetObject", + expectedSigningRegion: "us-west-2", + expectedSigningName: "s3", + }, + "client options dual-stack true, endpoint resolver dual-stack unset": { + options: s3.Options{ + Region: "us-west-2", + UseDualstack: true, + }, + expectedReqURL: "https://test-bucket.s3.dualstack.us-west-2.amazonaws.com/test-key?x-id=GetObject", + expectedSigningRegion: "us-west-2", + expectedSigningName: "s3", + }, + "client options dual-stack off, endpoint resolver dual-stack disabled": { + options: s3.Options{ + Region: "us-west-2", + EndpointOptions: s3.EndpointResolverOptions{ + UseDualStackEndpoint: aws.DualStackEndpointStateDisabled, + }, + }, + expectedReqURL: "https://test-bucket.s3.us-west-2.amazonaws.com/test-key?x-id=GetObject", + expectedSigningRegion: "us-west-2", + expectedSigningName: "s3", + }, + "client options dual-stack off, endpoint resolver dual-stack enabled": { + options: s3.Options{ + Region: "us-west-2", + EndpointOptions: s3.EndpointResolverOptions{ + UseDualStackEndpoint: aws.DualStackEndpointStateEnabled, + }, + }, + expectedReqURL: "https://test-bucket.s3.dualstack.us-west-2.amazonaws.com/test-key?x-id=GetObject", + expectedSigningRegion: "us-west-2", + expectedSigningName: "s3", + }, + "client options dual-stack on, endpoint resolver dual-stack disabled": { + options: s3.Options{ + Region: "us-west-2", + UseDualstack: true, + EndpointOptions: s3.EndpointResolverOptions{ + UseDualStackEndpoint: aws.DualStackEndpointStateDisabled, + }, + }, + expectedReqURL: "https://test-bucket.s3.us-west-2.amazonaws.com/test-key?x-id=GetObject", + expectedSigningRegion: "us-west-2", + expectedSigningName: "s3", + }, + "client options dual-stack off, endpoint resolver dual-stack on": { + options: s3.Options{ + Region: "us-west-2", + UseDualstack: false, + EndpointOptions: s3.EndpointResolverOptions{ + UseDualStackEndpoint: aws.DualStackEndpointStateEnabled, + }, + }, + expectedReqURL: "https://test-bucket.s3.dualstack.us-west-2.amazonaws.com/test-key?x-id=GetObject", + expectedSigningRegion: "us-west-2", + expectedSigningName: "s3", + }, + } + for name, tt := range cases { + t.Run(name, func(t *testing.T) { + runValidations(t, tt, func(ctx context.Context, client *s3.Client, retrieverMiddleware *requestRetriever) (interface{}, error) { + return client.GetObject(context.Background(), + &s3.GetObjectInput{ + Bucket: aws.String("test-bucket"), + Key: aws.String("test-key"), + }, addRequestRetriever(retrieverMiddleware)) + }) + }) + } +} + +func TestMultiRegionAccessPoints_UpdateEndpoint(t *testing.T) { + cases := map[string]testCaseForEndpointCustomization{ + "region as us-east-1": { + options: s3.Options{ + Region: "us-east-1", + }, + bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedReqURL: "https://mfzwi23gnjvgw.mrap.accesspoint.s3-global.amazonaws.com/", + expectedHeader: map[string]string{ + v4a.AmzRegionSetKey: "*", + }, + expectedSigningName: "s3", + expectedSigningRegion: "*", + }, + "region as us-west-2": { + options: s3.Options{ + Region: "us-west-2", + }, + bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedReqURL: "https://mfzwi23gnjvgw.mrap.accesspoint.s3-global.amazonaws.com/", + expectedHeader: map[string]string{ + v4a.AmzRegionSetKey: "*", + }, + expectedSigningName: "s3", + expectedSigningRegion: "*", + }, + "region as aws-global": { + options: s3.Options{ + Region: "aws-global", + }, + bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedReqURL: "https://mfzwi23gnjvgw.mrap.accesspoint.s3-global.amazonaws.com/", + expectedHeader: map[string]string{ + v4a.AmzRegionSetKey: "*", + }, + expectedSigningName: "s3", + expectedSigningRegion: "*", + }, + "cn partition": { + options: s3.Options{ + Region: "cn-north-1", + }, + bucket: "arn:aws-cn:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedReqURL: "https://mfzwi23gnjvgw.mrap.accesspoint.s3-global.amazonaws.com.cn/", + expectedHeader: map[string]string{ + v4a.AmzRegionSetKey: "*", + }, + expectedSigningName: "s3", + expectedSigningRegion: "*", + }, + "cn partition arn with cross partition client region": { + options: s3.Options{ + Region: "ap-north-1", + }, + bucket: "arn:aws-cn:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedErr: "Client was configured for partition `aws` but bucket referred to partition `aws-cn`", + }, + "region as us-west-2 with mrap disabled": { + options: s3.Options{ + Region: "us-west-2", + DisableMultiRegionAccessPoints: true, + }, + bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedErr: "Multi-Region Access Point ARNs are disabled", + }, + "region as aws-global with mrap disabled": { + options: s3.Options{ + Region: "aws-global", + DisableMultiRegionAccessPoints: true, + }, + bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedErr: "Multi-Region Access Point ARNs are disabled", + }, + "with dualstack": { + options: s3.Options{ + Region: "us-west-2", + UseDualstack: true, + }, + bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedErr: "S3 MRAP does not support dual-stack", + }, + "with accelerate": { + options: s3.Options{ + Region: "us-west-2", + UseAccelerate: true, + }, + bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedErr: "S3 MRAP does not support S3 Accelerate", + }, + "access point with no region and mrap disabled": { + options: s3.Options{ + Region: "us-west-2", + DisableMultiRegionAccessPoints: true, + }, + bucket: "arn:aws:s3::123456789012:accesspoint:myendpoint", + expectedErr: "Multi-Region Access Point ARNs are disabled", + }, + "endpoint with no region and disabled mrap": { + options: s3.Options{ + Region: "us-west-2", + DisableMultiRegionAccessPoints: true, + }, + bucket: "arn:aws:s3::123456789012:accesspoint:myendpoint", + expectedErr: "Multi-Region Access Point ARNs are disabled", + }, + "endpoint with no region": { + options: s3.Options{ + Region: "us-west-2", + }, + bucket: "arn:aws:s3::123456789012:accesspoint:myendpoint", + expectedReqURL: "https://myendpoint.accesspoint.s3-global.amazonaws.com/", + expectedHeader: map[string]string{ + v4a.AmzRegionSetKey: "*", + }, + expectedSigningName: "s3", + expectedSigningRegion: "*", + }, + "endpoint containing dot with no region": { + options: s3.Options{ + Region: "us-west-2", + }, + bucket: "arn:aws:s3::123456789012:accesspoint:my.bucket", + expectedReqURL: "https://my.bucket.accesspoint.s3-global.amazonaws.com/", + expectedHeader: map[string]string{ + v4a.AmzRegionSetKey: "*", + }, + expectedSigningName: "s3", + expectedSigningRegion: "*", + }, + "custom endpoint": { + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: s3.EndpointResolverFromURL("https://mockendpoint.amazonaws.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + }), + }, + bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedReqURL: "https://mfzwi23gnjvgw.mrap.accesspoint.s3-global.amazonaws.com/", + expectedHeader: map[string]string{ + v4a.AmzRegionSetKey: "*", + }, + expectedSigningName: "s3", + expectedSigningRegion: "*", + }, + "custom endpoint with hostname immutable": { + options: s3.Options{ + Region: "us-west-2", + EndpointResolver: s3.EndpointResolverFromURL("https://mockendpoint.amazonaws.com", func(endpoint *aws.Endpoint) { + endpoint.SigningRegion = "us-west-2" + endpoint.HostnameImmutable = true + }), + }, + bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedReqURL: "https://mockendpoint.amazonaws.com/arn%3Aaws%3As3%3A%3A123456789012%3Aaccesspoint%3Amfzwi23gnjvgw.mrap", + expectedHeader: map[string]string{ + v4a.AmzRegionSetKey: "*", + }, + expectedSigningName: "s3", + expectedSigningRegion: "*", + }, + "with fips client": { + options: s3.Options{ + Region: "us-west-2", + EndpointOptions: s3.EndpointResolverOptions{ + UseFIPSEndpoint: aws.FIPSEndpointStateEnabled, + }, + }, + bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedErr: "S3 MRAP does not support FIPS", + }, + "with fips (ResolvedRegion) client": { + options: s3.Options{ + Region: "fips-us-west-2", + }, + bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedErr: "S3 MRAP does not support FIPS", + }, + "Accesspoint ARN with region and MRAP disabled": { + options: s3.Options{ + Region: "us-west-2", + DisableMultiRegionAccessPoints: false, + }, + bucket: "arn:aws:s3:us-west-2:123456789012:accesspoint:mfzwi23gnjvgw.mrap", + expectedErr: "Invalid ARN: The access point name may only contain a-z, A-Z, 0-9 and `-`", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + runValidations(t, c, func(ctx context.Context, svc *s3.Client, reqRetriever *requestRetriever) (interface{}, error) { + if c.operation != nil { + return c.operation(ctx, svc, reqRetriever) + } + + return svc.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: ptr.String(c.bucket), + }, addRequestRetriever(reqRetriever)) + }) + }) + } +} + +// addRequestRetriever provides request retriever function - that can be used to fetch request from +// various build steps. Currently we support fetching after serializing and after finalized middlewares. +var addRequestRetriever = func(fm *requestRetriever) func(options *s3.Options) { + return func(options *s3.Options) { + // append request retriever middleware for request inspection + options.APIOptions = append(options.APIOptions, + func(stack *middleware.Stack) error { + // adds AFTER operation serializer middleware + return stack.Serialize.Insert(fm.serializedRequest, "OperationSerializer", middleware.After) + }, + func(stack *middleware.Stack) error { + // adds AFTER operation finalize middleware + return stack.Finalize.Add(fm.signedRequest, middleware.After) + }) + } +} + +// requestRetriever can be used to fetch request within various stages of request. +// currently we support fetching requests after serialization, and after signing. +type requestRetriever struct { + // serializedRequest retriver should be used to fetch request after Operation serializers are executed. + serializedRequest *requestRetrieverMiddleware + + // signedRequest retriever should be used to fetch request from Finalize step after + signedRequest *requestRetrieverMiddleware +} + +func runValidations(t *testing.T, c testCaseForEndpointCustomization, operation func( + context.Context, *s3.Client, *requestRetriever) (interface{}, error)) { + // options + opts := c.options.Copy() + opts.Credentials = unit.StubCredentialsProvider{} + opts.HTTPClient = smithyhttp.NopClient{} + opts.Retryer = aws.NopRetryer{} + + // build an s3 client + svc := s3.New(opts) + + // initialize request fetcher to fetch after input is serialized for request + serializedRequest := requestRetrieverMiddleware{} + + // initialize request fetcher to fetch request after it is signed + signedRequest := requestRetrieverMiddleware{} + + ctx := context.Background() + + // call an operation + _, err := operation(ctx, svc, &requestRetriever{ + serializedRequest: &serializedRequest, + signedRequest: &signedRequest, + }) + + // inspect any errors + if len(c.expectedErr) != 0 { + if err == nil { + t.Fatalf("expected error, got none") + } + if a, e := err.Error(), c.expectedErr; !strings.Contains(a, e) { + t.Fatalf("expect error code to contain %q, got %q", e, a) + } + return + } + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + + // build the captured request + req := serializedRequest.request.Build(ctx) + // verify the built request is as expected + if e, a := c.expectedReqURL, req.URL.String(); e != a { + t.Fatalf("expect url %s, got %s", e, a) + } + + if e, a := c.expectedSigningRegion, serializedRequest.signingRegion; !strings.EqualFold(e, a) { + t.Fatalf("expect signing region as %s, got %s", e, a) + } + + if e, a := c.expectedSigningName, serializedRequest.signingName; !strings.EqualFold(e, a) { + t.Fatalf("expect signing name as %s, got %s", e, a) + } + + // fetch signed request + signedReq := signedRequest.request + // validate if expected headers are present in request + for key, ev := range c.expectedHeader { + av := signedReq.Header.Get(key) + if len(av) == 0 { + t.Fatalf("expected header %v to be present in %v was not", key, req.Header) + } + if !strings.EqualFold(ev, av) { + t.Fatalf("expected header %v to be %v, got %v instead", key, ev, av) + } + } +} + +// request retriever middleware is used to fetch request within a stack step. +type requestRetrieverMiddleware struct { + request *smithyhttp.Request + signingRegion string + signingName string +} + +func (*requestRetrieverMiddleware) ID() string { return "S3:requestRetrieverMiddleware" } + +func (rm *requestRetrieverMiddleware) HandleSerialize( + ctx context.Context, in middleware.SerializeInput, next middleware.SerializeHandler, +) ( + out middleware.SerializeOutput, metadata middleware.Metadata, err error, +) { + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown request type %T", req) + } + rm.request = req + + rm.signingName = awsmiddleware.GetSigningName(ctx) + rm.signingRegion = awsmiddleware.GetSigningRegion(ctx) + + return next.HandleSerialize(ctx, in) +} + +func (rm *requestRetrieverMiddleware) HandleFinalize( + ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler, +) ( + out middleware.FinalizeOutput, metadata middleware.Metadata, err error, +) { + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown request type %T", req) + } + rm.request = req + + rm.signingName = awsmiddleware.GetSigningName(ctx) + rm.signingRegion = awsmiddleware.GetSigningRegion(ctx) + + return next.HandleFinalize(ctx, in) +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/write_get_object_response_test.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/write_get_object_response_test.go new file mode 100644 index 0000000000..3dfe256ef0 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/write_get_object_response_test.go @@ -0,0 +1,221 @@ +package customizations_test + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/go-cmp/cmp" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" +) + +type readSeeker struct { + br *bytes.Reader +} + +func (r *readSeeker) Read(p []byte) (int, error) { + return r.br.Read(p) +} + +func (r *readSeeker) Seek(offset int64, whence int) (int64, error) { + return r.br.Seek(offset, whence) +} + +type readOnlyReader struct { + br *bytes.Reader +} + +func (r *readOnlyReader) Read(p []byte) (int, error) { + return r.br.Read(p) +} + +type lenReader struct { + br *bytes.Reader +} + +func (r *lenReader) Read(p []byte) (int, error) { + return r.br.Read(p) +} + +func (r *lenReader) Len() int { + return r.br.Len() +} + +func TestWriteGetObjectResponse(t *testing.T) { + const contentLength = "Content-Length" + const contentSha256 = "X-Amz-Content-Sha256" + const unsignedPayload = "UNSIGNED-PAYLOAD" + + cases := map[string]struct { + Handler func(*testing.T) http.Handler + Input s3.WriteGetObjectResponseInput + }{ + "Content-Length seekable": { + Handler: func(t *testing.T) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + expectedInput := []byte("test input") + + if len(request.TransferEncoding) != 0 { + t.Errorf("expect no transfer-encoding") + } + + if diff := cmp.Diff(request.Header.Get(contentLength), fmt.Sprintf("%d", len(expectedInput))); len(diff) > 0 { + t.Error(diff) + } + + if diff := cmp.Diff(request.Header.Get(contentSha256), unsignedPayload); len(diff) > 0 { + t.Error(diff) + } + + all, err := ioutil.ReadAll(request.Body) + if err != nil { + t.Errorf("expect no error, got %v", err) + } + if diff := cmp.Diff(all, expectedInput); len(diff) > 0 { + t.Error(diff) + } + writer.WriteHeader(200) + }) + }, + Input: s3.WriteGetObjectResponseInput{ + RequestRoute: aws.String("route"), + RequestToken: aws.String("token"), + Body: &readSeeker{br: bytes.NewReader([]byte("test input"))}, + }, + }, + "Content-Length Len Interface": { + Handler: func(t *testing.T) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + expectedInput := []byte("test input") + + if len(request.TransferEncoding) != 0 { + t.Errorf("expect no transfer-encoding") + } + + if diff := cmp.Diff(request.Header.Get(contentLength), fmt.Sprintf("%d", len(expectedInput))); len(diff) > 0 { + t.Error(diff) + } + + if diff := cmp.Diff(request.Header.Get(contentSha256), unsignedPayload); len(diff) > 0 { + t.Error(diff) + } + + all, err := ioutil.ReadAll(request.Body) + if err != nil { + t.Errorf("expect no error, got %v", err) + } + if diff := cmp.Diff(all, expectedInput); len(diff) > 0 { + t.Error(diff) + } + writer.WriteHeader(200) + }) + }, + Input: s3.WriteGetObjectResponseInput{ + RequestRoute: aws.String("route"), + RequestToken: aws.String("token"), + Body: &lenReader{bytes.NewReader([]byte("test input"))}, + }, + }, + "Content-Length Input Parameter": { + Handler: func(t *testing.T) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + expectedInput := []byte("test input") + + if len(request.TransferEncoding) != 0 { + t.Errorf("expect no transfer-encoding") + } + + if diff := cmp.Diff(request.Header.Get(contentLength), fmt.Sprintf("%d", len(expectedInput))); len(diff) > 0 { + t.Error(diff) + } + + if diff := cmp.Diff(request.Header.Get(contentSha256), unsignedPayload); len(diff) > 0 { + t.Error(diff) + } + + all, err := ioutil.ReadAll(request.Body) + if err != nil { + t.Errorf("expect no error, got %v", err) + } + if diff := cmp.Diff(all, expectedInput); len(diff) > 0 { + t.Error(diff) + } + writer.WriteHeader(200) + }) + }, + Input: s3.WriteGetObjectResponseInput{ + RequestRoute: aws.String("route"), + RequestToken: aws.String("token"), + Body: &readOnlyReader{bytes.NewReader([]byte("test input"))}, + ContentLength: 10, + }, + }, + "Content-Length Not Provided": { + Handler: func(t *testing.T) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + expectedInput := []byte("test input") + + if diff := cmp.Diff(request.TransferEncoding, []string{"chunked"}); len(diff) > 0 { + t.Error(diff) + } + + if diff := cmp.Diff(request.Header.Get(contentLength), ""); len(diff) > 0 { + t.Error(diff) + } + + if diff := cmp.Diff(request.Header.Get(contentSha256), unsignedPayload); len(diff) > 0 { + t.Error(diff) + } + + all, err := ioutil.ReadAll(request.Body) + if err != nil { + t.Errorf("expect no error, got %v", err) + } + if diff := cmp.Diff(all, expectedInput); len(diff) > 0 { + t.Error(diff) + } + writer.WriteHeader(200) + }) + }, + Input: s3.WriteGetObjectResponseInput{ + RequestRoute: aws.String("route"), + RequestToken: aws.String("token"), + Body: &readOnlyReader{bytes.NewReader([]byte("test input"))}, + }, + }, + } + + for name, tt := range cases { + t.Run(name, func(t *testing.T) { + server := httptest.NewTLSServer(tt.Handler(t)) + defer server.Close() + client := s3.New(s3.Options{ + Region: "us-west-2", + HTTPClient: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + }, + EndpointResolver: s3.EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: server.URL, + SigningName: "s3-object-lambda", + SigningRegion: region, + Source: aws.EndpointSourceCustom, + HostnameImmutable: true, + }, nil + }), + }) + + _, err := client.WriteGetObjectResponse(context.Background(), &tt.Input) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + }) + } +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/ya.make new file mode 100644 index 0000000000..ec79e4a843 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations/ya.make @@ -0,0 +1,33 @@ +GO_LIBRARY() + +LICENSE(Apache-2.0) + +SRCS( + doc.go + handle_200_error.go + host.go + presigned_expires.go + process_arn_resource.go + remove_bucket_middleware.go + s3_object_lambda.go + signer_wrapper.go + update_endpoint.go +) + +GO_TEST_SRCS( + # update_endpoint_internal_test.go +) + +GO_XTEST_SRCS( + handle_200_error_test.go + presign_test.go + unit_test.go + update_endpoint_test.go + write_get_object_response_test.go +) + +END() + +RECURSE( + gotest +) diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/endpoints.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/endpoints.go new file mode 100644 index 0000000000..c7e5f6d2b2 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/endpoints.go @@ -0,0 +1,959 @@ +// Code generated by smithy-go-codegen DO NOT EDIT. + +package endpoints + +import ( + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + endpoints "github.com/aws/aws-sdk-go-v2/internal/endpoints/v2" + "github.com/aws/smithy-go/logging" + "regexp" + "strings" +) + +// Options is the endpoint resolver configuration options +type Options struct { + // Logger is a logging implementation that log events should be sent to. + Logger logging.Logger + + // LogDeprecated indicates that deprecated endpoints should be logged to the + // provided logger. + LogDeprecated bool + + // ResolvedRegion is used to override the region to be resolved, rather then the + // using the value passed to the ResolveEndpoint method. This value is used by the + // SDK to translate regions like fips-us-east-1 or us-east-1-fips to an alternative + // name. You must not set this value directly in your application. + ResolvedRegion string + + // DisableHTTPS informs the resolver to return an endpoint that does not use the + // HTTPS scheme. + DisableHTTPS bool + + // UseDualStackEndpoint specifies the resolver must resolve a dual-stack endpoint. + UseDualStackEndpoint aws.DualStackEndpointState + + // UseFIPSEndpoint specifies the resolver must resolve a FIPS endpoint. + UseFIPSEndpoint aws.FIPSEndpointState +} + +func (o Options) GetResolvedRegion() string { + return o.ResolvedRegion +} + +func (o Options) GetDisableHTTPS() bool { + return o.DisableHTTPS +} + +func (o Options) GetUseDualStackEndpoint() aws.DualStackEndpointState { + return o.UseDualStackEndpoint +} + +func (o Options) GetUseFIPSEndpoint() aws.FIPSEndpointState { + return o.UseFIPSEndpoint +} + +func transformToSharedOptions(options Options) endpoints.Options { + return endpoints.Options{ + Logger: options.Logger, + LogDeprecated: options.LogDeprecated, + ResolvedRegion: options.ResolvedRegion, + DisableHTTPS: options.DisableHTTPS, + UseDualStackEndpoint: options.UseDualStackEndpoint, + UseFIPSEndpoint: options.UseFIPSEndpoint, + } +} + +// Resolver S3 endpoint resolver +type Resolver struct { + partitions endpoints.Partitions +} + +// ResolveEndpoint resolves the service endpoint for the given region and options +func (r *Resolver) ResolveEndpoint(region string, options Options) (endpoint aws.Endpoint, err error) { + if len(region) == 0 { + return endpoint, &aws.MissingRegionError{} + } + + opt := transformToSharedOptions(options) + return r.partitions.ResolveEndpoint(region, opt) +} + +// New returns a new Resolver +func New() *Resolver { + return &Resolver{ + partitions: defaultPartitions, + } +} + +var partitionRegexp = struct { + Aws *regexp.Regexp + AwsCn *regexp.Regexp + AwsIso *regexp.Regexp + AwsIsoB *regexp.Regexp + AwsIsoE *regexp.Regexp + AwsIsoF *regexp.Regexp + AwsUsGov *regexp.Regexp +}{ + + Aws: regexp.MustCompile("^(us|eu|ap|sa|ca|me|af|il)\\-\\w+\\-\\d+$"), + AwsCn: regexp.MustCompile("^cn\\-\\w+\\-\\d+$"), + AwsIso: regexp.MustCompile("^us\\-iso\\-\\w+\\-\\d+$"), + AwsIsoB: regexp.MustCompile("^us\\-isob\\-\\w+\\-\\d+$"), + AwsIsoE: regexp.MustCompile("^eu\\-isoe\\-\\w+\\-\\d+$"), + AwsIsoF: regexp.MustCompile("^us\\-isof\\-\\w+\\-\\d+$"), + AwsUsGov: regexp.MustCompile("^us\\-gov\\-\\w+\\-\\d+$"), +} + +var defaultPartitions = endpoints.Partitions{ + { + ID: "aws", + Defaults: map[endpoints.DefaultKey]endpoints.Endpoint{ + { + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.{region}.amazonaws.com", + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + { + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.{region}.amazonaws.com", + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + { + Variant: endpoints.FIPSVariant | endpoints.DualStackVariant, + }: { + Hostname: "s3-fips.dualstack.{region}.amazonaws.com", + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + { + Variant: 0, + }: { + Hostname: "s3.{region}.amazonaws.com", + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + }, + RegionRegex: partitionRegexp.Aws, + IsRegionalized: true, + Endpoints: endpoints.Endpoints{ + endpoints.EndpointKey{ + Region: "af-south-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "af-south-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.af-south-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "ap-east-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "ap-east-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.ap-east-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "ap-northeast-1", + }: endpoints.Endpoint{ + Hostname: "s3.ap-northeast-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "ap-northeast-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.ap-northeast-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "ap-northeast-2", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "ap-northeast-2", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.ap-northeast-2.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "ap-northeast-3", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "ap-northeast-3", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.ap-northeast-3.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "ap-south-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "ap-south-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.ap-south-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "ap-south-2", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "ap-south-2", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.ap-south-2.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "ap-southeast-1", + }: endpoints.Endpoint{ + Hostname: "s3.ap-southeast-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "ap-southeast-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.ap-southeast-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "ap-southeast-2", + }: endpoints.Endpoint{ + Hostname: "s3.ap-southeast-2.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "ap-southeast-2", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.ap-southeast-2.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "ap-southeast-3", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "ap-southeast-3", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.ap-southeast-3.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "ap-southeast-4", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "ap-southeast-4", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.ap-southeast-4.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "aws-global", + }: endpoints.Endpoint{ + Hostname: "s3.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + CredentialScope: endpoints.CredentialScope{ + Region: "us-east-1", + }, + }, + endpoints.EndpointKey{ + Region: "ca-central-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "ca-central-1", + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.ca-central-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "ca-central-1", + Variant: endpoints.FIPSVariant | endpoints.DualStackVariant, + }: { + Hostname: "s3-fips.dualstack.ca-central-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "ca-central-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.ca-central-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "eu-central-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "eu-central-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.eu-central-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "eu-central-2", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "eu-central-2", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.eu-central-2.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "eu-north-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "eu-north-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.eu-north-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "eu-south-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "eu-south-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.eu-south-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "eu-south-2", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "eu-south-2", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.eu-south-2.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "eu-west-1", + }: endpoints.Endpoint{ + Hostname: "s3.eu-west-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "eu-west-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.eu-west-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "eu-west-2", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "eu-west-2", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.eu-west-2.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "eu-west-3", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "eu-west-3", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.eu-west-3.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "fips-ca-central-1", + }: endpoints.Endpoint{ + Hostname: "s3-fips.ca-central-1.amazonaws.com", + CredentialScope: endpoints.CredentialScope{ + Region: "ca-central-1", + }, + Deprecated: aws.TrueTernary, + }, + endpoints.EndpointKey{ + Region: "fips-us-east-1", + }: endpoints.Endpoint{ + Hostname: "s3-fips.us-east-1.amazonaws.com", + CredentialScope: endpoints.CredentialScope{ + Region: "us-east-1", + }, + Deprecated: aws.TrueTernary, + }, + endpoints.EndpointKey{ + Region: "fips-us-east-2", + }: endpoints.Endpoint{ + Hostname: "s3-fips.us-east-2.amazonaws.com", + CredentialScope: endpoints.CredentialScope{ + Region: "us-east-2", + }, + Deprecated: aws.TrueTernary, + }, + endpoints.EndpointKey{ + Region: "fips-us-west-1", + }: endpoints.Endpoint{ + Hostname: "s3-fips.us-west-1.amazonaws.com", + CredentialScope: endpoints.CredentialScope{ + Region: "us-west-1", + }, + Deprecated: aws.TrueTernary, + }, + endpoints.EndpointKey{ + Region: "fips-us-west-2", + }: endpoints.Endpoint{ + Hostname: "s3-fips.us-west-2.amazonaws.com", + CredentialScope: endpoints.CredentialScope{ + Region: "us-west-2", + }, + Deprecated: aws.TrueTernary, + }, + endpoints.EndpointKey{ + Region: "il-central-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "il-central-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.il-central-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "me-central-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "me-central-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.me-central-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "me-south-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "me-south-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.me-south-1.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "s3-external-1", + }: endpoints.Endpoint{ + Hostname: "s3-external-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + CredentialScope: endpoints.CredentialScope{ + Region: "us-east-1", + }, + }, + endpoints.EndpointKey{ + Region: "sa-east-1", + }: endpoints.Endpoint{ + Hostname: "s3.sa-east-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "sa-east-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.sa-east-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-east-1", + }: endpoints.Endpoint{ + Hostname: "s3.us-east-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-east-1", + Variant: endpoints.FIPSVariant | endpoints.DualStackVariant, + }: { + Hostname: "s3-fips.dualstack.us-east-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-east-1", + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.us-east-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-east-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.us-east-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-east-2", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "us-east-2", + Variant: endpoints.FIPSVariant | endpoints.DualStackVariant, + }: { + Hostname: "s3-fips.dualstack.us-east-2.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "us-east-2", + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.us-east-2.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "us-east-2", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.us-east-2.amazonaws.com", + }, + endpoints.EndpointKey{ + Region: "us-west-1", + }: endpoints.Endpoint{ + Hostname: "s3.us-west-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-west-1", + Variant: endpoints.FIPSVariant | endpoints.DualStackVariant, + }: { + Hostname: "s3-fips.dualstack.us-west-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-west-1", + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.us-west-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-west-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.us-west-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-west-2", + }: endpoints.Endpoint{ + Hostname: "s3.us-west-2.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-west-2", + Variant: endpoints.FIPSVariant | endpoints.DualStackVariant, + }: { + Hostname: "s3-fips.dualstack.us-west-2.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-west-2", + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.us-west-2.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-west-2", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.us-west-2.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, + }, + }, + { + ID: "aws-cn", + Defaults: map[endpoints.DefaultKey]endpoints.Endpoint{ + { + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.{region}.amazonaws.com.cn", + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + { + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.{region}.amazonaws.com.cn", + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + { + Variant: endpoints.FIPSVariant | endpoints.DualStackVariant, + }: { + Hostname: "s3-fips.{region}.api.amazonwebservices.com.cn", + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + { + Variant: 0, + }: { + Hostname: "s3.{region}.amazonaws.com.cn", + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + }, + RegionRegex: partitionRegexp.AwsCn, + IsRegionalized: true, + Endpoints: endpoints.Endpoints{ + endpoints.EndpointKey{ + Region: "cn-north-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "cn-north-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.cn-north-1.amazonaws.com.cn", + }, + endpoints.EndpointKey{ + Region: "cn-northwest-1", + }: endpoints.Endpoint{}, + endpoints.EndpointKey{ + Region: "cn-northwest-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.cn-northwest-1.amazonaws.com.cn", + }, + }, + }, + { + ID: "aws-iso", + Defaults: map[endpoints.DefaultKey]endpoints.Endpoint{ + { + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.{region}.c2s.ic.gov", + Protocols: []string{"https"}, + SignatureVersions: []string{"s3v4"}, + }, + { + Variant: 0, + }: { + Hostname: "s3.{region}.c2s.ic.gov", + Protocols: []string{"https"}, + SignatureVersions: []string{"s3v4"}, + }, + }, + RegionRegex: partitionRegexp.AwsIso, + IsRegionalized: true, + Endpoints: endpoints.Endpoints{ + endpoints.EndpointKey{ + Region: "us-iso-east-1", + }: endpoints.Endpoint{ + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + endpoints.EndpointKey{ + Region: "us-iso-west-1", + }: endpoints.Endpoint{}, + }, + }, + { + ID: "aws-iso-b", + Defaults: map[endpoints.DefaultKey]endpoints.Endpoint{ + { + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.{region}.sc2s.sgov.gov", + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + { + Variant: 0, + }: { + Hostname: "s3.{region}.sc2s.sgov.gov", + Protocols: []string{"http", "https"}, + SignatureVersions: []string{"s3v4"}, + }, + }, + RegionRegex: partitionRegexp.AwsIsoB, + IsRegionalized: true, + Endpoints: endpoints.Endpoints{ + endpoints.EndpointKey{ + Region: "us-isob-east-1", + }: endpoints.Endpoint{}, + }, + }, + { + ID: "aws-iso-e", + Defaults: map[endpoints.DefaultKey]endpoints.Endpoint{ + { + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.{region}.cloud.adc-e.uk", + Protocols: []string{"https"}, + SignatureVersions: []string{"v4"}, + }, + { + Variant: 0, + }: { + Hostname: "s3.{region}.cloud.adc-e.uk", + Protocols: []string{"https"}, + SignatureVersions: []string{"v4"}, + }, + }, + RegionRegex: partitionRegexp.AwsIsoE, + IsRegionalized: true, + }, + { + ID: "aws-iso-f", + Defaults: map[endpoints.DefaultKey]endpoints.Endpoint{ + { + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.{region}.csp.hci.ic.gov", + Protocols: []string{"https"}, + SignatureVersions: []string{"v4"}, + }, + { + Variant: 0, + }: { + Hostname: "s3.{region}.csp.hci.ic.gov", + Protocols: []string{"https"}, + SignatureVersions: []string{"v4"}, + }, + }, + RegionRegex: partitionRegexp.AwsIsoF, + IsRegionalized: true, + }, + { + ID: "aws-us-gov", + Defaults: map[endpoints.DefaultKey]endpoints.Endpoint{ + { + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.{region}.amazonaws.com", + Protocols: []string{"https"}, + SignatureVersions: []string{"s3", "s3v4"}, + }, + { + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.{region}.amazonaws.com", + Protocols: []string{"https"}, + SignatureVersions: []string{"s3", "s3v4"}, + }, + { + Variant: endpoints.FIPSVariant | endpoints.DualStackVariant, + }: { + Hostname: "s3-fips.dualstack.{region}.amazonaws.com", + Protocols: []string{"https"}, + SignatureVersions: []string{"s3", "s3v4"}, + }, + { + Variant: 0, + }: { + Hostname: "s3.{region}.amazonaws.com", + Protocols: []string{"https"}, + SignatureVersions: []string{"s3", "s3v4"}, + }, + }, + RegionRegex: partitionRegexp.AwsUsGov, + IsRegionalized: true, + Endpoints: endpoints.Endpoints{ + endpoints.EndpointKey{ + Region: "fips-us-gov-east-1", + }: endpoints.Endpoint{ + Hostname: "s3-fips.us-gov-east-1.amazonaws.com", + CredentialScope: endpoints.CredentialScope{ + Region: "us-gov-east-1", + }, + Deprecated: aws.TrueTernary, + }, + endpoints.EndpointKey{ + Region: "fips-us-gov-west-1", + }: endpoints.Endpoint{ + Hostname: "s3-fips.us-gov-west-1.amazonaws.com", + CredentialScope: endpoints.CredentialScope{ + Region: "us-gov-west-1", + }, + Deprecated: aws.TrueTernary, + }, + endpoints.EndpointKey{ + Region: "us-gov-east-1", + }: endpoints.Endpoint{ + Hostname: "s3.us-gov-east-1.amazonaws.com", + Protocols: []string{"http", "https"}, + }, + endpoints.EndpointKey{ + Region: "us-gov-east-1", + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.us-gov-east-1.amazonaws.com", + Protocols: []string{"http", "https"}, + }, + endpoints.EndpointKey{ + Region: "us-gov-east-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.us-gov-east-1.amazonaws.com", + Protocols: []string{"http", "https"}, + }, + endpoints.EndpointKey{ + Region: "us-gov-west-1", + }: endpoints.Endpoint{ + Hostname: "s3.us-gov-west-1.amazonaws.com", + Protocols: []string{"http", "https"}, + }, + endpoints.EndpointKey{ + Region: "us-gov-west-1", + Variant: endpoints.FIPSVariant, + }: { + Hostname: "s3-fips.us-gov-west-1.amazonaws.com", + Protocols: []string{"http", "https"}, + }, + endpoints.EndpointKey{ + Region: "us-gov-west-1", + Variant: endpoints.DualStackVariant, + }: { + Hostname: "s3.dualstack.us-gov-west-1.amazonaws.com", + Protocols: []string{"http", "https"}, + }, + }, + }, +} + +// GetDNSSuffix returns the dnsSuffix URL component for the given partition id +func GetDNSSuffix(id string, options Options) (string, error) { + variant := transformToSharedOptions(options).GetEndpointVariant() + switch { + case strings.EqualFold(id, "aws"): + switch variant { + case endpoints.DualStackVariant: + return "amazonaws.com", nil + + case endpoints.FIPSVariant: + return "amazonaws.com", nil + + case endpoints.FIPSVariant | endpoints.DualStackVariant: + return "amazonaws.com", nil + + case 0: + return "amazonaws.com", nil + + default: + return "", fmt.Errorf("unsupported endpoint variant %v, in partition %s", variant, id) + + } + + case strings.EqualFold(id, "aws-cn"): + switch variant { + case endpoints.DualStackVariant: + return "amazonaws.com.cn", nil + + case endpoints.FIPSVariant: + return "amazonaws.com.cn", nil + + case endpoints.FIPSVariant | endpoints.DualStackVariant: + return "api.amazonwebservices.com.cn", nil + + case 0: + return "amazonaws.com.cn", nil + + default: + return "", fmt.Errorf("unsupported endpoint variant %v, in partition %s", variant, id) + + } + + case strings.EqualFold(id, "aws-iso"): + switch variant { + case endpoints.FIPSVariant: + return "c2s.ic.gov", nil + + case 0: + return "c2s.ic.gov", nil + + default: + return "", fmt.Errorf("unsupported endpoint variant %v, in partition %s", variant, id) + + } + + case strings.EqualFold(id, "aws-iso-b"): + switch variant { + case endpoints.FIPSVariant: + return "sc2s.sgov.gov", nil + + case 0: + return "sc2s.sgov.gov", nil + + default: + return "", fmt.Errorf("unsupported endpoint variant %v, in partition %s", variant, id) + + } + + case strings.EqualFold(id, "aws-iso-e"): + switch variant { + case endpoints.FIPSVariant: + return "cloud.adc-e.uk", nil + + case 0: + return "cloud.adc-e.uk", nil + + default: + return "", fmt.Errorf("unsupported endpoint variant %v, in partition %s", variant, id) + + } + + case strings.EqualFold(id, "aws-iso-f"): + switch variant { + case endpoints.FIPSVariant: + return "csp.hci.ic.gov", nil + + case 0: + return "csp.hci.ic.gov", nil + + default: + return "", fmt.Errorf("unsupported endpoint variant %v, in partition %s", variant, id) + + } + + case strings.EqualFold(id, "aws-us-gov"): + switch variant { + case endpoints.DualStackVariant: + return "amazonaws.com", nil + + case endpoints.FIPSVariant: + return "amazonaws.com", nil + + case endpoints.FIPSVariant | endpoints.DualStackVariant: + return "amazonaws.com", nil + + case 0: + return "amazonaws.com", nil + + default: + return "", fmt.Errorf("unsupported endpoint variant %v, in partition %s", variant, id) + + } + + default: + return "", fmt.Errorf("unknown partition") + + } +} + +// GetDNSSuffixFromRegion returns the DNS suffix for the provided region and +// options. +func GetDNSSuffixFromRegion(region string, options Options) (string, error) { + switch { + case partitionRegexp.Aws.MatchString(region): + return GetDNSSuffix("aws", options) + + case partitionRegexp.AwsCn.MatchString(region): + return GetDNSSuffix("aws-cn", options) + + case partitionRegexp.AwsIso.MatchString(region): + return GetDNSSuffix("aws-iso", options) + + case partitionRegexp.AwsIsoB.MatchString(region): + return GetDNSSuffix("aws-iso-b", options) + + case partitionRegexp.AwsIsoE.MatchString(region): + return GetDNSSuffix("aws-iso-e", options) + + case partitionRegexp.AwsIsoF.MatchString(region): + return GetDNSSuffix("aws-iso-f", options) + + case partitionRegexp.AwsUsGov.MatchString(region): + return GetDNSSuffix("aws-us-gov", options) + + default: + return GetDNSSuffix("aws", options) + + } +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/endpoints_test.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/endpoints_test.go new file mode 100644 index 0000000000..08e5da2d83 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/endpoints_test.go @@ -0,0 +1,11 @@ +// Code generated by smithy-go-codegen DO NOT EDIT. + +package endpoints + +import ( + "testing" +) + +func TestRegexCompile(t *testing.T) { + _ = defaultPartitions +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/gotest/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/gotest/ya.make new file mode 100644 index 0000000000..eadac1b0f5 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/gotest/ya.make @@ -0,0 +1,5 @@ +GO_TEST_FOR(vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints) + +LICENSE(Apache-2.0) + +END() diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/ya.make new file mode 100644 index 0000000000..01b48bd1a7 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/endpoints/ya.make @@ -0,0 +1,15 @@ +GO_LIBRARY() + +LICENSE(Apache-2.0) + +SRCS( + endpoints.go +) + +GO_TEST_SRCS(endpoints_test.go) + +END() + +RECURSE( + gotest +) diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/s3testing/s3testing.go b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/s3testing/s3testing.go new file mode 100644 index 0000000000..c12e468718 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/s3testing/s3testing.go @@ -0,0 +1,27 @@ +package s3testing + +import ( + "fmt" + "math/rand" + + "github.com/aws/aws-sdk-go-v2/internal/sdkio" +) + +var randBytes = func() []byte { + b := make([]byte, 10*sdkio.MebiByte) + + if _, err := rand.Read(b); err != nil { + panic(fmt.Sprintf("failed to read random bytes, %v", err)) + } + return b +}() + +// GetTestBytes returns a pseudo-random []byte of length size +func GetTestBytes(size int) []byte { + if len(randBytes) >= size { + return randBytes[:size] + } + + b := append(randBytes, GetTestBytes(size-len(randBytes))...) + return b +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/s3testing/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/s3testing/ya.make new file mode 100644 index 0000000000..fbd1ea6138 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/s3testing/ya.make @@ -0,0 +1,9 @@ +GO_LIBRARY() + +LICENSE(Apache-2.0) + +SRCS( + s3testing.go +) + +END() diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/ya.make new file mode 100644 index 0000000000..77f2c4700b --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/s3/internal/ya.make @@ -0,0 +1,6 @@ +RECURSE( + arn + customizations + endpoints + s3testing +) |