diff options
author | vitalyisaev <vitalyisaev@ydb.tech> | 2023-12-12 21:55:07 +0300 |
---|---|---|
committer | vitalyisaev <vitalyisaev@ydb.tech> | 2023-12-12 22:25:10 +0300 |
commit | 4967f99474a4040ba150eb04995de06342252718 (patch) | |
tree | c9c118836513a8fab6e9fcfb25be5d404338bca7 /vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds | |
parent | 2ce9cccb9b0bdd4cd7a3491dc5cbf8687cda51de (diff) | |
download | ydb-4967f99474a4040ba150eb04995de06342252718.tar.gz |
YQ Connector: prepare code base for S3 integration
1. Кодовая база Коннектора переписана с помощью Go дженериков так, чтобы добавление нового источника данных (в частности S3 + csv) максимально переиспользовало имеющийся код (чтобы сохранялась логика нарезания на блоки данных, учёт трафика и пр.)
2. API Connector расширено для работы с S3, но ещё пока не протестировано.
Diffstat (limited to 'vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds')
5 files changed, 675 insertions, 0 deletions
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/doc.go b/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/doc.go new file mode 100644 index 0000000000..6ed71b42b2 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/doc.go @@ -0,0 +1,58 @@ +// Package ec2rolecreds provides the credentials provider implementation for +// retrieving AWS credentials from Amazon EC2 Instance Roles via Amazon EC2 IMDS. +// +// # Concurrency and caching +// +// The Provider is not safe to be used concurrently, and does not provide any +// caching of credentials retrieved. You should wrap the Provider with a +// `aws.CredentialsCache` to provide concurrency safety, and caching of +// credentials. +// +// # Loading credentials with the SDK's AWS Config +// +// The EC2 Instance role credentials provider will automatically be the resolved +// credential provider in the credential chain if no other credential provider is +// resolved first. +// +// To explicitly instruct the SDK's credentials resolving to use the EC2 Instance +// role for credentials, you specify a `credentials_source` property in the config +// profile the SDK will load. +// +// [default] +// credential_source = Ec2InstanceMetadata +// +// # Loading credentials with the Provider directly +// +// Another way to use the EC2 Instance role credentials provider is to create it +// directly and assign it as the credentials provider for an API client. +// +// The following example creates a credentials provider for a command, and wraps +// it with the CredentialsCache before assigning the provider to the Amazon S3 API +// client's Credentials option. +// +// provider := imds.New(imds.Options{}) +// +// // Create the service client value configured for credentials. +// svc := s3.New(s3.Options{ +// Credentials: aws.NewCredentialsCache(provider), +// }) +// +// If you need more control, you can set the configuration options on the +// credentials provider using the imds.Options type to configure the EC2 IMDS +// API Client and ExpiryWindow of the retrieved credentials. +// +// provider := imds.New(imds.Options{ +// // See imds.Options type's documentation for more options available. +// Client: imds.New(Options{ +// HTTPClient: customHTTPClient, +// }), +// +// // Modify how soon credentials expire prior to their original expiry time. +// ExpiryWindow: 5 * time.Minute, +// }) +// +// # EC2 IMDS API Client +// +// See the github.com/aws/aws-sdk-go-v2/feature/ec2/imds module for more details on +// configuring the client, and options available. +package ec2rolecreds diff --git a/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/gotest/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/gotest/ya.make new file mode 100644 index 0000000000..5bb848d140 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/gotest/ya.make @@ -0,0 +1,5 @@ +GO_TEST_FOR(vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds) + +LICENSE(Apache-2.0) + +END() diff --git a/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/provider.go b/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/provider.go new file mode 100644 index 0000000000..5c699f1665 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/provider.go @@ -0,0 +1,229 @@ +package ec2rolecreds + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "math" + "path" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + sdkrand "github.com/aws/aws-sdk-go-v2/internal/rand" + "github.com/aws/aws-sdk-go-v2/internal/sdk" + "github.com/aws/smithy-go" + "github.com/aws/smithy-go/logging" + "github.com/aws/smithy-go/middleware" +) + +// ProviderName provides a name of EC2Role provider +const ProviderName = "EC2RoleProvider" + +// GetMetadataAPIClient provides the interface for an EC2 IMDS API client for the +// GetMetadata operation. +type GetMetadataAPIClient interface { + GetMetadata(context.Context, *imds.GetMetadataInput, ...func(*imds.Options)) (*imds.GetMetadataOutput, error) +} + +// A Provider retrieves credentials from the EC2 service, and keeps track if +// those credentials are expired. +// +// The New function must be used to create the with a custom EC2 IMDS client. +// +// p := &ec2rolecreds.New(func(o *ec2rolecreds.Options{ +// o.Client = imds.New(imds.Options{/* custom options */}) +// }) +type Provider struct { + options Options +} + +// Options is a list of user settable options for setting the behavior of the Provider. +type Options struct { + // The API client that will be used by the provider to make GetMetadata API + // calls to EC2 IMDS. + // + // If nil, the provider will default to the EC2 IMDS client. + Client GetMetadataAPIClient +} + +// New returns an initialized Provider value configured to retrieve +// credentials from EC2 Instance Metadata service. +func New(optFns ...func(*Options)) *Provider { + options := Options{} + + for _, fn := range optFns { + fn(&options) + } + + if options.Client == nil { + options.Client = imds.New(imds.Options{}) + } + + return &Provider{ + options: options, + } +} + +// Retrieve retrieves credentials from the EC2 service. Error will be returned +// if the request fails, or unable to extract the desired credentials. +func (p *Provider) Retrieve(ctx context.Context) (aws.Credentials, error) { + credsList, err := requestCredList(ctx, p.options.Client) + if err != nil { + return aws.Credentials{Source: ProviderName}, err + } + + if len(credsList) == 0 { + return aws.Credentials{Source: ProviderName}, + fmt.Errorf("unexpected empty EC2 IMDS role list") + } + credsName := credsList[0] + + roleCreds, err := requestCred(ctx, p.options.Client, credsName) + if err != nil { + return aws.Credentials{Source: ProviderName}, err + } + + creds := aws.Credentials{ + AccessKeyID: roleCreds.AccessKeyID, + SecretAccessKey: roleCreds.SecretAccessKey, + SessionToken: roleCreds.Token, + Source: ProviderName, + + CanExpire: true, + Expires: roleCreds.Expiration, + } + + // Cap role credentials Expires to 1 hour so they can be refreshed more + // often. Jitter will be applied credentials cache if being used. + if anHour := sdk.NowTime().Add(1 * time.Hour); creds.Expires.After(anHour) { + creds.Expires = anHour + } + + return creds, nil +} + +// HandleFailToRefresh will extend the credentials Expires time if it it is +// expired. If the credentials will not expire within the minimum time, they +// will be returned. +// +// If the credentials cannot expire, the original error will be returned. +func (p *Provider) HandleFailToRefresh(ctx context.Context, prevCreds aws.Credentials, err error) ( + aws.Credentials, error, +) { + if !prevCreds.CanExpire { + return aws.Credentials{}, err + } + + if prevCreds.Expires.After(sdk.NowTime().Add(5 * time.Minute)) { + return prevCreds, nil + } + + newCreds := prevCreds + randFloat64, err := sdkrand.CryptoRandFloat64() + if err != nil { + return aws.Credentials{}, fmt.Errorf("failed to get random float, %w", err) + } + + // Random distribution of [5,15) minutes. + expireOffset := time.Duration(randFloat64*float64(10*time.Minute)) + 5*time.Minute + newCreds.Expires = sdk.NowTime().Add(expireOffset) + + logger := middleware.GetLogger(ctx) + logger.Logf(logging.Warn, "Attempting credential expiration extension due to a credential service availability issue. A refresh of these credentials will be attempted again in %v minutes.", math.Floor(expireOffset.Minutes())) + + return newCreds, nil +} + +// AdjustExpiresBy will adds the passed in duration to the passed in +// credential's Expires time, unless the time until Expires is less than 15 +// minutes. Returns the credentials, even if not updated. +func (p *Provider) AdjustExpiresBy(creds aws.Credentials, dur time.Duration) ( + aws.Credentials, error, +) { + if !creds.CanExpire { + return creds, nil + } + if creds.Expires.Before(sdk.NowTime().Add(15 * time.Minute)) { + return creds, nil + } + + creds.Expires = creds.Expires.Add(dur) + return creds, nil +} + +// ec2RoleCredRespBody provides the shape for unmarshaling credential +// request responses. +type ec2RoleCredRespBody struct { + // Success State + Expiration time.Time + AccessKeyID string + SecretAccessKey string + Token string + + // Error state + Code string + Message string +} + +const iamSecurityCredsPath = "/iam/security-credentials/" + +// requestCredList requests a list of credentials from the EC2 service. If +// there are no credentials, or there is an error making or receiving the +// request +func requestCredList(ctx context.Context, client GetMetadataAPIClient) ([]string, error) { + resp, err := client.GetMetadata(ctx, &imds.GetMetadataInput{ + Path: iamSecurityCredsPath, + }) + if err != nil { + return nil, fmt.Errorf("no EC2 IMDS role found, %w", err) + } + defer resp.Content.Close() + + credsList := []string{} + s := bufio.NewScanner(resp.Content) + for s.Scan() { + credsList = append(credsList, s.Text()) + } + + if err := s.Err(); err != nil { + return nil, fmt.Errorf("failed to read EC2 IMDS role, %w", err) + } + + return credsList, nil +} + +// requestCred requests the credentials for a specific credentials from the EC2 service. +// +// If the credentials cannot be found, or there is an error reading the response +// and error will be returned. +func requestCred(ctx context.Context, client GetMetadataAPIClient, credsName string) (ec2RoleCredRespBody, error) { + resp, err := client.GetMetadata(ctx, &imds.GetMetadataInput{ + Path: path.Join(iamSecurityCredsPath, credsName), + }) + if err != nil { + return ec2RoleCredRespBody{}, + fmt.Errorf("failed to get %s EC2 IMDS role credentials, %w", + credsName, err) + } + defer resp.Content.Close() + + var respCreds ec2RoleCredRespBody + if err := json.NewDecoder(resp.Content).Decode(&respCreds); err != nil { + return ec2RoleCredRespBody{}, + fmt.Errorf("failed to decode %s EC2 IMDS role credentials, %w", + credsName, err) + } + + if !strings.EqualFold(respCreds.Code, "Success") { + // If an error code was returned something failed requesting the role. + return ec2RoleCredRespBody{}, + fmt.Errorf("failed to get %s EC2 IMDS role credentials, %w", + credsName, + &smithy.GenericAPIError{Code: respCreds.Code, Message: respCreds.Message}) + } + + return respCreds, nil +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/provider_test.go b/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/provider_test.go new file mode 100644 index 0000000000..362b0b7536 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/provider_test.go @@ -0,0 +1,367 @@ +package ec2rolecreds + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/ioutil" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + sdkrand "github.com/aws/aws-sdk-go-v2/internal/rand" + "github.com/aws/aws-sdk-go-v2/internal/sdk" + "github.com/aws/smithy-go" + "github.com/aws/smithy-go/logging" + "github.com/aws/smithy-go/middleware" + "github.com/google/go-cmp/cmp" +) + +const credsRespTmpl = `{ + "Code": "Success", + "Type": "AWS-HMAC", + "AccessKeyId" : "accessKey", + "SecretAccessKey" : "secret", + "Token" : "token", + "Expiration" : "%s", + "LastUpdated" : "2009-11-23T00:00:00Z" +}` + +const credsFailRespTmpl = `{ + "Code": "ErrorCode", + "Message": "ErrorMsg", + "LastUpdated": "2009-11-23T00:00:00Z" +}` + +type mockClient struct { + t *testing.T + roleName string + failAssume bool + expireOn string +} + +func (c mockClient) GetMetadata( + ctx context.Context, params *imds.GetMetadataInput, optFns ...func(*imds.Options), +) ( + *imds.GetMetadataOutput, error, +) { + switch params.Path { + case iamSecurityCredsPath: + return &imds.GetMetadataOutput{ + Content: ioutil.NopCloser(strings.NewReader(c.roleName)), + }, nil + + case iamSecurityCredsPath + c.roleName: + var w strings.Builder + if c.failAssume { + fmt.Fprintf(&w, credsFailRespTmpl) + } else { + fmt.Fprintf(&w, credsRespTmpl, c.expireOn) + } + return &imds.GetMetadataOutput{ + Content: ioutil.NopCloser(strings.NewReader(w.String())), + }, nil + default: + return nil, fmt.Errorf("unexpected path, %v", params.Path) + } +} + +var ( + _ aws.AdjustExpiresByCredentialsCacheStrategy = (*Provider)(nil) + _ aws.HandleFailRefreshCredentialsCacheStrategy = (*Provider)(nil) +) + +func TestProvider(t *testing.T) { + orig := sdk.NowTime + defer func() { sdk.NowTime = orig }() + + p := New(func(options *Options) { + options.Client = mockClient{ + roleName: "RoleName", + failAssume: false, + expireOn: "2014-12-16T01:51:37Z", + } + }) + + creds, err := p.Retrieve(context.Background()) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + if e, a := "accessKey", creds.AccessKeyID; e != a { + t.Errorf("Expect access key ID to match") + } + if e, a := "secret", creds.SecretAccessKey; e != a { + t.Errorf("Expect secret access key to match") + } + if e, a := "token", creds.SessionToken; e != a { + t.Errorf("Expect session token to match") + } + + sdk.NowTime = func() time.Time { + return time.Date(2014, 12, 16, 0, 55, 37, 0, time.UTC) + } + + if creds.Expired() { + t.Errorf("Expect not expired") + } +} + +func TestProvider_FailAssume(t *testing.T) { + p := New(func(options *Options) { + options.Client = mockClient{ + roleName: "RoleName", + failAssume: true, + expireOn: "2014-12-16T01:51:37Z", + } + }) + + creds, err := p.Retrieve(context.Background()) + if err == nil { + t.Fatalf("expect error, got none") + } + + var apiErr smithy.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("expect %T error, got %v", apiErr, err) + } + if e, a := "ErrorCode", apiErr.ErrorCode(); e != a { + t.Errorf("expect %v code, got %v", e, a) + } + if e, a := "ErrorMsg", apiErr.ErrorMessage(); e != a { + t.Errorf("expect %v message, got %v", e, a) + } + + nestedErr := errors.Unwrap(apiErr) + if nestedErr != nil { + t.Fatalf("expect no nested error, got %v", err) + } + + if e, a := "", creds.AccessKeyID; e != a { + t.Errorf("Expect access key ID to match") + } + if e, a := "", creds.SecretAccessKey; e != a { + t.Errorf("Expect secret access key to match") + } + if e, a := "", creds.SessionToken; e != a { + t.Errorf("Expect session token to match") + } +} + +func TestProvider_IsExpired(t *testing.T) { + orig := sdk.NowTime + defer func() { sdk.NowTime = orig }() + + p := New(func(options *Options) { + options.Client = mockClient{ + roleName: "RoleName", + failAssume: false, + expireOn: "2014-12-16T01:51:37Z", + } + }) + + sdk.NowTime = func() time.Time { + return time.Date(2014, 12, 16, 0, 55, 37, 0, time.UTC) + } + + creds, err := p.Retrieve(context.Background()) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + if creds.Expired() { + t.Errorf("expect not to be expired") + } + + sdk.NowTime = func() time.Time { + return time.Date(2014, 12, 16, 1, 55, 37, 0, time.UTC) + } + + if !creds.Expired() { + t.Errorf("expect to be expired") + } +} + +type byteReader byte + +func (b byteReader) Read(p []byte) (int, error) { + for i := 0; i < len(p); i++ { + p[i] = byte(b) + } + return len(p), nil +} + +func TestProvider_HandleFailToRetrieve(t *testing.T) { + origTime := sdk.NowTime + defer func() { sdk.NowTime = origTime }() + sdk.NowTime = func() time.Time { + return time.Date(2014, 04, 04, 0, 1, 0, 0, time.UTC) + } + + origRand := sdkrand.Reader + defer func() { sdkrand.Reader = origRand }() + sdkrand.Reader = byteReader(0) + + cases := map[string]struct { + creds aws.Credentials + err error + randReader io.Reader + expectCreds aws.Credentials + expectErr string + expectLogged string + }{ + "expired low": { + randReader: byteReader(0), + creds: aws.Credentials{ + CanExpire: true, + Expires: sdk.NowTime().Add(-5 * time.Minute), + }, + err: fmt.Errorf("some error"), + expectCreds: aws.Credentials{ + CanExpire: true, + Expires: sdk.NowTime().Add(5 * time.Minute), + }, + expectLogged: fmt.Sprintf("again in 5 minutes"), + }, + "expired high": { + randReader: byteReader(0xFF), + creds: aws.Credentials{ + CanExpire: true, + Expires: sdk.NowTime().Add(-5 * time.Minute), + }, + err: fmt.Errorf("some error"), + expectCreds: aws.Credentials{ + CanExpire: true, + Expires: sdk.NowTime().Add(14*time.Minute + 59*time.Second), + }, + expectLogged: fmt.Sprintf("again in 14 minutes"), + }, + "not expired": { + randReader: byteReader(0xFF), + creds: aws.Credentials{ + CanExpire: true, + Expires: sdk.NowTime().Add(10 * time.Minute), + }, + err: fmt.Errorf("some error"), + expectCreds: aws.Credentials{ + CanExpire: true, + Expires: sdk.NowTime().Add(10 * time.Minute), + }, + }, + "cannot expire": { + randReader: byteReader(0xFF), + creds: aws.Credentials{ + CanExpire: false, + }, + err: fmt.Errorf("some error"), + expectErr: "some error", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + sdkrand.Reader = c.randReader + if sdkrand.Reader == nil { + sdkrand.Reader = byteReader(0) + } + + var logBuf bytes.Buffer + logger := logging.LoggerFunc(func(class logging.Classification, format string, args ...interface{}) { + fmt.Fprintf(&logBuf, string(class)+" "+format, args...) + }) + ctx := middleware.SetLogger(context.Background(), logger) + + p := New() + creds, err := p.HandleFailToRefresh(ctx, c.creds, c.err) + if err == nil && len(c.expectErr) != 0 { + t.Fatalf("expect error %v, got none", c.expectErr) + } + if err != nil && len(c.expectErr) == 0 { + t.Fatalf("expect no error, got %v", err) + } + if err != nil && !strings.Contains(err.Error(), c.expectErr) { + t.Fatalf("expect error to contain %v, got %v", c.expectErr, err) + } + if c.expectErr != "" { + return + } + + if len(c.expectLogged) != 0 && logBuf.Len() == 0 { + t.Errorf("expect %v logged, got none", c.expectLogged) + } + if e, a := c.expectLogged, logBuf.String(); !strings.Contains(a, e) { + t.Errorf("expect %v to be logged in %v", e, a) + } + + // Truncate time so it can be easily compared. + creds.Expires = creds.Expires.Truncate(time.Second) + + if diff := cmp.Diff(c.expectCreds, creds); diff != "" { + t.Errorf("expect creds match\n%s", diff) + } + }) + } +} + +func TestProvider_AdjustExpiresBy(t *testing.T) { + origTime := sdk.NowTime + defer func() { sdk.NowTime = origTime }() + sdk.NowTime = func() time.Time { + return time.Date(2014, 04, 04, 0, 1, 0, 0, time.UTC) + } + + cases := map[string]struct { + creds aws.Credentials + dur time.Duration + expectCreds aws.Credentials + }{ + "modify expires": { + creds: aws.Credentials{ + CanExpire: true, + Expires: sdk.NowTime().Add(1 * time.Hour), + }, + dur: -5 * time.Minute, + expectCreds: aws.Credentials{ + CanExpire: true, + Expires: sdk.NowTime().Add(55 * time.Minute), + }, + }, + "expiry too soon": { + creds: aws.Credentials{ + CanExpire: true, + Expires: sdk.NowTime().Add(14*time.Minute + 59*time.Second), + }, + dur: -5 * time.Minute, + expectCreds: aws.Credentials{ + CanExpire: true, + Expires: sdk.NowTime().Add(14*time.Minute + 59*time.Second), + }, + }, + "cannot expire": { + creds: aws.Credentials{ + CanExpire: false, + }, + dur: 10 * time.Minute, + expectCreds: aws.Credentials{ + CanExpire: false, + }, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + p := New() + creds, err := p.AdjustExpiresBy(c.creds, c.dur) + + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + + if diff := cmp.Diff(c.expectCreds, creds); diff != "" { + t.Errorf("expect creds match\n%s", diff) + } + }) + } +} diff --git a/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/ya.make new file mode 100644 index 0000000000..13dca6935a --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds/ya.make @@ -0,0 +1,16 @@ +GO_LIBRARY() + +LICENSE(Apache-2.0) + +SRCS( + doc.go + provider.go +) + +GO_TEST_SRCS(provider_test.go) + +END() + +RECURSE( + gotest +) |