YQ Connector: prepare code base for S3 integration

1. Кодовая база Коннектора переписана с помощью Go дженериков так, чтобы добавление нового источника данных (в частности S3 + csv) максимально переиспользовало имеющийся код (чтобы сохранялась логика нарезания на блоки данных, учёт трафика и пр.)
2. API Connector расширено для работы с S3, но ещё пока не протестировано.

7 files changed, 341 insertions, 0 deletions

diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/context.go b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/context.go
new file mode 100644
index 0000000000..cc919701a0
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/context.go
@@ -0,0 +1,48 @@
+package presignedurl
+
+import (
+	"context"
+
+	"github.com/aws/smithy-go/middleware"
+)
+
+// WithIsPresigning adds the isPresigning sentinel value to a context to signal
+// that the middleware stack is using the presign flow.
+//
+// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
+// to clear all stack values.
+func WithIsPresigning(ctx context.Context) context.Context {
+	return middleware.WithStackValue(ctx, isPresigningKey{}, true)
+}
+
+// GetIsPresigning returns if the context contains the isPresigning sentinel
+// value for presigning flows.
+//
+// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
+// to clear all stack values.
+func GetIsPresigning(ctx context.Context) bool {
+	v, _ := middleware.GetStackValue(ctx, isPresigningKey{}).(bool)
+	return v
+}
+
+type isPresigningKey struct{}
+
+// AddAsIsPresigingMiddleware adds a middleware to the head of the stack that
+// will update the stack's context to be flagged as being invoked for the
+// purpose of presigning.
+func AddAsIsPresigingMiddleware(stack *middleware.Stack) error {
+	return stack.Initialize.Add(asIsPresigningMiddleware{}, middleware.Before)
+}
+
+type asIsPresigningMiddleware struct{}
+
+func (asIsPresigningMiddleware) ID() string { return "AsIsPresigningMiddleware" }
+
+func (asIsPresigningMiddleware) HandleInitialize(
+	ctx context.Context, in middleware.InitializeInput, next middleware.InitializeHandler,
+) (
+	out middleware.InitializeOutput, metadata middleware.Metadata, err error,
+) {
+	ctx = WithIsPresigning(ctx)
+	return next.HandleInitialize(ctx, in)
+}
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/doc.go b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/doc.go
new file mode 100644
index 0000000000..1b85375cf8
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/doc.go
@@ -0,0 +1,3 @@
+// Package presignedurl provides the customizations for API clients to fill in
+// presigned URLs into input parameters.
+package presignedurl
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/go_module_metadata.go b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/go_module_metadata.go
new file mode 100644
index 0000000000..e7168c626e
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/go_module_metadata.go
@@ -0,0 +1,6 @@
+// Code generated by internal/repotools/cmd/updatemodulemeta DO NOT EDIT.
+
+package presignedurl
+
+// goModuleVersion is the tagged release for this module
+const goModuleVersion = "1.10.0"
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/gotest/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/gotest/ya.make
new file mode 100644
index 0000000000..715df102eb
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/gotest/ya.make
@@ -0,0 +1,5 @@
+GO_TEST_FOR(vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url)
+
+LICENSE(Apache-2.0)
+
+END()
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/middleware.go b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/middleware.go
new file mode 100644
index 0000000000..1e2f5c8122
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/middleware.go
@@ -0,0 +1,110 @@
+package presignedurl
+
+import (
+	"context"
+	"fmt"
+
+	awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+
+	"github.com/aws/smithy-go/middleware"
+)
+
+// URLPresigner provides the interface to presign the input parameters in to a
+// presigned URL.
+type URLPresigner interface {
+	// PresignURL presigns a URL.
+	PresignURL(ctx context.Context, srcRegion string, params interface{}) (*v4.PresignedHTTPRequest, error)
+}
+
+// ParameterAccessor provides an collection of accessor to for retrieving and
+// setting the values needed to PresignedURL generation
+type ParameterAccessor struct {
+	// GetPresignedURL accessor points to a function that retrieves a presigned url if present
+	GetPresignedURL func(interface{}) (string, bool, error)
+
+	// GetSourceRegion accessor points to a function that retrieves source region for presigned url
+	GetSourceRegion func(interface{}) (string, bool, error)
+
+	// CopyInput accessor points to a function that takes in an input, and returns a copy.
+	CopyInput func(interface{}) (interface{}, error)
+
+	// SetDestinationRegion accessor points to a function that sets destination region on api input struct
+	SetDestinationRegion func(interface{}, string) error
+
+	// SetPresignedURL accessor points to a function that sets presigned url on api input struct
+	SetPresignedURL func(interface{}, string) error
+}
+
+// Options provides the set of options needed by the presigned URL middleware.
+type Options struct {
+	// Accessor are the parameter accessors used by this middleware
+	Accessor ParameterAccessor
+
+	// Presigner is the URLPresigner used by the middleware
+	Presigner URLPresigner
+}
+
+// AddMiddleware adds the Presign URL middleware to the middleware stack.
+func AddMiddleware(stack *middleware.Stack, opts Options) error {
+	return stack.Initialize.Add(&presign{options: opts}, middleware.Before)
+}
+
+// RemoveMiddleware removes the Presign URL middleware from the stack.
+func RemoveMiddleware(stack *middleware.Stack) error {
+	_, err := stack.Initialize.Remove((*presign)(nil).ID())
+	return err
+}
+
+type presign struct {
+	options Options
+}
+
+func (m *presign) ID() string { return "Presign" }
+
+func (m *presign) HandleInitialize(
+	ctx context.Context, input middleware.InitializeInput, next middleware.InitializeHandler,
+) (
+	out middleware.InitializeOutput, metadata middleware.Metadata, err error,
+) {
+	// If PresignedURL is already set ignore middleware.
+	if _, ok, err := m.options.Accessor.GetPresignedURL(input.Parameters); err != nil {
+		return out, metadata, fmt.Errorf("presign middleware failed, %w", err)
+	} else if ok {
+		return next.HandleInitialize(ctx, input)
+	}
+
+	// If have source region is not set ignore middleware.
+	srcRegion, ok, err := m.options.Accessor.GetSourceRegion(input.Parameters)
+	if err != nil {
+		return out, metadata, fmt.Errorf("presign middleware failed, %w", err)
+	} else if !ok || len(srcRegion) == 0 {
+		return next.HandleInitialize(ctx, input)
+	}
+
+	// Create a copy of the original input so the destination region value can
+	// be added. This ensures that value does not leak into the original
+	// request parameters.
+	paramCpy, err := m.options.Accessor.CopyInput(input.Parameters)
+	if err != nil {
+		return out, metadata, fmt.Errorf("unable to create presigned URL, %w", err)
+	}
+
+	// Destination region is the API client's configured region.
+	dstRegion := awsmiddleware.GetRegion(ctx)
+	if err = m.options.Accessor.SetDestinationRegion(paramCpy, dstRegion); err != nil {
+		return out, metadata, fmt.Errorf("presign middleware failed, %w", err)
+	}
+
+	presignedReq, err := m.options.Presigner.PresignURL(ctx, srcRegion, paramCpy)
+	if err != nil {
+		return out, metadata, fmt.Errorf("unable to create presigned URL, %w", err)
+	}
+
+	// Update the original input with the presigned URL value.
+	if err = m.options.Accessor.SetPresignedURL(input.Parameters, presignedReq.URL); err != nil {
+		return out, metadata, fmt.Errorf("presign middleware failed, %w", err)
+	}
+
+	return next.HandleInitialize(ctx, input)
+}
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/middleware_test.go b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/middleware_test.go
new file mode 100644
index 0000000000..56ba1d9205
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/middleware_test.go
@@ -0,0 +1,151 @@
+package presignedurl
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+
+	awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+
+	"github.com/aws/smithy-go/middleware"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestPresignMiddleware(t *testing.T) {
+	cases := map[string]struct {
+		Input *mockURLPresignInput
+
+		ExpectInput *mockURLPresignInput
+		ExpectErr   string
+	}{
+		"no source": {
+			Input:       &mockURLPresignInput{},
+			ExpectInput: &mockURLPresignInput{},
+		},
+		"with presigned URL": {
+			Input: &mockURLPresignInput{
+				SourceRegion: "source-region",
+				PresignedURL: "https://example.amazonaws.com/someURL",
+			},
+			ExpectInput: &mockURLPresignInput{
+				SourceRegion: "source-region",
+				PresignedURL: "https://example.amazonaws.com/someURL",
+			},
+		},
+		"with source": {
+			Input: &mockURLPresignInput{
+				SourceRegion: "source-region",
+			},
+			ExpectInput: &mockURLPresignInput{
+				SourceRegion: "source-region",
+				PresignedURL: "https://example.source-region.amazonaws.com/?DestinationRegion=mock-region",
+			},
+		},
+		"matching source destination region": {
+			Input: &mockURLPresignInput{
+				SourceRegion: "mock-region",
+			},
+			ExpectInput: &mockURLPresignInput{
+				SourceRegion: "mock-region",
+				PresignedURL: "https://example.mock-region.amazonaws.com/?DestinationRegion=mock-region",
+			},
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			stack := middleware.NewStack(name, smithyhttp.NewStackRequest)
+
+			stack.Initialize.Add(&awsmiddleware.RegisterServiceMetadata{
+				Region: "mock-region",
+			}, middleware.After)
+
+			stack.Initialize.Add(&presign{options: getURLPresignMiddlewareOptions()}, middleware.After)
+
+			stack.Initialize.Add(middleware.InitializeMiddlewareFunc(name+"_verifyParams",
+				func(ctx context.Context, in middleware.InitializeInput, next middleware.InitializeHandler) (
+					out middleware.InitializeOutput, metadata middleware.Metadata, err error,
+				) {
+					input := in.Parameters.(*mockURLPresignInput)
+					if diff := cmp.Diff(c.ExpectInput, input); len(diff) != 0 {
+						t.Errorf("expect input to be updated\n%s", diff)
+					}
+
+					return next.HandleInitialize(ctx, in)
+				},
+			), middleware.After)
+
+			handler := middleware.DecorateHandler(smithyhttp.NewClientHandler(smithyhttp.NopClient{}), stack)
+			_, _, err := handler.Handle(context.Background(), c.Input)
+			if len(c.ExpectErr) != 0 {
+				if err == nil {
+					t.Fatalf("expect error, got none")
+				}
+				if e, a := c.ExpectErr, err.Error(); !strings.Contains(a, e) {
+					t.Fatalf("expect error to contain %v, got %v", e, a)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("expect no error, got %v", err)
+			}
+		})
+	}
+}
+
+func getURLPresignMiddlewareOptions() Options {
+	return Options{
+		Accessor: ParameterAccessor{
+			GetPresignedURL: func(c interface{}) (string, bool, error) {
+				presignURL := c.(*mockURLPresignInput).PresignedURL
+				if len(presignURL) != 0 {
+					return presignURL, true, nil
+				}
+				return "", false, nil
+			},
+			GetSourceRegion: func(c interface{}) (string, bool, error) {
+				srcRegion := c.(*mockURLPresignInput).SourceRegion
+				if len(srcRegion) != 0 {
+					return srcRegion, true, nil
+				}
+				return "", false, nil
+			},
+			CopyInput: func(c interface{}) (interface{}, error) {
+				input := *(c.(*mockURLPresignInput))
+				return &input, nil
+			},
+			SetDestinationRegion: func(c interface{}, v string) error {
+				c.(*mockURLPresignInput).DestinationRegion = v
+				return nil
+			},
+			SetPresignedURL: func(c interface{}, v string) error {
+				c.(*mockURLPresignInput).PresignedURL = v
+				return nil
+			},
+		},
+		Presigner: &mockURLPresigner{},
+	}
+}
+
+type mockURLPresignInput struct {
+	SourceRegion      string
+	DestinationRegion string
+	PresignedURL      string
+}
+
+type mockURLPresigner struct{}
+
+func (*mockURLPresigner) PresignURL(ctx context.Context, srcRegion string, params interface{}) (
+	req *v4.PresignedHTTPRequest, err error,
+) {
+	in := params.(*mockURLPresignInput)
+
+	return &v4.PresignedHTTPRequest{
+		URL:          "https://example." + srcRegion + ".amazonaws.com/?DestinationRegion=" + in.DestinationRegion,
+		Method:       "GET",
+		SignedHeader: http.Header{},
+	}, nil
+}
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/ya.make
new file mode 100644
index 0000000000..266a0d246b
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/ya.make
@@ -0,0 +1,18 @@
+GO_LIBRARY()
+
+LICENSE(Apache-2.0)
+
+SRCS(
+    context.go
+    doc.go
+    go_module_metadata.go
+    middleware.go
+)
+
+GO_TEST_SRCS(middleware_test.go)
+
+END()
+
+RECURSE(
+    gotest
+)