YQ Connector: prepare code base for S3 integration

1. Кодовая база Коннектора переписана с помощью Go дженериков так, чтобы добавление нового источника данных (в частности S3 + csv) максимально переиспользовало имеющийся код (чтобы сохранялась логика нарезания на блоки данных, учёт трафика и пр.)
2. API Connector расширено для работы с S3, но ещё пока не протестировано.

6 files changed, 441 insertions, 0 deletions

diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/accept_encoding_gzip.go b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/accept_encoding_gzip.go
new file mode 100644
index 0000000000..3f451fc9b4
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/accept_encoding_gzip.go
@@ -0,0 +1,176 @@
+package acceptencoding
+
+import (
+	"compress/gzip"
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/aws/smithy-go"
+	"github.com/aws/smithy-go/middleware"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+)
+
+const acceptEncodingHeaderKey = "Accept-Encoding"
+const contentEncodingHeaderKey = "Content-Encoding"
+
+// AddAcceptEncodingGzipOptions provides the options for the
+// AddAcceptEncodingGzip middleware setup.
+type AddAcceptEncodingGzipOptions struct {
+	Enable bool
+}
+
+// AddAcceptEncodingGzip explicitly adds handling for accept-encoding GZIP
+// middleware to the operation stack. This allows checksums to be correctly
+// computed without disabling GZIP support.
+func AddAcceptEncodingGzip(stack *middleware.Stack, options AddAcceptEncodingGzipOptions) error {
+	if options.Enable {
+		if err := stack.Finalize.Add(&EnableGzip{}, middleware.Before); err != nil {
+			return err
+		}
+		if err := stack.Deserialize.Insert(&DecompressGzip{}, "OperationDeserializer", middleware.After); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	return stack.Finalize.Add(&DisableGzip{}, middleware.Before)
+}
+
+// DisableGzip provides the middleware that will
+// disable the underlying http client automatically enabling for gzip
+// decompress content-encoding support.
+type DisableGzip struct{}
+
+// ID returns the id for the middleware.
+func (*DisableGzip) ID() string {
+	return "DisableAcceptEncodingGzip"
+}
+
+// HandleFinalize implements the FinalizeMiddleware interface.
+func (*DisableGzip) HandleFinalize(
+	ctx context.Context, input middleware.FinalizeInput, next middleware.FinalizeHandler,
+) (
+	output middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	req, ok := input.Request.(*smithyhttp.Request)
+	if !ok {
+		return output, metadata, &smithy.SerializationError{
+			Err: fmt.Errorf("unknown request type %T", input.Request),
+		}
+	}
+
+	// Explicitly enable gzip support, this will prevent the http client from
+	// auto extracting the zipped content.
+	req.Header.Set(acceptEncodingHeaderKey, "identity")
+
+	return next.HandleFinalize(ctx, input)
+}
+
+// EnableGzip provides a middleware to enable support for
+// gzip responses, with manual decompression. This prevents the underlying HTTP
+// client from performing the gzip decompression automatically.
+type EnableGzip struct{}
+
+// ID returns the id for the middleware.
+func (*EnableGzip) ID() string {
+	return "AcceptEncodingGzip"
+}
+
+// HandleFinalize implements the FinalizeMiddleware interface.
+func (*EnableGzip) HandleFinalize(
+	ctx context.Context, input middleware.FinalizeInput, next middleware.FinalizeHandler,
+) (
+	output middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+) {
+	req, ok := input.Request.(*smithyhttp.Request)
+	if !ok {
+		return output, metadata, &smithy.SerializationError{
+			Err: fmt.Errorf("unknown request type %T", input.Request),
+		}
+	}
+
+	// Explicitly enable gzip support, this will prevent the http client from
+	// auto extracting the zipped content.
+	req.Header.Set(acceptEncodingHeaderKey, "gzip")
+
+	return next.HandleFinalize(ctx, input)
+}
+
+// DecompressGzip provides the middleware for decompressing a gzip
+// response from the service.
+type DecompressGzip struct{}
+
+// ID returns the id for the middleware.
+func (*DecompressGzip) ID() string {
+	return "DecompressGzip"
+}
+
+// HandleDeserialize implements the DeserializeMiddlware interface.
+func (*DecompressGzip) HandleDeserialize(
+	ctx context.Context, input middleware.DeserializeInput, next middleware.DeserializeHandler,
+) (
+	output middleware.DeserializeOutput, metadata middleware.Metadata, err error,
+) {
+	output, metadata, err = next.HandleDeserialize(ctx, input)
+	if err != nil {
+		return output, metadata, err
+	}
+
+	resp, ok := output.RawResponse.(*smithyhttp.Response)
+	if !ok {
+		return output, metadata, &smithy.DeserializationError{
+			Err: fmt.Errorf("unknown response type %T", output.RawResponse),
+		}
+	}
+	if v := resp.Header.Get(contentEncodingHeaderKey); v != "gzip" {
+		return output, metadata, err
+	}
+
+	// Clear content length since it will no longer be valid once the response
+	// body is decompressed.
+	resp.Header.Del("Content-Length")
+	resp.ContentLength = -1
+
+	resp.Body = wrapGzipReader(resp.Body)
+
+	return output, metadata, err
+}
+
+type gzipReader struct {
+	reader io.ReadCloser
+	gzip   *gzip.Reader
+}
+
+func wrapGzipReader(reader io.ReadCloser) *gzipReader {
+	return &gzipReader{
+		reader: reader,
+	}
+}
+
+// Read wraps the gzip reader around the underlying io.Reader to extract the
+// response bytes on the fly.
+func (g *gzipReader) Read(b []byte) (n int, err error) {
+	if g.gzip == nil {
+		g.gzip, err = gzip.NewReader(g.reader)
+		if err != nil {
+			g.gzip = nil // ensure uninitialized gzip value isn't used in close.
+			return 0, fmt.Errorf("failed to decompress gzip response, %w", err)
+		}
+	}
+
+	return g.gzip.Read(b)
+}
+
+func (g *gzipReader) Close() error {
+	if g.gzip == nil {
+		return nil
+	}
+
+	if err := g.gzip.Close(); err != nil {
+		g.reader.Close()
+		return fmt.Errorf("failed to decompress gzip response, %w", err)
+	}
+
+	return g.reader.Close()
+}
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/accept_encoding_gzip_test.go b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/accept_encoding_gzip_test.go
new file mode 100644
index 0000000000..71fb7e26d5
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/accept_encoding_gzip_test.go
@@ -0,0 +1,215 @@
+package acceptencoding
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/hex"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/aws/smithy-go/middleware"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+)
+
+func TestAddAcceptEncodingGzip(t *testing.T) {
+	cases := map[string]struct {
+		Enable bool
+	}{
+		"disabled": {
+			Enable: false,
+		},
+		"enabled": {
+			Enable: true,
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			stack := middleware.NewStack("test", smithyhttp.NewStackRequest)
+
+			stack.Deserialize.Add(&stubOpDeserializer{}, middleware.After)
+
+			AddAcceptEncodingGzip(stack, AddAcceptEncodingGzipOptions{
+				Enable: c.Enable,
+			})
+
+			id := "OperationDeserializer"
+			if m, ok := stack.Deserialize.Get(id); !ok || m == nil {
+				t.Fatalf("expect %s not to be removed", id)
+			}
+
+			if c.Enable {
+				id = (*EnableGzip)(nil).ID()
+				if m, ok := stack.Finalize.Get(id); !ok || m == nil {
+					t.Fatalf("expect %s to be present.", id)
+				}
+
+				id = (*DecompressGzip)(nil).ID()
+				if m, ok := stack.Deserialize.Get(id); !ok || m == nil {
+					t.Fatalf("expect %s to be present.", id)
+				}
+				return
+			}
+			id = (*EnableGzip)(nil).ID()
+			if m, ok := stack.Finalize.Get(id); ok || m != nil {
+				t.Fatalf("expect %s not to be present.", id)
+			}
+
+			id = (*DecompressGzip)(nil).ID()
+			if m, ok := stack.Deserialize.Get(id); ok || m != nil {
+				t.Fatalf("expect %s not to be present.", id)
+			}
+		})
+	}
+}
+
+func TestAcceptEncodingGzipMiddleware(t *testing.T) {
+	m := &EnableGzip{}
+
+	_, _, err := m.HandleFinalize(context.Background(),
+		middleware.FinalizeInput{
+			Request: smithyhttp.NewStackRequest(),
+		},
+		middleware.FinalizeHandlerFunc(
+			func(ctx context.Context, input middleware.FinalizeInput) (
+				output middleware.FinalizeOutput, metadata middleware.Metadata, err error,
+			) {
+				req, ok := input.Request.(*smithyhttp.Request)
+				if !ok || req == nil {
+					t.Fatalf("expect smithy request, got %T", input.Request)
+				}
+
+				actual := req.Header.Get(acceptEncodingHeaderKey)
+				if e, a := "gzip", actual; e != a {
+					t.Errorf("expect %v accept-encoding, got %v", e, a)
+				}
+
+				return output, metadata, err
+			}),
+	)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+}
+
+func TestDecompressGzipMiddleware(t *testing.T) {
+	cases := map[string]struct {
+		Response            *smithyhttp.Response
+		ExpectBody          []byte
+		ExpectContentLength int64
+	}{
+		"not compressed": {
+			Response: &smithyhttp.Response{
+				Response: &http.Response{
+					StatusCode:    200,
+					Header:        http.Header{},
+					ContentLength: 2,
+					Body: &wasClosedReadCloser{
+						Reader: bytes.NewBuffer([]byte(`{}`)),
+					},
+				},
+			},
+			ExpectBody:          []byte(`{}`),
+			ExpectContentLength: 2,
+		},
+		"compressed": {
+			Response: &smithyhttp.Response{
+				Response: &http.Response{
+					StatusCode: 200,
+					Header: http.Header{
+						contentEncodingHeaderKey: []string{"gzip"},
+					},
+					ContentLength: 10,
+					Body: func() io.ReadCloser {
+						var buf bytes.Buffer
+						w := gzip.NewWriter(&buf)
+						w.Write([]byte(`{}`))
+						w.Close()
+
+						return &wasClosedReadCloser{Reader: &buf}
+					}(),
+				},
+			},
+			ExpectBody:          []byte(`{}`),
+			ExpectContentLength: -1, // Length empty because was decompressed
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			m := &DecompressGzip{}
+
+			var origRespBody io.Reader
+			output, _, err := m.HandleDeserialize(context.Background(),
+				middleware.DeserializeInput{},
+				middleware.DeserializeHandlerFunc(
+					func(ctx context.Context, input middleware.DeserializeInput) (
+						output middleware.DeserializeOutput, metadata middleware.Metadata, err error,
+					) {
+						output.RawResponse = c.Response
+						origRespBody = c.Response.Body
+						return output, metadata, err
+					}),
+			)
+			if err != nil {
+				t.Fatalf("expect no error, got %v", err)
+			}
+
+			resp, ok := output.RawResponse.(*smithyhttp.Response)
+			if !ok || resp == nil {
+				t.Fatalf("expect smithy request, got %T", output.RawResponse)
+			}
+
+			if e, a := c.ExpectContentLength, resp.ContentLength; e != a {
+				t.Errorf("expect %v content-length, got %v", e, a)
+			}
+
+			actual, err := ioutil.ReadAll(resp.Body)
+			if e, a := c.ExpectBody, actual; !bytes.Equal(e, a) {
+				t.Errorf("expect body equal\nexpect:\n%s\nactual:\n%s",
+					hex.Dump(e), hex.Dump(a))
+			}
+
+			if err := resp.Body.Close(); err != nil {
+				t.Fatalf("expect no close error, got %v", err)
+			}
+
+			if c, ok := origRespBody.(interface{ WasClosed() bool }); ok {
+				if !c.WasClosed() {
+					t.Errorf("expect original reader closed, but was not")
+				}
+			}
+		})
+	}
+}
+
+type stubOpDeserializer struct{}
+
+func (*stubOpDeserializer) ID() string { return "OperationDeserializer" }
+func (*stubOpDeserializer) HandleDeserialize(
+	ctx context.Context, input middleware.DeserializeInput, next middleware.DeserializeHandler,
+) (
+	output middleware.DeserializeOutput, metadata middleware.Metadata, err error,
+) {
+	return next.HandleDeserialize(ctx, input)
+}
+
+type wasClosedReadCloser struct {
+	io.Reader
+	closed bool
+}
+
+func (c *wasClosedReadCloser) WasClosed() bool {
+	return c.closed
+}
+
+func (c *wasClosedReadCloser) Close() error {
+	c.closed = true
+	if v, ok := c.Reader.(io.Closer); ok {
+		return v.Close()
+	}
+	return nil
+}
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/doc.go b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/doc.go
new file mode 100644
index 0000000000..7056d9bf6f
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/doc.go
@@ -0,0 +1,22 @@
+/*
+Package acceptencoding provides customizations associated with Accept Encoding Header.
+
+# Accept encoding gzip
+
+The Go HTTP client automatically supports accept-encoding and content-encoding
+gzip by default. This default behavior is not desired by the SDK, and prevents
+validating the response body's checksum. To prevent this the SDK must manually
+control usage of content-encoding gzip.
+
+To control content-encoding, the SDK must always set the `Accept-Encoding`
+header to a value. This prevents the HTTP client from using gzip automatically.
+When gzip is enabled on the API client, the SDK's customization will control
+decompressing the gzip data in order to not break the checksum validation. When
+gzip is disabled, the API client will disable gzip, preventing the HTTP
+client's default behavior.
+
+An `EnableAcceptEncodingGzip` option may or may not be present depending on the client using
+the below middleware. The option if present can be used to enable auto decompressing
+gzip by the SDK.
+*/
+package acceptencoding
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/go_module_metadata.go b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/go_module_metadata.go
new file mode 100644
index 0000000000..e57bbab9fe
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/go_module_metadata.go
@@ -0,0 +1,6 @@
+// Code generated by internal/repotools/cmd/updatemodulemeta DO NOT EDIT.
+
+package acceptencoding
+
+// goModuleVersion is the tagged release for this module
+const goModuleVersion = "1.10.0"
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/gotest/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/gotest/ya.make
new file mode 100644
index 0000000000..87dff7ba39
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/gotest/ya.make
@@ -0,0 +1,5 @@
+GO_TEST_FOR(vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding)
+
+LICENSE(Apache-2.0)
+
+END()
diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/ya.make b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/ya.make
new file mode 100644
index 0000000000..9e18eb7a5a
--- /dev/null
+++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding/ya.make
@@ -0,0 +1,17 @@
+GO_LIBRARY()
+
+LICENSE(Apache-2.0)
+
+SRCS(
+    accept_encoding_gzip.go
+    doc.go
+    go_module_metadata.go
+)
+
+GO_TEST_SRCS(accept_encoding_gzip_test.go)
+
+END()
+
+RECURSE(
+    gotest
+)