YQ Connector: prepare code base for S3 integration

1. Кодовая база Коннектора переписана с помощью Go дженериков так, чтобы добавление нового источника данных (в частности S3 + csv) максимально переиспользовало имеющийся код (чтобы сохранялась логика нарезания на блоки данных, учёт трафика и пр.)
2. API Connector расширено для работы с S3, но ещё пока не протестировано.

10 files changed, 495 insertions, 0 deletions

diff --git a/vendor/github.com/aws/smithy-go/endpoints/endpoint.go b/vendor/github.com/aws/smithy-go/endpoints/endpoint.go
new file mode 100644
index 0000000000..a935283974
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/endpoints/endpoint.go
@@ -0,0 +1,23 @@
+package transport
+
+import (
+	"net/http"
+	"net/url"
+
+	"github.com/aws/smithy-go"
+)
+
+// Endpoint is the endpoint object returned by Endpoint resolution V2
+type Endpoint struct {
+	// The complete URL minimally specfiying the scheme and host.
+	// May optionally specify the port and base path component.
+	URI url.URL
+
+	// An optional set of headers to be sent using transport layer headers.
+	Headers http.Header
+
+	// A grab-bag property map of endpoint attributes. The
+	// values present here are subject to change, or being add/removed at any
+	// time.
+	Properties smithy.Properties
+}
diff --git a/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/doc.go b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/doc.go
new file mode 100644
index 0000000000..e24e190dca
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/doc.go
@@ -0,0 +1,4 @@
+// Package rulesfn provides endpoint rule functions for evaluating endpoint
+// resolution rules.
+
+package rulesfn
diff --git a/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/gotest/ya.make b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/gotest/ya.make
new file mode 100644
index 0000000000..a6cfa83953
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/gotest/ya.make
@@ -0,0 +1,5 @@
+GO_TEST_FOR(vendor/github.com/aws/smithy-go/endpoints/private/rulesfn)
+
+LICENSE(Apache-2.0)
+
+END()
diff --git a/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/strings.go b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/strings.go
new file mode 100644
index 0000000000..8e230783e5
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/strings.go
@@ -0,0 +1,26 @@
+package rulesfn
+
+
+// Substring returns the substring of the input provided. If the start or stop
+// indexes are not valid for the input nil will be returned. If errors occur
+// they will be added to the provided [ErrorCollector].
+func SubString(input string, start, stop int, reverse bool) *string {
+	if start < 0 || stop < 1 || start >= stop || len(input) < stop {
+		return nil
+	}
+
+	for _, r := range input {
+		if r > 127 {
+			return nil
+		}
+	}
+
+	if !reverse {
+		v := input[start:stop]
+		return &v
+	}
+
+	rStart := len(input) - stop
+	rStop := len(input) - start
+	return SubString(input, rStart, rStop, false)
+}
diff --git a/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/strings_test.go b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/strings_test.go
new file mode 100644
index 0000000000..afa2ed5723
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/strings_test.go
@@ -0,0 +1,64 @@
+package rulesfn
+
+import (
+	"testing"
+	"github.com/aws/smithy-go/ptr"
+)
+
+func TestSubString(t *testing.T) {
+	cases := map[string]struct {
+		input       string
+		start, stop int
+		reverse     bool
+		expect      *string
+	}{
+		"prefix": {
+			input: "abcde", start: 0, stop: 3, reverse: false,
+			expect: ptr.String("abc"),
+		},
+		"prefix max-ascii": {
+			input: "abcde\u007F", start: 0, stop: 3, reverse: false,
+			expect: ptr.String("abc"),
+		},
+		"suffix reverse": {
+			input: "abcde", start: 0, stop: 3, reverse: true,
+			expect: ptr.String("cde"),
+		},
+		"too long": {
+			input: "ab", start: 0, stop: 3, reverse: false,
+			expect: nil,
+		},
+		"invalid start index": {
+			input: "ab", start: -1, stop: 3, reverse: false,
+			expect: nil,
+		},
+		"invalid stop index": {
+			input: "ab", start: 0, stop: 0, reverse: false,
+			expect: nil,
+		},
+		"non-ascii": {
+			input: "ab🐱", start: 0, stop: 1, reverse: false,
+			expect: nil,
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			actual := SubString(c.input, c.start, c.stop, c.reverse)
+			if c.expect == nil {
+				if actual != nil {
+					t.Fatalf("expect no result, got %v", *actual)
+				}
+				return
+			}
+
+			if actual == nil {
+				t.Fatalf("expect result, got none")
+			}
+
+			if e, a := *c.expect, *actual; e != a {
+				t.Errorf("expect %q, got %q", e, a)
+			}
+		})
+	}
+}
diff --git a/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/uri.go b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/uri.go
new file mode 100644
index 0000000000..0c11541276
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/uri.go
@@ -0,0 +1,130 @@
+package rulesfn
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"strings"
+
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+)
+
+// IsValidHostLabel returns if the input is a single valid [RFC 1123] host
+// label. If allowSubDomains is true, will allow validation to include nested
+// host labels. Returns false if the input is not a valid host label. If errors
+// occur they will be added to the provided [ErrorCollector].
+//
+// [RFC 1123]: https://www.ietf.org/rfc/rfc1123.txt
+func IsValidHostLabel(input string, allowSubDomains bool) bool {
+	var labels []string
+	if allowSubDomains {
+		labels = strings.Split(input, ".")
+	} else {
+		labels = []string{input}
+	}
+
+	for _, label := range labels {
+		if !smithyhttp.ValidHostLabel(label) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// ParseURL returns a [URL] if the provided string could be parsed. Returns nil
+// if the string could not be parsed. Any parsing error will be added to the
+// [ErrorCollector].
+//
+// If the input URL string contains an IP6 address with a zone index. The
+// returned [builtin.URL.Authority] value will contain the percent escaped (%)
+// zone index separator.
+func ParseURL(input string) *URL {
+	u, err := url.Parse(input)
+	if err != nil {
+		return nil
+	}
+
+	if u.RawQuery != "" {
+		return nil
+	}
+
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return nil
+	}
+
+	normalizedPath := u.Path
+	if !strings.HasPrefix(normalizedPath, "/") {
+		normalizedPath = "/" + normalizedPath
+	}
+	if !strings.HasSuffix(normalizedPath, "/") {
+		normalizedPath = normalizedPath + "/"
+	}
+
+	// IP6 hosts may have zone indexes that need to be escaped to be valid in a
+	// URI. The Go URL parser will unescape the `%25` into `%`. This needs to
+	// be reverted since the returned URL will be used in string builders.
+	authority := strings.ReplaceAll(u.Host, "%", "%25")
+
+	return &URL{
+		Scheme:         u.Scheme,
+		Authority:      authority,
+		Path:           u.Path,
+		NormalizedPath: normalizedPath,
+		IsIp:           net.ParseIP(hostnameWithoutZone(u)) != nil,
+	}
+}
+
+// URL provides the structure describing the parts of a parsed URL returned by
+// [ParseURL].
+type URL struct {
+	Scheme         string // https://www.rfc-editor.org/rfc/rfc3986#section-3.1
+	Authority      string // https://www.rfc-editor.org/rfc/rfc3986#section-3.2
+	Path           string // https://www.rfc-editor.org/rfc/rfc3986#section-3.3
+	NormalizedPath string // https://www.rfc-editor.org/rfc/rfc3986#section-6.2.3
+	IsIp           bool
+}
+
+// URIEncode returns an percent-encoded [RFC3986 section 2.1] version of the
+// input string.
+//
+// [RFC3986 section 2.1]: https://www.rfc-editor.org/rfc/rfc3986#section-2.1
+func URIEncode(input string) string {
+	var output strings.Builder
+	for _, c := range []byte(input) {
+		if validPercentEncodedChar(c) {
+			output.WriteByte(c)
+			continue
+		}
+
+		fmt.Fprintf(&output, "%%%X", c)
+	}
+
+	return output.String()
+}
+
+func validPercentEncodedChar(c byte) bool {
+	return (c >= 'a' && c <= 'z') ||
+		(c >= 'A' && c <= 'Z') ||
+		(c >= '0' && c <= '9') ||
+		c == '-' || c == '_' || c == '.' || c == '~'
+}
+
+// hostname implements u.Hostname() but strips the ipv6 zone ID (if present)
+// such that net.ParseIP can still recognize IPv6 addresses with zone IDs.
+//
+// FUTURE(10/2023): netip.ParseAddr handles this natively but we can't take
+// that package as a dependency yet due to our min go version (1.15, netip
+// starts in 1.18).  When we align with go runtime deprecation policy in
+// 10/2023, we can remove this.
+func hostnameWithoutZone(u *url.URL) string {
+	full := u.Hostname()
+
+	// this more or less mimics the internals of net/ (see unexported
+	// splitHostZone in that source) but throws the zone away because we don't
+	// need it
+	if i := strings.LastIndex(full, "%"); i > -1 {
+		return full[:i]
+	}
+	return full
+}
diff --git a/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/uri_test.go b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/uri_test.go
new file mode 100644
index 0000000000..baa9aaf936
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/uri_test.go
@@ -0,0 +1,207 @@
+package rulesfn
+
+import (
+	"testing"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestURIEncode(t *testing.T) {
+	cases := map[string]struct {
+		input  string
+		expect string
+	}{
+		"no encoding": {
+			input:  "a-zA-Z0-9-_.~",
+			expect: "a-zA-Z0-9-_.~",
+		},
+		"with encoding": {
+			input:  "🐛 becomes 🦋",
+			expect: "%F0%9F%90%9B%20becomes%20%F0%9F%A6%8B",
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			actual := URIEncode(c.input)
+			if e, a := c.expect, actual; e != a {
+				t.Errorf("expect `%v` encoding, got `%v`", e, a)
+			}
+		})
+	}
+}
+
+func TestParseURL(t *testing.T) {
+	cases := map[string]struct {
+		input     string
+		expect    *URL
+	}{
+		"https hostname with no path": {
+			input: "https://example.com",
+			expect: &URL{
+				Scheme:         "https",
+				Authority:      "example.com",
+				Path:           "",
+				NormalizedPath: "/",
+			},
+		},
+		"http hostname with no path": {
+			input: "http://example.com",
+			expect: &URL{
+				Scheme:         "http",
+				Authority:      "example.com",
+				Path:           "",
+				NormalizedPath: "/",
+			},
+		},
+		"https hostname with port with path": {
+			input: "https://example.com:80/foo/bar",
+			expect: &URL{
+				Scheme:         "https",
+				Authority:      "example.com:80",
+				Path:           "/foo/bar",
+				NormalizedPath: "/foo/bar/",
+			},
+		},
+		"invalid port": {
+			input:     "https://example.com:abc",
+			expect: nil,
+		},
+		"with query": {
+			input:     "https://example.com:8443?foo=bar&faz=baz",
+			expect: nil,
+		},
+		"ip4 URL": {
+			input: "https://127.0.0.1",
+			expect: &URL{
+				Scheme:         "https",
+				Authority:      "127.0.0.1",
+				Path:           "",
+				NormalizedPath: "/",
+				IsIp:           true,
+			},
+		},
+		"ip4 URL with port": {
+			input: "https://127.0.0.1:8443",
+			expect: &URL{
+				Scheme:         "https",
+				Authority:      "127.0.0.1:8443",
+				Path:           "",
+				NormalizedPath: "/",
+				IsIp:           true,
+			},
+		},
+		"ip6 short": {
+			input: "https://[fe80::1]",
+			expect: &URL{
+				Scheme:         "https",
+				Authority:      "[fe80::1]",
+				Path:           "",
+				NormalizedPath: "/",
+				IsIp:           true,
+			},
+		},
+		"ip6 short with interface": {
+			input: "https://[fe80::1%25en0]",
+			expect: &URL{
+				Scheme:         "https",
+				Authority:      "[fe80::1%25en0]",
+				Path:           "",
+				NormalizedPath: "/",
+				IsIp:           true,
+			},
+		},
+		"ip6 short with port": {
+			input: "https://[fe80::1]:8443",
+			expect: &URL{
+				Scheme:         "https",
+				Authority:      "[fe80::1]:8443",
+				Path:           "",
+				NormalizedPath: "/",
+				IsIp:           true,
+			},
+		},
+		"ip6 short with port with interface": {
+			input: "https://[fe80::1%25en0]:8443",
+			expect: &URL{
+				Scheme:         "https",
+				Authority:      "[fe80::1%25en0]:8443",
+				Path:           "",
+				NormalizedPath: "/",
+				IsIp:           true,
+			},
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			actual := ParseURL(c.input)
+			if c.expect == nil {
+				if actual != nil {
+					t.Fatalf("expect no result, got %v", *actual)
+				}
+				return
+			}
+
+			if actual == nil {
+				t.Fatalf("expect result, got none")
+			}
+
+			if diff := cmp.Diff(c.expect, actual); diff != "" {
+				t.Errorf("expect URL to match\n%s", diff)
+			}
+		})
+	}
+}
+
+func TestIsValidHostLabel(t *testing.T) {
+	cases := map[string]struct {
+		input           string
+		allowSubDomains bool
+		expect          bool
+	}{
+		"single label no split": {
+			input:  "abc123-",
+			expect: true,
+		},
+		"single label with split": {
+			input:           "abc123-",
+			allowSubDomains: true,
+			expect:          true,
+		},
+		"multiple labels no split": {
+			input:     "abc.123-",
+			expect: false,
+		},
+		"multiple labels with split": {
+			input:           "abc.123-",
+			allowSubDomains: true,
+			expect:          true,
+		},
+		"multiple labels with split invalid label": {
+			input:           "abc.123-...",
+			allowSubDomains: true,
+			expect: false,
+		},
+		"max length host label": {
+			input:  "012345678901234567890123456789012345678901234567890123456789123",
+			expect: true,
+		},
+		"too large host label": {
+			input:     "0123456789012345678901234567890123456789012345678901234567891234",
+			expect: false,
+		},
+		"too small host label": {
+			input:     "",
+			expect: false,
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			actual := IsValidHostLabel(c.input, c.allowSubDomains)
+			if e, a := c.expect, actual; e != a {
+				t.Fatalf("expect %v valid host label, got %v", e, a)
+			}
+		})
+	}
+}
diff --git a/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/ya.make b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/ya.make
new file mode 100644
index 0000000000..4b665b9ddd
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/endpoints/private/rulesfn/ya.make
@@ -0,0 +1,20 @@
+GO_LIBRARY()
+
+LICENSE(Apache-2.0)
+
+SRCS(
+    doc.go
+    strings.go
+    uri.go
+)
+
+GO_TEST_SRCS(
+    strings_test.go
+    uri_test.go
+)
+
+END()
+
+RECURSE(
+    gotest
+)
diff --git a/vendor/github.com/aws/smithy-go/endpoints/private/ya.make b/vendor/github.com/aws/smithy-go/endpoints/private/ya.make
new file mode 100644
index 0000000000..c1959ab928
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/endpoints/private/ya.make
@@ -0,0 +1,3 @@
+RECURSE(
+    rulesfn
+)
diff --git a/vendor/github.com/aws/smithy-go/endpoints/ya.make b/vendor/github.com/aws/smithy-go/endpoints/ya.make
new file mode 100644
index 0000000000..cc1698e9bb
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/endpoints/ya.make
@@ -0,0 +1,13 @@
+GO_LIBRARY()
+
+LICENSE(Apache-2.0)
+
+SRCS(
+    endpoint.go
+)
+
+END()
+
+RECURSE(
+    private
+)