YQ Connector: prepare code base for S3 integration

1. Кодовая база Коннектора переписана с помощью Go дженериков так, чтобы добавление нового источника данных (в частности S3 + csv) максимально переиспользовало имеющийся код (чтобы сохранялась логика нарезания на блоки данных, учёт трафика и пр.)
2. API Connector расширено для работы с S3, но ещё пока не протестировано.

4 files changed, 381 insertions, 0 deletions

diff --git a/vendor/github.com/aws/smithy-go/time/gotest/ya.make b/vendor/github.com/aws/smithy-go/time/gotest/ya.make
new file mode 100644
index 0000000000..c3db4332da
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/time/gotest/ya.make
@@ -0,0 +1,5 @@
+GO_TEST_FOR(vendor/github.com/aws/smithy-go/time)
+
+LICENSE(Apache-2.0)
+
+END()
diff --git a/vendor/github.com/aws/smithy-go/time/time.go b/vendor/github.com/aws/smithy-go/time/time.go
new file mode 100644
index 0000000000..b552a09f8a
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/time/time.go
@@ -0,0 +1,134 @@
+package time
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"strings"
+	"time"
+)
+
+const (
+	// dateTimeFormat is a IMF-fixdate formatted RFC3339 section 5.6
+	dateTimeFormatInput    = "2006-01-02T15:04:05.999999999Z"
+	dateTimeFormatInputNoZ = "2006-01-02T15:04:05.999999999"
+	dateTimeFormatOutput   = "2006-01-02T15:04:05.999Z"
+
+	// httpDateFormat is a date time defined by RFC 7231#section-7.1.1.1
+	// IMF-fixdate with no UTC offset.
+	httpDateFormat = "Mon, 02 Jan 2006 15:04:05 GMT"
+	// Additional formats needed for compatibility.
+	httpDateFormatSingleDigitDay             = "Mon, _2 Jan 2006 15:04:05 GMT"
+	httpDateFormatSingleDigitDayTwoDigitYear = "Mon, _2 Jan 06 15:04:05 GMT"
+)
+
+var millisecondFloat = big.NewFloat(1e3)
+
+// FormatDateTime formats value as a date-time, (RFC3339 section 5.6)
+//
+// Example: 1985-04-12T23:20:50.52Z
+func FormatDateTime(value time.Time) string {
+	return value.UTC().Format(dateTimeFormatOutput)
+}
+
+// ParseDateTime parses a string as a date-time, (RFC3339 section 5.6)
+//
+// Example: 1985-04-12T23:20:50.52Z
+func ParseDateTime(value string) (time.Time, error) {
+	return tryParse(value,
+		dateTimeFormatInput,
+		dateTimeFormatInputNoZ,
+		time.RFC3339Nano,
+		time.RFC3339,
+	)
+}
+
+// FormatHTTPDate formats value as a http-date, (RFC 7231#section-7.1.1.1 IMF-fixdate)
+//
+// Example: Tue, 29 Apr 2014 18:30:38 GMT
+func FormatHTTPDate(value time.Time) string {
+	return value.UTC().Format(httpDateFormat)
+}
+
+// ParseHTTPDate parses a string as a http-date, (RFC 7231#section-7.1.1.1 IMF-fixdate)
+//
+// Example: Tue, 29 Apr 2014 18:30:38 GMT
+func ParseHTTPDate(value string) (time.Time, error) {
+	return tryParse(value,
+		httpDateFormat,
+		httpDateFormatSingleDigitDay,
+		httpDateFormatSingleDigitDayTwoDigitYear,
+		time.RFC850,
+		time.ANSIC,
+	)
+}
+
+// FormatEpochSeconds returns value as a Unix time in seconds with with decimal precision
+//
+// Example: 1515531081.123
+func FormatEpochSeconds(value time.Time) float64 {
+	ms := value.UnixNano() / int64(time.Millisecond)
+	return float64(ms) / 1e3
+}
+
+// ParseEpochSeconds returns value as a Unix time in seconds with with decimal precision
+//
+// Example: 1515531081.123
+func ParseEpochSeconds(value float64) time.Time {
+	f := big.NewFloat(value)
+	f = f.Mul(f, millisecondFloat)
+	i, _ := f.Int64()
+	// Offset to `UTC` because time.Unix returns the time value based on system
+	// local setting.
+	return time.Unix(0, i*1e6).UTC()
+}
+
+func tryParse(v string, formats ...string) (time.Time, error) {
+	var errs parseErrors
+	for _, f := range formats {
+		t, err := time.Parse(f, v)
+		if err != nil {
+			errs = append(errs, parseError{
+				Format: f,
+				Err:    err,
+			})
+			continue
+		}
+		return t, nil
+	}
+
+	return time.Time{}, fmt.Errorf("unable to parse time string, %w", errs)
+}
+
+type parseErrors []parseError
+
+func (es parseErrors) Error() string {
+	var s strings.Builder
+	for _, e := range es {
+		fmt.Fprintf(&s, "\n * %q: %v", e.Format, e.Err)
+	}
+
+	return "parse errors:" + s.String()
+}
+
+type parseError struct {
+	Format string
+	Err    error
+}
+
+// SleepWithContext will wait for the timer duration to expire, or until the context
+// is canceled. Whichever happens first. If the context is canceled the
+// Context's error will be returned.
+func SleepWithContext(ctx context.Context, dur time.Duration) error {
+	t := time.NewTimer(dur)
+	defer t.Stop()
+
+	select {
+	case <-t.C:
+		break
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	return nil
+}
diff --git a/vendor/github.com/aws/smithy-go/time/time_test.go b/vendor/github.com/aws/smithy-go/time/time_test.go
new file mode 100644
index 0000000000..542e9ea552
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/time/time_test.go
@@ -0,0 +1,227 @@
+package time
+
+import (
+	"math"
+	"strconv"
+	"testing"
+	"time"
+)
+
+func TestDateTime(t *testing.T) {
+	cases := map[string]struct {
+		TimeString      string
+		TimeValue       time.Time
+		SymmetricString bool
+	}{
+		"no offset": {
+			TimeString: "1985-04-12T23:20:50.52Z",
+			TimeValue: time.Date(1985, 4, 12, 23, 20, 50, int(520*time.Millisecond),
+				time.UTC),
+			SymmetricString: true,
+		},
+		"no offset, no Z": {
+			TimeString: "1985-04-12T23:20:50.524",
+			TimeValue: time.Date(1985, 4, 12, 23, 20, 50, int(524*time.Millisecond),
+				time.UTC),
+			SymmetricString: false,
+		},
+		"with negative offset": {
+			TimeString: "1985-04-12T23:20:50.52-07:00",
+			TimeValue: time.Date(1985, 4, 12, 23, 20, 50, int(520*time.Millisecond),
+				time.FixedZone("-0700", -7*60*60)),
+			SymmetricString: false,
+		},
+		"with positive offset": {
+			TimeString: "1985-04-12T23:20:50.52+07:00",
+			TimeValue: time.Date(1985, 4, 12, 23, 20, 50, int(520*time.Millisecond),
+				time.FixedZone("-0700", +7*60*60)),
+			SymmetricString: false,
+		},
+		"UTC serialize": {
+			TimeString: "1985-04-13T06:20:50.52Z",
+			TimeValue: time.Date(1985, 4, 12, 23, 20, 50, int(520*time.Millisecond),
+				time.FixedZone("-0700", -7*60*60)),
+			SymmetricString: true,
+		},
+	}
+
+	for name, c := range cases {
+		t.Run(name, func(t *testing.T) {
+			formattedTimeValue := FormatDateTime(c.TimeValue)
+
+			// Round Trip time value ensure format and parse are compatible.
+			parsedTimeValue, err := ParseDateTime(formattedTimeValue)
+			if err != nil {
+				t.Fatalf("expected no error, got %v", err)
+			}
+
+			parsedTimeString, err := ParseDateTime(c.TimeString)
+			if err != nil {
+				t.Fatalf("expected no error, got %v", err)
+			}
+
+			// Ensure parsing date time from string matches expected time value.
+			if c.SymmetricString {
+				if e, a := c.TimeString, formattedTimeValue; e != a {
+					t.Errorf("expected %v, got %v", e, a)
+				}
+			}
+			if e, a := c.TimeValue, parsedTimeValue; !e.Equal(a) {
+				t.Errorf("expected %v, got %v", e, a)
+			}
+			if e, a := c.TimeValue, parsedTimeString; !e.Equal(a) {
+				t.Errorf("expected %v, got %v", e, a)
+			}
+		})
+	}
+}
+
+func TestHTTPDate(t *testing.T) {
+	refTime := time.Date(2014, 4, 29, 18, 30, 38, 0, time.UTC)
+
+	httpDate := FormatHTTPDate(refTime)
+	if e, a := "Tue, 29 Apr 2014 18:30:38 GMT", httpDate; e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	parseTime, err := ParseHTTPDate(httpDate)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	if e, a := refTime, parseTime; !e.Equal(a) {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+
+	// UTC serialize date time.
+	refTime = time.Date(2014, 4, 29, 18, 30, 38, 0, time.FixedZone("-700", -7*60*60))
+	httpDate = FormatHTTPDate(refTime)
+	if e, a := "Wed, 30 Apr 2014 01:30:38 GMT", httpDate; e != a {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+}
+
+func TestParseHTTPDate(t *testing.T) {
+	cases := map[string]struct {
+		date    string
+		expect  time.Time
+		wantErr bool
+	}{
+		"with leading zero on day": {
+			date:   "Fri, 05 Feb 2021 19:12:15 GMT",
+			expect: time.Date(2021, 2, 5, 19, 12, 15, 0, time.UTC),
+		},
+		"without leading zero on day": {
+			date:   "Fri, 5 Feb 2021 19:12:15 GMT",
+			expect: time.Date(2021, 2, 5, 19, 12, 15, 0, time.UTC),
+		},
+		"with double digit day": {
+			date:   "Fri, 15 Feb 2021 19:12:15 GMT",
+			expect: time.Date(2021, 2, 15, 19, 12, 15, 0, time.UTC),
+		},
+		"RFC850": {
+			date:   "Friday, 05-Feb-21 19:12:15 UTC",
+			expect: time.Date(2021, 2, 5, 19, 12, 15, 0, time.UTC),
+		},
+		"ANSIC with leading zero on day": {
+			date:   "Fri Feb 05 19:12:15 2021",
+			expect: time.Date(2021, 2, 5, 19, 12, 15, 0, time.UTC),
+		},
+		"ANSIC without leading zero on day": {
+			date:   "Fri Feb 5 19:12:15 2021",
+			expect: time.Date(2021, 2, 5, 19, 12, 15, 0, time.UTC),
+		},
+		"ANSIC with double digit day": {
+			date:   "Fri Feb 15 19:12:15 2021",
+			expect: time.Date(2021, 2, 15, 19, 12, 15, 0, time.UTC),
+		},
+		"invalid time format": {
+			date:    "1985-04-12T23:20:50.52Z",
+			wantErr: true,
+		},
+		"shortened year with double digit day": {
+			date:   "Thu, 11 Feb 21 11:04:03 GMT",
+			expect: time.Date(2021, 2, 11, 11, 04, 03, 0, time.UTC),
+		},
+		"shortened year without leading zero day": {
+			date:   "Thu, 5 Feb 21 11:04:03 GMT",
+			expect: time.Date(2021, 2, 5, 11, 04, 03, 0, time.UTC),
+		},
+		"shortened year with leading zero day": {
+			date:   "Thu, 05 Feb 21 11:04:03 GMT",
+			expect: time.Date(2021, 2, 5, 11, 04, 03, 0, time.UTC),
+		},
+	}
+
+	for name, tt := range cases {
+		t.Run(name, func(t *testing.T) {
+			result, err := ParseHTTPDate(tt.date)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("error = %v, wantErr = %v", err, tt.wantErr)
+			}
+			if err != nil {
+				return
+			}
+			if result.IsZero() {
+				t.Fatalf("expected non-zero timestamp")
+			}
+			if tt.expect != result {
+				t.Fatalf("expected %q, got %q", tt.expect, result)
+			}
+		})
+	}
+}
+
+func TestEpochSeconds(t *testing.T) {
+	cases := []struct {
+		reference    time.Time
+		expectedUnix float64
+		expectedTime time.Time
+	}{
+		{
+			reference:    time.Date(2018, 1, 9, 20, 51, 21, 123399936, time.UTC),
+			expectedUnix: 1515531081.123,
+			expectedTime: time.Date(2018, 1, 9, 20, 51, 21, 1.23e8, time.UTC),
+		},
+		{
+			reference:    time.Date(2018, 1, 9, 20, 51, 21, 1e8, time.UTC),
+			expectedUnix: 1515531081.1,
+			expectedTime: time.Date(2018, 1, 9, 20, 51, 21, 1e8, time.UTC),
+		},
+		{
+			reference:    time.Date(2018, 1, 9, 20, 51, 21, 123567891, time.UTC),
+			expectedUnix: 1515531081.123,
+			expectedTime: time.Date(2018, 1, 9, 20, 51, 21, 1.23e8, time.UTC),
+		},
+		{
+			reference:    time.Unix(0, math.MaxInt64).UTC(),
+			expectedUnix: 9223372036.854,
+			expectedTime: time.Date(2262, 04, 11, 23, 47, 16, 8.54e8, time.UTC),
+		},
+		{
+			reference:    time.Date(2018, 1, 9, 20, 51, 21, 123567891, time.FixedZone("-0700", -7*60*60)),
+			expectedUnix: 1515556281.123,
+			expectedTime: time.Date(2018, 1, 10, 03, 51, 21, 1.23e8, time.UTC),
+		},
+	}
+
+	for i, tt := range cases {
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			epochSeconds := FormatEpochSeconds(tt.reference)
+			if e, a := tt.expectedUnix, epochSeconds; e != a {
+				t.Errorf("expected %v, got %v", e, a)
+			}
+
+			parseTime := ParseEpochSeconds(epochSeconds)
+
+			if e, a := tt.expectedTime, parseTime; !e.Equal(a) {
+				t.Errorf("expected %v, got %v", e, a)
+			}
+		})
+	}
+
+	// Check an additional edge that higher precision values are truncated to milliseconds
+	if e, a := time.Date(2018, 1, 9, 20, 51, 21, 1.23e8, time.UTC), ParseEpochSeconds(1515531081.12356); !e.Equal(a) {
+		t.Errorf("expected %v, got %v", e, a)
+	}
+}
diff --git a/vendor/github.com/aws/smithy-go/time/ya.make b/vendor/github.com/aws/smithy-go/time/ya.make
new file mode 100644
index 0000000000..b9981746e5
--- /dev/null
+++ b/vendor/github.com/aws/smithy-go/time/ya.make
@@ -0,0 +1,15 @@
+GO_LIBRARY()
+
+LICENSE(Apache-2.0)
+
+SRCS(
+    time.go
+)
+
+GO_TEST_SRCS(time_test.go)
+
+END()
+
+RECURSE(
+    gotest
+)