diff options
author | uzhas <uzhas@ydb.tech> | 2023-08-24 17:35:21 +0300 |
---|---|---|
committer | uzhas <uzhas@ydb.tech> | 2023-08-24 17:53:39 +0300 |
commit | de6e39881d059d67cbcc978d076d9e3e5e9732fc (patch) | |
tree | cdf4e77c0156fe7f192d644883954f302fc56c01 /library/go/core | |
parent | b890c9f4f00efbc099a862b70c1dbc4c7db3dd2f (diff) | |
download | ydb-de6e39881d059d67cbcc978d076d9e3e5e9732fc.tar.gz |
move yql connector to ydb
move code
Diffstat (limited to 'library/go/core')
89 files changed, 7570 insertions, 0 deletions
diff --git a/library/go/core/log/compat/golog/log.go b/library/go/core/log/compat/golog/log.go new file mode 100644 index 0000000000..4ebe864234 --- /dev/null +++ b/library/go/core/log/compat/golog/log.go @@ -0,0 +1,21 @@ +package golog + +import ( + canal_log "github.com/siddontang/go-log/log" + "github.com/ydb-platform/ydb/library/go/core/log" +) + +func SetLevel(level log.Level) { + switch level { + case log.DebugLevel: + canal_log.SetLevel(canal_log.LevelDebug) + case log.ErrorLevel: + canal_log.SetLevel(canal_log.LevelError) + case log.FatalLevel: + canal_log.SetLevel(canal_log.LevelFatal) + case log.InfoLevel: + canal_log.SetLevel(canal_log.LevelInfo) + case log.TraceLevel: + canal_log.SetLevel(canal_log.LevelTrace) + } +} diff --git a/library/go/core/log/compat/golog/ya.make b/library/go/core/log/compat/golog/ya.make new file mode 100644 index 0000000000..19240f7378 --- /dev/null +++ b/library/go/core/log/compat/golog/ya.make @@ -0,0 +1,5 @@ +GO_LIBRARY() + +SRCS(log.go) + +END() diff --git a/library/go/core/log/compat/logrus/log.go b/library/go/core/log/compat/logrus/log.go new file mode 100644 index 0000000000..a92e52dda0 --- /dev/null +++ b/library/go/core/log/compat/logrus/log.go @@ -0,0 +1,202 @@ +package logrus + +import ( + "io" + "runtime" + "strings" + "sync" + + "github.com/sirupsen/logrus" + "github.com/ydb-platform/ydb/library/go/core/log" +) + +/* Call frame calculations are copied from logrus package */ +var ( + + // qualified package name, cached at first use + logrusPackage string + + // Positions in the call stack when tracing to report the calling method + minimumCallerDepth int + + // Used for caller information initialisation + callerInitOnce sync.Once +) + +const ( + maximumCallerDepth int = 25 + knownLogrusFrames int = 4 +) + +func init() { + // start at the bottom of the stack before the package-name cache is primed + minimumCallerDepth = 1 +} + +// getPackageName reduces a fully qualified function name to the package name +// There really ought to be to be a better way... +func getPackageName(f string) string { + for { + lastPeriod := strings.LastIndex(f, ".") + lastSlash := strings.LastIndex(f, "/") + if lastPeriod > lastSlash { + f = f[:lastPeriod] + } else { + break + } + } + + return f +} + +func getCallerDepth() int { + // cache this package's fully-qualified name + callerInitOnce.Do(func() { + pcs := make([]uintptr, maximumCallerDepth) + _ = runtime.Callers(0, pcs) + + // dynamic get the package name and the minimum caller depth + logrusIsNext := false + for i := 0; i < maximumCallerDepth; i++ { + funcName := runtime.FuncForPC(pcs[i]).Name() + if logrusIsNext { + logrusPackage = getPackageName(funcName) + break + } + if strings.Contains(funcName, "LogrusAdapter") { + logrusIsNext = true + continue + } + } + + minimumCallerDepth = knownLogrusFrames + }) + + // Restrict the lookback frames to avoid runaway lookups + pcs := make([]uintptr, maximumCallerDepth) + depth := runtime.Callers(minimumCallerDepth, pcs) + frames := runtime.CallersFrames(pcs[:depth]) + callerDepth := minimumCallerDepth + + for f, again := frames.Next(); again; f, again = frames.Next() { + pkg := getPackageName(f.Function) + + // If the caller isn't part of this package, we're done + if pkg != logrusPackage { + return callerDepth - 2 + } + callerDepth++ + } + + // if we got here, we failed to find the caller's context + return 0 +} + +func convertLevel(level log.Level) logrus.Level { + switch level { + case log.TraceLevel: + return logrus.TraceLevel + case log.DebugLevel: + return logrus.DebugLevel + case log.InfoLevel: + return logrus.InfoLevel + case log.WarnLevel: + return logrus.WarnLevel + case log.ErrorLevel: + return logrus.ErrorLevel + case log.FatalLevel: + return logrus.FatalLevel + } + + return logrus.PanicLevel +} + +func SetLevel(level log.Level) { + logrus.SetLevel(convertLevel(level)) +} + +type LogrusAdapter struct { + logger log.Logger + adaptCallstack bool + convertPrefix bool +} + +func (a *LogrusAdapter) Format(entry *logrus.Entry) ([]byte, error) { + var name *string + fields := make([]log.Field, 0, len(entry.Data)) + for key, val := range entry.Data { + skip := false + if a.convertPrefix && key == "prefix" { + if w, ok := val.(string); ok { + name = &w + skip = true + } + } + if !skip { + fields = append(fields, log.Any(key, val)) + } + } + + var logger log.Logger + if a.adaptCallstack { + logger = log.AddCallerSkip(a.logger, getCallerDepth()) + } else { + logger = a.logger + } + + if a.convertPrefix && name != nil { + logger = logger.WithName(*name) + } + + switch entry.Level { + case logrus.TraceLevel: + logger.Trace(entry.Message, fields...) + case logrus.DebugLevel: + logger.Debug(entry.Message, fields...) + case logrus.InfoLevel: + logger.Info(entry.Message, fields...) + case logrus.WarnLevel: + logger.Warn(entry.Message, fields...) + case logrus.ErrorLevel: + logger.Error(entry.Message, fields...) + case logrus.FatalLevel: + logger.Fatal(entry.Message, fields...) + case logrus.PanicLevel: + logger.Fatal(entry.Message, fields...) + } + + return nil, nil +} + +type Option func(*LogrusAdapter) + +func DontAdaptCallstack() Option { + return func(adapter *LogrusAdapter) { + adapter.adaptCallstack = false + } +} + +func ConvertPrefix() Option { + return func(adapter *LogrusAdapter) { + adapter.convertPrefix = true + } +} + +// AdaptLogrus replaces logr formatter by wrapped logger +func AdaptLogrus(logr *logrus.Logger, logger log.Logger, level log.Level, opts ...Option) { + logr.SetLevel(convertLevel(level)) + + adapter := &LogrusAdapter{logger, true, false} + + for _, opt := range opts { + opt(adapter) + } + + logr.SetFormatter(adapter) + logr.SetOutput(io.Discard) +} + +// AdaptStandardLogger replaces logrus.StandardLogger() formatter by wrapped logger +func AdaptStandardLogger(logger log.Logger, level log.Level, opts ...Option) { + AdaptLogrus(logrus.StandardLogger(), logger, level, opts...) +} diff --git a/library/go/core/log/compat/logrus/ya.make b/library/go/core/log/compat/logrus/ya.make new file mode 100644 index 0000000000..19240f7378 --- /dev/null +++ b/library/go/core/log/compat/logrus/ya.make @@ -0,0 +1,5 @@ +GO_LIBRARY() + +SRCS(log.go) + +END() diff --git a/library/go/core/log/compat/pion/log.go b/library/go/core/log/compat/pion/log.go new file mode 100644 index 0000000000..cf93e549c8 --- /dev/null +++ b/library/go/core/log/compat/pion/log.go @@ -0,0 +1,76 @@ +package pion + +import ( + "github.com/pion/logging" + "github.com/ydb-platform/ydb/library/go/core/log" +) + +type LoggerFactory struct { + StandardLogger log.Logger +} + +func (l LoggerFactory) NewLogger(scope string) logging.LeveledLogger { + return LoggerAdapter{ + standardLogger: l.StandardLogger, + scope: scope, + } +} + +type LoggerAdapter struct { + standardLogger log.Logger + scope string +} + +func (a LoggerAdapter) Trace(msg string) { + log.AddCallerSkip(a.standardLogger, 1) + a.standardLogger.Trace(a.addScope(msg)) +} + +func (a LoggerAdapter) Tracef(format string, args ...interface{}) { + log.AddCallerSkip(a.standardLogger, 1) + a.standardLogger.Tracef(a.addScope(format), args...) +} + +func (a LoggerAdapter) Debug(msg string) { + log.AddCallerSkip(a.standardLogger, 1) + a.standardLogger.Debug(a.addScope(msg)) +} + +func (a LoggerAdapter) Debugf(format string, args ...interface{}) { + log.AddCallerSkip(a.standardLogger, 1) + a.standardLogger.Debugf(a.addScope(format), args...) +} + +func (a LoggerAdapter) Info(msg string) { + log.AddCallerSkip(a.standardLogger, 1) + a.standardLogger.Info(a.addScope(msg)) +} + +func (a LoggerAdapter) Infof(format string, args ...interface{}) { + log.AddCallerSkip(a.standardLogger, 1) + a.standardLogger.Infof(a.addScope(format), args...) +} + +func (a LoggerAdapter) Warn(msg string) { + log.AddCallerSkip(a.standardLogger, 1) + a.standardLogger.Warn(a.addScope(msg)) +} + +func (a LoggerAdapter) Warnf(format string, args ...interface{}) { + log.AddCallerSkip(a.standardLogger, 1) + a.standardLogger.Warnf(a.addScope(format), args...) +} + +func (a LoggerAdapter) Error(msg string) { + log.AddCallerSkip(a.standardLogger, 1) + a.standardLogger.Error(a.addScope(msg)) +} + +func (a LoggerAdapter) Errorf(format string, args ...interface{}) { + log.AddCallerSkip(a.standardLogger, 1) + a.standardLogger.Errorf(a.addScope(format), args...) +} + +func (a LoggerAdapter) addScope(s string) string { + return a.scope + ": " + s +} diff --git a/library/go/core/log/compat/pion/ya.make b/library/go/core/log/compat/pion/ya.make new file mode 100644 index 0000000000..19240f7378 --- /dev/null +++ b/library/go/core/log/compat/pion/ya.make @@ -0,0 +1,5 @@ +GO_LIBRARY() + +SRCS(log.go) + +END() diff --git a/library/go/core/log/compat/stdlog/stdlog.go b/library/go/core/log/compat/stdlog/stdlog.go new file mode 100644 index 0000000000..f812cd10b7 --- /dev/null +++ b/library/go/core/log/compat/stdlog/stdlog.go @@ -0,0 +1,54 @@ +package stdlog + +import ( + "bytes" + "fmt" + stdlog "log" + + "github.com/ydb-platform/ydb/library/go/core/log" +) + +func levelToFunc(logger log.Logger, lvl log.Level) (func(msg string, fields ...log.Field), error) { + switch lvl { + case log.DebugLevel: + return logger.Debug, nil + case log.TraceLevel: + return logger.Trace, nil + case log.InfoLevel: + return logger.Info, nil + case log.WarnLevel: + return logger.Warn, nil + case log.ErrorLevel: + return logger.Error, nil + case log.FatalLevel: + return logger.Fatal, nil + } + + return nil, fmt.Errorf("unknown log level: %v", lvl) +} + +type loggerWriter struct { + logFunc func(msg string, fields ...log.Field) +} + +func (w *loggerWriter) Write(p []byte) (int, error) { + p = bytes.TrimSpace(p) + w.logFunc(string(p)) + return len(p), nil +} + +// New creates stdlib log.Logger that writes to provided logger on Error level +func New(logger log.Logger) *stdlog.Logger { + l := log.AddCallerSkip(logger, 3) + return stdlog.New(&loggerWriter{logFunc: l.Error}, "", 0) +} + +// NewAt creates stdlib log.Logger that writes to provided logger on specified level +func NewAt(logger log.Logger, lvl log.Level) (*stdlog.Logger, error) { + l := log.AddCallerSkip(logger, 3) + logFunc, err := levelToFunc(l, lvl) + if err != nil { + return nil, err + } + return stdlog.New(&loggerWriter{logFunc: logFunc}, "", 0), nil +} diff --git a/library/go/core/log/compat/stdlog/ya.make b/library/go/core/log/compat/stdlog/ya.make new file mode 100644 index 0000000000..44456e973a --- /dev/null +++ b/library/go/core/log/compat/stdlog/ya.make @@ -0,0 +1,5 @@ +GO_LIBRARY() + +SRCS(stdlog.go) + +END() diff --git a/library/go/core/log/compat/ya.make b/library/go/core/log/compat/ya.make new file mode 100644 index 0000000000..7f371862bd --- /dev/null +++ b/library/go/core/log/compat/ya.make @@ -0,0 +1,6 @@ +RECURSE( + golog + logrus + pion + stdlog +) diff --git a/library/go/core/log/ctxlog/ctxlog.go b/library/go/core/log/ctxlog/ctxlog.go new file mode 100644 index 0000000000..e054e9c2ed --- /dev/null +++ b/library/go/core/log/ctxlog/ctxlog.go @@ -0,0 +1,124 @@ +package ctxlog + +import ( + "context" + "fmt" + + "github.com/ydb-platform/ydb/library/go/core/log" +) + +type ctxKey struct{} + +// ContextFields returns log.Fields bound with ctx. +// If no fields are bound, it returns nil. +func ContextFields(ctx context.Context) []log.Field { + fs, _ := ctx.Value(ctxKey{}).([]log.Field) + return fs +} + +// WithFields returns a new context that is bound with given fields and based +// on parent ctx. +func WithFields(ctx context.Context, fields ...log.Field) context.Context { + if len(fields) == 0 { + return ctx + } + + return context.WithValue(ctx, ctxKey{}, mergeFields(ContextFields(ctx), fields)) +} + +// Trace logs at Trace log level using fields both from arguments and ones that +// are bound to ctx. +func Trace(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { + log.AddCallerSkip(l, 1).Trace(msg, mergeFields(ContextFields(ctx), fields)...) +} + +// Debug logs at Debug log level using fields both from arguments and ones that +// are bound to ctx. +func Debug(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { + log.AddCallerSkip(l, 1).Debug(msg, mergeFields(ContextFields(ctx), fields)...) +} + +// Info logs at Info log level using fields both from arguments and ones that +// are bound to ctx. +func Info(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { + log.AddCallerSkip(l, 1).Info(msg, mergeFields(ContextFields(ctx), fields)...) +} + +// Warn logs at Warn log level using fields both from arguments and ones that +// are bound to ctx. +func Warn(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { + log.AddCallerSkip(l, 1).Warn(msg, mergeFields(ContextFields(ctx), fields)...) +} + +// Error logs at Error log level using fields both from arguments and ones that +// are bound to ctx. +func Error(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { + log.AddCallerSkip(l, 1).Error(msg, mergeFields(ContextFields(ctx), fields)...) +} + +// Fatal logs at Fatal log level using fields both from arguments and ones that +// are bound to ctx. +func Fatal(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { + log.AddCallerSkip(l, 1).Fatal(msg, mergeFields(ContextFields(ctx), fields)...) +} + +// Tracef logs at Trace log level using fields that are bound to ctx. +// The message is formatted using provided arguments. +func Tracef(ctx context.Context, l log.Logger, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + log.AddCallerSkip(l, 1).Trace(msg, ContextFields(ctx)...) +} + +// Debugf logs at Debug log level using fields that are bound to ctx. +// The message is formatted using provided arguments. +func Debugf(ctx context.Context, l log.Logger, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + log.AddCallerSkip(l, 1).Debug(msg, ContextFields(ctx)...) +} + +// Infof logs at Info log level using fields that are bound to ctx. +// The message is formatted using provided arguments. +func Infof(ctx context.Context, l log.Logger, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + log.AddCallerSkip(l, 1).Info(msg, ContextFields(ctx)...) +} + +// Warnf logs at Warn log level using fields that are bound to ctx. +// The message is formatted using provided arguments. +func Warnf(ctx context.Context, l log.Logger, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + log.AddCallerSkip(l, 1).Warn(msg, ContextFields(ctx)...) +} + +// Errorf logs at Error log level using fields that are bound to ctx. +// The message is formatted using provided arguments. +func Errorf(ctx context.Context, l log.Logger, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + log.AddCallerSkip(l, 1).Error(msg, ContextFields(ctx)...) +} + +// Fatalf logs at Fatal log level using fields that are bound to ctx. +// The message is formatted using provided arguments. +func Fatalf(ctx context.Context, l log.Logger, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + log.AddCallerSkip(l, 1).Fatal(msg, ContextFields(ctx)...) +} + +func mergeFields(a, b []log.Field) []log.Field { + if a == nil { + return b + } + if b == nil { + return a + } + + // NOTE: just append() here is unsafe. If a caller passed slice of fields + // followed by ... with capacity greater than length, then simultaneous + // logging will lead to a data race condition. + // + // See https://golang.org/ref/spec#Passing_arguments_to_..._parameters + c := make([]log.Field, len(a)+len(b)) + n := copy(c, a) + copy(c[n:], b) + return c +} diff --git a/library/go/core/log/ctxlog/ctxlog_test.go b/library/go/core/log/ctxlog/ctxlog_test.go new file mode 100644 index 0000000000..c3fbedfc09 --- /dev/null +++ b/library/go/core/log/ctxlog/ctxlog_test.go @@ -0,0 +1,66 @@ +package ctxlog + +import ( + "context" + "reflect" + "testing" + + "github.com/ydb-platform/ydb/library/go/core/log" +) + +func TestContextFields(t *testing.T) { + for _, test := range []struct { + ctx context.Context + exp []log.Field + }{ + { + ctx: context.Background(), + exp: nil, + }, + { + ctx: contextWithFields( + log.String("foo", "bar"), + log.String("bar", "baz"), + ), + exp: []log.Field{ + log.String("foo", "bar"), + log.String("bar", "baz"), + }, + }, + } { + t.Run("", func(t *testing.T) { + act := ContextFields(test.ctx) + if exp := test.exp; !reflect.DeepEqual(act, exp) { + t.Fatalf( + "ContextFields() = %v; want %v", + act, exp, + ) + } + }) + } +} + +// TestWithFields tests the case when race condition may occur on adding fields +// to a bound field slice capable enough to store additional ones. +func TestWithFields(t *testing.T) { + fs := make([]log.Field, 2, 4) + fs[0] = log.String("a", "a") + fs[1] = log.String("b", "b") + + // Bind to ctx1 field slice with cap(fs) = 2. + ctx1 := WithFields(context.Background(), fs...) + + // Bind additional two fields to ctx2 that are able to fit the parent's + // ctx1 bound fields. + _ = WithFields(ctx1, log.String("c", "c"), log.String("d", "d")) + + var act, exp [2]log.Field // Expect to zero-values of Field. + copy(act[:], fs[2:4]) // Check the tail of initial slice. + if act != exp { + t.Fatalf("fields tail is non-empty: %v", act) + } +} + +func contextWithFields(fs ...log.Field) context.Context { + return context.WithValue(context.Background(), ctxKey{}, fs) +} diff --git a/library/go/core/log/ctxlog/gotest/ya.make b/library/go/core/log/ctxlog/gotest/ya.make new file mode 100644 index 0000000000..74138e031c --- /dev/null +++ b/library/go/core/log/ctxlog/gotest/ya.make @@ -0,0 +1,3 @@ +GO_TEST_FOR(library/go/core/log/ctxlog) + +END() diff --git a/library/go/core/log/ctxlog/ya.make b/library/go/core/log/ctxlog/ya.make new file mode 100644 index 0000000000..61b48a372f --- /dev/null +++ b/library/go/core/log/ctxlog/ya.make @@ -0,0 +1,9 @@ +GO_LIBRARY() + +SRCS(ctxlog.go) + +GO_TEST_SRCS(ctxlog_test.go) + +END() + +RECURSE(gotest) diff --git a/library/go/core/log/fields.go b/library/go/core/log/fields.go new file mode 100644 index 0000000000..afd41c197e --- /dev/null +++ b/library/go/core/log/fields.go @@ -0,0 +1,446 @@ +package log + +import ( + "fmt" + "time" +) + +const ( + // DefaultErrorFieldName is the default field name used for errors + DefaultErrorFieldName = "error" +) + +// FieldType is a type of data Field can represent +type FieldType int + +const ( + // FieldTypeNil is for a pure nil + FieldTypeNil FieldType = iota + // FieldTypeString is for a string + FieldTypeString + // FieldTypeBinary is for a binary array + FieldTypeBinary + // FieldTypeBoolean is for boolean + FieldTypeBoolean + // FieldTypeSigned is for signed integers + FieldTypeSigned + // FieldTypeUnsigned is for unsigned integers + FieldTypeUnsigned + // FieldTypeFloat is for float + FieldTypeFloat + // FieldTypeTime is for time.Time + FieldTypeTime + // FieldTypeDuration is for time.Duration + FieldTypeDuration + // FieldTypeError is for an error + FieldTypeError + // FieldTypeArray is for an array of any type + FieldTypeArray + // FieldTypeAny is for any type + FieldTypeAny + // FieldTypeReflect is for unknown types + FieldTypeReflect + // FieldTypeByteString is for a bytes that can be represented as UTF-8 string + FieldTypeByteString +) + +// Field stores one structured logging field +type Field struct { + key string + ftype FieldType + string string + signed int64 + unsigned uint64 + float float64 + iface interface{} +} + +// Key returns field key +func (f Field) Key() string { + return f.key +} + +// Type returns field type +func (f Field) Type() FieldType { + return f.ftype +} + +// String returns field string +func (f Field) String() string { + return f.string +} + +// Binary constructs field of []byte +func (f Field) Binary() []byte { + if f.iface == nil { + return nil + } + return f.iface.([]byte) +} + +// Bool returns field bool +func (f Field) Bool() bool { + return f.Signed() != 0 +} + +// Signed returns field int64 +func (f Field) Signed() int64 { + return f.signed +} + +// Unsigned returns field uint64 +func (f Field) Unsigned() uint64 { + return f.unsigned +} + +// Float returns field float64 +func (f Field) Float() float64 { + return f.float +} + +// Time returns field time.Time +func (f Field) Time() time.Time { + return time.Unix(0, f.signed) +} + +// Duration returns field time.Duration +func (f Field) Duration() time.Duration { + return time.Nanosecond * time.Duration(f.signed) +} + +// Error constructs field of error type +func (f Field) Error() error { + if f.iface == nil { + return nil + } + return f.iface.(error) +} + +// Interface returns field interface +func (f Field) Interface() interface{} { + return f.iface +} + +// Any returns contained data as interface{} +// nolint: gocyclo +func (f Field) Any() interface{} { + switch f.Type() { + case FieldTypeNil: + return nil + case FieldTypeString: + return f.String() + case FieldTypeBinary: + return f.Interface() + case FieldTypeBoolean: + return f.Bool() + case FieldTypeSigned: + return f.Signed() + case FieldTypeUnsigned: + return f.Unsigned() + case FieldTypeFloat: + return f.Float() + case FieldTypeTime: + return f.Time() + case FieldTypeDuration: + return f.Duration() + case FieldTypeError: + return f.Error() + case FieldTypeArray: + return f.Interface() + case FieldTypeAny: + return f.Interface() + case FieldTypeReflect: + return f.Interface() + case FieldTypeByteString: + return f.Interface() + default: + // For when new field type is not added to this func + panic(fmt.Sprintf("unknown field type: %d", f.Type())) + } +} + +// Nil constructs field of nil type +func Nil(key string) Field { + return Field{key: key, ftype: FieldTypeNil} +} + +// String constructs field of string type +func String(key, value string) Field { + return Field{key: key, ftype: FieldTypeString, string: value} +} + +// Sprintf constructs field of string type with formatting +func Sprintf(key, format string, args ...interface{}) Field { + return Field{key: key, ftype: FieldTypeString, string: fmt.Sprintf(format, args...)} +} + +// Strings constructs Field from []string +func Strings(key string, value []string) Field { + return Array(key, value) +} + +// Binary constructs field of []byte type +func Binary(key string, value []byte) Field { + return Field{key: key, ftype: FieldTypeBinary, iface: value} +} + +// Bool constructs field of bool type +func Bool(key string, value bool) Field { + field := Field{key: key, ftype: FieldTypeBoolean} + if value { + field.signed = 1 + } else { + field.signed = 0 + } + + return field +} + +// Bools constructs Field from []bool +func Bools(key string, value []bool) Field { + return Array(key, value) +} + +// Int constructs Field from int +func Int(key string, value int) Field { + return Int64(key, int64(value)) +} + +// Ints constructs Field from []int +func Ints(key string, value []int) Field { + return Array(key, value) +} + +// Int8 constructs Field from int8 +func Int8(key string, value int8) Field { + return Int64(key, int64(value)) +} + +// Int8s constructs Field from []int8 +func Int8s(key string, value []int8) Field { + return Array(key, value) +} + +// Int16 constructs Field from int16 +func Int16(key string, value int16) Field { + return Int64(key, int64(value)) +} + +// Int16s constructs Field from []int16 +func Int16s(key string, value []int16) Field { + return Array(key, value) +} + +// Int32 constructs Field from int32 +func Int32(key string, value int32) Field { + return Int64(key, int64(value)) +} + +// Int32s constructs Field from []int32 +func Int32s(key string, value []int32) Field { + return Array(key, value) +} + +// Int64 constructs Field from int64 +func Int64(key string, value int64) Field { + return Field{key: key, ftype: FieldTypeSigned, signed: value} +} + +// Int64s constructs Field from []int64 +func Int64s(key string, value []int64) Field { + return Array(key, value) +} + +// UInt constructs Field from uint +func UInt(key string, value uint) Field { + return UInt64(key, uint64(value)) +} + +// UInts constructs Field from []uint +func UInts(key string, value []uint) Field { + return Array(key, value) +} + +// UInt8 constructs Field from uint8 +func UInt8(key string, value uint8) Field { + return UInt64(key, uint64(value)) +} + +// UInt8s constructs Field from []uint8 +func UInt8s(key string, value []uint8) Field { + return Array(key, value) +} + +// UInt16 constructs Field from uint16 +func UInt16(key string, value uint16) Field { + return UInt64(key, uint64(value)) +} + +// UInt16s constructs Field from []uint16 +func UInt16s(key string, value []uint16) Field { + return Array(key, value) +} + +// UInt32 constructs Field from uint32 +func UInt32(key string, value uint32) Field { + return UInt64(key, uint64(value)) +} + +// UInt32s constructs Field from []uint32 +func UInt32s(key string, value []uint32) Field { + return Array(key, value) +} + +// UInt64 constructs Field from uint64 +func UInt64(key string, value uint64) Field { + return Field{key: key, ftype: FieldTypeUnsigned, unsigned: value} +} + +// UInt64s constructs Field from []uint64 +func UInt64s(key string, value []uint64) Field { + return Array(key, value) +} + +// Float32 constructs Field from float32 +func Float32(key string, value float32) Field { + return Float64(key, float64(value)) +} + +// Float32s constructs Field from []float32 +func Float32s(key string, value []float32) Field { + return Array(key, value) +} + +// Float64 constructs Field from float64 +func Float64(key string, value float64) Field { + return Field{key: key, ftype: FieldTypeFloat, float: value} +} + +// Float64s constructs Field from []float64 +func Float64s(key string, value []float64) Field { + return Array(key, value) +} + +// Time constructs field of time.Time type +func Time(key string, value time.Time) Field { + return Field{key: key, ftype: FieldTypeTime, signed: value.UnixNano()} +} + +// Times constructs Field from []time.Time +func Times(key string, value []time.Time) Field { + return Array(key, value) +} + +// Duration constructs field of time.Duration type +func Duration(key string, value time.Duration) Field { + return Field{key: key, ftype: FieldTypeDuration, signed: value.Nanoseconds()} +} + +// Durations constructs Field from []time.Duration +func Durations(key string, value []time.Duration) Field { + return Array(key, value) +} + +// NamedError constructs field of error type +func NamedError(key string, value error) Field { + return Field{key: key, ftype: FieldTypeError, iface: value} +} + +// Error constructs field of error type with default field name +func Error(value error) Field { + return NamedError(DefaultErrorFieldName, value) +} + +// Errors constructs Field from []error +func Errors(key string, value []error) Field { + return Array(key, value) +} + +// Array constructs field of array type +func Array(key string, value interface{}) Field { + return Field{key: key, ftype: FieldTypeArray, iface: value} +} + +// Reflect constructs field of unknown type +func Reflect(key string, value interface{}) Field { + return Field{key: key, ftype: FieldTypeReflect, iface: value} +} + +// ByteString constructs field of bytes that could represent UTF-8 string +func ByteString(key string, value []byte) Field { + return Field{key: key, ftype: FieldTypeByteString, iface: value} +} + +// Any tries to deduce interface{} underlying type and constructs Field from it. +// Use of this function is ok only for the sole purpose of not repeating its entire code +// or parts of it in user's code (when you need to log interface{} types with unknown content). +// Otherwise please use specialized functions. +// nolint: gocyclo +func Any(key string, value interface{}) Field { + switch val := value.(type) { + case bool: + return Bool(key, val) + case float64: + return Float64(key, val) + case float32: + return Float32(key, val) + case int: + return Int(key, val) + case []int: + return Ints(key, val) + case int64: + return Int64(key, val) + case []int64: + return Int64s(key, val) + case int32: + return Int32(key, val) + case []int32: + return Int32s(key, val) + case int16: + return Int16(key, val) + case []int16: + return Int16s(key, val) + case int8: + return Int8(key, val) + case []int8: + return Int8s(key, val) + case string: + return String(key, val) + case []string: + return Strings(key, val) + case uint: + return UInt(key, val) + case []uint: + return UInts(key, val) + case uint64: + return UInt64(key, val) + case []uint64: + return UInt64s(key, val) + case uint32: + return UInt32(key, val) + case []uint32: + return UInt32s(key, val) + case uint16: + return UInt16(key, val) + case []uint16: + return UInt16s(key, val) + case uint8: + return UInt8(key, val) + case []byte: + return Binary(key, val) + case time.Time: + return Time(key, val) + case []time.Time: + return Times(key, val) + case time.Duration: + return Duration(key, val) + case []time.Duration: + return Durations(key, val) + case error: + return NamedError(key, val) + case []error: + return Errors(key, val) + default: + return Field{key: key, ftype: FieldTypeAny, iface: value} + } +} diff --git a/library/go/core/log/fields_test.go b/library/go/core/log/fields_test.go new file mode 100644 index 0000000000..ff6890b46a --- /dev/null +++ b/library/go/core/log/fields_test.go @@ -0,0 +1,32 @@ +package log + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Simple test, that all type of fields are correctly zapified. +// Maybe we also need some test that checks resulting zap.Field type also. +func TestFieldAny(t *testing.T) { + for typ := FieldType(0); typ <= FieldTypeReflect; typ++ { + field := Field{ftype: typ} + assert.NotPanics(t, func() { + field.Any() + }) + } +} + +func TestAny(t *testing.T) { + var v struct{ A int } + field := Any("test", &v) + assert.Equal(t, field.ftype, FieldTypeAny) +} + +func TestReflect(t *testing.T) { + field := Reflect("test", 1) + assert.Equal(t, field.ftype, FieldTypeReflect) +} + +// TODO: test fields +// TODO: test field converters diff --git a/library/go/core/log/gotest/ya.make b/library/go/core/log/gotest/ya.make new file mode 100644 index 0000000000..84907af36d --- /dev/null +++ b/library/go/core/log/gotest/ya.make @@ -0,0 +1,3 @@ +GO_TEST_FOR(library/go/core/log) + +END() diff --git a/library/go/core/log/levels.go b/library/go/core/log/levels.go new file mode 100644 index 0000000000..54810410b9 --- /dev/null +++ b/library/go/core/log/levels.go @@ -0,0 +1,108 @@ +package log + +import ( + "fmt" + "strings" +) + +// Level of logging +type Level int + +// MarshalText marshals level to text +func (l Level) MarshalText() ([]byte, error) { + if l >= maxLevel || l < 0 { + return nil, fmt.Errorf("failed to marshal log level: level value (%d) is not in the allowed range (0-%d)", l, maxLevel-1) + } + return []byte(l.String()), nil +} + +// UnmarshalText unmarshals level from text +func (l *Level) UnmarshalText(text []byte) error { + level, err := ParseLevel(string(text)) + if err != nil { + return err + } + + *l = level + return nil +} + +// Standard log levels +const ( + TraceLevel Level = iota + DebugLevel + InfoLevel + WarnLevel + ErrorLevel + FatalLevel + maxLevel +) + +func Levels() (l []Level) { + for i := 0; i < int(maxLevel); i++ { + l = append(l, Level(i)) + } + return +} + +// String values for standard log levels +const ( + TraceString = "trace" + DebugString = "debug" + InfoString = "info" + WarnString = "warn" + ErrorString = "error" + FatalString = "fatal" +) + +// String implements Stringer interface for Level +func (l Level) String() string { + switch l { + case TraceLevel: + return TraceString + case DebugLevel: + return DebugString + case InfoLevel: + return InfoString + case WarnLevel: + return WarnString + case ErrorLevel: + return ErrorString + case FatalLevel: + return FatalString + default: + // For when new log level is not added to this func (most likely never). + panic(fmt.Sprintf("unknown log level: %d", l)) + } +} + +// Set implements flag.Value interface +func (l *Level) Set(v string) error { + lvl, err := ParseLevel(v) + if err != nil { + return err + } + + *l = lvl + return nil +} + +// ParseLevel parses log level from string. Returns ErrUnknownLevel for unknown log level. +func ParseLevel(l string) (Level, error) { + switch strings.ToLower(l) { + case TraceString: + return TraceLevel, nil + case DebugString: + return DebugLevel, nil + case InfoString: + return InfoLevel, nil + case WarnString: + return WarnLevel, nil + case ErrorString: + return ErrorLevel, nil + case FatalString: + return FatalLevel, nil + default: + return FatalLevel, fmt.Errorf("unknown log level: %s", l) + } +} diff --git a/library/go/core/log/levels_test.go b/library/go/core/log/levels_test.go new file mode 100644 index 0000000000..3181b91268 --- /dev/null +++ b/library/go/core/log/levels_test.go @@ -0,0 +1,51 @@ +package log_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/ydb-platform/ydb/library/go/core/log" +) + +var levelsToTest = []struct { + name string + level log.Level +}{ + {name: log.TraceString, level: log.TraceLevel}, + {name: log.DebugString, level: log.DebugLevel}, + {name: log.InfoString, level: log.InfoLevel}, + {name: log.WarnString, level: log.WarnLevel}, + {name: log.ErrorString, level: log.ErrorLevel}, + {name: log.FatalString, level: log.FatalLevel}, +} + +func TestLevels(t *testing.T) { + for _, levelInput := range levelsToTest { + t.Run("Convert "+levelInput.name, func(t *testing.T) { + levelFromLevelString, err := log.ParseLevel(levelInput.name) + require.NoError(t, err) + require.Equal(t, levelInput.level, levelFromLevelString) + + levelStringFromLevel := levelInput.level.String() + require.Equal(t, levelInput.name, levelStringFromLevel) + + levelFromLevelStringFromLevel, err := log.ParseLevel(levelStringFromLevel) + require.NoError(t, err) + require.Equal(t, levelInput.level, levelFromLevelStringFromLevel) + }) + } +} + +func TestLevel_MarshalText(t *testing.T) { + level := log.DebugLevel + _, err := level.MarshalText() + require.NoError(t, err) + + level = log.Level(100500) + _, err = level.MarshalText() + require.Error(t, err) + + level = log.Level(-1) + _, err = level.MarshalText() + require.Error(t, err) +} diff --git a/library/go/core/log/log.go b/library/go/core/log/log.go new file mode 100644 index 0000000000..3e1f76e870 --- /dev/null +++ b/library/go/core/log/log.go @@ -0,0 +1,134 @@ +package log + +import "errors" + +// Logger is the universal logger that can do everything. +type Logger interface { + loggerStructured + loggerFmt + toStructured + toFmt + withName +} + +type withName interface { + WithName(name string) Logger +} + +type toLogger interface { + // Logger returns general logger + Logger() Logger +} + +// Structured provides interface for logging using fields. +type Structured interface { + loggerStructured + toFmt + toLogger +} + +type loggerStructured interface { + // Trace logs at Trace log level using fields + Trace(msg string, fields ...Field) + // Debug logs at Debug log level using fields + Debug(msg string, fields ...Field) + // Info logs at Info log level using fields + Info(msg string, fields ...Field) + // Warn logs at Warn log level using fields + Warn(msg string, fields ...Field) + // Error logs at Error log level using fields + Error(msg string, fields ...Field) + // Fatal logs at Fatal log level using fields + Fatal(msg string, fields ...Field) +} + +type toFmt interface { + // Fmt returns fmt logger + Fmt() Fmt +} + +// Fmt provides interface for logging using fmt formatter. +type Fmt interface { + loggerFmt + toStructured + toLogger +} + +type loggerFmt interface { + // Tracef logs at Trace log level using fmt formatter + Tracef(format string, args ...interface{}) + // Debugf logs at Debug log level using fmt formatter + Debugf(format string, args ...interface{}) + // Infof logs at Info log level using fmt formatter + Infof(format string, args ...interface{}) + // Warnf logs at Warn log level using fmt formatter + Warnf(format string, args ...interface{}) + // Errorf logs at Error log level using fmt formatter + Errorf(format string, args ...interface{}) + // Fatalf logs at Fatal log level using fmt formatter + Fatalf(format string, args ...interface{}) +} + +type toStructured interface { + // Structured returns structured logger + Structured() Structured +} + +// LoggerWith is an interface for 'With' function +// LoggerWith provides interface for logger modifications. +type LoggerWith interface { + // With implements 'With' + With(fields ...Field) Logger +} + +// With for loggers that implement LoggerWith interface, returns logger that +// always adds provided key/value to every log entry. Otherwise returns same logger. +func With(l Logger, fields ...Field) Logger { + e, ok := l.(LoggerWith) + if !ok { + return l + } + + return e.With(fields...) +} + +// LoggerAddCallerSkip is an interface for 'AddCallerSkip' function +type LoggerAddCallerSkip interface { + // AddCallerSkip implements 'AddCallerSkip' + AddCallerSkip(skip int) Logger +} + +// AddCallerSkip for loggers that implement LoggerAddCallerSkip interface, returns logger that +// adds caller skip to each log entry. Otherwise returns same logger. +func AddCallerSkip(l Logger, skip int) Logger { + e, ok := l.(LoggerAddCallerSkip) + if !ok { + return l + } + + return e.AddCallerSkip(skip) +} + +// WriteAt is a helper method that checks logger and writes message at given level +func WriteAt(l Structured, lvl Level, msg string, fields ...Field) error { + if l == nil { + return errors.New("nil logger given") + } + + switch lvl { + case DebugLevel: + l.Debug(msg, fields...) + case TraceLevel: + l.Trace(msg, fields...) + case InfoLevel: + l.Info(msg, fields...) + case WarnLevel: + l.Warn(msg, fields...) + case ErrorLevel: + l.Error(msg, fields...) + case FatalLevel: + l.Fatal(msg, fields...) + } + + return nil +} diff --git a/library/go/core/log/nop/nop.go b/library/go/core/log/nop/nop.go new file mode 100644 index 0000000000..950742878a --- /dev/null +++ b/library/go/core/log/nop/nop.go @@ -0,0 +1,73 @@ +package nop + +import ( + "os" + + "github.com/ydb-platform/ydb/library/go/core/log" +) + +// Logger that does nothing +type Logger struct{} + +var _ log.Logger = &Logger{} +var _ log.Structured = &Logger{} +var _ log.Fmt = &Logger{} + +// Logger returns general logger +func (l *Logger) Logger() log.Logger { + return l +} + +// Fmt returns fmt logger +func (l *Logger) Fmt() log.Fmt { + return l +} + +// Structured returns structured logger +func (l *Logger) Structured() log.Structured { + return l +} + +// Trace implements Trace method of log.Logger interface +func (l *Logger) Trace(msg string, fields ...log.Field) {} + +// Tracef implements Tracef method of log.Logger interface +func (l *Logger) Tracef(format string, args ...interface{}) {} + +// Debug implements Debug method of log.Logger interface +func (l *Logger) Debug(msg string, fields ...log.Field) {} + +// Debugf implements Debugf method of log.Logger interface +func (l *Logger) Debugf(format string, args ...interface{}) {} + +// Info implements Info method of log.Logger interface +func (l *Logger) Info(msg string, fields ...log.Field) {} + +// Infof implements Infof method of log.Logger interface +func (l *Logger) Infof(format string, args ...interface{}) {} + +// Warn implements Warn method of log.Logger interface +func (l *Logger) Warn(msg string, fields ...log.Field) {} + +// Warnf implements Warnf method of log.Logger interface +func (l *Logger) Warnf(format string, args ...interface{}) {} + +// Error implements Error method of log.Logger interface +func (l *Logger) Error(msg string, fields ...log.Field) {} + +// Errorf implements Errorf method of log.Logger interface +func (l *Logger) Errorf(format string, args ...interface{}) {} + +// Fatal implements Fatal method of log.Logger interface +func (l *Logger) Fatal(msg string, fields ...log.Field) { + os.Exit(1) +} + +// Fatalf implements Fatalf method of log.Logger interface +func (l *Logger) Fatalf(format string, args ...interface{}) { + os.Exit(1) +} + +func (l *Logger) WithName(name string) log.Logger { + return l +} diff --git a/library/go/core/log/nop/ya.make b/library/go/core/log/nop/ya.make new file mode 100644 index 0000000000..5d9e935c04 --- /dev/null +++ b/library/go/core/log/nop/ya.make @@ -0,0 +1,5 @@ +GO_LIBRARY() + +SRCS(nop.go) + +END() diff --git a/library/go/core/log/test/log_bench_test.go b/library/go/core/log/test/log_bench_test.go new file mode 100644 index 0000000000..c75e3dc2d6 --- /dev/null +++ b/library/go/core/log/test/log_bench_test.go @@ -0,0 +1,39 @@ +package test + +import ( + "fmt" + "testing" + + "github.com/ydb-platform/ydb/library/go/core/log" +) + +func BenchmarkOutput(b *testing.B) { + for _, loggerInput := range loggersToTest { + for _, count := range []int{0, 1, 2, 5} { + logger, err := loggerInput.factory(log.DebugLevel) + if err != nil { + b.Fatalf("failed to create logger: %s", b.Name()) + } + b.Run(fmt.Sprintf("%s fields %d", loggerInput.name, count), func(b *testing.B) { + benchmarkFields(b, logger, count) + }) + } + } +} + +func benchmarkFields(b *testing.B, logger log.Logger, count int) { + flds := genFields(count) + + for n := 0; n < b.N; n++ { + logger.Debug(msg, flds...) + } +} + +func genFields(count int) []log.Field { + flds := make([]log.Field, 0, count) + for ; count > 0; count-- { + flds = append(flds, log.String(key, value)) + } + + return flds +} diff --git a/library/go/core/log/test/log_test.go b/library/go/core/log/test/log_test.go new file mode 100644 index 0000000000..b839127a32 --- /dev/null +++ b/library/go/core/log/test/log_test.go @@ -0,0 +1,120 @@ +package test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/ydb-platform/ydb/library/go/core/log" + "github.com/ydb-platform/ydb/library/go/core/log/nop" + "github.com/ydb-platform/ydb/library/go/core/log/zap" + uzap "go.uber.org/zap" +) + +var ( + msg = "msg" + msgfmt = "%s %s" + msgfmtargs = []interface{}{"hello", "world"} + key = "key" + value = "value" + withKey = "withKey" + withValue = "withValue" +) + +var loggersToTest = []struct { + name string + factory func(level log.Level) (log.Logger, error) +}{ + { + name: "Zap", + factory: func(level log.Level) (log.Logger, error) { + cfg := zap.JSONConfig(level) + // Disable output + cfg.OutputPaths = []string{} + cfg.ErrorOutputPaths = []string{} + return zap.New(cfg) + }, + }, + { + name: "ZapNop", + factory: func(level log.Level) (log.Logger, error) { + return &zap.Logger{ + L: uzap.NewNop(), + }, nil + }, + }, + { + name: "Nop", + factory: func(level log.Level) (log.Logger, error) { + return &nop.Logger{}, nil + }, + }, +} + +func TestLoggers(t *testing.T) { + for _, loggerInput := range loggersToTest { + for _, level := range log.Levels() { + t.Run("Construct "+loggerInput.name+level.String(), func(t *testing.T) { + logger, err := loggerInput.factory(level) + require.NoError(t, err) + require.NotNil(t, logger) + + lfmt := logger.Fmt() + require.NotNil(t, lfmt) + + l := lfmt.Structured() + require.NotNil(t, l) + require.Equal(t, logger, l) + }) + + t.Run("With "+loggerInput.name+level.String(), func(t *testing.T) { + logger, err := loggerInput.factory(level) + require.NoError(t, err) + require.NotNil(t, logger) + + withField := log.String(withKey, withValue) + loggerWith := log.With(logger, withField) + require.NotNil(t, loggerWith) + }) + + t.Run("AddCallerSkip "+loggerInput.name+level.String(), func(t *testing.T) { + logger, err := loggerInput.factory(level) + require.NoError(t, err) + require.NotNil(t, logger) + + loggerCallerSkip := log.AddCallerSkip(logger, 1) + require.NotNil(t, loggerCallerSkip) + }) + + // TODO: validate log output + t.Run("Logger "+loggerInput.name+level.String(), func(t *testing.T) { + logger, err := loggerInput.factory(level) + require.NoError(t, err) + require.NotNil(t, logger) + + logger.Trace(msg, log.String(key, value)) + logger.Debug(msg, log.String(key, value)) + logger.Info(msg, log.String(key, value)) + logger.Warn(msg, log.String(key, value)) + logger.Error(msg, log.String(key, value)) + // TODO: test fatal + }) + + // TODO: validate log output + t.Run("LoggerFMT "+loggerInput.name+level.String(), func(t *testing.T) { + logger, err := loggerInput.factory(level) + require.NoError(t, err) + require.NotNil(t, logger) + + lfmt := logger.Fmt() + require.NotNil(t, lfmt) + + lfmt.Tracef(msgfmt, msgfmtargs...) + lfmt.Debugf(msgfmt, msgfmtargs...) + lfmt.Infof(msgfmt, msgfmtargs...) + lfmt.Warnf(msgfmt, msgfmtargs...) + lfmt.Errorf(msgfmt, msgfmtargs...) + // TODO: test fatal + }) + } + } +} diff --git a/library/go/core/log/test/ya.make b/library/go/core/log/test/ya.make new file mode 100644 index 0000000000..be231ce558 --- /dev/null +++ b/library/go/core/log/test/ya.make @@ -0,0 +1,8 @@ +GO_TEST() + +GO_TEST_SRCS( + log_bench_test.go + log_test.go +) + +END() diff --git a/library/go/core/log/ya.make b/library/go/core/log/ya.make new file mode 100644 index 0000000000..588e736521 --- /dev/null +++ b/library/go/core/log/ya.make @@ -0,0 +1,22 @@ +GO_LIBRARY() + +SRCS( + fields.go + levels.go + log.go +) + +GO_TEST_SRCS(fields_test.go) + +GO_XTEST_SRCS(levels_test.go) + +END() + +RECURSE( + compat + ctxlog + gotest + nop + test + zap +) diff --git a/library/go/core/log/zap/asynczap/background.go b/library/go/core/log/zap/asynczap/background.go new file mode 100644 index 0000000000..5af635dfe3 --- /dev/null +++ b/library/go/core/log/zap/asynczap/background.go @@ -0,0 +1,155 @@ +package asynczap + +import ( + "bytes" + "errors" + "sync" + "sync/atomic" + "time" + + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" +) + +// background is a single object shared by all clones of core. +type background struct { + options Options + + q queue + out zapcore.WriteSyncer + + // use manual buffering instead of bufio background to preserve write atomicity. + // + // bufio.Writer might split log lines at arbitrary position. + writeBuffer bytes.Buffer + + wg sync.WaitGroup + mu sync.Mutex + cond *sync.Cond + stopped bool + iter int64 + lastErr error + forceFlush chan struct{} + + droppedRecords int64 + writeErrors int64 + reportOverflow int64 +} + +func newBackground(options Options, out zapcore.WriteSyncer) *background { + b := &background{ + options: options, + out: out, + forceFlush: make(chan struct{}), + } + b.cond = sync.NewCond(&b.mu) + return b +} + +func (b *background) flush() { + _, err := b.out.Write(b.writeBuffer.Bytes()) + if err != nil { + b.onError(err) + } + b.writeBuffer.Reset() +} + +func (b *background) onError(err error) { + atomic.AddInt64(&b.writeErrors, 1) + + b.lastErr = err +} + +func (b *background) stop() { + b.mu.Lock() + b.stopped = true + b.mu.Unlock() + + b.wg.Wait() +} + +func (b *background) finishIter() (stop bool) { + b.mu.Lock() + stop = b.stopped + b.mu.Unlock() + + atomic.StoreInt64(&b.reportOverflow, 0) + b.cond.Broadcast() + return +} + +func (b *background) run() { + defer b.wg.Done() + + flush := time.NewTicker(b.options.FlushInterval) + defer flush.Stop() + + var bufs []*buffer.Buffer + for { + bufs = bufs[:0] + b.mu.Lock() + + bufs = b.q.dequeueAll(bufs) + for _, buf := range bufs { + b.writeBuffer.Write(buf.Bytes()) + buf.Free() + + if b.writeBuffer.Len() > b.options.WriteBufferSize { + b.flush() + } + } + + if b.writeBuffer.Len() != 0 { + b.flush() + } + + b.iter++ + b.mu.Unlock() + + if b.finishIter() { + return + } + + select { + case <-flush.C: + case <-b.forceFlush: + flush.Reset(b.options.FlushInterval) + } + + } +} + +func (b *background) checkQueueSize() (size int, ok, shouldReport bool) { + size = int(b.q.loadSize()) + if size >= b.options.MaxMemoryUsage { + atomic.AddInt64(&b.droppedRecords, 1) + + old := atomic.SwapInt64(&b.reportOverflow, 1) + return size, false, old == 0 + } + + return 0, true, false +} + +func (b *background) sync() error { + b.mu.Lock() + defer b.mu.Unlock() + + select { + case b.forceFlush <- struct{}{}: + default: + } + + now := b.iter + for { + if b.iter >= now+1 { + return b.lastErr + } + + if b.stopped { + return errors.New("core has stopped") + } + + b.cond.Wait() + } +} diff --git a/library/go/core/log/zap/asynczap/core.go b/library/go/core/log/zap/asynczap/core.go new file mode 100644 index 0000000000..11acd24fba --- /dev/null +++ b/library/go/core/log/zap/asynczap/core.go @@ -0,0 +1,113 @@ +// Package asynczap implements asynchronous core for zap. +// +// By default, zap writes every log line synchronously and without buffering. This behaviour +// is completely inadequate for high-rate logging. +// +// This implementation of zap.Core moves file write to background goroutine, while carefully +// monitoring memory consumption. +// +// When background goroutine can't keep up with logging rate, log records are dropped. +package asynczap + +import ( + "fmt" + "sync/atomic" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +type ( + Core struct { + zapcore.LevelEnabler + enc zapcore.Encoder + w *background + options Options + } + + Stats struct { + // Number of records dropped during memory overflow. + DroppedRecords int + + // Number of errors returned from underlying writer. + WriteErrors int + } +) + +// NewCore creates a Core that writes logs to a WriteSyncer. +func NewCore(enc zapcore.Encoder, ws zapcore.WriteSyncer, enab zapcore.LevelEnabler, options Options) *Core { + options.setDefault() + + w := newBackground(options, ws) + w.wg.Add(1) + go w.run() + + return &Core{ + LevelEnabler: enab, + enc: enc, + w: w, + } +} + +func (c *Core) Stop() { + _ = c.Sync() + c.w.stop() +} + +func (c *Core) With(fields []zap.Field) zapcore.Core { + clone := c.clone() + for i := range fields { + fields[i].AddTo(clone.enc) + } + return clone +} + +func (c *Core) Check(ent zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if c.Enabled(ent.Level) { + return ce.AddCore(ent, c) + } + return ce +} + +func (c *Core) Write(ent zapcore.Entry, fields []zap.Field) error { + if size, ok, shouldReport := c.w.checkQueueSize(); !ok { + if shouldReport { + // Report overflow error only once per background iteration, to avoid spamming error output. + return fmt.Errorf("logger queue overflow: %d >= %d", size, c.options.MaxMemoryUsage) + } else { + return nil + } + } + + buf, err := c.enc.EncodeEntry(ent, fields) + if err != nil { + return err + } + + c.w.q.enqueue(buf) + if ent.Level > zap.ErrorLevel { + // Since we may be crashing the program, sync the output. + _ = c.Sync() + } + return nil +} + +func (c *Core) Sync() error { + return c.w.sync() +} + +func (c *Core) Stat() Stats { + return Stats{ + DroppedRecords: int(atomic.LoadInt64(&c.w.droppedRecords)), + WriteErrors: int(atomic.LoadInt64(&c.w.writeErrors)), + } +} + +func (c *Core) clone() *Core { + return &Core{ + LevelEnabler: c.LevelEnabler, + enc: c.enc.Clone(), + w: c.w, + options: c.options, + } +} diff --git a/library/go/core/log/zap/asynczap/core_test.go b/library/go/core/log/zap/asynczap/core_test.go new file mode 100644 index 0000000000..35ae245678 --- /dev/null +++ b/library/go/core/log/zap/asynczap/core_test.go @@ -0,0 +1,123 @@ +package asynczap + +import ( + "bytes" + "os" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func TestCompareToDefault(t *testing.T) { + var buf0, buf1 bytes.Buffer + out0 := zapcore.AddSync(&buf0) + out1 := zapcore.AddSync(&buf1) + + format := zap.NewProductionEncoderConfig() + format.EncodeTime = func(t time.Time, e zapcore.PrimitiveArrayEncoder) { + e.AppendString("10:00") + } + + asyncCore := NewCore( + zapcore.NewJSONEncoder(format), + out0, + zap.DebugLevel, + Options{}) + + log0 := zap.New(asyncCore) + log0.Error("foo") + + require.NoError(t, asyncCore.Sync()) + asyncCore.Stop() + + syncCore := zapcore.NewCore( + zapcore.NewJSONEncoder(format), + out1, + zap.DebugLevel) + + log1 := zap.New(syncCore) + log1.Error("foo") + + require.Equal(t, buf0.String(), buf1.String()) +} + +type countWriteSyncer int32 + +func (c *countWriteSyncer) Write(b []byte) (int, error) { + atomic.AddInt32((*int32)(c), 1) + return len(b), nil +} + +func (c *countWriteSyncer) Sync() error { + return nil +} + +func TestSync(t *testing.T) { + var c countWriteSyncer + out0 := &c + + format := zap.NewProductionEncoderConfig() + format.EncodeTime = func(t time.Time, e zapcore.PrimitiveArrayEncoder) { + e.AppendString("10:00") + } + + asyncCore := NewCore( + zapcore.NewJSONEncoder(format), + out0, + zap.DebugLevel, + Options{FlushInterval: 10 * time.Nanosecond}) + + log0 := zap.New(asyncCore) + + for i := 0; i < 100000; i++ { + log0.Error("123") + _ = log0.Sync() + require.EqualValues(t, i+1, atomic.LoadInt32((*int32)(&c))) + } +} + +type lockWriter struct { + c chan struct{} +} + +func (w *lockWriter) Write(b []byte) (int, error) { + <-w.c + return 0, nil +} + +func TestDropsRecordsOnOverflow(t *testing.T) { + go func() { + time.Sleep(time.Second * 15) + + buf := make([]byte, 1024*1024) + n := runtime.Stack(buf, true) + _, _ = os.Stderr.Write(buf[:n]) + }() + + w := &lockWriter{c: make(chan struct{})} + + asyncCore := NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(w), + zap.DebugLevel, + Options{ + MaxMemoryUsage: 100, + }) + defer asyncCore.Stop() + + log := zap.New(asyncCore) + + for i := 0; i < 1000; i++ { + log.Error("foobar", zap.String("key", strings.Repeat("x", 1000))) + } + + assert.Greater(t, asyncCore.Stat().DroppedRecords, 990) + close(w.c) +} diff --git a/library/go/core/log/zap/asynczap/gotest/ya.make b/library/go/core/log/zap/asynczap/gotest/ya.make new file mode 100644 index 0000000000..30e947f6fc --- /dev/null +++ b/library/go/core/log/zap/asynczap/gotest/ya.make @@ -0,0 +1,3 @@ +GO_TEST_FOR(library/go/core/log/zap/asynczap) + +END() diff --git a/library/go/core/log/zap/asynczap/options.go b/library/go/core/log/zap/asynczap/options.go new file mode 100644 index 0000000000..9b7f241fe9 --- /dev/null +++ b/library/go/core/log/zap/asynczap/options.go @@ -0,0 +1,34 @@ +package asynczap + +import "time" + +const ( + defaultMaxMemoryUsage = 1 << 26 // 64MB + defaultWriteBufferSize = 1 << 20 // 1MB + defaultFlushInterval = time.Millisecond * 100 +) + +type Options struct { + // MaxMemoryUsage is maximum amount of memory that will be used by in-flight log records. + MaxMemoryUsage int + + // WriteBufferSize specifies size of the buffer used for writes to underlying file. + WriteBufferSize int + + // FlushInterval specifies how often background goroutine would wake up. + FlushInterval time.Duration +} + +func (o *Options) setDefault() { + if o.MaxMemoryUsage == 0 { + o.MaxMemoryUsage = defaultMaxMemoryUsage + } + + if o.WriteBufferSize == 0 { + o.WriteBufferSize = defaultWriteBufferSize + } + + if o.FlushInterval == 0 { + o.FlushInterval = defaultFlushInterval + } +} diff --git a/library/go/core/log/zap/asynczap/queue.go b/library/go/core/log/zap/asynczap/queue.go new file mode 100644 index 0000000000..a37e87ef47 --- /dev/null +++ b/library/go/core/log/zap/asynczap/queue.go @@ -0,0 +1,83 @@ +package asynczap + +import ( + "sync" + "sync/atomic" + "unsafe" + + "go.uber.org/zap/buffer" +) + +var entryPool sync.Pool + +func newEntry() *entry { + pooled := entryPool.Get() + if pooled != nil { + return pooled.(*entry) + } else { + return new(entry) + } +} + +func putEntry(e *entry) { + entryPool.Put(e) +} + +type entry struct { + next *entry + buf *buffer.Buffer +} + +type queue struct { + size int64 + head unsafe.Pointer +} + +func (q *queue) loadHead() *entry { + return (*entry)(atomic.LoadPointer(&q.head)) +} + +func (q *queue) casHead(old, new *entry) (swapped bool) { + return atomic.CompareAndSwapPointer(&q.head, unsafe.Pointer(old), unsafe.Pointer(new)) +} + +func (q *queue) swapHead() *entry { + return (*entry)(atomic.SwapPointer(&q.head, nil)) +} + +func (q *queue) loadSize() int64 { + return atomic.LoadInt64(&q.size) +} + +func (q *queue) enqueue(buf *buffer.Buffer) { + e := newEntry() + e.buf = buf + + atomic.AddInt64(&q.size, int64(buf.Cap())) + for { + e.next = q.loadHead() + if q.casHead(e.next, e) { + break + } + } +} + +func (q *queue) dequeueAll(to []*buffer.Buffer) []*buffer.Buffer { + head := q.swapHead() + + for head != nil { + atomic.AddInt64(&q.size, -int64(head.buf.Cap())) + to = append(to, head.buf) + + next := head.next + putEntry(head) + head = next + } + + for i := 0; i < len(to)/2; i++ { + j := len(to) - i - 1 + to[i], to[j] = to[j], to[i] + } + + return to +} diff --git a/library/go/core/log/zap/asynczap/queue_test.go b/library/go/core/log/zap/asynczap/queue_test.go new file mode 100644 index 0000000000..25e9e62a1e --- /dev/null +++ b/library/go/core/log/zap/asynczap/queue_test.go @@ -0,0 +1,59 @@ +package asynczap + +import ( + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/buffer" +) + +func BenchmarkQueue(b *testing.B) { + var q queue + + go func() { + var buf []*buffer.Buffer + + for range time.Tick(10 * time.Millisecond) { + buf = q.dequeueAll(buf) + buf = buf[:0] + } + }() + + p := &buffer.Buffer{} + + b.ReportAllocs() + b.SetParallelism(runtime.NumCPU() - 1) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + q.enqueue(p) + } + }) +} + +func TestQueue(t *testing.T) { + var b0, b1, b2, b3 *buffer.Buffer + b0 = &buffer.Buffer{} + b0.AppendString("b0") + b1 = &buffer.Buffer{} + b1.AppendString("b1") + b2 = &buffer.Buffer{} + b2.AppendString("b2") + b3 = &buffer.Buffer{} + b3.AppendString("b3") + + var q queue + q.enqueue(b0) + q.enqueue(b1) + q.enqueue(b2) + + require.Equal(t, []*buffer.Buffer{b0, b1, b2}, q.dequeueAll(nil)) + + q.enqueue(b0) + q.enqueue(b1) + q.enqueue(b2) + q.enqueue(b3) + + require.Equal(t, []*buffer.Buffer{b0, b1, b2, b3}, q.dequeueAll(nil)) +} diff --git a/library/go/core/log/zap/asynczap/ya.make b/library/go/core/log/zap/asynczap/ya.make new file mode 100644 index 0000000000..30e06b4d1b --- /dev/null +++ b/library/go/core/log/zap/asynczap/ya.make @@ -0,0 +1,17 @@ +GO_LIBRARY() + +SRCS( + background.go + core.go + options.go + queue.go +) + +GO_TEST_SRCS( + core_test.go + queue_test.go +) + +END() + +RECURSE(gotest) diff --git a/library/go/core/log/zap/benchmark_test.go b/library/go/core/log/zap/benchmark_test.go new file mode 100644 index 0000000000..bb3102e3a8 --- /dev/null +++ b/library/go/core/log/zap/benchmark_test.go @@ -0,0 +1,131 @@ +package zap + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/ydb-platform/ydb/library/go/core/log" + "go.uber.org/zap" +) + +func BenchmarkZapLogger(b *testing.B) { + // use config for both loggers + cfg := NewDeployConfig() + cfg.OutputPaths = nil + cfg.ErrorOutputPaths = nil + + b.Run("stock", func(b *testing.B) { + for _, level := range log.Levels() { + b.Run(level.String(), func(b *testing.B) { + cfg.Level = zap.NewAtomicLevelAt(ZapifyLevel(level)) + + logger, err := cfg.Build() + require.NoError(b, err) + + funcs := []func(string, ...zap.Field){ + logger.Debug, + logger.Info, + logger.Warn, + logger.Error, + logger.Fatal, + } + + message := "test" + fields := []zap.Field{ + zap.String("test", "test"), + zap.Bool("test", true), + zap.Int("test", 42), + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + funcs[i%(len(funcs)-1)](message, fields...) + } + }) + } + }) + + b.Run("wrapped", func(b *testing.B) { + for _, level := range log.Levels() { + b.Run(level.String(), func(b *testing.B) { + cfg.Level = zap.NewAtomicLevelAt(ZapifyLevel(level)) + logger, err := New(cfg) + require.NoError(b, err) + + funcs := []func(string, ...log.Field){ + logger.Debug, + logger.Info, + logger.Warn, + logger.Error, + logger.Fatal, + } + + message := "test" + fields := []log.Field{ + log.String("test", "test"), + log.Bool("test", true), + log.Int("test", 42), + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + funcs[i%(len(funcs)-1)](message, fields...) + } + }) + } + }) +} + +func BenchmarkZapifyField(b *testing.B) { + fields := []log.Field{ + log.Nil("test"), + log.String("test", "test"), + log.Binary("test", []byte("test")), + log.Bool("test", true), + log.Int("test", 42), + log.UInt("test", 42), + log.Float64("test", 42), + log.Time("test", time.Now()), + log.Duration("test", time.Second), + log.NamedError("test", errors.New("test")), + log.Strings("test", []string{"test"}), + log.Any("test", "test"), + log.Reflect("test", "test"), + log.ByteString("test", []byte("test")), + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + zapifyField(fields[i%(len(fields)-1)]) + } +} + +func BenchmarkZapifyFields(b *testing.B) { + fields := []log.Field{ + log.Nil("test"), + log.String("test", "test"), + log.Binary("test", []byte("test")), + log.Bool("test", true), + log.Int("test", 42), + log.UInt("test", 42), + log.Float64("test", 42), + log.Time("test", time.Now()), + log.Duration("test", time.Second), + log.NamedError("test", errors.New("test")), + log.Strings("test", []string{"test"}), + log.Any("test", "test"), + log.Reflect("test", "test"), + log.ByteString("test", []byte("test")), + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + zapifyFields(fields...) + } +} diff --git a/library/go/core/log/zap/deploy.go b/library/go/core/log/zap/deploy.go new file mode 100644 index 0000000000..f86cfbab31 --- /dev/null +++ b/library/go/core/log/zap/deploy.go @@ -0,0 +1,132 @@ +package zap + +import ( + "github.com/ydb-platform/ydb/library/go/core/log" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// NewDeployEncoderConfig returns an opinionated EncoderConfig for +// deploy environment. +func NewDeployEncoderConfig() zapcore.EncoderConfig { + return zapcore.EncoderConfig{ + MessageKey: "msg", + LevelKey: "levelStr", + StacktraceKey: "stackTrace", + TimeKey: "@timestamp", + CallerKey: "", + NameKey: "loggerName", + EncodeLevel: zapcore.CapitalLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeDuration: zapcore.StringDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + } +} + +type cfgOption func(cfg *zap.Config) + +// WithSampling sets sampling settings initial and thereafter +func WithSampling(initial int, thereafter int) cfgOption { + return cfgOption(func(cfg *zap.Config) { + cfg.Sampling = &zap.SamplingConfig{ + Initial: initial, + Thereafter: thereafter, + } + }) +} + +// SetOutputPaths sets OutputPaths (stdout by default) +func SetOutputPaths(paths []string) cfgOption { + return cfgOption(func(cfg *zap.Config) { + cfg.OutputPaths = paths + }) +} + +// WithDevelopment sets Development option of zap.Config +func WithDevelopment(enabled bool) cfgOption { + return cfgOption(func(cfg *zap.Config) { + cfg.Development = enabled + }) +} + +// WithLevel sets level of logging +func WithLevel(level log.Level) cfgOption { + return cfgOption(func(cfg *zap.Config) { + cfg.Level = zap.NewAtomicLevelAt(ZapifyLevel(level)) + }) +} + +// NewDeployConfig returns default configuration (with no sampling). +// Not recommended for production use. +func NewDeployConfig(opts ...cfgOption) zap.Config { + cfg := zap.Config{ + Level: zap.NewAtomicLevelAt(zap.DebugLevel), + Encoding: "json", + OutputPaths: []string{"stdout"}, + ErrorOutputPaths: []string{"stderr"}, + EncoderConfig: NewDeployEncoderConfig(), + } + + for _, opt := range opts { + opt(&cfg) + } + + return cfg +} + +// NewCustomDeployLogger constructs new logger by config cfg +func NewCustomDeployLogger(cfg zap.Config, opts ...zap.Option) (*Logger, error) { + zl, err := cfg.Build(opts...) + if err != nil { + return nil, err + } + + return &Logger{ + L: addDeployContext(zl).(*zap.Logger), + }, nil +} + +// NewDeployLogger constructs fully-fledged Deploy compatible logger +// based on predefined config. See https://deploy.yandex-team.ru/docs/concepts/pod/sidecars/logs/logs#format +// for more information +func NewDeployLogger(level log.Level, opts ...zap.Option) (*Logger, error) { + return NewCustomDeployLogger( + NewDeployConfig( + WithLevel(level), + ), + opts..., + ) +} + +// NewProductionDeployConfig returns configuration, suitable for production use. +// +// It uses a JSON encoder, writes to standard error, and enables sampling. +// Stacktraces are automatically included on logs of ErrorLevel and above. +func NewProductionDeployConfig() zap.Config { + return NewDeployConfig( + WithDevelopment(false), + WithSampling(100, 100), + ) +} + +// Same as NewDeployLogger, but with sampling +func NewProductionDeployLogger(level log.Level, opts ...zap.Option) (*Logger, error) { + return NewCustomDeployLogger( + NewDeployConfig( + WithLevel(level), + WithDevelopment(false), + WithSampling(100, 100), + ), + opts..., + ) +} + +func addDeployContext(i interface{}) interface{} { + switch c := i.(type) { + case *zap.Logger: + return c.With(zap.Namespace("@fields")) + case zapcore.Core: + return c.With([]zapcore.Field{zap.Namespace("@fields")}) + } + return i +} diff --git a/library/go/core/log/zap/encoders/cli.go b/library/go/core/log/zap/encoders/cli.go new file mode 100644 index 0000000000..f19d8527df --- /dev/null +++ b/library/go/core/log/zap/encoders/cli.go @@ -0,0 +1,78 @@ +package encoders + +import ( + "sync" + + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" +) + +const ( + // EncoderNameCli is the encoder name to use for zap config + EncoderNameCli = "cli" +) + +var cliPool = sync.Pool{New: func() interface{} { + return &cliEncoder{} +}} + +func getCliEncoder() *cliEncoder { + return cliPool.Get().(*cliEncoder) +} + +type cliEncoder struct { + *kvEncoder +} + +// NewCliEncoder constructs cli encoder +func NewCliEncoder(cfg zapcore.EncoderConfig) (zapcore.Encoder, error) { + return newCliEncoder(cfg), nil +} + +func newCliEncoder(cfg zapcore.EncoderConfig) *cliEncoder { + return &cliEncoder{ + kvEncoder: newKVEncoder(cfg), + } +} + +func (enc *cliEncoder) Clone() zapcore.Encoder { + clone := enc.clone() + _, _ = clone.buf.Write(enc.buf.Bytes()) + return clone +} + +func (enc *cliEncoder) clone() *cliEncoder { + clone := getCliEncoder() + clone.kvEncoder = getKVEncoder() + clone.cfg = enc.cfg + clone.openNamespaces = enc.openNamespaces + clone.pool = enc.pool + clone.buf = enc.pool.Get() + return clone +} + +func (enc *cliEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { + final := enc.clone() + + // Direct write because we do not want to quote message in cli mode + final.buf.AppendString(ent.Message) + + // Add any structured context. + for _, f := range fields { + f.AddTo(final) + } + + // If there's no stacktrace key, honor that; this allows users to force + // single-line output. + if ent.Stack != "" && final.cfg.StacktraceKey != "" { + final.buf.AppendByte('\n') + final.AppendString(ent.Stack) + } + + if final.cfg.LineEnding != "" { + final.AppendString(final.cfg.LineEnding) + } else { + final.AppendString(zapcore.DefaultLineEnding) + } + return final.buf, nil +} diff --git a/library/go/core/log/zap/encoders/gotest/ya.make b/library/go/core/log/zap/encoders/gotest/ya.make new file mode 100644 index 0000000000..8f23accf05 --- /dev/null +++ b/library/go/core/log/zap/encoders/gotest/ya.make @@ -0,0 +1,3 @@ +GO_TEST_FOR(library/go/core/log/zap/encoders) + +END() diff --git a/library/go/core/log/zap/encoders/kv.go b/library/go/core/log/zap/encoders/kv.go new file mode 100644 index 0000000000..8fd6c607c6 --- /dev/null +++ b/library/go/core/log/zap/encoders/kv.go @@ -0,0 +1,386 @@ +package encoders + +import ( + "encoding/base64" + "encoding/json" + "math" + "strings" + "sync" + "time" + + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" +) + +const ( + // EncoderNameKV is the encoder name to use for zap config + EncoderNameKV = "kv" +) + +const ( + // We use ' for quote symbol instead of " so that it doesn't interfere with %q of fmt package + stringQuoteSymbol = '\'' + kvArraySeparator = ',' +) + +var kvPool = sync.Pool{New: func() interface{} { + return &kvEncoder{} +}} + +func getKVEncoder() *kvEncoder { + return kvPool.Get().(*kvEncoder) +} + +type kvEncoder struct { + cfg zapcore.EncoderConfig + pool buffer.Pool + buf *buffer.Buffer + openNamespaces int + + // for encoding generic values by reflection + reflectBuf *buffer.Buffer + reflectEnc *json.Encoder +} + +// NewKVEncoder constructs kv encoder +func NewKVEncoder(cfg zapcore.EncoderConfig) (zapcore.Encoder, error) { + return newKVEncoder(cfg), nil +} + +func newKVEncoder(cfg zapcore.EncoderConfig) *kvEncoder { + pool := buffer.NewPool() + return &kvEncoder{ + cfg: cfg, + pool: pool, + buf: pool.Get(), + } +} + +func (enc *kvEncoder) addElementSeparator() { + if enc.buf.Len() == 0 { + return + } + + enc.buf.AppendByte(' ') +} + +func (enc *kvEncoder) addKey(key string) { + enc.addElementSeparator() + enc.buf.AppendString(key) + enc.buf.AppendByte('=') +} + +func (enc *kvEncoder) appendFloat(val float64, bitSize int) { + enc.appendArrayItemSeparator() + switch { + case math.IsNaN(val): + enc.buf.AppendString(`"NaN"`) + case math.IsInf(val, 1): + enc.buf.AppendString(`"+Inf"`) + case math.IsInf(val, -1): + enc.buf.AppendString(`"-Inf"`) + default: + enc.buf.AppendFloat(val, bitSize) + } +} + +func (enc *kvEncoder) AddArray(key string, arr zapcore.ArrayMarshaler) error { + enc.addKey(key) + return enc.AppendArray(arr) +} + +func (enc *kvEncoder) AddObject(key string, obj zapcore.ObjectMarshaler) error { + enc.addKey(key) + return enc.AppendObject(obj) +} + +func (enc *kvEncoder) AddBinary(key string, val []byte) { + enc.AddString(key, base64.StdEncoding.EncodeToString(val)) +} + +func (enc *kvEncoder) AddByteString(key string, val []byte) { + enc.addKey(key) + enc.AppendByteString(val) +} + +func (enc *kvEncoder) AddBool(key string, val bool) { + enc.addKey(key) + enc.AppendBool(val) +} + +func (enc *kvEncoder) AddComplex128(key string, val complex128) { + enc.addKey(key) + enc.AppendComplex128(val) +} + +func (enc *kvEncoder) AddDuration(key string, val time.Duration) { + enc.addKey(key) + enc.AppendDuration(val) +} + +func (enc *kvEncoder) AddFloat64(key string, val float64) { + enc.addKey(key) + enc.AppendFloat64(val) +} + +func (enc *kvEncoder) AddInt64(key string, val int64) { + enc.addKey(key) + enc.AppendInt64(val) +} + +func (enc *kvEncoder) resetReflectBuf() { + if enc.reflectBuf == nil { + enc.reflectBuf = enc.pool.Get() + enc.reflectEnc = json.NewEncoder(enc.reflectBuf) + } else { + enc.reflectBuf.Reset() + } +} + +func (enc *kvEncoder) AddReflected(key string, obj interface{}) error { + enc.resetReflectBuf() + err := enc.reflectEnc.Encode(obj) + if err != nil { + return err + } + enc.reflectBuf.TrimNewline() + enc.addKey(key) + _, err = enc.buf.Write(enc.reflectBuf.Bytes()) + return err +} + +func (enc *kvEncoder) OpenNamespace(key string) { + enc.addKey(key) + enc.buf.AppendByte('{') + enc.openNamespaces++ +} + +func (enc *kvEncoder) AddString(key, val string) { + enc.addKey(key) + enc.AppendString(val) +} + +func (enc *kvEncoder) AddTime(key string, val time.Time) { + enc.addKey(key) + enc.AppendTime(val) +} + +func (enc *kvEncoder) AddUint64(key string, val uint64) { + enc.addKey(key) + enc.AppendUint64(val) +} + +func (enc *kvEncoder) appendArrayItemSeparator() { + last := enc.buf.Len() - 1 + if last < 0 { + return + } + + switch enc.buf.Bytes()[last] { + case '[', '{', '=': + return + default: + enc.buf.AppendByte(kvArraySeparator) + } +} + +func (enc *kvEncoder) AppendArray(arr zapcore.ArrayMarshaler) error { + enc.appendArrayItemSeparator() + enc.buf.AppendByte('[') + err := arr.MarshalLogArray(enc) + enc.buf.AppendByte(']') + return err +} + +func (enc *kvEncoder) AppendObject(obj zapcore.ObjectMarshaler) error { + enc.appendArrayItemSeparator() + enc.buf.AppendByte('{') + err := obj.MarshalLogObject(enc) + enc.buf.AppendByte('}') + return err +} + +func (enc *kvEncoder) AppendBool(val bool) { + enc.appendArrayItemSeparator() + enc.buf.AppendBool(val) +} + +func (enc *kvEncoder) AppendByteString(val []byte) { + enc.appendArrayItemSeparator() + _, _ = enc.buf.Write(val) +} + +func (enc *kvEncoder) AppendComplex128(val complex128) { + enc.appendArrayItemSeparator() + r, i := real(val), imag(val) + + enc.buf.AppendByte('"') + // Because we're always in a quoted string, we can use strconv without + // special-casing NaN and +/-Inf. + enc.buf.AppendFloat(r, 64) + enc.buf.AppendByte('+') + enc.buf.AppendFloat(i, 64) + enc.buf.AppendByte('i') + enc.buf.AppendByte('"') +} + +func (enc *kvEncoder) AppendDuration(val time.Duration) { + cur := enc.buf.Len() + enc.cfg.EncodeDuration(val, enc) + if cur == enc.buf.Len() { + // User-supplied EncodeDuration is a no-op. Fall back to nanoseconds to keep + // JSON valid. + enc.AppendInt64(int64(val)) + } +} + +func (enc *kvEncoder) AppendInt64(val int64) { + enc.appendArrayItemSeparator() + enc.buf.AppendInt(val) +} + +func (enc *kvEncoder) AppendReflected(val interface{}) error { + enc.appendArrayItemSeparator() + enc.resetReflectBuf() + err := enc.reflectEnc.Encode(val) + if err != nil { + return err + } + enc.reflectBuf.TrimNewline() + enc.addElementSeparator() + _, err = enc.buf.Write(enc.reflectBuf.Bytes()) + return err +} + +func (enc *kvEncoder) AppendString(val string) { + enc.appendArrayItemSeparator() + var quotes bool + if strings.ContainsAny(val, " =[]{}") { + quotes = true + } + + if quotes { + enc.buf.AppendByte(stringQuoteSymbol) + } + enc.buf.AppendString(val) + if quotes { + enc.buf.AppendByte(stringQuoteSymbol) + } +} + +func (enc *kvEncoder) AppendTime(val time.Time) { + cur := enc.buf.Len() + enc.cfg.EncodeTime(val, enc) + if cur == enc.buf.Len() { + // User-supplied EncodeTime is a no-op. Fall back to nanos since epoch to keep + // output JSON valid. + enc.AppendInt64(val.UnixNano()) + } +} + +func (enc *kvEncoder) AppendUint64(val uint64) { + enc.appendArrayItemSeparator() + enc.buf.AppendUint(val) +} + +func (enc *kvEncoder) AddComplex64(k string, v complex64) { enc.AddComplex128(k, complex128(v)) } +func (enc *kvEncoder) AddFloat32(k string, v float32) { enc.AddFloat64(k, float64(v)) } +func (enc *kvEncoder) AddInt(k string, v int) { enc.AddInt64(k, int64(v)) } +func (enc *kvEncoder) AddInt32(k string, v int32) { enc.AddInt64(k, int64(v)) } +func (enc *kvEncoder) AddInt16(k string, v int16) { enc.AddInt64(k, int64(v)) } +func (enc *kvEncoder) AddInt8(k string, v int8) { enc.AddInt64(k, int64(v)) } +func (enc *kvEncoder) AddUint(k string, v uint) { enc.AddUint64(k, uint64(v)) } +func (enc *kvEncoder) AddUint32(k string, v uint32) { enc.AddUint64(k, uint64(v)) } +func (enc *kvEncoder) AddUint16(k string, v uint16) { enc.AddUint64(k, uint64(v)) } +func (enc *kvEncoder) AddUint8(k string, v uint8) { enc.AddUint64(k, uint64(v)) } +func (enc *kvEncoder) AddUintptr(k string, v uintptr) { enc.AddUint64(k, uint64(v)) } +func (enc *kvEncoder) AppendComplex64(v complex64) { enc.AppendComplex128(complex128(v)) } +func (enc *kvEncoder) AppendFloat64(v float64) { enc.appendFloat(v, 64) } +func (enc *kvEncoder) AppendFloat32(v float32) { enc.appendFloat(float64(v), 32) } +func (enc *kvEncoder) AppendInt(v int) { enc.AppendInt64(int64(v)) } +func (enc *kvEncoder) AppendInt32(v int32) { enc.AppendInt64(int64(v)) } +func (enc *kvEncoder) AppendInt16(v int16) { enc.AppendInt64(int64(v)) } +func (enc *kvEncoder) AppendInt8(v int8) { enc.AppendInt64(int64(v)) } +func (enc *kvEncoder) AppendUint(v uint) { enc.AppendUint64(uint64(v)) } +func (enc *kvEncoder) AppendUint32(v uint32) { enc.AppendUint64(uint64(v)) } +func (enc *kvEncoder) AppendUint16(v uint16) { enc.AppendUint64(uint64(v)) } +func (enc *kvEncoder) AppendUint8(v uint8) { enc.AppendUint64(uint64(v)) } +func (enc *kvEncoder) AppendUintptr(v uintptr) { enc.AppendUint64(uint64(v)) } + +func (enc *kvEncoder) Clone() zapcore.Encoder { + clone := enc.clone() + _, _ = clone.buf.Write(enc.buf.Bytes()) + return clone +} + +func (enc *kvEncoder) clone() *kvEncoder { + clone := getKVEncoder() + clone.cfg = enc.cfg + clone.openNamespaces = enc.openNamespaces + clone.pool = enc.pool + clone.buf = enc.pool.Get() + return clone +} + +// nolint: gocyclo +func (enc *kvEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { + final := enc.clone() + if final.cfg.TimeKey != "" && final.cfg.EncodeTime != nil { + final.addElementSeparator() + final.buf.AppendString(final.cfg.TimeKey + "=") + final.cfg.EncodeTime(ent.Time, final) + } + if final.cfg.LevelKey != "" && final.cfg.EncodeLevel != nil { + final.addElementSeparator() + final.buf.AppendString(final.cfg.LevelKey + "=") + final.cfg.EncodeLevel(ent.Level, final) + } + if ent.LoggerName != "" && final.cfg.NameKey != "" { + nameEncoder := final.cfg.EncodeName + + if nameEncoder == nil { + // Fall back to FullNameEncoder for backward compatibility. + nameEncoder = zapcore.FullNameEncoder + } + + final.addElementSeparator() + final.buf.AppendString(final.cfg.NameKey + "=") + nameEncoder(ent.LoggerName, final) + } + if ent.Caller.Defined && final.cfg.CallerKey != "" && final.cfg.EncodeCaller != nil { + final.addElementSeparator() + final.buf.AppendString(final.cfg.CallerKey + "=") + final.cfg.EncodeCaller(ent.Caller, final) + } + + if enc.buf.Len() > 0 { + final.addElementSeparator() + _, _ = final.buf.Write(enc.buf.Bytes()) + } + + // Add the message itself. + if final.cfg.MessageKey != "" { + final.addElementSeparator() + final.buf.AppendString(final.cfg.MessageKey + "=") + final.AppendString(ent.Message) + } + + // Add any structured context. + for _, f := range fields { + f.AddTo(final) + } + + // If there's no stacktrace key, honor that; this allows users to force + // single-line output. + if ent.Stack != "" && final.cfg.StacktraceKey != "" { + final.buf.AppendByte('\n') + final.buf.AppendString(ent.Stack) + } + + if final.cfg.LineEnding != "" { + final.buf.AppendString(final.cfg.LineEnding) + } else { + final.buf.AppendString(zapcore.DefaultLineEnding) + } + return final.buf, nil +} diff --git a/library/go/core/log/zap/encoders/kv_test.go b/library/go/core/log/zap/encoders/kv_test.go new file mode 100644 index 0000000000..85778e888f --- /dev/null +++ b/library/go/core/log/zap/encoders/kv_test.go @@ -0,0 +1,121 @@ +package encoders + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func TestKVEncodeEntry(t *testing.T) { + type bar struct { + Key string `json:"key"` + + Val float64 `json:"val"` + } + + type foo struct { + A string `json:"aee"` + B int `json:"bee"` + C float64 `json:"cee"` + D []bar `json:"dee"` + } + + tests := []struct { + desc string + expected string + ent zapcore.Entry + fields []zapcore.Field + }{ + { + desc: "info entry with some fields", + expected: `T=2018-06-19T16:33:42.000Z L=info N=bob M='lob law' so=passes answer=42 common_pie=3.14 ` + + `such={"aee":"lol","bee":123,"cee":0.9999,"dee":[{"key":"pi","val":3.141592653589793},` + + `{"key":"tau","val":6.283185307179586}]} +`, + ent: zapcore.Entry{ + Level: zapcore.InfoLevel, + Time: time.Date(2018, 6, 19, 16, 33, 42, 99, time.UTC), + LoggerName: "bob", + Message: "lob law", + }, + fields: []zapcore.Field{ + zap.String("so", "passes"), + zap.Int("answer", 42), + zap.Float64("common_pie", 3.14), + zap.Reflect("such", foo{ + A: "lol", + B: 123, + C: 0.9999, + D: []bar{ + {"pi", 3.141592653589793}, + {"tau", 6.283185307179586}, + }, + }), + }, + }, + { + desc: "info entry with array fields", + expected: `T=2020-06-26T11:13:42.000Z L=info N=alice M='str array' env=test ` + + `intarray=[-5,-7,0,-12] ` + + `uintarray=[1,2,3,4,5] ` + + `strarray=[funny,bunny] ` + + `book=['Alice's Adventures in Wonderland','Lewis Carroll',26-11-1865] ` + + `floatarray=[3.14,-2.17,0.0000000000000000000000000000000000662607]` + "\n", + ent: zapcore.Entry{ + Level: zapcore.InfoLevel, + Time: time.Date(2020, 6, 26, 11, 13, 42, 0, time.UTC), + LoggerName: "alice", + Message: "str array", + }, + fields: []zapcore.Field{ + zap.String("env", "test"), + zap.Ints("intarray", []int{-5, -7, 0, -12}), + zap.Uints("uintarray", []uint{1, 2, 3, 4, 5}), + zap.Strings("strarray", []string{"funny", "bunny"}), + zap.Strings("book", []string{"Alice's Adventures in Wonderland", "Lewis Carroll", "26-11-1865"}), + zap.Float32s("floatarray", []float32{3.14, -2.17, 0.662607015e-34}), + }, + }, + { + desc: "corner cases of arrays", + expected: "T=2020-06-26T12:13:42.000Z L=info N=zorg M='str array' cornerequal=['hello=',world] cornerbracket=['is[',jail,']'] cornerbraces=['is{',exit,'}']\n", + ent: zapcore.Entry{ + Level: zapcore.InfoLevel, + Time: time.Date(2020, 6, 26, 12, 13, 42, 0, time.UTC), + LoggerName: "zorg", + Message: "str array", + }, + fields: []zapcore.Field{ + zap.Strings("cornerequal", []string{"hello=", "world"}), + zap.Strings("cornerbracket", []string{"is[", "jail", "]"}), + zap.Strings("cornerbraces", []string{"is{", "exit", "}"}), + }, + }, + } + + enc, _ := NewKVEncoder(zapcore.EncoderConfig{ + MessageKey: "M", + LevelKey: "L", + TimeKey: "T", + NameKey: "N", + CallerKey: "C", + StacktraceKey: "S", + EncodeLevel: zapcore.LowercaseLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeDuration: zapcore.SecondsDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + }) + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + buf, err := enc.EncodeEntry(tt.ent, tt.fields) + if assert.NoError(t, err, "Unexpected KV encoding error.") { + assert.Equal(t, tt.expected, buf.String(), "Incorrect encoded KV entry.") + } + buf.Free() + }) + } +} diff --git a/library/go/core/log/zap/encoders/tskv.go b/library/go/core/log/zap/encoders/tskv.go new file mode 100644 index 0000000000..75fafa1e20 --- /dev/null +++ b/library/go/core/log/zap/encoders/tskv.go @@ -0,0 +1,442 @@ +package encoders + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "math" + "strings" + "sync" + "time" + + "github.com/ydb-platform/ydb/library/go/core/xerrors" + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" +) + +const ( + // EncoderNameKV is the encoder name to use for zap config + EncoderNameTSKV = "tskv" +) + +const ( + tskvLineEnding = '\n' + tskvElementSeparator = '\t' + tskvKVSeparator = '=' + tskvMark = "tskv" + tskvArrayStart = '[' + tskvArrayEnd = ']' + tskvArraySeparator = ',' +) + +var tskvKeyEscapeRules = []string{ + `\`, `\\`, + "\t", "\\t", + "\n", "\\n", + "\r", `\r`, + "\x00", `\0`, + "=", `\=`, +} + +var tskvValueEscapeRules = []string{ + `\`, `\\`, + "\t", "\\t", + "\n", `\n`, + "\r", `\r`, + "\x00", `\0`, +} + +type tskvEscaper struct { + keyReplacer *strings.Replacer + valueReplacer *strings.Replacer +} + +func newTSKVEscaper() tskvEscaper { + return tskvEscaper{ + keyReplacer: strings.NewReplacer(tskvKeyEscapeRules...), + valueReplacer: strings.NewReplacer(tskvValueEscapeRules...), + } +} + +func (esc *tskvEscaper) escapeKey(key string) string { + return esc.keyReplacer.Replace(key) +} + +func (esc *tskvEscaper) escapeValue(val string) string { + return esc.valueReplacer.Replace(val) +} + +func hexEncode(val []byte) []byte { + dst := make([]byte, hex.EncodedLen(len(val))) + hex.Encode(dst, val) + return dst +} + +var tskvPool = sync.Pool{New: func() interface{} { + return &tskvEncoder{} +}} + +func getTSKVEncoder() *tskvEncoder { + return tskvPool.Get().(*tskvEncoder) +} + +type tskvEncoder struct { + cfg zapcore.EncoderConfig + pool buffer.Pool + buf *buffer.Buffer + + // for encoding generic values by reflection + reflectBuf *buffer.Buffer + reflectEnc *json.Encoder + + tskvEscaper tskvEscaper +} + +// NewKVEncoder constructs tskv encoder +func NewTSKVEncoder(cfg zapcore.EncoderConfig) (zapcore.Encoder, error) { + return newTSKVEncoder(cfg), nil +} + +func newTSKVEncoder(cfg zapcore.EncoderConfig) *tskvEncoder { + pool := buffer.NewPool() + return &tskvEncoder{ + cfg: cfg, + pool: pool, + buf: pool.Get(), + tskvEscaper: newTSKVEscaper(), + } +} + +func (enc *tskvEncoder) appendElementSeparator() { + if enc.buf.Len() == 0 { + return + } + + enc.buf.AppendByte(tskvElementSeparator) +} + +func (enc *tskvEncoder) appendArrayItemSeparator() { + last := enc.buf.Len() - 1 + if last < 0 { + return + } + + switch enc.buf.Bytes()[last] { + case tskvArrayStart, tskvKVSeparator: + return + default: + enc.buf.AppendByte(tskvArraySeparator) + } +} + +func (enc *tskvEncoder) safeAppendKey(key string) { + enc.appendElementSeparator() + enc.buf.AppendString(enc.tskvEscaper.escapeKey(key)) + enc.buf.AppendByte(tskvKVSeparator) +} + +func (enc *tskvEncoder) safeAppendString(val string) { + enc.buf.AppendString(enc.tskvEscaper.escapeValue(val)) +} + +func (enc *tskvEncoder) appendFloat(val float64, bitSize int) { + enc.appendArrayItemSeparator() + switch { + case math.IsNaN(val): + enc.buf.AppendString(`"NaN"`) + case math.IsInf(val, 1): + enc.buf.AppendString(`"+Inf"`) + case math.IsInf(val, -1): + enc.buf.AppendString(`"-Inf"`) + default: + enc.buf.AppendFloat(val, bitSize) + } +} + +func (enc *tskvEncoder) AddArray(key string, arr zapcore.ArrayMarshaler) error { + enc.safeAppendKey(key) + return enc.AppendArray(arr) +} + +func (enc *tskvEncoder) AddObject(key string, obj zapcore.ObjectMarshaler) error { + enc.safeAppendKey(key) + return enc.AppendObject(obj) +} + +func (enc *tskvEncoder) AddBinary(key string, val []byte) { + enc.AddByteString(key, val) +} + +func (enc *tskvEncoder) AddByteString(key string, val []byte) { + enc.safeAppendKey(key) + enc.AppendByteString(val) +} + +func (enc *tskvEncoder) AddBool(key string, val bool) { + enc.safeAppendKey(key) + enc.AppendBool(val) +} + +func (enc *tskvEncoder) AddComplex128(key string, val complex128) { + enc.safeAppendKey(key) + enc.AppendComplex128(val) +} + +func (enc *tskvEncoder) AddDuration(key string, val time.Duration) { + enc.safeAppendKey(key) + enc.AppendDuration(val) +} + +func (enc *tskvEncoder) AddFloat64(key string, val float64) { + enc.safeAppendKey(key) + enc.AppendFloat64(val) +} + +func (enc *tskvEncoder) AddInt64(key string, val int64) { + enc.safeAppendKey(key) + enc.AppendInt64(val) +} + +func (enc *tskvEncoder) resetReflectBuf() { + if enc.reflectBuf == nil { + enc.reflectBuf = enc.pool.Get() + enc.reflectEnc = json.NewEncoder(enc.reflectBuf) + } else { + enc.reflectBuf.Reset() + } +} + +func (enc *tskvEncoder) AddReflected(key string, obj interface{}) error { + enc.resetReflectBuf() + err := enc.reflectEnc.Encode(obj) + if err != nil { + return err + } + enc.reflectBuf.TrimNewline() + enc.safeAppendKey(key) + enc.safeAppendString(enc.reflectBuf.String()) + return err +} + +// OpenNamespace is not supported due to tskv format design +// See AppendObject() for more details +func (enc *tskvEncoder) OpenNamespace(key string) { + panic("TSKV encoder does not support namespaces") +} + +func (enc *tskvEncoder) AddString(key, val string) { + enc.safeAppendKey(key) + enc.safeAppendString(val) +} + +func (enc *tskvEncoder) AddTime(key string, val time.Time) { + enc.safeAppendKey(key) + enc.AppendTime(val) +} + +func (enc *tskvEncoder) AddUint64(key string, val uint64) { + enc.safeAppendKey(key) + enc.AppendUint64(val) +} + +func (enc *tskvEncoder) AppendArray(arr zapcore.ArrayMarshaler) error { + enc.appendArrayItemSeparator() + enc.buf.AppendByte(tskvArrayStart) + err := arr.MarshalLogArray(enc) + enc.buf.AppendByte(tskvArrayEnd) + return err +} + +// TSKV format does not support hierarchy data so we can't log Objects here +// The only thing we can do is to implicitly use fmt.Stringer interface +// +// ObjectMarshaler interface requires MarshalLogObject method +// from within MarshalLogObject you only have access to ObjectEncoder methods (AddString, AddBool ...) +// so if you call AddString then object log will be split by \t sign +// but \t is key-value separator and tskv doesn't have another separators +// e.g +// json encoded: objLogFieldName={"innerObjKey1":{"innerObjKey2":"value"}} +// tskv encoded: objLogFieldName={ \tinnerObjKey1={ \tinnerObjKey2=value}} +func (enc *tskvEncoder) AppendObject(obj zapcore.ObjectMarshaler) error { + var err error + + enc.appendArrayItemSeparator() + enc.buf.AppendByte('{') + stringerObj, ok := obj.(fmt.Stringer) + if !ok { + err = xerrors.Errorf("fmt.Stringer implementation required due to marshall into tskv format") + } else { + enc.safeAppendString(stringerObj.String()) + } + enc.buf.AppendByte('}') + + return err +} + +func (enc *tskvEncoder) AppendBool(val bool) { + enc.appendArrayItemSeparator() + enc.buf.AppendBool(val) +} + +func (enc *tskvEncoder) AppendByteString(val []byte) { + enc.appendArrayItemSeparator() + _, _ = enc.buf.Write(hexEncode(val)) +} + +func (enc *tskvEncoder) AppendComplex128(val complex128) { // TODO + enc.appendArrayItemSeparator() + + r, i := real(val), imag(val) + enc.buf.AppendByte('"') + // Because we're always in a quoted string, we can use strconv without + // special-casing NaN and +/-Inf. + enc.buf.AppendFloat(r, 64) + enc.buf.AppendByte('+') + enc.buf.AppendFloat(i, 64) + enc.buf.AppendByte('i') + enc.buf.AppendByte('"') +} + +func (enc *tskvEncoder) AppendDuration(val time.Duration) { + cur := enc.buf.Len() + enc.cfg.EncodeDuration(val, enc) + if cur == enc.buf.Len() { + // User-supplied EncodeDuration is a no-op. Fall back to nanoseconds + enc.AppendInt64(int64(val)) + } +} + +func (enc *tskvEncoder) AppendInt64(val int64) { + enc.appendArrayItemSeparator() + enc.buf.AppendInt(val) +} + +func (enc *tskvEncoder) AppendReflected(val interface{}) error { + enc.appendArrayItemSeparator() + + enc.resetReflectBuf() + err := enc.reflectEnc.Encode(val) + if err != nil { + return err + } + enc.reflectBuf.TrimNewline() + enc.safeAppendString(enc.reflectBuf.String()) + return nil +} + +func (enc *tskvEncoder) AppendString(val string) { + enc.appendArrayItemSeparator() + enc.safeAppendString(val) +} + +func (enc *tskvEncoder) AppendTime(val time.Time) { + cur := enc.buf.Len() + enc.cfg.EncodeTime(val, enc) + if cur == enc.buf.Len() { + // User-supplied EncodeTime is a no-op. Fall back to nanos since epoch to keep output tskv valid. + enc.AppendInt64(val.Unix()) + } +} + +func (enc *tskvEncoder) AppendUint64(val uint64) { + enc.appendArrayItemSeparator() + enc.buf.AppendUint(val) +} + +func (enc *tskvEncoder) AddComplex64(k string, v complex64) { enc.AddComplex128(k, complex128(v)) } +func (enc *tskvEncoder) AddFloat32(k string, v float32) { enc.AddFloat64(k, float64(v)) } +func (enc *tskvEncoder) AddInt(k string, v int) { enc.AddInt64(k, int64(v)) } +func (enc *tskvEncoder) AddInt32(k string, v int32) { enc.AddInt64(k, int64(v)) } +func (enc *tskvEncoder) AddInt16(k string, v int16) { enc.AddInt64(k, int64(v)) } +func (enc *tskvEncoder) AddInt8(k string, v int8) { enc.AddInt64(k, int64(v)) } +func (enc *tskvEncoder) AddUint(k string, v uint) { enc.AddUint64(k, uint64(v)) } +func (enc *tskvEncoder) AddUint32(k string, v uint32) { enc.AddUint64(k, uint64(v)) } +func (enc *tskvEncoder) AddUint16(k string, v uint16) { enc.AddUint64(k, uint64(v)) } +func (enc *tskvEncoder) AddUint8(k string, v uint8) { enc.AddUint64(k, uint64(v)) } +func (enc *tskvEncoder) AddUintptr(k string, v uintptr) { enc.AddUint64(k, uint64(v)) } +func (enc *tskvEncoder) AppendComplex64(v complex64) { enc.AppendComplex128(complex128(v)) } +func (enc *tskvEncoder) AppendFloat64(v float64) { enc.appendFloat(v, 64) } +func (enc *tskvEncoder) AppendFloat32(v float32) { enc.appendFloat(float64(v), 32) } +func (enc *tskvEncoder) AppendInt(v int) { enc.AppendInt64(int64(v)) } +func (enc *tskvEncoder) AppendInt32(v int32) { enc.AppendInt64(int64(v)) } +func (enc *tskvEncoder) AppendInt16(v int16) { enc.AppendInt64(int64(v)) } +func (enc *tskvEncoder) AppendInt8(v int8) { enc.AppendInt64(int64(v)) } +func (enc *tskvEncoder) AppendUint(v uint) { enc.AppendUint64(uint64(v)) } +func (enc *tskvEncoder) AppendUint32(v uint32) { enc.AppendUint64(uint64(v)) } +func (enc *tskvEncoder) AppendUint16(v uint16) { enc.AppendUint64(uint64(v)) } +func (enc *tskvEncoder) AppendUint8(v uint8) { enc.AppendUint64(uint64(v)) } +func (enc *tskvEncoder) AppendUintptr(v uintptr) { enc.AppendUint64(uint64(v)) } + +func (enc *tskvEncoder) Clone() zapcore.Encoder { + clone := enc.clone() + _, _ = clone.buf.Write(enc.buf.Bytes()) + return clone +} + +func (enc *tskvEncoder) clone() *tskvEncoder { + clone := getTSKVEncoder() + clone.cfg = enc.cfg + clone.pool = enc.pool + clone.buf = enc.pool.Get() + clone.tskvEscaper = enc.tskvEscaper + return clone +} + +// nolint: gocyclo +func (enc *tskvEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { + final := enc.clone() + final.AppendString(tskvMark) + + if final.cfg.TimeKey != "" && final.cfg.EncodeTime != nil { + final.safeAppendKey(final.cfg.TimeKey) + final.cfg.EncodeTime(ent.Time, final) + } + if final.cfg.LevelKey != "" && final.cfg.EncodeLevel != nil { + final.safeAppendKey(final.cfg.LevelKey) + final.cfg.EncodeLevel(ent.Level, final) + } + if ent.LoggerName != "" && final.cfg.NameKey != "" { + nameEncoder := final.cfg.EncodeName + + if nameEncoder == nil { + // Fall back to FullNameEncoder for backward compatibility. + nameEncoder = zapcore.FullNameEncoder + } + + final.safeAppendKey(final.cfg.NameKey) + nameEncoder(ent.LoggerName, final) + } + if ent.Caller.Defined && final.cfg.CallerKey != "" && final.cfg.EncodeCaller != nil { + final.safeAppendKey(final.cfg.CallerKey) + final.cfg.EncodeCaller(ent.Caller, final) + } + + if enc.buf.Len() > 0 { + final.appendElementSeparator() + _, _ = final.buf.Write(enc.buf.Bytes()) + } + + // Add the message itself. + if final.cfg.MessageKey != "" { + final.safeAppendKey(final.cfg.MessageKey) + final.safeAppendString(ent.Message) + } + + // Add any structured context. + for _, f := range fields { + f.AddTo(final) + } + + if ent.Stack != "" && final.cfg.StacktraceKey != "" { + final.safeAppendKey(final.cfg.StacktraceKey) + final.safeAppendString(ent.Stack) + } + + if final.cfg.LineEnding != "" { + final.buf.AppendString(final.cfg.LineEnding) + } else { + final.buf.AppendByte(tskvLineEnding) + } + + return final.buf, nil +} diff --git a/library/go/core/log/zap/encoders/tskv_test.go b/library/go/core/log/zap/encoders/tskv_test.go new file mode 100644 index 0000000000..44a74111a3 --- /dev/null +++ b/library/go/core/log/zap/encoders/tskv_test.go @@ -0,0 +1,600 @@ +package encoders + +import ( + "errors" + "fmt" + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" +) + +func TestTSKVEscaper(t *testing.T) { + tests := []struct { + input string + expectedKey string + expectedValue string + desc string + }{ + { + input: "plain text$ no need to escape", + expectedKey: "plain text$ no need to escape", + expectedValue: "plain text$ no need to escape", + desc: "test without escape", + }, { + input: "test escape\tab", + expectedKey: `test escape\tab`, + expectedValue: `test escape\tab`, + desc: "escape tab", + }, + { + input: "\ntest es\\cape\t\t a\rll char\x00s in string=", + expectedKey: `\ntest es\\cape\t\t a\rll char\0s in string\=`, + expectedValue: `\ntest es\\cape\t\t a\rll char\0s in string=`, + desc: "escape all chars", + }, + } + esc := newTSKVEscaper() + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + cur := esc.escapeKey(tt.input) + assert.Equal(t, tt.expectedKey, cur, "Incorrect escaped TSKV key.") + }) + + t.Run(tt.desc, func(t *testing.T) { + cur := esc.escapeValue(tt.input) + assert.Equal(t, tt.expectedValue, cur, "Incorrect escaped TSKV value.") + }) + } +} + +type noJSON struct{} + +func (nj noJSON) MarshalJSON() ([]byte, error) { + return nil, errors.New("no") +} + +type nonloggable struct{} + +func (l nonloggable) MarshalLogObject(enc zapcore.ObjectEncoder) error { + return nil +} + +type loggable struct { + bool bool + spec string +} + +func (l loggable) MarshalLogObject(enc zapcore.ObjectEncoder) error { + return nil +} + +func (l loggable) String() string { + + return fmt.Sprintf("loggable%s=%t%s", l.spec, l.bool, l.spec) +} + +func (l loggable) MarshalLogArray(enc zapcore.ArrayEncoder) error { + if !l.bool { + return errors.New("can't marshal") + } + enc.AppendBool(l.bool) + return nil +} + +type loggables int + +func (ls loggables) MarshalLogArray(enc zapcore.ArrayEncoder) error { + l := loggable{true, ""} + for i := 0; i < int(ls); i++ { + if err := enc.AppendObject(l); err != nil { + return err + } + } + return nil +} + +func getCommonTestConfig() zapcore.EncoderConfig { + return zapcore.EncoderConfig{ + MessageKey: "M", + LevelKey: "L", + TimeKey: "T", + NameKey: "N", + CallerKey: "C", + StacktraceKey: "S", + EncodeLevel: zapcore.LowercaseLevelEncoder, + EncodeTime: zapcore.EpochTimeEncoder, + EncodeDuration: zapcore.SecondsDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + } +} + +func getSpecCharsTestConfig() zapcore.EncoderConfig { + return zapcore.EncoderConfig{ + MessageKey: "M\t", + LevelKey: "L\n", + TimeKey: "T\r", + NameKey: "N\x00", + CallerKey: "C=", + StacktraceKey: "S\\", + EncodeLevel: zapcore.LowercaseLevelEncoder, + EncodeTime: zapcore.EpochTimeEncoder, + EncodeDuration: zapcore.SecondsDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + } +} + +func getRawTskvEncoder() *tskvEncoder { + pool := buffer.NewPool() + cfg := getCommonTestConfig() + return &tskvEncoder{ + cfg: cfg, + pool: pool, + buf: pool.Get(), + tskvEscaper: newTSKVEscaper(), + } + +} + +func assertOutput(t testing.TB, expected string, f func(encoder zapcore.Encoder)) { + enc := getRawTskvEncoder() + f(enc) + assert.Equal(t, expected, enc.buf.String(), "Unexpected encoder output after adding.") + + enc.buf.Reset() + enc.AddString("foo", "bar") + f(enc) + expectedPrefix := `foo=bar` + if expected != "" { + // If we expect output, it should be tab-separated from the previous + // field. + expectedPrefix += "\t" + } + assert.Equal(t, expectedPrefix+expected, enc.buf.String(), "Unexpected encoder output after adding as a second field.") +} + +func TestTSKVEncoderObjectFields(t *testing.T) { + tests := []struct { + desc string + expected string + f func(encoder zapcore.Encoder) + }{ + {"binary", `k=61623132`, func(e zapcore.Encoder) { e.AddBinary("k", []byte("ab12")) }}, + {"binary esc ", `k\n=61623132`, func(e zapcore.Encoder) { e.AddBinary("k\n", []byte("ab12")) }}, + {"bool", `k=true`, func(e zapcore.Encoder) { e.AddBool("k", true) }}, + {"bool", `k\t=false`, func(e zapcore.Encoder) { e.AddBool("k\t", false) }}, + + {"byteString", `k=765c`, func(e zapcore.Encoder) { e.AddByteString(`k`, []byte(`v\`)) }}, + {"byteString esc", `k\t=61623132`, func(e zapcore.Encoder) { e.AddByteString("k\t", []byte("ab12")) }}, + {"byteString empty val", `k=`, func(e zapcore.Encoder) { e.AddByteString("k", []byte{}) }}, + {"byteString nil val", `k=`, func(e zapcore.Encoder) { e.AddByteString("k", nil) }}, + + {"complex128", `k="1+2i"`, func(e zapcore.Encoder) { e.AddComplex128("k", 1+2i) }}, + {"complex128 esc", `k\t="1+2i"`, func(e zapcore.Encoder) { e.AddComplex128("k\t", 1+2i) }}, + {"complex64", `k="1+2i"`, func(e zapcore.Encoder) { e.AddComplex64("k", 1+2i) }}, + {"complex64 esc", `k\t="1+2i"`, func(e zapcore.Encoder) { e.AddComplex64("k\t", 1+2i) }}, + + {"duration", `k$=0.000000001`, func(e zapcore.Encoder) { e.AddDuration("k$", 1) }}, + {"duration esc", `k\t=0.000000001`, func(e zapcore.Encoder) { e.AddDuration("k\t", 1) }}, + + {"float64", `k=1`, func(e zapcore.Encoder) { e.AddFloat64("k", 1.0) }}, + {"float64 esc", `k\t=1`, func(e zapcore.Encoder) { e.AddFloat64("k\t", 1.0) }}, + {"float64", `k=10000000000`, func(e zapcore.Encoder) { e.AddFloat64("k", 1e10) }}, + {"float64", `k="NaN"`, func(e zapcore.Encoder) { e.AddFloat64("k", math.NaN()) }}, + {"float64", `k="+Inf"`, func(e zapcore.Encoder) { e.AddFloat64("k", math.Inf(1)) }}, + {"float64", `k="-Inf"`, func(e zapcore.Encoder) { e.AddFloat64("k", math.Inf(-1)) }}, + + {"float32", `k=1`, func(e zapcore.Encoder) { e.AddFloat32("k", 1.0) }}, + {"float32", `k\t=1`, func(e zapcore.Encoder) { e.AddFloat32("k\t", 1.0) }}, + {"float32", `k=10000000000`, func(e zapcore.Encoder) { e.AddFloat32("k", 1e10) }}, + {"float32", `k="NaN"`, func(e zapcore.Encoder) { e.AddFloat32("k", float32(math.NaN())) }}, + {"float32", `k="+Inf"`, func(e zapcore.Encoder) { e.AddFloat32("k", float32(math.Inf(1))) }}, + {"float32", `k="-Inf"`, func(e zapcore.Encoder) { e.AddFloat32("k", float32(math.Inf(-1))) }}, + + {"int", `k=42`, func(e zapcore.Encoder) { e.AddInt("k", 42) }}, + {"int esc", `k\t=42`, func(e zapcore.Encoder) { e.AddInt("k\t", 42) }}, + {"int64", `k=42`, func(e zapcore.Encoder) { e.AddInt64("k", 42) }}, + {"int32", `k=42`, func(e zapcore.Encoder) { e.AddInt32("k", 42) }}, + {"int16", `k=42`, func(e zapcore.Encoder) { e.AddInt16("k", 42) }}, + {"int8", `k=42`, func(e zapcore.Encoder) { e.AddInt8("k", 42) }}, + + {"string", `k=v$`, func(e zapcore.Encoder) { e.AddString("k", "v$") }}, + {"string esc", `k\t=v\\`, func(e zapcore.Encoder) { e.AddString("k\t", `v\`) }}, + {"string", `k=`, func(e zapcore.Encoder) { e.AddString("k", "") }}, + + {"time", `k=1`, func(e zapcore.Encoder) { e.AddTime("k", time.Unix(1, 0)) }}, + {"time esc", `k\t=1`, func(e zapcore.Encoder) { e.AddTime("k\t", time.Unix(1, 0)) }}, + + {"uint", `k=42`, func(e zapcore.Encoder) { e.AddUint("k", 42) }}, + {"uint esc", `k\t=42`, func(e zapcore.Encoder) { e.AddUint("k\t", 42) }}, + {"uint64", `k=42`, func(e zapcore.Encoder) { e.AddUint64("k", 42) }}, + {"uint32", `k=42`, func(e zapcore.Encoder) { e.AddUint32("k", 42) }}, + {"uint16", `k=42`, func(e zapcore.Encoder) { e.AddUint16("k", 42) }}, + {"uint8", `k=42`, func(e zapcore.Encoder) { e.AddUint8("k", 42) }}, + {"uintptr", `k=42`, func(e zapcore.Encoder) { e.AddUintptr("k", 42) }}, + { + desc: "object (success)", + expected: `k={loggable=true}`, + f: func(e zapcore.Encoder) { + assert.NoError(t, e.AddObject("k", loggable{true, ""}), "Unexpected error calling AddObject.") + }, + }, + { + desc: "object esc (success)", + expected: `k={loggable\t=true\t}`, + f: func(e zapcore.Encoder) { + assert.NoError(t, e.AddObject("k", loggable{true, "\t"}), "Unexpected error calling AddObject.") + }, + }, + { + desc: "object (error)", + expected: `k={}`, + f: func(e zapcore.Encoder) { + assert.Error(t, e.AddObject("k", nonloggable{}), "Expected an error calling AddObject.") + }, + }, + { + desc: "array (with nested object)", + expected: `loggables=[{loggable=true},{loggable=true}]`, + f: func(e zapcore.Encoder) { + assert.NoError( + t, + e.AddArray("loggables", loggables(2)), + "Unexpected error calling AddObject with nested ArrayMarshalers.", + ) + }, + }, + { + desc: "array (success)", + expected: `k=[true]`, + f: func(e zapcore.Encoder) { + assert.NoError(t, e.AddArray(`k`, loggable{true, ""}), "Unexpected error calling MarshalLogArray.") + }, + }, + { + desc: "array esc (success)", + expected: `k\t=[true]`, + f: func(e zapcore.Encoder) { + assert.NoError(t, e.AddArray("k\t", loggable{true, ""}), "Unexpected error calling MarshalLogArray.") + }, + }, + { + desc: "array (error)", + expected: `k=[]`, + f: func(e zapcore.Encoder) { + assert.Error(t, e.AddArray("k", loggable{false, ""}), "Expected an error calling MarshalLogArray.") + }, + }, + { + desc: "reflect enc (success)", + expected: `k\t={"aee":"l=l","bee":123,"cee":0.9999,"dee":[{"key":"p\\ni","val":3.141592653589793},{"key":"tau=","val":6.283185307179586}]}`, + f: func(e zapcore.Encoder) { + type bar struct { + Key string `json:"key"` + Val float64 `json:"val"` + } + + type foo struct { + A string `json:"aee"` + B int `json:"bee"` + C float64 `json:"cee"` + D []bar `json:"dee"` + } + + assert.NoError(t, e.AddReflected("k\t", foo{ + A: "l=l", + B: 123, + C: 0.9999, + D: []bar{ + {"p\ni", 3.141592653589793}, + {"tau=", 6.283185307179586}, + }, + }), "Unexpected error JSON-serializing a map.") + }, + }, + { + desc: "reflect (failure)", + expected: "", + f: func(e zapcore.Encoder) { + assert.Error(t, e.AddReflected("k", noJSON{}), "Unexpected success JSON-serializing a noJSON.") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + assertOutput(t, tt.expected, tt.f) + }) + } +} + +func TestTskvEncoderOpenNamespace(t *testing.T) { + enc := getRawTskvEncoder() + assert.PanicsWithValue(t, "TSKV encoder does not support namespaces", func() { enc.OpenNamespace("test") }) +} + +func TestTSKVEncoderArrays(t *testing.T) { + tests := []struct { + desc string + expected string // expect f to be called twice + f func(zapcore.ArrayEncoder) + }{ + {"bool", `[true,true]`, func(e zapcore.ArrayEncoder) { e.AppendBool(true) }}, + {"byteString", `[6b,6b]`, func(e zapcore.ArrayEncoder) { e.AppendByteString([]byte("k")) }}, + {"byteString", `[6b5c,6b5c]`, func(e zapcore.ArrayEncoder) { e.AppendByteString([]byte(`k\`)) }}, + {"complex128", `["1+2i","1+2i"]`, func(e zapcore.ArrayEncoder) { e.AppendComplex128(1 + 2i) }}, + {"complex64", `["1+2i","1+2i"]`, func(e zapcore.ArrayEncoder) { e.AppendComplex64(1 + 2i) }}, + {"durations", `[0.000000002,0.000000002]`, func(e zapcore.ArrayEncoder) { e.AppendDuration(2) }}, + {"float64", `[3.14,3.14]`, func(e zapcore.ArrayEncoder) { e.AppendFloat64(3.14) }}, + {"float32", `[3.14,3.14]`, func(e zapcore.ArrayEncoder) { e.AppendFloat32(3.14) }}, + {"int", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendInt(42) }}, + {"int64", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendInt64(42) }}, + {"int32", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendInt32(42) }}, + {"int16", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendInt16(42) }}, + {"int8", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendInt8(42) }}, + {"string", `[k,k]`, func(e zapcore.ArrayEncoder) { e.AppendString("k") }}, + {"string", `[k\\,k\\]`, func(e zapcore.ArrayEncoder) { e.AppendString(`k\`) }}, + {"times", `[1,1]`, func(e zapcore.ArrayEncoder) { e.AppendTime(time.Unix(1, 0)) }}, + {"uint", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUint(42) }}, + {"uint64", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUint64(42) }}, + {"uint32", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUint32(42) }}, + {"uint16", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUint16(42) }}, + {"uint8", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUint8(42) }}, + {"uintptr", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUintptr(42) }}, + { + desc: "arrays (success)", + expected: `[[true],[true]]`, + f: func(arr zapcore.ArrayEncoder) { + assert.NoError(t, arr.AppendArray(zapcore.ArrayMarshalerFunc(func(inner zapcore.ArrayEncoder) error { + inner.AppendBool(true) + return nil + })), "Unexpected error appending an array.") + }, + }, + { + desc: "arrays (error)", + expected: `[[true],[true]]`, + f: func(arr zapcore.ArrayEncoder) { + assert.Error(t, arr.AppendArray(zapcore.ArrayMarshalerFunc(func(inner zapcore.ArrayEncoder) error { + inner.AppendBool(true) + return errors.New("fail") + })), "Expected an error appending an array.") + }, + }, + { + desc: "objects (success)", + expected: `[{loggable=true},{loggable=true}]`, + f: func(arr zapcore.ArrayEncoder) { + assert.NoError(t, arr.AppendObject(loggable{true, ""}), "Unexpected error appending an object.") + }, + }, + { + desc: "objects esc (success)", + expected: `[{loggable\t=true\t},{loggable\t=true\t}]`, + f: func(arr zapcore.ArrayEncoder) { + assert.NoError(t, arr.AppendObject(loggable{true, "\t"}), "Unexpected error appending an object.") + }, + }, + { + desc: "objects (error: fmt.Stringer not implemented)", + expected: `[{},{}]`, + f: func(arr zapcore.ArrayEncoder) { + assert.Error(t, arr.AppendObject(nonloggable{}), "Expected an error appending an object.") + }, + }, + { + desc: "reflect (success)", + expected: `[{"foo":5},{"foo":5}]`, + f: func(arr zapcore.ArrayEncoder) { + assert.NoError( + t, + arr.AppendReflected(map[string]int{"foo": 5}), + "Unexpected an error appending an object with reflection.", + ) + }, + }, + { + desc: "reflect esc (success)", + expected: `[{"foo\\t":5},{"foo\\t":5}]`, + f: func(arr zapcore.ArrayEncoder) { + assert.NoError( + t, + arr.AppendReflected(map[string]int{"foo\t": 5}), + "Unexpected an error appending an object with reflection.", + ) + }, + }, + { + desc: "reflect (error)", + expected: `[]`, + f: func(arr zapcore.ArrayEncoder) { + assert.Error( + t, + arr.AppendReflected(noJSON{}), + "Unexpected an error appending an object with reflection.", + ) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + f := func(enc zapcore.Encoder) error { + return enc.AddArray("array", zapcore.ArrayMarshalerFunc(func(arr zapcore.ArrayEncoder) error { + tt.f(arr) + tt.f(arr) + return nil + })) + } + assertOutput(t, `array=`+tt.expected, func(enc zapcore.Encoder) { + err := f(enc) + assert.NoError(t, err, "Unexpected error adding array to JSON encoder.") + }) + }) + } +} + +func TestTSKVEncodeEntry(t *testing.T) { + entryTime := time.Date(2019, 7, 13, 15, 33, 42, 99, time.UTC) + + tests := []struct { + desc string + expected string + cnf zapcore.EncoderConfig + ent zapcore.Entry + fields []zapcore.Field + }{ + { + desc: "entry without escape", + expected: `tskv T=1563032022 L=info M=text here +`, + cnf: getCommonTestConfig(), + ent: zapcore.Entry{ + Time: entryTime, + Message: "text here", + }, + fields: []zapcore.Field{}, + }, + { + desc: "all fields entry without escape", + expected: `tskv T=1563032022 L=debug N=bob C=foo.go:42 M=text here S=fake-stack +`, + cnf: getCommonTestConfig(), + ent: zapcore.Entry{ + Level: zapcore.DebugLevel, + Time: entryTime, + LoggerName: "bob", + Message: "text here", + Caller: zapcore.EntryCaller{Defined: true, File: "foo.go", Line: 42}, + Stack: "fake-stack", + }, + fields: []zapcore.Field{}, + }, + { + desc: "entry with escaped field names", + expected: `tskv T\r=1563032022 L\n=debug N\0=bob C\==foo.go:42 M\t=text here S\\=fake-stack +`, + cnf: getSpecCharsTestConfig(), + ent: zapcore.Entry{ + Level: zapcore.DebugLevel, + Time: entryTime, + LoggerName: "bob", + Message: "text here", + Caller: zapcore.EntryCaller{Defined: true, File: "foo.go", Line: 42}, + Stack: "fake-stack", + }, + fields: []zapcore.Field{}, + }, + { + desc: "entry message escape", + expected: `tskv T=1563032022 L=info M=t\\ex=t\0he\r\tre\n +`, + cnf: getCommonTestConfig(), + ent: zapcore.Entry{ + Time: entryTime, + Message: "t\\ex=t\x00he\r\tre\n", + }, + fields: []zapcore.Field{}, + }, + { + desc: "entry multi-line stack escape", + expected: `tskv T=1563032022 L=info M= S=fake-st\rack\n\tlevel2\n\tlevel1 +`, + cnf: getCommonTestConfig(), + ent: zapcore.Entry{ + Time: entryTime, + Stack: "fake-st\rack\n\tlevel2\n\tlevel1", + }, + fields: []zapcore.Field{}, + }, + { + desc: "entry multi-line caller escape", + expected: `tskv T=1563032022 L=info C=fo\to.go:42 M= +`, + cnf: getCommonTestConfig(), + ent: zapcore.Entry{ + Time: entryTime, + Caller: zapcore.EntryCaller{Defined: true, File: "fo\to.go", Line: 42}, + }, + fields: []zapcore.Field{}, + }, + { + desc: "entry multi-line logger escape", + expected: `tskv T=1563032022 L=info N=b\0b M= +`, + cnf: getCommonTestConfig(), + ent: zapcore.Entry{ + Time: entryTime, + LoggerName: "b\x00b", + }, + fields: []zapcore.Field{}, + }, + { + desc: "entry with additional zap fields", + expected: `tskv T=1563032022 L=info M= so=passes answer=42 common_pie=3.14 ` + + `reflect={"loggable":"yes"} bytes_array=0001020309 bool=true complex="0+1i" +`, + cnf: getCommonTestConfig(), + ent: zapcore.Entry{ + Time: entryTime, + }, + fields: []zapcore.Field{ + zap.String("so", "passes"), + zap.Int("answer", 42), + zap.Float64("common_pie", 3.14), + zap.Reflect("reflect", map[string]string{"loggable": "yes"}), + zap.Binary("bytes_array", []byte{0, 1, 2, 3, '\t'}), + zap.Bool("bool", true), + zap.Complex128("complex", 1i)}, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + enc, err := NewTSKVEncoder(tt.cnf) + if err != nil { + panic(err) + } + + buf, err := enc.EncodeEntry(tt.ent, tt.fields) + if assert.NoError(t, err, "Unexpected TSKV encoding error.") { + assert.Equal(t, tt.expected, buf.String(), "Incorrect encoded TSKV entry.") + } + buf.Free() + }) + } +} + +func TestTskvEncoderLoggerWithMethod(t *testing.T) { + entryTime := time.Date(2019, 7, 13, 15, 33, 42, 99, time.UTC) + + enc := getRawTskvEncoder() + enc.AddString("Permanent", "message") + enc.Clone() + tt := struct { + desc string + expected string + ent zapcore.Entry + }{ + desc: "entry without escape", + expected: `tskv T=1563032022 L=info Permanent=message M=text here +`, + ent: zapcore.Entry{ + Time: entryTime, + Message: "text here", + }, + } + + for i := 0; i < 3; i++ { + t.Run(tt.desc, func(t *testing.T) { + buf, err := enc.EncodeEntry(tt.ent, []zapcore.Field{}) + if assert.NoError(t, err, "Unexpected TSKV encoding error.") { + assert.Equal(t, tt.expected, buf.String(), "Incorrect encoded TSKV entry.") + } + }) + } +} diff --git a/library/go/core/log/zap/encoders/ya.make b/library/go/core/log/zap/encoders/ya.make new file mode 100644 index 0000000000..8d71ae0b1b --- /dev/null +++ b/library/go/core/log/zap/encoders/ya.make @@ -0,0 +1,16 @@ +GO_LIBRARY() + +SRCS( + cli.go + kv.go + tskv.go +) + +GO_TEST_SRCS( + kv_test.go + tskv_test.go +) + +END() + +RECURSE(gotest) diff --git a/library/go/core/log/zap/gotest/ya.make b/library/go/core/log/zap/gotest/ya.make new file mode 100644 index 0000000000..f971bead7d --- /dev/null +++ b/library/go/core/log/zap/gotest/ya.make @@ -0,0 +1,3 @@ +GO_TEST_FOR(library/go/core/log/zap) + +END() diff --git a/library/go/core/log/zap/logrotate/error.go b/library/go/core/log/zap/logrotate/error.go new file mode 100644 index 0000000000..f59b322591 --- /dev/null +++ b/library/go/core/log/zap/logrotate/error.go @@ -0,0 +1,5 @@ +package logrotate + +import "errors" + +var ErrNotSupported = errors.New("logrotate sink is not supported on your platform") diff --git a/library/go/core/log/zap/logrotate/example_sink_test.go b/library/go/core/log/zap/logrotate/example_sink_test.go new file mode 100644 index 0000000000..871081439c --- /dev/null +++ b/library/go/core/log/zap/logrotate/example_sink_test.go @@ -0,0 +1,54 @@ +//go:build linux || darwin +// +build linux darwin + +package logrotate_test + +import ( + "net/url" + "path/filepath" + "syscall" + + "github.com/ydb-platform/ydb/library/go/core/log" + "github.com/ydb-platform/ydb/library/go/core/log/zap" + "github.com/ydb-platform/ydb/library/go/core/log/zap/logrotate" + uberzap "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func Example_simpleUsage() { + // Basic usage, when you don't need any custom preferences is quite easy. + // register our logrotate sink and force it to reopen files on sighup(remember to check for errors) + _ = logrotate.RegisterLogrotateSink(syscall.SIGHUP) + // create zap logger as usual, using `logrotate://` instead of omitting it or using `file://` + cfg := zap.JSONConfig(log.DebugLevel) + logPath, _ := filepath.Abs("./example.log") + cfg.OutputPaths = []string{"logrotate://" + logPath} + logger, _ := zap.New(cfg) + // That's all, when your process receives SIGHUP file will be reopened + logger.Debug("this log should be reopened by SIGHUP") +} + +func Example_namedUsage() { + // Note: each scheme can be registered only once and can not be unregistered + // If you want to provide custom unused scheme name(remember to check for errors): + _ = logrotate.RegisterNamedLogrotateSink("rotate-usr1", syscall.SIGUSR1) + // Now we create logger using that cheme + cfg := zap.JSONConfig(log.DebugLevel) + logPath, _ := filepath.Abs("./example.log") + cfg.OutputPaths = []string{"rotate-usr1://" + logPath} + logger, _ := zap.New(cfg) + // Now file will be reopened by SIGUSR1 + logger.Debug("this log should be reopened by SIGHUP") +} + +func Example_standaloneUsage() { + // If you don't want to register scheme, or use custom logging core you can do this(remember to check for errors): + u, _ := url.ParseRequestURI("/tmp/example.log") + sink, _ := logrotate.NewLogrotateSink(u, syscall.SIGHUP) + + encoder := zapcore.NewConsoleEncoder(zapcore.EncoderConfig{MessageKey: "msg"}) + core := zapcore.NewCore(encoder, sink, uberzap.NewAtomicLevel()) + logger := uberzap.New(core) + // Now file will be reopened by SIGHUP + logger.Debug("this log should be reopened by SIGHUP") +} diff --git a/library/go/core/log/zap/logrotate/gotest/ya.make b/library/go/core/log/zap/logrotate/gotest/ya.make new file mode 100644 index 0000000000..03fe7d8484 --- /dev/null +++ b/library/go/core/log/zap/logrotate/gotest/ya.make @@ -0,0 +1,3 @@ +GO_TEST_FOR(library/go/core/log/zap/logrotate) + +END() diff --git a/library/go/core/log/zap/logrotate/sink.go b/library/go/core/log/zap/logrotate/sink.go new file mode 100644 index 0000000000..c8f3f14a3f --- /dev/null +++ b/library/go/core/log/zap/logrotate/sink.go @@ -0,0 +1,121 @@ +//go:build darwin || freebsd || linux +// +build darwin freebsd linux + +package logrotate + +import ( + "fmt" + "net/url" + "os" + "os/signal" + "sync/atomic" + "unsafe" + + "github.com/ydb-platform/ydb/library/go/core/xerrors" + "go.uber.org/zap" +) + +const defaultSchemeName = "logrotate" + +// Register logrotate sink in zap sink registry. +// This sink internally is like file sink, but listens to provided logrotate signal +// and reopens file when that signal is delivered +// This can be called only once. Any future calls will result in an error +func RegisterLogrotateSink(sig ...os.Signal) error { + return RegisterNamedLogrotateSink(defaultSchemeName, sig...) +} + +// Same as RegisterLogrotateSink, but use provided schemeName instead of default `logrotate` +// Can be useful in special cases for registering different types of sinks for different signal +func RegisterNamedLogrotateSink(schemeName string, sig ...os.Signal) error { + factory := func(url *url.URL) (sink zap.Sink, e error) { + return NewLogrotateSink(url, sig...) + } + return zap.RegisterSink(schemeName, factory) +} + +// sink itself, use RegisterLogrotateSink to register it in zap machinery +type sink struct { + path string + notifier chan os.Signal + file unsafe.Pointer +} + +// Factory for logrotate sink, which accepts os.Signals to listen to for reloading +// Generally if you don't build your own core it is used by zap machinery. +// See RegisterLogrotateSink. +func NewLogrotateSink(u *url.URL, sig ...os.Signal) (zap.Sink, error) { + notifier := make(chan os.Signal, 1) + signal.Notify(notifier, sig...) + + if u.User != nil { + return nil, fmt.Errorf("user and password not allowed with logrotate file URLs: got %v", u) + } + if u.Fragment != "" { + return nil, fmt.Errorf("fragments not allowed with logrotate file URLs: got %v", u) + } + // Error messages are better if we check hostname and port separately. + if u.Port() != "" { + return nil, fmt.Errorf("ports not allowed with logrotate file URLs: got %v", u) + } + if hn := u.Hostname(); hn != "" && hn != "localhost" { + return nil, fmt.Errorf("logrotate file URLs must leave host empty or use localhost: got %v", u) + } + + sink := &sink{ + path: u.Path, + notifier: notifier, + } + if err := sink.reopen(); err != nil { + return nil, err + } + go sink.listenToSignal() + return sink, nil +} + +// wait for signal delivery or chanel close +func (m *sink) listenToSignal() { + for { + _, ok := <-m.notifier + if !ok { + return + } + if err := m.reopen(); err != nil { + // Last chance to signalize about an error + _, _ = fmt.Fprintf(os.Stderr, "%s", err) + } + } +} + +func (m *sink) reopen() error { + file, err := os.OpenFile(m.path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + if err != nil { + return xerrors.Errorf("failed to open log file on %s: %w", m.path, err) + } + old := (*os.File)(m.file) + atomic.StorePointer(&m.file, unsafe.Pointer(file)) + if old != nil { + if err := old.Close(); err != nil { + return xerrors.Errorf("failed to close old file: %w", err) + } + } + return nil +} + +func (m *sink) getFile() *os.File { + return (*os.File)(atomic.LoadPointer(&m.file)) +} + +func (m *sink) Close() error { + signal.Stop(m.notifier) + close(m.notifier) + return m.getFile().Close() +} + +func (m *sink) Write(p []byte) (n int, err error) { + return m.getFile().Write(p) +} + +func (m *sink) Sync() error { + return m.getFile().Sync() +} diff --git a/library/go/core/log/zap/logrotate/sink_stub.go b/library/go/core/log/zap/logrotate/sink_stub.go new file mode 100644 index 0000000000..389a55b812 --- /dev/null +++ b/library/go/core/log/zap/logrotate/sink_stub.go @@ -0,0 +1,23 @@ +//go:build !darwin && !freebsd && !linux +// +build !darwin,!freebsd,!linux + +package logrotate + +import ( + "net/url" + "os" + + "go.uber.org/zap" +) + +func RegisterLogrotateSink(sig ...os.Signal) error { + return ErrNotSupported +} + +func RegisterNamedLogrotateSink(schemeName string, sig ...os.Signal) error { + return ErrNotSupported +} + +func NewLogrotateSink(u *url.URL, sig ...os.Signal) (zap.Sink, error) { + return nil, ErrNotSupported +} diff --git a/library/go/core/log/zap/logrotate/sink_test.go b/library/go/core/log/zap/logrotate/sink_test.go new file mode 100644 index 0000000000..e434a0ef8a --- /dev/null +++ b/library/go/core/log/zap/logrotate/sink_test.go @@ -0,0 +1,86 @@ +//go:build linux || darwin +// +build linux darwin + +package logrotate + +import ( + "io" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/ydb-platform/ydb/library/go/core/log" + "github.com/ydb-platform/ydb/library/go/core/log/zap" +) + +func TestLogrotateSink(t *testing.T) { + testLogFilename := "test.log" + testDir := "testLogrotate" + + // use test dir in default temp files location + tempDir, err := os.MkdirTemp("", testDir) + require.NoError(t, err, "failed to create temporary directory %s", testDir) + + testLogPath := filepath.Join(tempDir, testLogFilename) + + defer func() { + _ = os.RemoveAll(tempDir) + }() // clean up + + err = RegisterLogrotateSink(syscall.SIGUSR1) + require.NoError(t, err, "failed to register sink") + + // Double registration is not allowed + err = RegisterLogrotateSink(syscall.SIGUSR1) + require.Error(t, err) + + cfg := zap.JSONConfig(log.DebugLevel) + cfg.OutputPaths = []string{"logrotate://" + testLogPath} + logger, err := zap.New(cfg) + require.NoError(t, err, "failed to create logger") + + testLogFile, err := os.OpenFile(testLogPath, os.O_RDONLY, 0) + require.NoError(t, err, "expected logger to create file: %v", err) + defer func() { + _ = testLogFile.Close() + }() + + // test write to file + logger.Debug("test") + logger.Debug("test") + + err = os.Rename(testLogPath, testLogPath+".rotated") + require.NoError(t, err, "failed to rename file") + + err = syscall.Kill(syscall.Getpid(), syscall.SIGUSR1) + require.NoError(t, err, "failed to send signal to self, %v", err) + + // There is an essential race that we can not control of delivering signal, + // so we just wait enough here + time.Sleep(time.Second) + + logger.Debug("test") + logger.Debug("test") + logger.Debug("test") + + // Reopen file to sync content + err = syscall.Kill(syscall.Getpid(), syscall.SIGUSR1) + require.NoError(t, err, "failed to send signal to self, %v", err) + time.Sleep(time.Second) + + requireLineCount(t, testLogPath, 3) + requireLineCount(t, testLogPath+".rotated", 2) +} + +func requireLineCount(t *testing.T, path string, lines int) { + file, err := os.OpenFile(path, os.O_RDONLY, 0) + require.NoError(t, err, "failed to open log file for reading") + defer func() { _ = file.Close() }() + dataRead, err := io.ReadAll(file) + require.NoError(t, err, "failed to read log file") + require.Equal(t, lines, strings.Count(string(dataRead), "\n")) +} diff --git a/library/go/core/log/zap/logrotate/ya.make b/library/go/core/log/zap/logrotate/ya.make new file mode 100644 index 0000000000..3171836441 --- /dev/null +++ b/library/go/core/log/zap/logrotate/ya.make @@ -0,0 +1,42 @@ +GO_LIBRARY() + +IF (OS_LINUX) + SRCS( + error.go + sink.go + ) + + GO_TEST_SRCS(sink_test.go) + + GO_XTEST_SRCS(example_sink_test.go) +ENDIF() + +IF (OS_DARWIN) + SRCS( + error.go + sink.go + ) + + GO_TEST_SRCS(sink_test.go) + + GO_XTEST_SRCS(example_sink_test.go) +ENDIF() + +IF (OS_WINDOWS) + SRCS( + error.go + sink_stub.go + ) +ENDIF() + +END() + +IF ( + OS_DARWIN + OR + OS_FREEBSD + OR + OS_LINUX +) + RECURSE_FOR_TESTS(gotest) +ENDIF() diff --git a/library/go/core/log/zap/qloud.go b/library/go/core/log/zap/qloud.go new file mode 100644 index 0000000000..1f3c90a964 --- /dev/null +++ b/library/go/core/log/zap/qloud.go @@ -0,0 +1,49 @@ +package zap + +import ( + "github.com/ydb-platform/ydb/library/go/core/log" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// NewQloudLogger constructs fully-fledged Qloud compatible logger +// based on predefined config. See https://wiki.yandex-team.ru/qloud/doc/logs +// for more information +func NewQloudLogger(level log.Level, opts ...zap.Option) (*Logger, error) { + cfg := zap.Config{ + Level: zap.NewAtomicLevelAt(ZapifyLevel(level)), + Encoding: "json", + OutputPaths: []string{"stdout"}, + ErrorOutputPaths: []string{"stderr"}, + EncoderConfig: zapcore.EncoderConfig{ + MessageKey: "msg", + LevelKey: "level", + StacktraceKey: "stackTrace", + TimeKey: "", + CallerKey: "", + EncodeLevel: zapcore.LowercaseLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeDuration: zapcore.StringDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + }, + } + + zl, err := cfg.Build(opts...) + if err != nil { + return nil, err + } + + return &Logger{ + L: addQloudContext(zl).(*zap.Logger), + }, nil +} + +func addQloudContext(i interface{}) interface{} { + switch c := i.(type) { + case *zap.Logger: + return c.With(zap.Namespace("@fields")) + case zapcore.Core: + return c.With([]zapcore.Field{zap.Namespace("@fields")}) + } + return i +} diff --git a/library/go/core/log/zap/ya.make b/library/go/core/log/zap/ya.make new file mode 100644 index 0000000000..bc41967e07 --- /dev/null +++ b/library/go/core/log/zap/ya.make @@ -0,0 +1,23 @@ +GO_LIBRARY() + +SRCS( + deploy.go + qloud.go + zap.go + zapify.go +) + +GO_TEST_SRCS( + benchmark_test.go + zap_test.go + zapify_test.go +) + +END() + +RECURSE( + asynczap + encoders + gotest + logrotate +) diff --git a/library/go/core/log/zap/zap.go b/library/go/core/log/zap/zap.go new file mode 100644 index 0000000000..e3274cf372 --- /dev/null +++ b/library/go/core/log/zap/zap.go @@ -0,0 +1,252 @@ +package zap + +import ( + "fmt" + + "github.com/ydb-platform/ydb/library/go/core/log" + "github.com/ydb-platform/ydb/library/go/core/log/zap/encoders" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +const ( + // callerSkip is number of stack frames to skip when logging caller + callerSkip = 1 +) + +func init() { + if err := zap.RegisterEncoder(encoders.EncoderNameKV, encoders.NewKVEncoder); err != nil { + panic(err) + } + if err := zap.RegisterEncoder(encoders.EncoderNameCli, encoders.NewCliEncoder); err != nil { + panic(err) + } + if err := zap.RegisterEncoder(encoders.EncoderNameTSKV, encoders.NewTSKVEncoder); err != nil { + panic(err) + } +} + +// Logger implements log.Logger interface +type Logger struct { + L *zap.Logger +} + +var _ log.Logger = &Logger{} +var _ log.Structured = &Logger{} +var _ log.Fmt = &Logger{} +var _ log.LoggerWith = &Logger{} +var _ log.LoggerAddCallerSkip = &Logger{} + +// New constructs zap-based logger from provided config +func New(cfg zap.Config) (*Logger, error) { + zl, err := cfg.Build(zap.AddCallerSkip(callerSkip)) + if err != nil { + return nil, err + } + + return &Logger{ + L: zl, + }, nil +} + +// NewWithCore constructs zap-based logger from provided core +func NewWithCore(core zapcore.Core, options ...zap.Option) *Logger { + options = append(options, zap.AddCallerSkip(callerSkip)) + return &Logger{L: zap.New(core, options...)} +} + +// Must constructs zap-based logger from provided config and panics on error +func Must(cfg zap.Config) *Logger { + l, err := New(cfg) + if err != nil { + panic(fmt.Sprintf("failed to construct zap logger: %v", err)) + } + return l +} + +// JSONConfig returns zap config for structured logging (zap's json encoder) +func JSONConfig(level log.Level) zap.Config { + return StandardConfig("json", level) +} + +// ConsoleConfig returns zap config for logging to console (zap's console encoder) +func ConsoleConfig(level log.Level) zap.Config { + return StandardConfig("console", level) +} + +// CLIConfig returns zap config for cli logging (custom cli encoder) +func CLIConfig(level log.Level) zap.Config { + return StandardConfig("cli", level) +} + +// KVConfig returns zap config for logging to kv (custom kv encoder) +func KVConfig(level log.Level) zap.Config { + return StandardConfig("kv", level) +} + +// TSKVConfig returns zap config for logging to tskv (custom tskv encoder) +func TSKVConfig(level log.Level) zap.Config { + return zap.Config{ + Level: zap.NewAtomicLevelAt(ZapifyLevel(level)), + Encoding: "tskv", + OutputPaths: []string{"stdout"}, + ErrorOutputPaths: []string{"stderr"}, + EncoderConfig: zapcore.EncoderConfig{ + MessageKey: "message", + LevelKey: "levelname", + TimeKey: "unixtime", + CallerKey: "caller", + NameKey: "name", + EncodeLevel: zapcore.CapitalLevelEncoder, + EncodeTime: zapcore.EpochTimeEncoder, + EncodeDuration: zapcore.StringDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + }, + } +} + +// StandardConfig returns default zap config with specified encoding and level +func StandardConfig(encoding string, level log.Level) zap.Config { + return zap.Config{ + Level: zap.NewAtomicLevelAt(ZapifyLevel(level)), + Encoding: encoding, + OutputPaths: []string{"stdout"}, + ErrorOutputPaths: []string{"stderr"}, + EncoderConfig: zapcore.EncoderConfig{ + MessageKey: "msg", + LevelKey: "level", + TimeKey: "ts", + CallerKey: "caller", + NameKey: "name", + EncodeLevel: zapcore.CapitalLevelEncoder, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeDuration: zapcore.StringDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + }, + } +} + +// Logger returns general logger +func (l *Logger) Logger() log.Logger { + return l +} + +// Fmt returns fmt logger +func (l *Logger) Fmt() log.Fmt { + return l +} + +// Structured returns structured logger +func (l *Logger) Structured() log.Structured { + return l +} + +// With returns logger that always adds provided key/value to every log entry +func (l *Logger) With(fields ...log.Field) log.Logger { + return &Logger{ + L: l.L.With(zapifyFields(fields...)...), + } +} + +func (l *Logger) AddCallerSkip(skip int) log.Logger { + return &Logger{ + L: l.L.WithOptions(zap.AddCallerSkip(skip)), + } +} + +// Trace logs at Trace log level using fields +func (l *Logger) Trace(msg string, fields ...log.Field) { + if ce := l.L.Check(zap.DebugLevel, msg); ce != nil { + ce.Write(zapifyFields(fields...)...) + } +} + +// Tracef logs at Trace log level using fmt formatter +func (l *Logger) Tracef(msg string, args ...interface{}) { + if ce := l.L.Check(zap.DebugLevel, ""); ce != nil { + ce.Message = fmt.Sprintf(msg, args...) + ce.Write() + } +} + +// Debug logs at Debug log level using fields +func (l *Logger) Debug(msg string, fields ...log.Field) { + if ce := l.L.Check(zap.DebugLevel, msg); ce != nil { + ce.Write(zapifyFields(fields...)...) + } +} + +// Debugf logs at Debug log level using fmt formatter +func (l *Logger) Debugf(msg string, args ...interface{}) { + if ce := l.L.Check(zap.DebugLevel, ""); ce != nil { + ce.Message = fmt.Sprintf(msg, args...) + ce.Write() + } +} + +// Info logs at Info log level using fields +func (l *Logger) Info(msg string, fields ...log.Field) { + if ce := l.L.Check(zap.InfoLevel, msg); ce != nil { + ce.Write(zapifyFields(fields...)...) + } +} + +// Infof logs at Info log level using fmt formatter +func (l *Logger) Infof(msg string, args ...interface{}) { + if ce := l.L.Check(zap.InfoLevel, ""); ce != nil { + ce.Message = fmt.Sprintf(msg, args...) + ce.Write() + } +} + +// Warn logs at Warn log level using fields +func (l *Logger) Warn(msg string, fields ...log.Field) { + if ce := l.L.Check(zap.WarnLevel, msg); ce != nil { + ce.Write(zapifyFields(fields...)...) + } +} + +// Warnf logs at Warn log level using fmt formatter +func (l *Logger) Warnf(msg string, args ...interface{}) { + if ce := l.L.Check(zap.WarnLevel, ""); ce != nil { + ce.Message = fmt.Sprintf(msg, args...) + ce.Write() + } +} + +// Error logs at Error log level using fields +func (l *Logger) Error(msg string, fields ...log.Field) { + if ce := l.L.Check(zap.ErrorLevel, msg); ce != nil { + ce.Write(zapifyFields(fields...)...) + } +} + +// Errorf logs at Error log level using fmt formatter +func (l *Logger) Errorf(msg string, args ...interface{}) { + if ce := l.L.Check(zap.ErrorLevel, ""); ce != nil { + ce.Message = fmt.Sprintf(msg, args...) + ce.Write() + } +} + +// Fatal logs at Fatal log level using fields +func (l *Logger) Fatal(msg string, fields ...log.Field) { + if ce := l.L.Check(zap.FatalLevel, msg); ce != nil { + ce.Write(zapifyFields(fields...)...) + } +} + +// Fatalf logs at Fatal log level using fmt formatter +func (l *Logger) Fatalf(msg string, args ...interface{}) { + if ce := l.L.Check(zap.FatalLevel, ""); ce != nil { + ce.Message = fmt.Sprintf(msg, args...) + ce.Write() + } +} + +// WithName adds name to logger +func (l *Logger) WithName(name string) log.Logger { + return &Logger{ + L: l.L.Named(name), + } +} diff --git a/library/go/core/log/zap/zap_test.go b/library/go/core/log/zap/zap_test.go new file mode 100644 index 0000000000..514a1fb89f --- /dev/null +++ b/library/go/core/log/zap/zap_test.go @@ -0,0 +1,113 @@ +package zap + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/ydb-platform/ydb/library/go/core/log" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func TestNewQloudLogger(t *testing.T) { + logger, err := NewQloudLogger(log.DebugLevel) + assert.NoError(t, err) + + core, logs := observer.New(zap.DebugLevel) + + logger.L = logger.L.WithOptions(zap.WrapCore(func(c zapcore.Core) zapcore.Core { + return addQloudContext(core).(zapcore.Core) + })) + + expectedMessage := "test message" + + logger.Info(expectedMessage, log.String("package", "zap")) + assert.Equal(t, 1, logs.Len()) + + loggedEntry := logs.AllUntimed()[0] + assert.Equal(t, zap.InfoLevel, loggedEntry.Level) + assert.Equal(t, expectedMessage, loggedEntry.Message) + assert.Equal(t, + map[string]interface{}{ + "@fields": map[string]interface{}{ + "package": "zap", + }, + }, + loggedEntry.ContextMap(), + ) +} + +func TestLogger_FormattedMethods(t *testing.T) { + testCases := []struct { + lvl log.Level + expectLogged []zapcore.Entry + }{ + {log.TraceLevel, []zapcore.Entry{ + {Level: zap.DebugLevel, Message: "test at trace"}, + {Level: zap.DebugLevel, Message: "test at debug"}, + {Level: zap.InfoLevel, Message: "test at info"}, + {Level: zap.WarnLevel, Message: "test at warn"}, + {Level: zap.ErrorLevel, Message: "test at error"}, + }}, + {log.DebugLevel, []zapcore.Entry{ + {Level: zap.DebugLevel, Message: "test at trace"}, + {Level: zap.DebugLevel, Message: "test at debug"}, + {Level: zap.InfoLevel, Message: "test at info"}, + {Level: zap.WarnLevel, Message: "test at warn"}, + {Level: zap.ErrorLevel, Message: "test at error"}, + }}, + {log.InfoLevel, []zapcore.Entry{ + {Level: zap.InfoLevel, Message: "test at info"}, + {Level: zap.WarnLevel, Message: "test at warn"}, + {Level: zap.ErrorLevel, Message: "test at error"}, + }}, + {log.WarnLevel, []zapcore.Entry{ + {Level: zap.WarnLevel, Message: "test at warn"}, + {Level: zap.ErrorLevel, Message: "test at error"}, + }}, + {log.ErrorLevel, []zapcore.Entry{ + {Level: zap.ErrorLevel, Message: "test at error"}, + }}, + } + + for _, tc := range testCases { + t.Run(tc.lvl.String(), func(t *testing.T) { + logger, err := New(ConsoleConfig(tc.lvl)) + assert.NoError(t, err) + + core, logs := observer.New(ZapifyLevel(tc.lvl)) + + logger.L = logger.L.WithOptions(zap.WrapCore(func(_ zapcore.Core) zapcore.Core { + return core + })) + + for _, lvl := range log.Levels() { + switch lvl { + case log.TraceLevel: + logger.Tracef("test at %s", lvl.String()) + case log.DebugLevel: + logger.Debugf("test at %s", lvl.String()) + case log.InfoLevel: + logger.Infof("test at %s", lvl.String()) + case log.WarnLevel: + logger.Warnf("test at %s", lvl.String()) + case log.ErrorLevel: + logger.Errorf("test at %s", lvl.String()) + case log.FatalLevel: + // skipping fatal + } + } + + loggedEntries := logs.AllUntimed() + + assert.Equal(t, len(tc.expectLogged), logs.Len(), cmp.Diff(tc.expectLogged, loggedEntries)) + + for i, le := range loggedEntries { + assert.Equal(t, tc.expectLogged[i].Level, le.Level) + assert.Equal(t, tc.expectLogged[i].Message, le.Message) + } + }) + } +} diff --git a/library/go/core/log/zap/zapify.go b/library/go/core/log/zap/zapify.go new file mode 100644 index 0000000000..5fd6ffb1be --- /dev/null +++ b/library/go/core/log/zap/zapify.go @@ -0,0 +1,95 @@ +package zap + +import ( + "fmt" + + "github.com/ydb-platform/ydb/library/go/core/log" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// ZapifyLevel turns interface log level to zap log level +func ZapifyLevel(level log.Level) zapcore.Level { + switch level { + case log.TraceLevel: + return zapcore.DebugLevel + case log.DebugLevel: + return zapcore.DebugLevel + case log.InfoLevel: + return zapcore.InfoLevel + case log.WarnLevel: + return zapcore.WarnLevel + case log.ErrorLevel: + return zapcore.ErrorLevel + case log.FatalLevel: + return zapcore.FatalLevel + default: + // For when new log level is not added to this func (most likely never). + panic(fmt.Sprintf("unknown log level: %d", level)) + } +} + +// UnzapifyLevel turns zap log level to interface log level. +func UnzapifyLevel(level zapcore.Level) log.Level { + switch level { + case zapcore.DebugLevel: + return log.DebugLevel + case zapcore.InfoLevel: + return log.InfoLevel + case zapcore.WarnLevel: + return log.WarnLevel + case zapcore.ErrorLevel: + return log.ErrorLevel + case zapcore.FatalLevel, zapcore.DPanicLevel, zapcore.PanicLevel: + return log.FatalLevel + default: + // For when new log level is not added to this func (most likely never). + panic(fmt.Sprintf("unknown log level: %d", level)) + } +} + +// nolint: gocyclo +func zapifyField(field log.Field) zap.Field { + switch field.Type() { + case log.FieldTypeNil: + return zap.Reflect(field.Key(), nil) + case log.FieldTypeString: + return zap.String(field.Key(), field.String()) + case log.FieldTypeBinary: + return zap.Binary(field.Key(), field.Binary()) + case log.FieldTypeBoolean: + return zap.Bool(field.Key(), field.Bool()) + case log.FieldTypeSigned: + return zap.Int64(field.Key(), field.Signed()) + case log.FieldTypeUnsigned: + return zap.Uint64(field.Key(), field.Unsigned()) + case log.FieldTypeFloat: + return zap.Float64(field.Key(), field.Float()) + case log.FieldTypeTime: + return zap.Time(field.Key(), field.Time()) + case log.FieldTypeDuration: + return zap.Duration(field.Key(), field.Duration()) + case log.FieldTypeError: + return zap.NamedError(field.Key(), field.Error()) + case log.FieldTypeArray: + return zap.Any(field.Key(), field.Interface()) + case log.FieldTypeAny: + return zap.Any(field.Key(), field.Interface()) + case log.FieldTypeReflect: + return zap.Reflect(field.Key(), field.Interface()) + case log.FieldTypeByteString: + return zap.ByteString(field.Key(), field.Binary()) + default: + // For when new field type is not added to this func + panic(fmt.Sprintf("unknown field type: %d", field.Type())) + } +} + +func zapifyFields(fields ...log.Field) []zapcore.Field { + zapFields := make([]zapcore.Field, 0, len(fields)) + for _, field := range fields { + zapFields = append(zapFields, zapifyField(field)) + } + + return zapFields +} diff --git a/library/go/core/log/zap/zapify_test.go b/library/go/core/log/zap/zapify_test.go new file mode 100644 index 0000000000..b11b1dc261 --- /dev/null +++ b/library/go/core/log/zap/zapify_test.go @@ -0,0 +1,60 @@ +package zap + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/ydb-platform/ydb/library/go/core/log" + "go.uber.org/zap/zapcore" +) + +// Simple test, that all type of fields are correctly zapified. +// Maybe we also need some test that checks resulting zap.Field type also. +func TestZapifyField(t *testing.T) { + fileds := []log.Field{ + log.Nil("test"), + log.String("test", "test"), + log.Binary("test", []byte("test")), + log.Bool("test", true), + log.Int("test", -42), + log.UInt("test", 42), + log.Float64("test", 0.42), + log.Time("test", time.Now()), + log.Duration("test", time.Second), + log.Error(fmt.Errorf("test")), + log.Array("test", []uint32{42}), + log.Any("test", struct{ ID uint32 }{ID: 42}), + log.Reflect("test", struct{ ID uint32 }{ID: 42}), + } + for _, field := range fileds { + assert.NotPanics(t, func() { + zapifyField(field) + }) + } +} + +func TestZapifyAny(t *testing.T) { + f := zapifyField(log.Any("test", struct{ ID uint32 }{ID: 42})) + assert.Equal(t, zapcore.ReflectType, f.Type) +} + +func TestZapifyReflect(t *testing.T) { + f := zapifyField(log.Any("test", struct{ ID uint32 }{ID: 42})) + assert.Equal(t, zapcore.ReflectType, f.Type) +} + +type stringer struct{} + +func (*stringer) String() string { + return "hello" +} + +func TestZapifyStringer(t *testing.T) { + f0 := zapifyField(log.Any("test", &stringer{})) + assert.Equal(t, zapcore.StringerType, f0.Type) + + f1 := zapifyField(log.Reflect("test", &stringer{})) + assert.Equal(t, zapcore.ReflectType, f1.Type) +} diff --git a/library/go/core/xerrors/assertxerrors/assertxerrors.go b/library/go/core/xerrors/assertxerrors/assertxerrors.go new file mode 100644 index 0000000000..593fc1595a --- /dev/null +++ b/library/go/core/xerrors/assertxerrors/assertxerrors.go @@ -0,0 +1,87 @@ +package assertxerrors + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/ydb-platform/ydb/library/go/core/xerrors/internal/modes" + "github.com/ydb-platform/ydb/library/go/test/testhelpers" +) + +func RunTestsPerMode(t *testing.T, expected Expectations, constructor func(t *testing.T) error) { + for _, mode := range modes.KnownStackTraceModes() { + t.Run(fmt.Sprintf("Mode%s", mode), func(t *testing.T) { + modes.SetStackTraceMode(mode) + err := constructor(t) + expected.Assert(t, err) + }) + } +} + +type StackTraceModeExpectation struct { + expectedPlusV string + lines []int +} + +func NewStackTraceModeExpectation(plusv string, lines ...int) StackTraceModeExpectation { + return StackTraceModeExpectation{expectedPlusV: plusv, lines: lines} +} + +type Expectations struct { + ExpectedS string + ExpectedV string + Frames StackTraceModeExpectation + Stacks StackTraceModeExpectation + StackThenFrames StackTraceModeExpectation + StackThenNothing StackTraceModeExpectation + Nothing StackTraceModeExpectation +} + +func (e Expectations) Assert(t *testing.T, err error) { + assert.Equal(t, e.ExpectedS, fmt.Sprintf("%s", err)) + assert.Equal(t, e.ExpectedV, fmt.Sprintf("%v", err)) + + var expected StackTraceModeExpectation + switch modes.GetStackTraceMode() { + case modes.StackTraceModeFrames: + expected = e.Frames + case modes.StackTraceModeStacks: + expected = e.Stacks + case modes.StackTraceModeStackThenFrames: + expected = e.StackThenFrames + case modes.StackTraceModeStackThenNothing: + expected = e.StackThenNothing + case modes.StackTraceModeNothing: + expected = e.Nothing + } + + assertErrorOutput(t, expected, err) +} + +func assertErrorOutput(t *testing.T, expected StackTraceModeExpectation, err error) { + // Cut starting \n's if needed (we use `` notation with newlines for expected error messages) + preparedExpected := strings.TrimPrefix(expected.expectedPlusV, "\n") + actual := fmt.Sprintf("%+v", err) + + var e error + preparedExpected, e = testhelpers.RemoveLines(preparedExpected, expected.lines...) + if !assert.NoErrorf(t, e, "lines removal from expected:\n%s", preparedExpected) { + t.Logf("initial expected:\n%s", expected.expectedPlusV) + t.Logf("initial actual:\n%s", actual) + return + } + + preparedActual, e := testhelpers.RemoveLines(actual, expected.lines...) + if !assert.NoErrorf(t, e, "lines removal from actual:\n%s", actual) { + t.Logf("initial expected:\n%s", expected.expectedPlusV) + t.Logf("initial actual:\n%s", actual) + return + } + + if !assert.Equal(t, preparedExpected, preparedActual) { + t.Logf("initial expected:\n%s", expected.expectedPlusV) + t.Logf("initial actual:\n%s", actual) + } +} diff --git a/library/go/core/xerrors/assertxerrors/ya.make b/library/go/core/xerrors/assertxerrors/ya.make new file mode 100644 index 0000000000..c61fc38f34 --- /dev/null +++ b/library/go/core/xerrors/assertxerrors/ya.make @@ -0,0 +1,5 @@ +GO_LIBRARY() + +SRCS(assertxerrors.go) + +END() diff --git a/library/go/core/xerrors/benchmark_test.go b/library/go/core/xerrors/benchmark_test.go new file mode 100644 index 0000000000..c573808eef --- /dev/null +++ b/library/go/core/xerrors/benchmark_test.go @@ -0,0 +1,154 @@ +package xerrors + +import ( + "errors" + "fmt" + "testing" + + pkgerrors "github.com/pkg/errors" + "github.com/ydb-platform/ydb/library/go/core/xerrors/benchxerrors" + "github.com/ydb-platform/ydb/library/go/test/testhelpers" + "golang.org/x/xerrors" +) + +const ( + benchNewMsg = "foo" + benchErrorfMsg = "bar: %w" +) + +func BenchmarkNewStd(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = errors.New(benchNewMsg) + } +} + +func BenchmarkNewPkg(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = pkgerrors.New(benchNewMsg) + } +} + +func BenchmarkNewXerrors(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = xerrors.New(benchNewMsg) + } +} + +func BenchmarkNewV2(b *testing.B) { + benchxerrors.RunPerMode(b, func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = New(benchNewMsg) + } + }) +} +func BenchmarkErrorfStd(b *testing.B) { + err := errors.New(benchNewMsg) + for i := 0; i < b.N; i++ { + _ = fmt.Errorf(benchErrorfMsg, err) + } +} + +func BenchmarkErrorfPkg(b *testing.B) { + err := errors.New(benchNewMsg) + for i := 0; i < b.N; i++ { + _ = pkgerrors.Wrap(err, benchErrorfMsg) + } +} + +func BenchmarkErrorfXerrors(b *testing.B) { + err := errors.New(benchNewMsg) + for i := 0; i < b.N; i++ { + _ = xerrors.Errorf(benchErrorfMsg, err) + } +} + +func BenchmarkErrorfV2(b *testing.B) { + err := errors.New(benchNewMsg) + benchxerrors.RunPerMode(b, func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = Errorf(benchErrorfMsg, err) + } + }) +} + +func BenchmarkNewErrorfStd(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = fmt.Errorf(benchErrorfMsg, errors.New(benchNewMsg)) + } +} + +func BenchmarkNewErrorfPkg(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = pkgerrors.Wrap(pkgerrors.New(benchNewMsg), benchErrorfMsg) + } +} + +func BenchmarkNewErrorfXerrors(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = xerrors.Errorf(benchErrorfMsg, xerrors.New(benchNewMsg)) + } +} + +func BenchmarkNewErrorfV2(b *testing.B) { + benchxerrors.RunPerMode(b, func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = Errorf(benchErrorfMsg, New(benchNewMsg)) + } + }) +} + +func BenchmarkNewErrorfErrorfStd(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = fmt.Errorf(benchErrorfMsg, fmt.Errorf(benchErrorfMsg, errors.New(benchNewMsg))) + } +} + +func BenchmarkNewErrorfErrorfPkg(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = pkgerrors.Wrap(pkgerrors.Wrap(pkgerrors.New(benchNewMsg), benchErrorfMsg), benchErrorfMsg) + } +} + +func BenchmarkNewErrorfErrorfXerrors(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = xerrors.Errorf(benchErrorfMsg, xerrors.Errorf(benchErrorfMsg, xerrors.New(benchNewMsg))) + } +} + +func BenchmarkNewErrorfErrorfV2(b *testing.B) { + benchxerrors.RunPerMode(b, func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = Errorf(benchErrorfMsg, Errorf(benchErrorfMsg, New(benchNewMsg))) + } + }) +} + +func recurse(f func()) { + testhelpers.Recurse(256, f) +} + +func BenchmarkBigStackNewErrorfErrorfStd(b *testing.B) { + for i := 0; i < b.N; i++ { + recurse(func() { _ = fmt.Errorf(benchErrorfMsg, fmt.Errorf(benchErrorfMsg, errors.New(benchNewMsg))) }) + } +} + +func BenchmarkBigStackNewErrorfErrorfPkg(b *testing.B) { + for i := 0; i < b.N; i++ { + recurse(func() { _ = pkgerrors.Wrap(pkgerrors.Wrap(pkgerrors.New(benchNewMsg), benchErrorfMsg), benchErrorfMsg) }) + } +} + +func BenchmarkBigStackNewErrorfErrorfXerrors(b *testing.B) { + for i := 0; i < b.N; i++ { + recurse(func() { _ = xerrors.Errorf(benchErrorfMsg, xerrors.Errorf(benchErrorfMsg, xerrors.New(benchNewMsg))) }) + } +} + +func BenchmarkBigStackNewErrorfErrorfV2(b *testing.B) { + benchxerrors.RunPerMode(b, func(b *testing.B) { + for i := 0; i < b.N; i++ { + recurse(func() { _ = Errorf(benchErrorfMsg, Errorf(benchErrorfMsg, New(benchNewMsg))) }) + } + }) +} diff --git a/library/go/core/xerrors/benchxerrors/benchxerrors.go b/library/go/core/xerrors/benchxerrors/benchxerrors.go new file mode 100644 index 0000000000..d17286e9fb --- /dev/null +++ b/library/go/core/xerrors/benchxerrors/benchxerrors.go @@ -0,0 +1,17 @@ +package benchxerrors + +import ( + "fmt" + "testing" + + "github.com/ydb-platform/ydb/library/go/core/xerrors/internal/modes" +) + +func RunPerMode(b *testing.B, bench func(b *testing.B)) { + for _, mode := range modes.KnownStackTraceModes() { + b.Run(fmt.Sprintf("Mode%s", mode), func(b *testing.B) { + modes.SetStackTraceMode(mode) + bench(b) + }) + } +} diff --git a/library/go/core/xerrors/benchxerrors/ya.make b/library/go/core/xerrors/benchxerrors/ya.make new file mode 100644 index 0000000000..07de188bb3 --- /dev/null +++ b/library/go/core/xerrors/benchxerrors/ya.make @@ -0,0 +1,5 @@ +GO_LIBRARY() + +SRCS(benchxerrors.go) + +END() diff --git a/library/go/core/xerrors/doc.go b/library/go/core/xerrors/doc.go new file mode 100644 index 0000000000..de06dd15d2 --- /dev/null +++ b/library/go/core/xerrors/doc.go @@ -0,0 +1,2 @@ +// package xerrors is a drop in replacement for errors and golang.org/x/xerrors packages and functionally for github.com/pkg/errors. +package xerrors diff --git a/library/go/core/xerrors/errorf.go b/library/go/core/xerrors/errorf.go new file mode 100644 index 0000000000..0ed8541f28 --- /dev/null +++ b/library/go/core/xerrors/errorf.go @@ -0,0 +1,92 @@ +package xerrors + +import ( + "fmt" + "io" + "strings" + + "github.com/ydb-platform/ydb/library/go/x/xruntime" +) + +type wrappedErrorf struct { + err error + stacktrace *xruntime.StackTrace +} + +var _ ErrorStackTrace = &wrappedErrorf{} + +func Errorf(format string, a ...interface{}) error { + err := fmt.Errorf(format, a...) + return &wrappedErrorf{ + err: err, + stacktrace: newStackTrace(1, err), + } +} + +func SkipErrorf(skip int, format string, a ...interface{}) error { + err := fmt.Errorf(format, a...) + return &wrappedErrorf{ + err: err, + stacktrace: newStackTrace(skip+1, err), + } +} + +func (e *wrappedErrorf) Format(s fmt.State, v rune) { + switch v { + case 'v': + if s.Flag('+') { + msg := e.err.Error() + inner := Unwrap(e.err) + // If Errorf wrapped another error then it will be our message' suffix. If so, cut it since otherwise we will + // print it again as part of formatting that error. + if inner != nil { + if strings.HasSuffix(msg, inner.Error()) { + msg = msg[:len(msg)-len(inner.Error())] + // Cut last space if needed but only if there is stacktrace present (very likely) + if e.stacktrace != nil && strings.HasSuffix(msg, ": ") { + msg = msg[:len(msg)-1] + } + } + } + + _, _ = io.WriteString(s, msg) + if e.stacktrace != nil { + // New line is useful only when printing frames, otherwise it is better to print next error in the chain + // right after we print this one + _, _ = io.WriteString(s, "\n") + writeStackTrace(s, e.stacktrace) + } + + // Print next error down the chain if there is one + if inner != nil { + _, _ = fmt.Fprintf(s, "%+v", inner) + } + + return + } + fallthrough + case 's': + _, _ = io.WriteString(s, e.err.Error()) + case 'q': + _, _ = fmt.Fprintf(s, "%q", e.err.Error()) + } +} + +func (e *wrappedErrorf) Error() string { + // Wrapped error has correct formatting + return e.err.Error() +} + +func (e *wrappedErrorf) Unwrap() error { + // Skip wrapped error and return whatever it is wrapping if inner error contains single error + // TODO: test for correct unwrap + if _, ok := e.err.(interface{ Unwrap() []error }); ok { + return e.err + } + + return Unwrap(e.err) +} + +func (e *wrappedErrorf) StackTrace() *xruntime.StackTrace { + return e.stacktrace +} diff --git a/library/go/core/xerrors/errorf_formatting_with_error_test.go b/library/go/core/xerrors/errorf_formatting_with_error_test.go new file mode 100644 index 0000000000..c6bf47aac2 --- /dev/null +++ b/library/go/core/xerrors/errorf_formatting_with_error_test.go @@ -0,0 +1,72 @@ +package xerrors + +import ( + "testing" + + "github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors" +) + +func TestErrorfFormattingWithError(t *testing.T) { + constructor := func(t *testing.T) error { + err := New("new") + return Errorf("errorf: %w", err) + } + expected := assertxerrors.Expectations{ + ExpectedS: "errorf: new", + ExpectedV: "errorf: new", + Frames: assertxerrors.NewStackTraceModeExpectation(` +errorf: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithError.func1 + library/go/core/xerrors/errorf_formatting_with_error_test.go:12 +new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithError.func1 + library/go/core/xerrors/errorf_formatting_with_error_test.go:11 +`, + ), + Stacks: assertxerrors.NewStackTraceModeExpectation(` +errorf: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithError.func1 + library/go/core/xerrors/errorf_formatting_with_error_test.go:12 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithError.func1 + library/go/core/xerrors/errorf_formatting_with_error_test.go:11 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, 10, 11, 12, 13, + ), + StackThenFrames: assertxerrors.NewStackTraceModeExpectation(` +errorf: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithError.func1 + library/go/core/xerrors/errorf_formatting_with_error_test.go:12 +new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithError.func1 + library/go/core/xerrors/errorf_formatting_with_error_test.go:11 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 6, 7, 8, 9, + ), + StackThenNothing: assertxerrors.NewStackTraceModeExpectation(` +errorf: new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithError.func1 + library/go/core/xerrors/errorf_formatting_with_error_test.go:11 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + Nothing: assertxerrors.NewStackTraceModeExpectation("errorf: new"), + } + assertxerrors.RunTestsPerMode(t, expected, constructor) +} diff --git a/library/go/core/xerrors/errorf_formatting_with_std_error_test.go b/library/go/core/xerrors/errorf_formatting_with_std_error_test.go new file mode 100644 index 0000000000..c8d3153890 --- /dev/null +++ b/library/go/core/xerrors/errorf_formatting_with_std_error_test.go @@ -0,0 +1,60 @@ +package xerrors + +import ( + "errors" + "testing" + + "github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors" +) + +func TestErrorfFormattingWithStdError(t *testing.T) { + constructor := func(t *testing.T) error { + err := errors.New("new") + return Errorf("errorf: %w", err) + } + expected := assertxerrors.Expectations{ + ExpectedS: "errorf: new", + ExpectedV: "errorf: new", + Frames: assertxerrors.NewStackTraceModeExpectation(` +errorf: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithStdError.func1 + library/go/core/xerrors/errorf_formatting_with_std_error_test.go:13 +new`, + ), + Stacks: assertxerrors.NewStackTraceModeExpectation(` +errorf: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithStdError.func1 + library/go/core/xerrors/errorf_formatting_with_std_error_test.go:13 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +new`, + 3, 4, 5, 6, + ), + StackThenFrames: assertxerrors.NewStackTraceModeExpectation(` +errorf: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithStdError.func1 + library/go/core/xerrors/errorf_formatting_with_std_error_test.go:13 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +new`, + 3, 4, 5, 6, + ), + StackThenNothing: assertxerrors.NewStackTraceModeExpectation(` +errorf: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithStdError.func1 + library/go/core/xerrors/errorf_formatting_with_std_error_test.go:13 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +new`, + 3, 4, 5, 6, + ), + Nothing: assertxerrors.NewStackTraceModeExpectation("errorf: new"), + } + assertxerrors.RunTestsPerMode(t, expected, constructor) +} diff --git a/library/go/core/xerrors/errorf_formatting_without_error_test.go b/library/go/core/xerrors/errorf_formatting_without_error_test.go new file mode 100644 index 0000000000..602804f97f --- /dev/null +++ b/library/go/core/xerrors/errorf_formatting_without_error_test.go @@ -0,0 +1,58 @@ +package xerrors + +import ( + "testing" + + "github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors" +) + +func TestErrorfFormattingWithoutError(t *testing.T) { + constructor := func(t *testing.T) error { + return Errorf("errorf: %s", "not an error") + } + expected := assertxerrors.Expectations{ + ExpectedS: "errorf: not an error", + ExpectedV: "errorf: not an error", + Frames: assertxerrors.NewStackTraceModeExpectation(` +errorf: not an error + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithoutError.func1 + library/go/core/xerrors/errorf_formatting_without_error_test.go:11 +`, + ), + Stacks: assertxerrors.NewStackTraceModeExpectation(` +errorf: not an error + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithoutError.func1 + library/go/core/xerrors/errorf_formatting_without_error_test.go:11 +github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + StackThenFrames: assertxerrors.NewStackTraceModeExpectation(` +errorf: not an error + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithoutError.func1 + library/go/core/xerrors/errorf_formatting_without_error_test.go:11 +github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + StackThenNothing: assertxerrors.NewStackTraceModeExpectation(` +errorf: not an error + github.com/ydb-platform/ydb/library/go/core/xerrors.TestErrorfFormattingWithoutError.func1 + library/go/core/xerrors/errorf_formatting_without_error_test.go:11 +github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + Nothing: assertxerrors.NewStackTraceModeExpectation("errorf: not an error"), + } + assertxerrors.RunTestsPerMode(t, expected, constructor) +} diff --git a/library/go/core/xerrors/errorf_multiple_errors_test.go b/library/go/core/xerrors/errorf_multiple_errors_test.go new file mode 100644 index 0000000000..844b71355e --- /dev/null +++ b/library/go/core/xerrors/errorf_multiple_errors_test.go @@ -0,0 +1,19 @@ +package xerrors + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestErrorfMultipleErrors(t *testing.T) { + err1 := New("error1") + err2 := New("error2") + err3 := New("error3") + + compositeErr := Errorf("errorf: %w, %w", err1, err2) + + require.True(t, Is(compositeErr, err1)) + require.True(t, Is(compositeErr, err2)) + require.False(t, Is(compositeErr, err3)) +} diff --git a/library/go/core/xerrors/forward.go b/library/go/core/xerrors/forward.go new file mode 100644 index 0000000000..aaa900133c --- /dev/null +++ b/library/go/core/xerrors/forward.go @@ -0,0 +1,56 @@ +package xerrors + +import "errors" + +// Unwrap returns the result of calling the Unwrap method on err, if err's +// type contains an Unwrap method returning error. +// Otherwise, Unwrap returns nil. +func Unwrap(err error) error { + return errors.Unwrap(err) +} + +// Is reports whether any error in err's chain matches target. +// +// The chain consists of err itself followed by the sequence of errors obtained by +// repeatedly calling Unwrap. +// +// An error is considered to match a target if it is equal to that target or if +// it implements a method Is(error) bool such that Is(target) returns true. +// +// An error type might provide an Is method so it can be treated as equivalent +// to an existing error. For example, if MyError defines +// +// func (m MyError) Is(target error) bool { return target == os.ErrExist } +// +// then Is(MyError{}, os.ErrExist) returns true. See syscall.Errno.Is for +// an example in the standard library. +func Is(err, target error) bool { + return errors.Is(err, target) +} + +// As finds the first error in err's chain that matches target, and if so, sets +// target to that error value and returns true. Otherwise, it returns false. +// +// The chain consists of err itself followed by the sequence of errors obtained by +// repeatedly calling Unwrap. +// +// An error matches target if the error's concrete value is assignable to the value +// pointed to by target, or if the error has a method As(interface{}) bool such that +// As(target) returns true. In the latter case, the As method is responsible for +// setting target. +// +// An error type might provide an As method so it can be treated as if it were a +// different error type. +// +// As panics if target is not a non-nil pointer to either a type that implements +// error, or to any interface type. +func As(err error, target interface{}) bool { + return errors.As(err, target) +} + +// Wrapper provides context around another error. +type Wrapper interface { + // Unwrap returns the next error in the error chain. + // If there is no next error, Unwrap returns nil. + Unwrap() error +} diff --git a/library/go/core/xerrors/gotest/ya.make b/library/go/core/xerrors/gotest/ya.make new file mode 100644 index 0000000000..1525c5623a --- /dev/null +++ b/library/go/core/xerrors/gotest/ya.make @@ -0,0 +1,5 @@ +GO_TEST_FOR(library/go/core/xerrors) + +ENABLE(GO_DEBUG_PATH_RELATIVE) + +END() diff --git a/library/go/core/xerrors/internal/modes/stack_frames_count.go b/library/go/core/xerrors/internal/modes/stack_frames_count.go new file mode 100644 index 0000000000..c117becf6a --- /dev/null +++ b/library/go/core/xerrors/internal/modes/stack_frames_count.go @@ -0,0 +1,22 @@ +package modes + +import "sync/atomic" + +type StackFramesCount = int32 + +const ( + StackFramesCount16 StackFramesCount = 16 + StackFramesCount32 StackFramesCount = 32 + StackFramesCount64 StackFramesCount = 64 + StackFramesCount128 StackFramesCount = 128 +) + +var StackFramesCountMax = StackFramesCount32 + +func SetStackFramesCountMax(count StackFramesCount) { + atomic.StoreInt32(&StackFramesCountMax, count) +} + +func GetStackFramesCountMax() StackFramesCount { + return atomic.LoadInt32(&StackFramesCountMax) +} diff --git a/library/go/core/xerrors/internal/modes/stack_trace_mode.go b/library/go/core/xerrors/internal/modes/stack_trace_mode.go new file mode 100644 index 0000000000..04f78ffd3d --- /dev/null +++ b/library/go/core/xerrors/internal/modes/stack_trace_mode.go @@ -0,0 +1,48 @@ +package modes + +import "sync/atomic" + +type StackTraceMode int32 + +const ( + StackTraceModeFrames StackTraceMode = iota + StackTraceModeStacks + StackTraceModeStackThenFrames + StackTraceModeStackThenNothing + StackTraceModeNothing +) + +func (m StackTraceMode) String() string { + return []string{"Frames", "Stacks", "StackThenFrames", "StackThenNothing", "Nothing"}[m] +} + +const defaultStackTraceMode = StackTraceModeFrames + +var ( + // Default mode + stackTraceMode = defaultStackTraceMode + // Known modes (used in tests) + knownStackTraceModes = []StackTraceMode{ + StackTraceModeFrames, + StackTraceModeStacks, + StackTraceModeStackThenFrames, + StackTraceModeStackThenNothing, + StackTraceModeNothing, + } +) + +func SetStackTraceMode(v StackTraceMode) { + atomic.StoreInt32((*int32)(&stackTraceMode), int32(v)) +} + +func GetStackTraceMode() StackTraceMode { + return StackTraceMode(atomic.LoadInt32((*int32)(&stackTraceMode))) +} + +func DefaultStackTraceMode() { + SetStackTraceMode(defaultStackTraceMode) +} + +func KnownStackTraceModes() []StackTraceMode { + return knownStackTraceModes +} diff --git a/library/go/core/xerrors/internal/modes/ya.make b/library/go/core/xerrors/internal/modes/ya.make new file mode 100644 index 0000000000..51342e3b12 --- /dev/null +++ b/library/go/core/xerrors/internal/modes/ya.make @@ -0,0 +1,8 @@ +GO_LIBRARY() + +SRCS( + stack_frames_count.go + stack_trace_mode.go +) + +END() diff --git a/library/go/core/xerrors/internal/ya.make b/library/go/core/xerrors/internal/ya.make new file mode 100644 index 0000000000..49349fb87b --- /dev/null +++ b/library/go/core/xerrors/internal/ya.make @@ -0,0 +1 @@ +RECURSE(modes) diff --git a/library/go/core/xerrors/mode.go b/library/go/core/xerrors/mode.go new file mode 100644 index 0000000000..c6b16bed76 --- /dev/null +++ b/library/go/core/xerrors/mode.go @@ -0,0 +1,93 @@ +package xerrors + +import ( + "fmt" + + "github.com/ydb-platform/ydb/library/go/core/xerrors/internal/modes" + "github.com/ydb-platform/ydb/library/go/x/xruntime" +) + +func DefaultStackTraceMode() { + modes.DefaultStackTraceMode() +} + +func EnableFrames() { + modes.SetStackTraceMode(modes.StackTraceModeFrames) +} + +func EnableStacks() { + modes.SetStackTraceMode(modes.StackTraceModeStacks) +} + +func EnableStackThenFrames() { + modes.SetStackTraceMode(modes.StackTraceModeStackThenFrames) +} + +func EnableStackThenNothing() { + modes.SetStackTraceMode(modes.StackTraceModeStackThenNothing) +} + +func DisableStackTraces() { + modes.SetStackTraceMode(modes.StackTraceModeNothing) +} + +// newStackTrace returns stacktrace based on current mode and frames count +func newStackTrace(skip int, err error) *xruntime.StackTrace { + skip++ + m := modes.GetStackTraceMode() + switch m { + case modes.StackTraceModeFrames: + return xruntime.NewFrame(skip) + case modes.StackTraceModeStackThenFrames: + if err != nil && StackTraceOfEffect(err) != nil { + return xruntime.NewFrame(skip) + } + + return _newStackTrace(skip) + case modes.StackTraceModeStackThenNothing: + if err != nil && StackTraceOfEffect(err) != nil { + return nil + } + + return _newStackTrace(skip) + case modes.StackTraceModeStacks: + return _newStackTrace(skip) + case modes.StackTraceModeNothing: + return nil + } + + panic(fmt.Sprintf("unknown stack trace mode %d", m)) +} + +func MaxStackFrames16() { + modes.SetStackFramesCountMax(modes.StackFramesCount16) +} + +func MaxStackFrames32() { + modes.SetStackFramesCountMax(modes.StackFramesCount32) +} + +func MaxStackFrames64() { + modes.SetStackFramesCountMax(modes.StackFramesCount64) +} + +func MaxStackFrames128() { + modes.SetStackFramesCountMax(modes.StackFramesCount128) +} + +func _newStackTrace(skip int) *xruntime.StackTrace { + skip++ + count := modes.GetStackFramesCountMax() + switch count { + case 16: + return xruntime.NewStackTrace16(skip) + case 32: + return xruntime.NewStackTrace32(skip) + case 64: + return xruntime.NewStackTrace64(skip) + case 128: + return xruntime.NewStackTrace128(skip) + } + + panic(fmt.Sprintf("unknown stack frames count %d", count)) +} diff --git a/library/go/core/xerrors/multierr/error.go b/library/go/core/xerrors/multierr/error.go new file mode 100644 index 0000000000..443fa177bc --- /dev/null +++ b/library/go/core/xerrors/multierr/error.go @@ -0,0 +1,277 @@ +package multierr + +import ( + "bytes" + "fmt" + "io" + "strings" + "sync/atomic" + + "github.com/ydb-platform/ydb/library/go/core/xerrors" +) + +type errorGroup interface { + Errors() []error +} + +// Errors returns a slice containing zero or more errors the supplied +// error is composed of. If the error is nil, a nil slice is returned. +// +// If the error is not composed of the errors (do not implement errorGroup +// interface with Errors method), the returned slice contains just the error +// that was padded in. +// +// Callers of this function are free to modify the returned slice. +func Errors(err error) []error { + if err == nil { + return nil + } + + eg, ok := err.(errorGroup) + if !ok { + return []error{err} + } + + errs := eg.Errors() + + result := make([]error, len(errs)) + copy(result, errs) + + return result +} + +// Append appends the given errors together. Either value may be nil. +// +// This function is a specialization of Combine for the common case where +// there are only two errors. +func Append(left, right error) error { + switch { + case left == nil: + return right + case right == nil: + return left + } + + if _, ok := right.(*multiError); !ok { + if l, ok := left.(*multiError); ok && atomic.SwapUint32(&l.copyNeeded, 1) == 0 { + errors := append(l.Errors(), right) + + return &multiError{errors: errors} + } else if !ok { + return &multiError{errors: []error{left, right}} + } + } + + return fromSlice([]error{left, right}) +} + +// Combine combines the passed errors into a single error. +// +// If zero arguments were passed or if all items are nil, +// a nil error is returned. +// +// If only a single error was passed, it is returned as-is. +// +// Combine skips over nil arguments so this function may be +// used to combine errors from operations that fail independently +// of each other. +// +// If any of the passed errors is an errorGroup error, it will be +// flattened along with the other errors. +func Combine(errors ...error) error { + return fromSlice(errors) +} + +func fromSlice(errors []error) error { + inspection := inspect(errors) + + switch inspection.topLevelErrorsCount { + case 0: + return nil + case 1: + return errors[inspection.firstErrorIdx] + case len(errors): + if !inspection.containsErrorGroup { + return &multiError{errors: errors} + } + } + + nonNilErrs := make([]error, 0, inspection.errorsCapacity) + + for _, err := range errors[inspection.firstErrorIdx:] { + if err == nil { + continue + } + + if eg, ok := err.(errorGroup); ok { + nonNilErrs = append(nonNilErrs, eg.Errors()...) + + continue + } + + nonNilErrs = append(nonNilErrs, err) + } + + return &multiError{errors: nonNilErrs} +} + +type errorsInspection struct { + topLevelErrorsCount int + errorsCapacity int + firstErrorIdx int + containsErrorGroup bool +} + +func inspect(errors []error) errorsInspection { + var inspection errorsInspection + + first := true + + for i, err := range errors { + if err == nil { + continue + } + + inspection.topLevelErrorsCount++ + if first { + first = false + inspection.firstErrorIdx = i + } + + if eg, ok := err.(errorGroup); ok { + inspection.containsErrorGroup = true + inspection.errorsCapacity += len(eg.Errors()) + + continue + } + + inspection.errorsCapacity++ + } + + return inspection +} + +type multiError struct { + copyNeeded uint32 + errors []error +} + +// As attempts to find the first error in the error list +// that matched the type of the value that target points to. +// +// This function allows errors.As to traverse the values stored on the +// multiError error. +func (e *multiError) As(target interface{}) bool { + for _, err := range e.Errors() { + if xerrors.As(err, target) { + return true + } + } + + return false +} + +// Is attempts to match the provided error against +// errors in the error list. +// +// This function allows errors.Is to traverse the values stored on the +// multiError error. +func (e *multiError) Is(target error) bool { + for _, err := range e.Errors() { + if xerrors.Is(err, target) { + return true + } + } + + return false +} + +func (e *multiError) Error() string { + if e == nil { + return "" + } + + var buff bytes.Buffer + + e.writeSingleLine(&buff) + + return buff.String() +} + +// Errors returns the list of underlying errors. +// +// This slice MUST NOT be modified. +func (e *multiError) Errors() []error { + if e == nil { + return nil + } + + return e.errors +} + +var ( + singleLineSeparator = []byte("; ") + + multiLineSeparator = []byte("\n") + multiLineIndent = []byte(" ") +) + +func (e *multiError) writeSingleLine(w io.Writer) { + first := true + + for _, err := range e.Errors() { + if first { + first = false + } else { + _, _ = w.Write(singleLineSeparator) + } + + _, _ = io.WriteString(w, err.Error()) + } +} + +func (e *multiError) Format(f fmt.State, c rune) { + if c == 'v' && f.Flag('+') { + e.writeMultiLine(f) + + return + } + + e.writeSingleLine(f) +} + +func (e *multiError) writeMultiLine(w io.Writer) { + var ( + errors = e.Errors() + lastIdx = len(errors) - 1 + ) + + for _, err := range errors[:lastIdx] { + writePrefixLine(w, multiLineIndent, fmt.Sprintf("%+v", err)) + + _, _ = w.Write(multiLineSeparator) + } + + writePrefixLine(w, multiLineIndent, fmt.Sprintf("%+v", errors[lastIdx])) +} + +func writePrefixLine(w io.Writer, prefix []byte, s string) { + first := true + + for len(s) > 0 { + if first { + first = false + } else { + _, _ = w.Write(prefix) + } + + idx := strings.IndexByte(s, '\n') + if idx < 0 { + idx = len(s) - 1 + } + + _, _ = io.WriteString(w, s[:idx+1]) + + s = s[idx+1:] + } +} diff --git a/library/go/core/xerrors/multierr/error_test.go b/library/go/core/xerrors/multierr/error_test.go new file mode 100644 index 0000000000..ac58434948 --- /dev/null +++ b/library/go/core/xerrors/multierr/error_test.go @@ -0,0 +1,647 @@ +package multierr + +import ( + "errors" + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/ydb-platform/ydb/library/go/core/xerrors" +) + +func TestCombine(t *testing.T) { + t.Parallel() + + testCases := []struct { + GivenErrors []error + ExpectedError error + }{ + { + GivenErrors: nil, + ExpectedError: nil, + }, + { + GivenErrors: []error{}, + ExpectedError: nil, + }, + { + GivenErrors: []error{ + errors.New("foo"), + nil, + errors.New("bar"), + nil, + }, + ExpectedError: newMultiError( + errors.New("foo"), + errors.New("bar"), + ), + }, + { + GivenErrors: []error{ + errors.New("foo"), + newMultiError(errors.New("bar")), + }, + ExpectedError: newMultiError( + errors.New("foo"), + errors.New("bar"), + ), + }, + { + GivenErrors: []error{errors.New("something wrong")}, + ExpectedError: errors.New("something wrong"), + }, + { + GivenErrors: []error{ + errors.New("foo"), + errors.New("bar"), + }, + ExpectedError: newMultiError( + errors.New("foo"), + errors.New("bar"), + ), + }, + { + GivenErrors: []error{ + errors.New("something"), + errors.New("O\n O\n P\n S\n"), + errors.New("wrong"), + }, + ExpectedError: newMultiError( + errors.New("something"), + errors.New("O\n O\n P\n S\n"), + errors.New("wrong"), + ), + }, + { + GivenErrors: []error{ + errors.New("foo"), + newMultiError( + errors.New("bar"), + errors.New("baz"), + ), + errors.New("qyz"), + }, + ExpectedError: newMultiError( + errors.New("foo"), + errors.New("bar"), + errors.New("baz"), + errors.New("qyz"), + ), + }, + { + GivenErrors: []error{ + errors.New("foo"), + nil, + newMultiError( + errors.New("bar"), + ), + nil, + }, + ExpectedError: newMultiError( + errors.New("foo"), + errors.New("bar"), + ), + }, + { + GivenErrors: []error{ + errors.New("foo"), + newMultiError( + errors.New("bar"), + ), + }, + ExpectedError: newMultiError( + errors.New("foo"), + errors.New("bar"), + ), + }, + } + + for i, c := range testCases { + c := c + + t.Run(fmt.Sprint(i), func(t *testing.T) { + t.Parallel() + + require.Equal(t, c.ExpectedError, Combine(c.GivenErrors...)) + }) + } +} + +func TestFormatWithoutTraces(t *testing.T) { + t.Parallel() + + testCases := []struct { + GivenError error + ExpectedSingleLineError string + ExpectedMultiLineError string + }{ + { + GivenError: Combine(errors.New("foo")), + ExpectedSingleLineError: "foo", + ExpectedMultiLineError: "foo", + }, + { + GivenError: Combine( + errors.New("foo"), + errors.New("bar"), + ), + ExpectedSingleLineError: "foo; bar", + ExpectedMultiLineError: "" + + "foo\n" + + "bar", + }, + { + GivenError: Combine( + errors.New("foo"), + errors.New("bar"), + errors.New("baz"), + errors.New("qyz"), + ), + ExpectedSingleLineError: "foo; bar; baz; qyz", + ExpectedMultiLineError: "" + + "foo\n" + + "bar\n" + + "baz\n" + + "qyz", + }, + { + GivenError: Combine( + errors.New("something"), + errors.New("O\n O\n P\n S\n"), + errors.New("wrong"), + ), + ExpectedSingleLineError: "something; O\n O\n P\n S\n; wrong", + ExpectedMultiLineError: "" + + "something\n" + + "O\n" + + " O\n" + + " P\n" + + " S\n\n" + + "wrong", + }, + } + + for i, c := range testCases { + c := c + + t.Run(fmt.Sprint(i), func(t *testing.T) { + t.Parallel() + + t.Run("sprint/single_line", func(t *testing.T) { + assert.Equal(t, c.ExpectedSingleLineError, fmt.Sprintf("%v", c.GivenError)) + }) + + t.Run("error", func(t *testing.T) { + assert.Equal(t, c.ExpectedSingleLineError, c.GivenError.Error()) + }) + + t.Run("sprintf/multi_line", func(t *testing.T) { + assert.Equal(t, c.ExpectedMultiLineError, fmt.Sprintf("%+v", c.GivenError)) + }) + }) + } +} + +func TestCombineDoesNotModifySlice(t *testing.T) { + t.Parallel() + + errs := []error{ + errors.New("foo"), + nil, + errors.New("bar"), + } + + assert.Error(t, Combine(errs...)) + assert.Len(t, errs, 3) + assert.NoError(t, errs[1]) +} + +func TestAppend(t *testing.T) { + t.Parallel() + + testCases := []struct { + GivenLeftError error + GivenRightError error + ExpectedError error + }{ + { + GivenLeftError: nil, + GivenRightError: nil, + ExpectedError: nil, + }, + { + GivenLeftError: nil, + GivenRightError: errors.New("something wrong"), + ExpectedError: errors.New("something wrong"), + }, + { + GivenLeftError: errors.New("something wrong"), + GivenRightError: nil, + ExpectedError: errors.New("something wrong"), + }, + { + GivenLeftError: errors.New("foo"), + GivenRightError: errors.New("bar"), + ExpectedError: newMultiError( + errors.New("foo"), + errors.New("bar"), + ), + }, + { + GivenLeftError: newMultiError( + errors.New("foo"), + errors.New("bar"), + ), + GivenRightError: errors.New("baz"), + ExpectedError: newMultiError( + errors.New("foo"), + errors.New("bar"), + errors.New("baz"), + ), + }, + { + GivenLeftError: errors.New("baz"), + GivenRightError: newMultiError( + errors.New("foo"), + errors.New("bar"), + ), + ExpectedError: newMultiError( + errors.New("baz"), + errors.New("foo"), + errors.New("bar"), + ), + }, + { + GivenLeftError: newMultiError( + errors.New("foo"), + ), + GivenRightError: newMultiError( + errors.New("bar"), + ), + ExpectedError: newMultiError( + errors.New("foo"), + errors.New("bar"), + ), + }, + } + + for i, c := range testCases { + c := c + t.Run(fmt.Sprint(i), func(t *testing.T) { + t.Parallel() + + err := Append(c.GivenLeftError, c.GivenRightError) + + require.Equal(t, c.ExpectedError, err) + }) + } +} + +func TestAppendDoesNotModify(t *testing.T) { + t.Parallel() + + var ( + initial = newMultiErrorWithCapacity() + foo = Append(initial, errors.New("foo")) + bar = Append(initial, errors.New("bar")) + ) + + t.Run("initial_not_modified", func(t *testing.T) { + t.Parallel() + + assert.EqualError(t, initial, newMultiErrorWithCapacity().Error()) + }) + + t.Run("errors_appended", func(t *testing.T) { + t.Parallel() + + assert.EqualError(t, bar, Append(newMultiErrorWithCapacity(), errors.New("bar")).Error()) + assert.EqualError(t, foo, Append(newMultiErrorWithCapacity(), errors.New("foo")).Error()) + }) + + t.Run("errors_len_equal", func(t *testing.T) { + t.Parallel() + + assert.Len(t, Errors(foo), len(Errors(bar))) + assert.Len(t, Errors(foo), len(Errors(initial))+1) + }) +} + +func TestErrors(t *testing.T) { + t.Parallel() + + testCases := []struct { + Given error + Expected []error + Cast bool + }{ + { + Given: nil, + Expected: nil, + }, + { + Given: errors.New("go"), + Expected: []error{errors.New("go")}, + }, + { + Given: groupNotMultiError{}, + Expected: groupNotMultiError{}.Errors(), + }, + { + Given: Combine( + errors.New("foo"), + errors.New("bar"), + ), + Expected: []error{ + errors.New("foo"), + errors.New("bar"), + }, + Cast: true, + }, + { + Given: Append( + errors.New("foo"), + errors.New("bar"), + ), + Expected: []error{ + errors.New("foo"), + errors.New("bar"), + }, + Cast: true, + }, + { + Given: Append( + errors.New("foo"), + Combine( + errors.New("bar"), + ), + ), + Expected: []error{ + errors.New("foo"), + errors.New("bar"), + }, + }, + { + Given: Combine( + errors.New("foo"), + Append( + errors.New("bar"), + errors.New("baz"), + ), + errors.New("qux"), + ), + Expected: []error{ + errors.New("foo"), + errors.New("bar"), + errors.New("baz"), + errors.New("qux"), + }, + }, + } + + for i, c := range testCases { + c := c + + t.Run(fmt.Sprint(i), func(t *testing.T) { + t.Parallel() + + t.Run("errors", func(t *testing.T) { + require.Equal(t, c.Expected, Errors(c.Given)) + }) + + if !c.Cast { + return + } + + t.Run("multiError/errors", func(t *testing.T) { + require.Equal(t, c.Expected, c.Given.(*multiError).Errors()) + }) + + t.Run("errorGroup/errors", func(t *testing.T) { + require.Equal(t, c.Expected, c.Given.(errorGroup).Errors()) + }) + }) + } +} + +func TestAppendRace(t *testing.T) { + t.Parallel() + + initial := newMultiErrorWithCapacity() + + require.NotPanics(t, func() { + var wg sync.WaitGroup + + for i := 0; i < 10; i++ { + wg.Add(1) + + go func() { + defer wg.Done() + + err := initial + for j := 0; j < 10; j++ { + err = Append(err, errors.New("foo")) + } + }() + } + + wg.Wait() + }) +} + +func TestErrorsSliceIsImmutable(t *testing.T) { + t.Parallel() + + var ( + foo = errors.New("foo") + bar = errors.New("bar") + ) + + err := Append(foo, bar) + actualErrors := Errors(err) + require.Equal(t, []error{foo, bar}, actualErrors) + + actualErrors[0] = nil + actualErrors[1] = errors.New("bax") + + require.Equal(t, []error{foo, bar}, Errors(err)) +} + +func TestNilMultiError(t *testing.T) { + t.Parallel() + + var err *multiError + + require.Empty(t, err.Error()) + require.Empty(t, err.Errors()) +} + +var ( + errFoo = errors.New("foo") + errBar = errors.New("bar") + errAbsent = errors.New("absent") +) + +func TestIsMultiError(t *testing.T) { + t.Parallel() + + testCases := []struct { + GivenError error + GivenTarget error + ExpectedIs bool + }{ + { + GivenError: nil, + GivenTarget: nil, + ExpectedIs: true, + }, + { + GivenError: nil, + GivenTarget: errFoo, + ExpectedIs: false, + }, + { + GivenError: Combine(errFoo), + GivenTarget: nil, + ExpectedIs: false, + }, + { + GivenError: Combine(errFoo), + GivenTarget: errFoo, + ExpectedIs: true, + }, + { + GivenError: Append(errFoo, errBar), + GivenTarget: errFoo, + ExpectedIs: true, + }, + { + GivenError: Append(errFoo, errBar), + GivenTarget: errBar, + ExpectedIs: true, + }, + { + GivenError: Append(errFoo, errBar), + GivenTarget: errAbsent, + ExpectedIs: false, + }, + } + + for i, c := range testCases { + c := c + + t.Run(fmt.Sprint(i), func(t *testing.T) { + t.Parallel() + + if err, ok := c.GivenError.(*multiError); ok { + t.Run("is", func(t *testing.T) { + assert.Equal(t, c.ExpectedIs, err.Is(c.GivenTarget)) + }) + } + + t.Run("errors", func(t *testing.T) { + assert.Equal(t, c.ExpectedIs, errors.Is(c.GivenError, c.GivenTarget)) + }) + + t.Run("xerrors", func(t *testing.T) { + assert.Equal(t, c.ExpectedIs, xerrors.Is(c.GivenError, c.GivenTarget)) + }) + }) + } +} + +func TestAsMultiError(t *testing.T) { + t.Parallel() + + testCases := []struct { + GivenError error + ExpectedAs bool + }{ + { + GivenError: nil, + ExpectedAs: false, + }, + { + GivenError: Combine(targetError{}), + ExpectedAs: true, + }, + { + GivenError: Combine(mockedError{}), + ExpectedAs: false, + }, + { + GivenError: Append(mockedError{}, targetError{}), + ExpectedAs: true, + }, + { + GivenError: Append(mockedError{}, groupNotMultiError{}), + ExpectedAs: false, + }, + } + + for i, c := range testCases { + c := c + + t.Run(fmt.Sprint(i), func(t *testing.T) { + t.Parallel() + + var target targetError + + if err, ok := c.GivenError.(*multiError); ok { + t.Run("as", func(t *testing.T) { + assert.Equal(t, c.ExpectedAs, err.As(&target)) + }) + } + + t.Run("errors", func(t *testing.T) { + assert.Equal(t, c.ExpectedAs, errors.As(c.GivenError, &target)) + }) + + t.Run("xerrors", func(t *testing.T) { + assert.Equal(t, c.ExpectedAs, xerrors.As(c.GivenError, &target)) + }) + }) + } +} + +func newMultiError(errors ...error) error { + return &multiError{errors: errors} +} + +func newMultiErrorWithCapacity() error { + return appendN(nil, errors.New("append"), 50) +} + +func appendN(initial, err error, repeat int) error { + errs := initial + + for i := 0; i < repeat; i++ { + errs = Append(errs, err) + } + + return errs +} + +type groupNotMultiError struct{} + +func (e groupNotMultiError) Error() string { + return "something wrong" +} + +func (e groupNotMultiError) Errors() []error { + return []error{errors.New("something wrong")} +} + +type mockedError struct{} + +func (e mockedError) Error() string { + return "mocked" +} + +type targetError struct{} + +func (e targetError) Error() string { + return "target" +} diff --git a/library/go/core/xerrors/multierr/gotest/ya.make b/library/go/core/xerrors/multierr/gotest/ya.make new file mode 100644 index 0000000000..f1881bfbe6 --- /dev/null +++ b/library/go/core/xerrors/multierr/gotest/ya.make @@ -0,0 +1,5 @@ +GO_TEST_FOR(library/go/core/xerrors/multierr) + +ENABLE(GO_DEBUG_PATH_RELATIVE) + +END() diff --git a/library/go/core/xerrors/multierr/ya.make b/library/go/core/xerrors/multierr/ya.make new file mode 100644 index 0000000000..b3741d43f6 --- /dev/null +++ b/library/go/core/xerrors/multierr/ya.make @@ -0,0 +1,15 @@ +GO_LIBRARY() + +SRCS( + error.go +) + +GO_TEST_SRCS( + error_test.go +) + +END() + +RECURSE( + gotest +) diff --git a/library/go/core/xerrors/new.go b/library/go/core/xerrors/new.go new file mode 100644 index 0000000000..0749f125a7 --- /dev/null +++ b/library/go/core/xerrors/new.go @@ -0,0 +1,48 @@ +package xerrors + +import ( + "fmt" + "io" + + "github.com/ydb-platform/ydb/library/go/x/xruntime" +) + +type newError struct { + msg string + stacktrace *xruntime.StackTrace +} + +var _ ErrorStackTrace = &newError{} + +func New(text string) error { + return &newError{ + msg: text, + stacktrace: newStackTrace(1, nil), + } +} + +func (e *newError) Error() string { + return e.msg +} + +func (e *newError) Format(s fmt.State, v rune) { + switch v { + case 'v': + if s.Flag('+') && e.stacktrace != nil { + _, _ = io.WriteString(s, e.msg) + _, _ = io.WriteString(s, "\n") + writeStackTrace(s, e.stacktrace) + return + } + + fallthrough + case 's': + _, _ = io.WriteString(s, e.msg) + case 'q': + _, _ = fmt.Fprintf(s, "%q", e.msg) + } +} + +func (e *newError) StackTrace() *xruntime.StackTrace { + return e.stacktrace +} diff --git a/library/go/core/xerrors/new_formatting_test.go b/library/go/core/xerrors/new_formatting_test.go new file mode 100644 index 0000000000..30daa409f8 --- /dev/null +++ b/library/go/core/xerrors/new_formatting_test.go @@ -0,0 +1,58 @@ +package xerrors + +import ( + "testing" + + "github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors" +) + +func TestNewFormatting(t *testing.T) { + constructor := func(t *testing.T) error { + return New("new") + } + expected := assertxerrors.Expectations{ + ExpectedS: "new", + ExpectedV: "new", + Frames: assertxerrors.NewStackTraceModeExpectation(` +new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestNewFormatting.func1 + library/go/core/xerrors/new_formatting_test.go:11 +`, + ), + Stacks: assertxerrors.NewStackTraceModeExpectation(` +new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestNewFormatting.func1 + library/go/core/xerrors/new_formatting_test.go:11 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:83 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + StackThenFrames: assertxerrors.NewStackTraceModeExpectation(` +new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestNewFormatting.func1 + library/go/core/xerrors/new_formatting_test.go:11 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:83 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + StackThenNothing: assertxerrors.NewStackTraceModeExpectation(` +new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestNewFormatting.func1 + library/go/core/xerrors/new_formatting_test.go:11 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:83 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + Nothing: assertxerrors.NewStackTraceModeExpectation("new"), + } + assertxerrors.RunTestsPerMode(t, expected, constructor) +} diff --git a/library/go/core/xerrors/sentinel.go b/library/go/core/xerrors/sentinel.go new file mode 100644 index 0000000000..6651588619 --- /dev/null +++ b/library/go/core/xerrors/sentinel.go @@ -0,0 +1,150 @@ +package xerrors + +import ( + "errors" + "fmt" + "io" + "strings" + + "github.com/ydb-platform/ydb/library/go/x/xreflect" + "github.com/ydb-platform/ydb/library/go/x/xruntime" +) + +// NewSentinel acts as New but does not add stack frame +func NewSentinel(text string) *Sentinel { + return &Sentinel{error: errors.New(text)} +} + +// Sentinel error +type Sentinel struct { + error +} + +// WithFrame adds stack frame to sentinel error (DEPRECATED) +func (s *Sentinel) WithFrame() error { + return &sentinelWithStackTrace{ + err: s, + stacktrace: newStackTrace(1, nil), + } +} + +func (s *Sentinel) WithStackTrace() error { + return &sentinelWithStackTrace{ + err: s, + stacktrace: newStackTrace(1, nil), + } +} + +// Wrap error with this sentinel error. Adds stack frame. +func (s *Sentinel) Wrap(err error) error { + if err == nil { + panic("tried to wrap a nil error") + } + + return &sentinelWrapper{ + err: s, + wrapped: err, + stacktrace: newStackTrace(1, err), + } +} + +type sentinelWithStackTrace struct { + err error + stacktrace *xruntime.StackTrace +} + +func (e *sentinelWithStackTrace) Error() string { + return e.err.Error() +} + +func (e *sentinelWithStackTrace) Format(s fmt.State, v rune) { + switch v { + case 'v': + if s.Flag('+') && e.stacktrace != nil { + msg := e.err.Error() + _, _ = io.WriteString(s, msg) + writeMsgAndStackTraceSeparator(s, msg) + writeStackTrace(s, e.stacktrace) + return + } + fallthrough + case 's': + _, _ = io.WriteString(s, e.err.Error()) + case 'q': + _, _ = fmt.Fprintf(s, "%q", e.err.Error()) + } +} + +func writeMsgAndStackTraceSeparator(w io.Writer, msg string) { + separator := "\n" + if !strings.HasSuffix(msg, ":") { + separator = ":\n" + } + + _, _ = io.WriteString(w, separator) +} + +// Is checks if e holds the specified error. Checks only immediate error. +func (e *sentinelWithStackTrace) Is(target error) bool { + return e.err == target +} + +// As checks if ew holds the specified error type. Checks only immediate error. +// It does NOT perform target checks as it relies on errors.As to do it +func (e *sentinelWithStackTrace) As(target interface{}) bool { + return xreflect.Assign(e.err, target) +} + +type sentinelWrapper struct { + err error + wrapped error + stacktrace *xruntime.StackTrace +} + +func (e *sentinelWrapper) Error() string { + return fmt.Sprintf("%s", e) +} + +func (e *sentinelWrapper) Format(s fmt.State, v rune) { + switch v { + case 'v': + if s.Flag('+') { + if e.stacktrace != nil { + msg := e.err.Error() + _, _ = io.WriteString(s, msg) + writeMsgAndStackTraceSeparator(s, msg) + writeStackTrace(s, e.stacktrace) + _, _ = fmt.Fprintf(s, "%+v", e.wrapped) + } else { + _, _ = io.WriteString(s, e.err.Error()) + _, _ = io.WriteString(s, ": ") + _, _ = fmt.Fprintf(s, "%+v", e.wrapped) + } + + return + } + fallthrough + case 's': + _, _ = io.WriteString(s, e.err.Error()) + _, _ = io.WriteString(s, ": ") + _, _ = io.WriteString(s, e.wrapped.Error()) + case 'q': + _, _ = fmt.Fprintf(s, "%q", fmt.Sprintf("%s: %s", e.err.Error(), e.wrapped.Error())) + } +} + +// Unwrap implements Wrapper interface +func (e *sentinelWrapper) Unwrap() error { + return e.wrapped +} + +// Is checks if ew holds the specified error. Checks only immediate error. +func (e *sentinelWrapper) Is(target error) bool { + return e.err == target +} + +// As checks if error holds the specified error type. Checks only immediate error. +// It does NOT perform target checks as it relies on errors.As to do it +func (e *sentinelWrapper) As(target interface{}) bool { + return xreflect.Assign(e.err, target) +} diff --git a/library/go/core/xerrors/sentinel_test.go b/library/go/core/xerrors/sentinel_test.go new file mode 100644 index 0000000000..dd566ea366 --- /dev/null +++ b/library/go/core/xerrors/sentinel_test.go @@ -0,0 +1,92 @@ +package xerrors + +import ( + "fmt" + "io" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSentinelWrapNil(t *testing.T) { + sentinel := NewSentinel("sentinel") + assert.Panics(t, func() { assert.NoError(t, sentinel.Wrap(nil)) }) +} + +func TestSentinelWrap(t *testing.T) { + sentinel := NewSentinel("sentinel") + assert.EqualError(t, sentinel.Wrap(New("err")), "sentinel: err") +} + +func TestSentinelMultiWrap(t *testing.T) { + top := NewSentinel("top") + middle := NewSentinel("middle") + err := top.Wrap(middle.Wrap(New("bottom"))) + assert.EqualError(t, err, "top: middle: bottom") +} + +func TestSentinelIs(t *testing.T) { + sentinel := NewSentinel("sentinel") + assert.True(t, Is(sentinel, sentinel)) + assert.True(t, Is(sentinel.Wrap(New("err")), sentinel)) + assert.True(t, Is(NewSentinel("err").Wrap(sentinel), sentinel)) + assert.True(t, Is(Errorf("wrapper: %w", sentinel), sentinel)) + assert.True(t, Is(sentinel.WithStackTrace(), sentinel)) + assert.True(t, Is(Errorf("wrapper: %w", sentinel.WithStackTrace()), sentinel)) +} + +func TestSentinelMultiWrapIs(t *testing.T) { + top := NewSentinel("top") + middle := NewSentinel("middle") + err := top.Wrap(middle.Wrap(io.EOF)) + assert.True(t, Is(err, top)) + assert.True(t, Is(err, middle)) + assert.True(t, Is(err, io.EOF)) + assert.False(t, Is(err, New("random"))) +} + +func TestSentinelAs(t *testing.T) { + sentinel := NewSentinel("sentinel") + var target *Sentinel + + assert.True(t, As(sentinel, &target)) + assert.NotNil(t, target) + target = nil + + assert.True(t, As(sentinel.Wrap(New("err")), &target)) + assert.NotNil(t, target) + target = nil + + assert.True(t, As(NewSentinel("err").Wrap(sentinel), &target)) + assert.NotNil(t, target) + target = nil + + assert.True(t, As(Errorf("wrapper: %w", sentinel), &target)) + assert.NotNil(t, target) + target = nil + + assert.True(t, As(sentinel.WithStackTrace(), &target)) + assert.NotNil(t, target) + target = nil + + assert.True(t, As(Errorf("wrapper: %w", sentinel.WithStackTrace()), &target)) + assert.NotNil(t, target) + target = nil +} + +func TestSentinelMultiWrapAs(t *testing.T) { + top := NewSentinel("top") + middle := NewSentinel("middle") + err := top.Wrap(middle.Wrap(io.EOF)) + + var target *Sentinel + assert.True(t, As(err, &target)) + assert.NotNil(t, target) +} + +func TestSentinelFormatting(t *testing.T) { + sentinel := NewSentinel("sentinel") + assert.Equal(t, "sentinel", fmt.Sprintf("%s", sentinel)) + assert.Equal(t, "sentinel", fmt.Sprintf("%v", sentinel)) + assert.Equal(t, "sentinel", fmt.Sprintf("%+v", sentinel)) +} diff --git a/library/go/core/xerrors/sentinel_with_stack_formatting_with_colon_test.go b/library/go/core/xerrors/sentinel_with_stack_formatting_with_colon_test.go new file mode 100644 index 0000000000..4ad87ff91b --- /dev/null +++ b/library/go/core/xerrors/sentinel_with_stack_formatting_with_colon_test.go @@ -0,0 +1,59 @@ +package xerrors + +import ( + "testing" + + "github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors" +) + +func TestSentinelWithStackTraceFormattingWithColon(t *testing.T) { + constructor := func(t *testing.T) error { + err := NewSentinel("sentinel:") + return err.WithStackTrace() + } + expected := assertxerrors.Expectations{ + ExpectedS: "sentinel:", + ExpectedV: "sentinel:", + Frames: assertxerrors.NewStackTraceModeExpectation(` +sentinel: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWithStackTraceFormattingWithColon.func1 + library/go/core/xerrors/sentinel_with_stack_formatting_with_colon_test.go:12 +`, + ), + Stacks: assertxerrors.NewStackTraceModeExpectation(` +sentinel: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWithStackTraceFormattingWithColon.func1 + library/go/core/xerrors/sentinel_with_stack_formatting_with_colon_test.go:12 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + StackThenFrames: assertxerrors.NewStackTraceModeExpectation(` +sentinel: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWithStackTraceFormattingWithColon.func1 + library/go/core/xerrors/sentinel_with_stack_formatting_with_colon_test.go:12 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + StackThenNothing: assertxerrors.NewStackTraceModeExpectation(` +sentinel: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWithStackTraceFormattingWithColon.func1 + library/go/core/xerrors/sentinel_with_stack_formatting_with_colon_test.go:12 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + Nothing: assertxerrors.NewStackTraceModeExpectation("sentinel:"), + } + assertxerrors.RunTestsPerMode(t, expected, constructor) +} diff --git a/library/go/core/xerrors/sentinel_with_stack_formatting_without_colon_test.go b/library/go/core/xerrors/sentinel_with_stack_formatting_without_colon_test.go new file mode 100644 index 0000000000..f5d057a2ef --- /dev/null +++ b/library/go/core/xerrors/sentinel_with_stack_formatting_without_colon_test.go @@ -0,0 +1,59 @@ +package xerrors + +import ( + "testing" + + "github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors" +) + +func TestSentinelWithStackTraceFormattingWithoutColon(t *testing.T) { + constructor := func(t *testing.T) error { + err := NewSentinel("sentinel") + return err.WithStackTrace() + } + expected := assertxerrors.Expectations{ + ExpectedS: "sentinel", + ExpectedV: "sentinel", + Frames: assertxerrors.NewStackTraceModeExpectation(` +sentinel: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWithStackTraceFormattingWithoutColon.func1 + library/go/core/xerrors/sentinel_with_stack_formatting_without_colon_test.go:12 +`, + ), + Stacks: assertxerrors.NewStackTraceModeExpectation(` +sentinel: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWithStackTraceFormattingWithoutColon.func1 + library/go/core/xerrors/sentinel_with_stack_formatting_without_colon_test.go:12 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + StackThenFrames: assertxerrors.NewStackTraceModeExpectation(` +sentinel: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWithStackTraceFormattingWithoutColon.func1 + library/go/core/xerrors/sentinel_with_stack_formatting_without_colon_test.go:12 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + StackThenNothing: assertxerrors.NewStackTraceModeExpectation(` +sentinel: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWithStackTraceFormattingWithoutColon.func1 + library/go/core/xerrors/sentinel_with_stack_formatting_without_colon_test.go:12 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + Nothing: assertxerrors.NewStackTraceModeExpectation("sentinel"), + } + assertxerrors.RunTestsPerMode(t, expected, constructor) +} diff --git a/library/go/core/xerrors/sentinel_wrap_formatting_with_colon_test.go b/library/go/core/xerrors/sentinel_wrap_formatting_with_colon_test.go new file mode 100644 index 0000000000..d5d96ecab0 --- /dev/null +++ b/library/go/core/xerrors/sentinel_wrap_formatting_with_colon_test.go @@ -0,0 +1,60 @@ +package xerrors + +import ( + "testing" + + "github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors" +) + +func TestSentinelWrapFormattingWithColon(t *testing.T) { + constructor := func(t *testing.T) error { + sentinel := NewSentinel("sntnl_wrapper:") + err := NewSentinel("sentinel") + return sentinel.Wrap(err) + } + expected := assertxerrors.Expectations{ + ExpectedS: "sntnl_wrapper:: sentinel", + ExpectedV: "sntnl_wrapper:: sentinel", + Frames: assertxerrors.NewStackTraceModeExpectation(` +sntnl_wrapper: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapFormattingWithColon.func1 + library/go/core/xerrors/sentinel_wrap_formatting_with_colon_test.go:13 +sentinel`, + ), + Stacks: assertxerrors.NewStackTraceModeExpectation(` +sntnl_wrapper: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapFormattingWithColon.func1 + library/go/core/xerrors/sentinel_wrap_formatting_with_colon_test.go:13 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +sentinel`, + 3, 4, 5, 6, + ), + StackThenFrames: assertxerrors.NewStackTraceModeExpectation(` +sntnl_wrapper: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapFormattingWithColon.func1 + library/go/core/xerrors/sentinel_wrap_formatting_with_colon_test.go:13 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +sentinel`, + 3, 4, 5, 6, + ), + StackThenNothing: assertxerrors.NewStackTraceModeExpectation(` +sntnl_wrapper: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapFormattingWithColon.func1 + library/go/core/xerrors/sentinel_wrap_formatting_with_colon_test.go:13 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +sentinel`, + 3, 4, 5, 6, + ), + Nothing: assertxerrors.NewStackTraceModeExpectation("sntnl_wrapper:: sentinel"), + } + assertxerrors.RunTestsPerMode(t, expected, constructor) +} diff --git a/library/go/core/xerrors/sentinel_wrap_formatting_without_colon_test.go b/library/go/core/xerrors/sentinel_wrap_formatting_without_colon_test.go new file mode 100644 index 0000000000..f5936b1835 --- /dev/null +++ b/library/go/core/xerrors/sentinel_wrap_formatting_without_colon_test.go @@ -0,0 +1,60 @@ +package xerrors + +import ( + "testing" + + "github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors" +) + +func TestSentinelWrapFormattingWithoutColon(t *testing.T) { + constructor := func(t *testing.T) error { + sentinel := NewSentinel("sntnl_wrapper") + err := NewSentinel("sentinel") + return sentinel.Wrap(err) + } + expected := assertxerrors.Expectations{ + ExpectedS: "sntnl_wrapper: sentinel", + ExpectedV: "sntnl_wrapper: sentinel", + Frames: assertxerrors.NewStackTraceModeExpectation(` +sntnl_wrapper: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapFormattingWithoutColon.func1 + library/go/core/xerrors/sentinel_wrap_formatting_without_colon_test.go:13 +sentinel`, + ), + Stacks: assertxerrors.NewStackTraceModeExpectation(` +sntnl_wrapper: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapFormattingWithoutColon.func1 + library/go/core/xerrors/sentinel_wrap_formatting_without_colon_test.go:13 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +sentinel`, + 3, 4, 5, 6, + ), + StackThenFrames: assertxerrors.NewStackTraceModeExpectation(` +sntnl_wrapper: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapFormattingWithoutColon.func1 + library/go/core/xerrors/sentinel_wrap_formatting_without_colon_test.go:13 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +sentinel`, + 3, 4, 5, 6, + ), + StackThenNothing: assertxerrors.NewStackTraceModeExpectation(` +sntnl_wrapper: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapFormattingWithoutColon.func1 + library/go/core/xerrors/sentinel_wrap_formatting_without_colon_test.go:13 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +sentinel`, + 3, 4, 5, 6, + ), + Nothing: assertxerrors.NewStackTraceModeExpectation("sntnl_wrapper: sentinel"), + } + assertxerrors.RunTestsPerMode(t, expected, constructor) +} diff --git a/library/go/core/xerrors/sentinel_wrap_new_formatting_test.go b/library/go/core/xerrors/sentinel_wrap_new_formatting_test.go new file mode 100644 index 0000000000..5514f0c27f --- /dev/null +++ b/library/go/core/xerrors/sentinel_wrap_new_formatting_test.go @@ -0,0 +1,73 @@ +package xerrors + +import ( + "testing" + + "github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors" +) + +func TestSentinelWrapNewFormatting(t *testing.T) { + constructor := func(t *testing.T) error { + err := New("new") + sentinel := NewSentinel("sentinel") + return sentinel.Wrap(err) + } + expected := assertxerrors.Expectations{ + ExpectedS: "sentinel: new", + ExpectedV: "sentinel: new", + Frames: assertxerrors.NewStackTraceModeExpectation(` +sentinel: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapNewFormatting.func1 + library/go/core/xerrors/sentinel_wrap_new_formatting_test.go:13 +new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapNewFormatting.func1 + library/go/core/xerrors/sentinel_wrap_new_formatting_test.go:11 +`, + ), + Stacks: assertxerrors.NewStackTraceModeExpectation(` +sentinel: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapNewFormatting.func1 + library/go/core/xerrors/sentinel_wrap_new_formatting_test.go:13 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapNewFormatting.func1 + library/go/core/xerrors/sentinel_wrap_new_formatting_test.go:11 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, 10, 11, 12, 13, + ), + StackThenFrames: assertxerrors.NewStackTraceModeExpectation(` +sentinel: + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapNewFormatting.func1 + library/go/core/xerrors/sentinel_wrap_new_formatting_test.go:13 +new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapNewFormatting.func1 + library/go/core/xerrors/sentinel_wrap_new_formatting_test.go:11 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 6, 7, 8, 9, + ), + StackThenNothing: assertxerrors.NewStackTraceModeExpectation(` +sentinel: new + github.com/ydb-platform/ydb/library/go/core/xerrors.TestSentinelWrapNewFormatting.func1 + library/go/core/xerrors/sentinel_wrap_new_formatting_test.go:11 + github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors.RunTestsPerMode.func1 + /home/sidh/devel/go/src/github.com/ydb-platform/ydb/library/go/core/xerrors/assertxerrors/assertxerrors.go:18 + testing.tRunner + /home/sidh/.ya/tools/v4/774223543/src/testing/testing.go:1127 +`, + 3, 4, 5, 6, + ), + Nothing: assertxerrors.NewStackTraceModeExpectation("sentinel: new"), + } + assertxerrors.RunTestsPerMode(t, expected, constructor) +} diff --git a/library/go/core/xerrors/stacktrace.go b/library/go/core/xerrors/stacktrace.go new file mode 100644 index 0000000000..84a00b14f4 --- /dev/null +++ b/library/go/core/xerrors/stacktrace.go @@ -0,0 +1,80 @@ +package xerrors + +import ( + "errors" + "fmt" + "io" + + "github.com/ydb-platform/ydb/library/go/x/xruntime" +) + +func writeStackTrace(w io.Writer, stacktrace *xruntime.StackTrace) { + for _, frame := range stacktrace.Frames() { + if frame.Function != "" { + _, _ = fmt.Fprintf(w, " %s\n ", frame.Function) + } + + if frame.File != "" { + _, _ = fmt.Fprintf(w, " %s:%d\n", frame.File, frame.Line) + } + } +} + +type ErrorStackTrace interface { + StackTrace() *xruntime.StackTrace +} + +// StackTraceOfEffect returns last stacktrace that was added to error chain (furthest from the root error). +// Guarantees that returned value has valid StackTrace object (but not that there are any frames). +func StackTraceOfEffect(err error) ErrorStackTrace { + var st ErrorStackTrace + for { + if !As(err, &st) { + return nil + } + + if st.StackTrace() != nil { + return st + } + + err = st.(error) + err = errors.Unwrap(err) + } +} + +// StackTraceOfCause returns first stacktrace that was added to error chain (closest to the root error). +// Guarantees that returned value has valid StackTrace object (but not that there are any frames). +func StackTraceOfCause(err error) ErrorStackTrace { + var res ErrorStackTrace + var st ErrorStackTrace + for { + if !As(err, &st) { + return res + } + + if st.StackTrace() != nil { + res = st + } + + err = st.(error) + err = errors.Unwrap(err) + } +} + +// NextStackTracer returns next error with stack trace. +// Guarantees that returned value has valid StackTrace object (but not that there are any frames). +func NextStackTrace(st ErrorStackTrace) ErrorStackTrace { + var res ErrorStackTrace + for { + err := st.(error) + err = errors.Unwrap(err) + + if !As(err, &res) { + return nil + } + + if res.StackTrace() != nil { + return res + } + } +} diff --git a/library/go/core/xerrors/ya.make b/library/go/core/xerrors/ya.make new file mode 100644 index 0000000000..48ded42a2f --- /dev/null +++ b/library/go/core/xerrors/ya.make @@ -0,0 +1,36 @@ +GO_LIBRARY() + +SRCS( + doc.go + errorf.go + forward.go + mode.go + new.go + sentinel.go + stacktrace.go +) + +GO_TEST_SRCS( + benchmark_test.go + errorf_formatting_with_error_test.go + errorf_formatting_with_std_error_test.go + errorf_formatting_without_error_test.go + errorf_multiple_errors_test.go + new_formatting_test.go + sentinel_test.go + sentinel_with_stack_formatting_with_colon_test.go + sentinel_with_stack_formatting_without_colon_test.go + sentinel_wrap_formatting_with_colon_test.go + sentinel_wrap_formatting_without_colon_test.go + sentinel_wrap_new_formatting_test.go +) + +END() + +RECURSE( + assertxerrors + benchxerrors + gotest + internal + multierr +) |