diff --git a/.goxc.json b/.goxc.json index adc9960bb..16491bd6c 100644 --- a/.goxc.json +++ b/.goxc.json @@ -5,7 +5,7 @@ ], "Arch": "amd64", "Os": "linux darwin windows", - "PackageVersion": "0.1.3", + "PackageVersion": "0.3.1", "TaskSettings": { "publish-github": { "owner": "yandex", diff --git a/.travis.yml b/.travis.yml index 9679cc1b5..2fbdf73d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: go go: - - 1.11.x + - 1.15.x env: - GO111MODULE=on diff --git a/Makefile b/Makefile index 20726e4b8..553118b5b 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ fmt: tools: @echo "$(OK_COLOR)Install tools$(NO_COLOR)" - go install golang.org/x/tools/cmd/goimports + go install golang.org/x/tools/cmd/goimports@latest go get golang.org/x/tools/cmd/cover go get github.com/modocache/gover go get github.com/mattn/goveralls diff --git a/README.md b/README.md index b02541986..85616d1ed 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ [![Join the chat at https://gitter.im/yandex/pandora](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/yandex/pandora?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://travis-ci.org/yandex/pandora.svg)](https://travis-ci.org/yandex/pandora) [![Coverage Status](https://coveralls.io/repos/yandex/pandora/badge.svg?branch=develop&service=github)](https://coveralls.io/github/yandex/pandora?branch=develop) -[![Read the Docs](https://readthedocs.org/projects/yandexpandora/badge/)](https://readthedocs.org/projects/yandexpandora/) +[![Documentation Status](https://readthedocs.org/projects/yandexpandora/badge/?version=develop)](https://yandexpandora.readthedocs.io/en/develop/?badge=develop) -Pandora is a high-performance load generator in Go language. It has built-in HTTP(S) and HTTP/2 support and you can write your own load scenarios in Go, compiling them just before your test. +Pandora is a high-performance load generator in Go language. It has built-in HTTP(S) and HTTP/2 support and you can write your own load scenarios in Go, compiling them just before your test. ## How to start @@ -42,5 +42,9 @@ Run the binary with your config (see config examples at [examples](https://githu pandora myconfig.yaml ``` -Or use Pandora with [Yandex.Tank](http://yandextank.readthedocs.org/en/latest/configuration.html#pandora) and +Or use Pandora with [Yandex.Tank](https://yandextank.readthedocs.io/en/latest/core_and_modules.html#pandora) and [Overload](https://overload.yandex.net). + +### Documentation +[ReadTheDocs](https://yandexpandora.readthedocs.io/en/develop/) + diff --git a/acceptance_tests/acceptance_suite_test.go b/acceptance_tests/acceptance_suite_test.go index d3d22e4fb..6f4ac8764 100644 --- a/acceptance_tests/acceptance_suite_test.go +++ b/acceptance_tests/acceptance_suite_test.go @@ -14,10 +14,9 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" - "go.uber.org/zap" - "github.com/yandex/pandora/lib/ginkgoutil" "github.com/yandex/pandora/lib/tag" + "go.uber.org/zap" ) var pandoraBin string @@ -112,7 +111,7 @@ func NewInstansePoolConfig() *InstancePoolConfig { } type InstancePoolConfig struct { - Id string + ID string Provider map[string]interface{} `json:"ammo"` Aggregator map[string]interface{} `json:"result"` Gun map[string]interface{} `json:"gun"` @@ -185,5 +184,5 @@ func (pt *PandoraTester) ExitCode() int { func (pt *PandoraTester) Close() { pt.Terminate() - os.RemoveAll(pt.TestDir) + _ = os.RemoveAll(pt.TestDir) } diff --git a/acceptance_tests/http_test.go b/acceptance_tests/http_test.go index cfdccd44c..fea9dc757 100644 --- a/acceptance_tests/http_test.go +++ b/acceptance_tests/http_test.go @@ -2,14 +2,12 @@ package acceptance import ( "net/http" - - "golang.org/x/net/http2" - "net/http/httptest" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "go.uber.org/atomic" + "golang.org/x/net/http2" ) var _ = Describe("http", func() { @@ -133,7 +131,7 @@ var _ = Describe("http", func() { }) func startHTTP2(server *httptest.Server) { - http2.ConfigureServer(server.Config, nil) + _ = http2.ConfigureServer(server.Config, nil) server.TLS = server.Config.TLSConfig server.StartTLS() } diff --git a/cli/cli.go b/cli/cli.go index cfc0317ff..68ee43150 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -17,19 +17,17 @@ import ( "time" "github.com/spf13/viper" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "github.com/yandex/pandora/core/config" "github.com/yandex/pandora/core/engine" "github.com/yandex/pandora/lib/zaputil" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) -const Version = "0.3.0" +const Version = "0.5.1" const defaultConfigFile = "load" const stdinConfigSelector = "-" -var useStdinConfig = false var configSearchDirs = []string{"./", "./config", "/etc/pandora"} type cliConfig struct { @@ -90,15 +88,28 @@ func Run() { } var ( example bool + expvar bool + version bool ) flag.BoolVar(&example, "example", false, "print example config to STDOUT and exit") + flag.BoolVar(&version, "version", false, "print pandora core version") + flag.BoolVar(&expvar, "expvar", false, "enable expvar service (DEPRECATED, use monitoring config section instead)") flag.Parse() + if expvar { + fmt.Fprintf(os.Stderr, "-expvar flag is DEPRECATED. Use monitoring config section instead\n") + } + if example { panic("Not implemented yet") // TODO: print example config file content } + if version { + fmt.Fprintf(os.Stderr, "Pandora core/%s\n", Version) + return + } + conf := readConfig() log := newLogger(conf.Log) zap.ReplaceGlobals(log) @@ -117,30 +128,41 @@ func Run() { errs := make(chan error) go runEngine(ctx, pandora, errs) + // waiting for signal or error message from engine + awaitPandoraTermination(pandora, cancel, errs, log) + log.Info("Engine run successfully finished") +} + +// helper function that awaits pandora run +func awaitPandoraTermination(pandora *engine.Engine, gracefulShutdown func(), errs chan error, log *zap.Logger) { sigs := make(chan os.Signal, 2) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - // waiting for signal or error message from engine select { case sig := <-sigs: + var interruptTimeout = 3 * time.Second switch sig { case syscall.SIGINT: - const interruptTimeout = 5 * time.Second - log.Info("SIGINT received. Trying to stop gracefully.", zap.Duration("timeout", interruptTimeout)) - cancel() - select { - case <-time.After(interruptTimeout): - log.Fatal("Interrupt timeout exceeded") - case sig := <-sigs: - log.Fatal("Another signal received. Quiting.", zap.Stringer("signal", sig)) - case err := <-errs: - log.Fatal("Engine interrupted", zap.Error(err)) - } + // await gun timeout but no longer than 30 sec. + interruptTimeout = 30 * time.Second + log.Info("SIGINT received. Graceful shutdown.", zap.Duration("timeout", interruptTimeout)) + gracefulShutdown() case syscall.SIGTERM: - log.Fatal("SIGTERM received. Quiting.") + log.Info("SIGTERM received. Trying to stop gracefully.", zap.Duration("timeout", interruptTimeout)) + gracefulShutdown() default: log.Fatal("Unexpected signal received. Quiting.", zap.Stringer("signal", sig)) } + + select { + case <-time.After(interruptTimeout): + log.Fatal("Interrupt timeout exceeded") + case sig := <-sigs: + log.Fatal("Another signal received. Quiting.", zap.Stringer("signal", sig)) + case err := <-errs: + log.Fatal("Engine interrupted", zap.Error(err)) + } + case err := <-errs: switch err { case nil: @@ -148,7 +170,7 @@ func Run() { case err: const awaitTimeout = 3 * time.Second log.Error("Engine run failed. Awaiting started tasks.", zap.Error(err), zap.Duration("timeout", awaitTimeout)) - cancel() + gracefulShutdown() time.AfterFunc(awaitTimeout, func() { log.Fatal("Engine tasks timeout exceeded.") }) @@ -156,7 +178,6 @@ func Run() { log.Fatal("Engine run failed. Pandora graceful shutdown successfully finished") } } - log.Info("Engine run successfully finished") } func runEngine(ctx context.Context, engine *engine.Engine, errs chan error) { @@ -176,6 +197,7 @@ func readConfig() *cliConfig { v := newViper() + var useStdinConfig = false args := flag.Args() if len(args) > 0 { switch { @@ -262,10 +284,16 @@ func startMonitoring(conf monitoringConfig) (stop func()) { zap.L().Fatal("CPU profile file create fail", zap.Error(err)) } zap.L().Info("Starting CPU profiling") - pprof.StartCPUProfile(f) + err = pprof.StartCPUProfile(f) + if err != nil { + zap.L().Info("CPU profiling is already enabled") + } stops = append(stops, func() { pprof.StopCPUProfile() - f.Close() + err := f.Close() + if err != nil { + zap.L().Info("Error closing CPUProfile file") + } }) } if conf.MemProfile.Enabled { @@ -276,8 +304,14 @@ func startMonitoring(conf monitoringConfig) (stop func()) { stops = append(stops, func() { zap.L().Info("Writing memory profile") runtime.GC() - pprof.WriteHeapProfile(f) - f.Close() + err := pprof.WriteHeapProfile(f) + if err != nil { + zap.L().Info("Error writing HeapProfile file") + } + err = f.Close() + if err != nil { + zap.L().Info("Error closing HeapProfile file") + } }) } stop = func() { diff --git a/cli/expvar.go b/cli/expvar.go index ed8f436d9..054ffe9f1 100644 --- a/cli/expvar.go +++ b/cli/expvar.go @@ -3,10 +3,9 @@ package cli import ( "time" - "go.uber.org/zap" - "github.com/yandex/pandora/core/engine" "github.com/yandex/pandora/lib/monitoring" + "go.uber.org/zap" ) func newEngineMetrics() engine.Metrics { diff --git a/components/example/example.go b/components/example/example.go index f6dab2222..c5515f787 100644 --- a/components/example/example.go +++ b/components/example/example.go @@ -10,10 +10,9 @@ import ( "fmt" "sync" - "go.uber.org/zap" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/aggregator/netsample" + "go.uber.org/zap" ) type Ammo struct { diff --git a/components/example/import/import.go b/components/example/import/import.go index c5c99c73d..d7c622c0d 100644 --- a/components/example/import/import.go +++ b/components/example/import/import.go @@ -6,11 +6,11 @@ package example import ( - . "github.com/yandex/pandora/components/example" + "github.com/yandex/pandora/components/example" "github.com/yandex/pandora/core/register" ) func Import() { - register.Provider("example", NewProvider, DefaultProviderConfig) - register.Gun("example", NewGun) + register.Provider("example", example.NewProvider, example.DefaultProviderConfig) + register.Gun("example", example.NewGun) } diff --git a/components/grpc/ammo/ammo.go b/components/grpc/ammo/ammo.go new file mode 100644 index 000000000..3defba11a --- /dev/null +++ b/components/grpc/ammo/ammo.go @@ -0,0 +1,39 @@ +// Copyright (c) 2017 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package ammo + +type Ammo struct { + Tag string `json:"tag"` + Call string `json:"call"` + Metadata map[string]string `json:"metadata"` + Payload map[string]interface{} `json:"payload"` + id int + isInvalid bool +} + +func (a *Ammo) Reset(tag string, call string, metadata map[string]string, payload map[string]interface{}) { + *a = Ammo{tag, call, metadata, payload, -1, false} +} + +func (a *Ammo) SetID(id int) { + a.id = id +} + +func (a *Ammo) ID() int { + return a.id +} + +func (a *Ammo) Invalidate() { + a.isInvalid = true +} + +func (a *Ammo) IsInvalid() bool { + return a.isInvalid +} + +func (a *Ammo) IsValid() bool { + return !a.isInvalid +} diff --git a/components/grpc/ammo/grpcjson/provider.go b/components/grpc/ammo/grpcjson/provider.go new file mode 100644 index 000000000..067012fec --- /dev/null +++ b/components/grpc/ammo/grpcjson/provider.go @@ -0,0 +1,105 @@ +// Copyright (c) 2017 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package grpcjson + +import ( + "bufio" + "context" + + jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" + "github.com/spf13/afero" + "github.com/yandex/pandora/components/grpc/ammo" + "github.com/yandex/pandora/lib/confutil" + "go.uber.org/zap" +) + +func NewProvider(fs afero.Fs, conf Config) *Provider { + var p Provider + if conf.Source.Path != "" { + conf.File = conf.Source.Path + } + p = Provider{ + Provider: ammo.NewProvider(fs, conf.File, p.start), + Config: conf, + } + return &p +} + +type Provider struct { + ammo.Provider + Config + log *zap.Logger +} + +type Source struct { + Type string + Path string +} + +type Config struct { + File string //`validate:"required"` + // Limit limits total num of ammo. Unlimited if zero. + Limit int `validate:"min=0"` + // Passes limits ammo file passes. Unlimited if zero. + Passes int `validate:"min=0"` + ContinueOnError bool + //Maximum number of byte in an ammo. Default is bufio.MaxScanTokenSize + MaxAmmoSize int + Source Source `config:"source"` + ChosenCases []string +} + +func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { + var ammoNum, passNum int + for { + passNum++ + scanner := bufio.NewScanner(ammoFile) + if p.Config.MaxAmmoSize != 0 { + var buffer []byte + scanner.Buffer(buffer, p.Config.MaxAmmoSize) + } + for line := 1; scanner.Scan() && (p.Limit == 0 || ammoNum < p.Limit); line++ { + data := scanner.Bytes() + a, err := decodeAmmo(data, p.Pool.Get().(*ammo.Ammo)) + if err != nil { + if p.Config.ContinueOnError { + a.Invalidate() + } else { + return errors.Wrapf(err, "failed to decode ammo at line: %v; data: %q", line, data) + } + } + if !confutil.IsChosenCase(a.Tag, p.Config.ChosenCases) { + continue + } + ammoNum++ + select { + case p.Sink <- a: + case <-ctx.Done(): + return nil + } + } + if p.Passes != 0 && passNum >= p.Passes { + break + } + _, err := ammoFile.Seek(0, 0) + if err != nil { + return errors.Wrap(err, "Failed to seek ammo file") + } + } + return nil +} + +func decodeAmmo(jsonDoc []byte, am *ammo.Ammo) (*ammo.Ammo, error) { + var ammo ammo.Ammo + err := jsoniter.Unmarshal(jsonDoc, &ammo) + if err != nil { + return am, errors.WithStack(err) + } + + am.Reset(ammo.Tag, ammo.Call, ammo.Metadata, ammo.Payload) + return am, nil +} diff --git a/components/grpc/ammo/provider.go b/components/grpc/ammo/provider.go new file mode 100644 index 000000000..120a1258c --- /dev/null +++ b/components/grpc/ammo/provider.go @@ -0,0 +1,62 @@ +// Copyright (c) 2017 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package ammo + +import ( + "context" + "sync" + + "github.com/pkg/errors" + "github.com/spf13/afero" + "github.com/yandex/pandora/core" + "go.uber.org/atomic" +) + +func NewProvider(fs afero.Fs, fileName string, start func(ctx context.Context, file afero.File) error) Provider { + return Provider{ + fs: fs, + fileName: fileName, + start: start, + Sink: make(chan *Ammo, 128), + Pool: sync.Pool{New: func() interface{} { return &Ammo{} }}, + Close: func() {}, + } +} + +type Provider struct { + fs afero.Fs + fileName string + start func(ctx context.Context, file afero.File) error + Sink chan *Ammo + Pool sync.Pool + idCounter atomic.Int64 + Close func() + core.ProviderDeps +} + +func (p *Provider) Acquire() (core.Ammo, bool) { + ammo, ok := <-p.Sink + if ok { + ammo.SetID(int(p.idCounter.Inc() - 1)) + } + return ammo, ok +} + +func (p *Provider) Release(a core.Ammo) { + p.Pool.Put(a) +} + +func (p *Provider) Run(ctx context.Context, deps core.ProviderDeps) error { + defer p.Close() + p.ProviderDeps = deps + defer close(p.Sink) + file, err := p.fs.Open(p.fileName) + if err != nil { + return errors.Wrap(err, "failed to open ammo file") + } + defer file.Close() + return p.start(ctx, file) +} diff --git a/components/grpc/core.go b/components/grpc/core.go new file mode 100644 index 000000000..55afab67e --- /dev/null +++ b/components/grpc/core.go @@ -0,0 +1,233 @@ +package grpc + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "log" + "time" + + "github.com/jhump/protoreflect/desc" + "github.com/jhump/protoreflect/dynamic" + "github.com/jhump/protoreflect/dynamic/grpcdynamic" + "github.com/jhump/protoreflect/grpcreflect" + "github.com/yandex/pandora/components/grpc/ammo" + "github.com/yandex/pandora/core" + "github.com/yandex/pandora/core/aggregator/netsample" + "github.com/yandex/pandora/core/warmup" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + reflectpb "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" + "google.golang.org/grpc/status" +) + +const defaultTimeout = time.Second * 15 + +type Sample struct { + URL string + ShootTimeSeconds float64 +} + +type grpcDialOptions struct { + Authority string `config:"authority"` + Timeout time.Duration `config:"timeout"` +} + +type GunConfig struct { + Target string `validate:"required"` + Timeout time.Duration `config:"timeout"` // grpc request timeout + TLS bool `config:"tls"` + DialOptions grpcDialOptions `config:"dial_options"` +} + +type Gun struct { + DebugLog bool + client *grpc.ClientConn + conf GunConfig + aggr core.Aggregator + core.GunDeps + + stub grpcdynamic.Stub + services map[string]desc.MethodDescriptor +} + +func (g *Gun) WarmUp(opts *warmup.Options) (interface{}, error) { + conn, err := makeGRPCConnect(g.conf.Target, g.conf.TLS, g.conf.DialOptions) + if err != nil { + return nil, fmt.Errorf("failed to connect to target: %w", err) + } + defer conn.Close() + + meta := make(metadata.MD) + refCtx := metadata.NewOutgoingContext(context.Background(), meta) + refClient := grpcreflect.NewClient(refCtx, reflectpb.NewServerReflectionClient(conn)) + listServices, err := refClient.ListServices() + if err != nil { + opts.Log.Fatal("Fatal: failed to get services list\n %s\n", zap.Error(err)) + } + services := make(map[string]desc.MethodDescriptor) + for _, s := range listServices { + service, err := refClient.ResolveService(s) + if err != nil { + if grpcreflect.IsElementNotFoundError(err) { + continue + } + opts.Log.Fatal("FATAL ResolveService: %s", zap.Error(err)) + } + listMethods := service.GetMethods() + for _, m := range listMethods { + services[m.GetFullyQualifiedName()] = *m + } + } + return services, nil +} + +func (g *Gun) AcceptWarmUpResult(i interface{}) error { + services, ok := i.(map[string]desc.MethodDescriptor) + if !ok { + return fmt.Errorf("grpc WarmUp result should be services: map[string]desc.MethodDescriptor") + } + g.services = services + return nil +} + +func NewGun(conf GunConfig) *Gun { + return &Gun{conf: conf} +} + +func (g *Gun) Bind(aggr core.Aggregator, deps core.GunDeps) error { + conn, err := makeGRPCConnect(g.conf.Target, g.conf.TLS, g.conf.DialOptions) + if err != nil { + log.Fatalf("FATAL: grpc.Dial failed\n %s\n", err) + } + g.client = conn + g.aggr = aggr + g.GunDeps = deps + g.stub = grpcdynamic.NewStub(conn) + + if ent := deps.Log.Check(zap.DebugLevel, "Gun bind"); ent != nil { + // Enable debug level logging during shooting. Creating log entries isn't free. + g.DebugLog = true + } + + return nil +} + +func (g *Gun) Shoot(am core.Ammo) { + customAmmo := am.(*ammo.Ammo) + g.shoot(customAmmo) +} + +func (g *Gun) shoot(ammo *ammo.Ammo) { + + code := 0 + sample := netsample.Acquire(ammo.Tag) + defer func() { + sample.SetProtoCode(code) + g.aggr.Report(sample) + }() + + method, ok := g.services[ammo.Call] + if !ok { + log.Fatalf("Fatal: No such method %s\n", ammo.Call) + return + } + + payloadJSON, err := json.Marshal(ammo.Payload) + if err != nil { + log.Fatalf("FATAL: Payload parsing error %s\n", err) + return + } + md := method.GetInputType() + message := dynamic.NewMessage(md) + err = message.UnmarshalJSON(payloadJSON) + if err != nil { + code = 400 + log.Printf("BAD REQUEST: %s\n", err) + return + } + + timeout := defaultTimeout + if g.conf.Timeout != 0 { + timeout = time.Second * g.conf.Timeout + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, metadata.New(ammo.Metadata)) + out, err := g.stub.InvokeRpc(ctx, &method, message) + code = convertGrpcStatus(err) + + if err != nil { + log.Printf("Response error: %s\n", err) + } + + if g.DebugLog { + g.Log.Debug("Request:", zap.Stringer("method", &method), zap.Stringer("message", message)) + g.Log.Debug("Response:", zap.Stringer("resp", out)) + } + +} + +func makeGRPCConnect(target string, isTLS bool, dialOptions grpcDialOptions) (conn *grpc.ClientConn, err error) { + opts := []grpc.DialOption{} + if isTLS { + opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}))) + } else { + opts = append(opts, grpc.WithInsecure()) + } + timeout := time.Second + if dialOptions.Timeout != 0 { + timeout = dialOptions.Timeout + } + opts = append(opts, grpc.WithTimeout(timeout)) + opts = append(opts, grpc.WithUserAgent("load test, pandora universal grpc shooter")) + + if dialOptions.Authority != "" { + opts = append(opts, grpc.WithAuthority(dialOptions.Authority)) + } + + conn, err = grpc.Dial(target, opts...) + return +} + +func convertGrpcStatus(err error) int { + s := status.Convert(err) + + switch s.Code() { + case codes.OK: + return 200 + case codes.Canceled: + return 499 + case codes.InvalidArgument: + return 400 + case codes.DeadlineExceeded: + return 504 + case codes.NotFound: + return 404 + case codes.AlreadyExists: + return 409 + case codes.PermissionDenied: + return 403 + case codes.ResourceExhausted: + return 429 + case codes.FailedPrecondition: + return 400 + case codes.Aborted: + return 409 + case codes.OutOfRange: + return 400 + case codes.Unimplemented: + return 501 + case codes.Unavailable: + return 503 + case codes.Unauthenticated: + return 401 + default: + return 500 + } +} diff --git a/components/grpc/core_tests/core_test.go b/components/grpc/core_tests/core_test.go new file mode 100644 index 000000000..e355c4adb --- /dev/null +++ b/components/grpc/core_tests/core_test.go @@ -0,0 +1,12 @@ +package core + +import ( + "testing" + + "github.com/yandex/pandora/components/grpc" + "github.com/yandex/pandora/core/warmup" +) + +func TestGrpcGunImplementsWarmedUp(t *testing.T) { + _ = warmup.WarmedUp(&grpc.Gun{}) +} diff --git a/components/grpc/import/import.go b/components/grpc/import/import.go new file mode 100644 index 000000000..77c362ac3 --- /dev/null +++ b/components/grpc/import/import.go @@ -0,0 +1,27 @@ +// Copyright (c) 2017 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package example + +import ( + "github.com/spf13/afero" + "github.com/yandex/pandora/components/grpc" + "github.com/yandex/pandora/components/grpc/ammo/grpcjson" + "github.com/yandex/pandora/core" + "github.com/yandex/pandora/core/register" +) + +func Import(fs afero.Fs) { + + register.Provider("grpc/json", func(conf grpcjson.Config) core.Provider { + return grpcjson.NewProvider(fs, conf) + }) + + register.Gun("grpc", grpc.NewGun, func() grpc.GunConfig { + return grpc.GunConfig{ + Target: "default target", + } + }) +} diff --git a/components/phttp/ammo/simple/ammo.go b/components/phttp/ammo/simple/ammo.go index 4c01e8575..5fdc5e756 100644 --- a/components/phttp/ammo/simple/ammo.go +++ b/components/phttp/ammo/simple/ammo.go @@ -15,27 +15,44 @@ import ( type Ammo struct { // OPTIMIZE(skipor): reuse *http.Request. // Need to research is it possible. http.Transport can hold reference to http.Request. - req *http.Request - tag string - id int + req *http.Request + tag string + id int + isInvalid bool } func (a *Ammo) Request() (*http.Request, *netsample.Sample) { sample := netsample.Acquire(a.tag) - sample.SetId(a.id) + sample.SetID(a.id) return a.req, sample } func (a *Ammo) Reset(req *http.Request, tag string) { - *a = Ammo{req, tag, -1} + *a = Ammo{req, tag, -1, false} } -func (a *Ammo) SetId(id int) { +func (a *Ammo) SetID(id int) { a.id = id } -func (a *Ammo) Id() int { +func (a *Ammo) ID() int { return a.id } +func (a *Ammo) Invalidate() { + a.isInvalid = true +} + +func (a *Ammo) IsInvalid() bool { + return a.isInvalid +} + +func (a *Ammo) IsValid() bool { + return !a.isInvalid +} + +func (a *Ammo) Tag() string { + return a.tag +} + var _ phttp.Ammo = (*Ammo)(nil) diff --git a/components/phttp/ammo/simple/jsonline/data.go b/components/phttp/ammo/simple/jsonline/data.go index c95b0b802..b0b256a98 100644 --- a/components/phttp/ammo/simple/jsonline/data.go +++ b/components/phttp/ammo/simple/jsonline/data.go @@ -8,7 +8,7 @@ type data struct { // Request endpoint is defied by gun config. Host string `json:"host"` Method string `json:"method"` - Uri string `json:"uri"` + URI string `json:"uri"` // Headers defines headers to send. // NOTE: Host header will be silently ignored. Headers map[string]string `json:"headers"` diff --git a/components/phttp/ammo/simple/jsonline/data_ffjson.go b/components/phttp/ammo/simple/jsonline/data_ffjson.go index 56f4327f6..c2102b56b 100644 --- a/components/phttp/ammo/simple/jsonline/data_ffjson.go +++ b/components/phttp/ammo/simple/jsonline/data_ffjson.go @@ -302,7 +302,7 @@ handle_Uri: outBuf := fs.Output.Bytes() - j.Uri = string(string(outBuf)) + j.URI = string(string(outBuf)) } } diff --git a/components/phttp/ammo/simple/jsonline/jsonline_suite_test.go b/components/phttp/ammo/simple/jsonline/jsonline_suite_test.go index 3f22584a4..bcdc60ec4 100644 --- a/components/phttp/ammo/simple/jsonline/jsonline_suite_test.go +++ b/components/phttp/ammo/simple/jsonline/jsonline_suite_test.go @@ -17,7 +17,6 @@ import ( . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" "github.com/spf13/afero" - "github.com/yandex/pandora/components/phttp/ammo/simple" "github.com/yandex/pandora/core" "github.com/yandex/pandora/lib/ginkgoutil" @@ -34,13 +33,13 @@ var testData = []data{ { Host: "example.com", Method: "GET", - Uri: "/00", + URI: "/00", Headers: map[string]string{"Accept": "*/*", "Accept-Encoding": "gzip, deflate", "User-Agent": "Pandora/0.0.1"}, }, { Host: "ya.ru", Method: "HEAD", - Uri: "/01", + URI: "/01", Headers: map[string]string{"Accept": "*/*", "Accept-Encoding": "gzip, brotli", "User-Agent": "YaBro/0.1"}, Tag: "head", }, @@ -73,7 +72,7 @@ var _ = Describe("data", func() { data := data{ Host: "ya.ru", Method: "GET", - Uri: "/00", + URI: "/00", Headers: map[string]string{"A": "a", "B": "b"}, Tag: "tag", } @@ -86,7 +85,7 @@ var _ = Describe("data", func() { "URL": PointTo(MatchFields(IgnoreExtras, Fields{ "Scheme": Equal("http"), "Host": Equal(data.Host), - "Path": Equal(data.Uri), + "Path": Equal(data.URI), })), "Header": Equal(http.Header{ "A": []string{"a"}, @@ -227,13 +226,13 @@ func Benchmark(b *testing.B) { } b.Run("Decode", func(b *testing.B) { for n := 0; n < b.N; n++ { - decodeAmmo(jsonDoc, &simple.Ammo{}) + _, _ = decodeAmmo(jsonDoc, &simple.Ammo{}) } }) b.Run("DecodeWithPool", func(b *testing.B) { for n := 0; n < b.N; n++ { h := pool.Get().(*simple.Ammo) - decodeAmmo(jsonDoc, h) + _, _ = decodeAmmo(jsonDoc, h) pool.Put(h) } }) diff --git a/components/phttp/ammo/simple/jsonline/provider.go b/components/phttp/ammo/simple/jsonline/provider.go index 98810f4ec..25c5f540f 100644 --- a/components/phttp/ammo/simple/jsonline/provider.go +++ b/components/phttp/ammo/simple/jsonline/provider.go @@ -14,6 +14,8 @@ import ( "github.com/pkg/errors" "github.com/spf13/afero" "github.com/yandex/pandora/components/phttp/ammo/simple" + "github.com/yandex/pandora/lib/confutil" + "go.uber.org/zap" ) func NewProvider(fs afero.Fs, conf Config) *Provider { @@ -28,6 +30,7 @@ func NewProvider(fs afero.Fs, conf Config) *Provider { type Provider struct { simple.Provider Config + log *zap.Logger } type Config struct { @@ -35,19 +38,35 @@ type Config struct { // Limit limits total num of ammo. Unlimited if zero. Limit int `validate:"min=0"` // Passes limits ammo file passes. Unlimited if zero. - Passes int `validate:"min=0"` + Passes int `validate:"min=0"` + ContinueOnError bool + //Maximum number of byte in an ammo. Default is bufio.MaxScanTokenSize + MaxAmmoSize int + ChosenCases []string } func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { var ammoNum, passNum int + for { passNum++ scanner := bufio.NewScanner(ammoFile) + if p.Config.MaxAmmoSize != 0 { + var buffer []byte + scanner.Buffer(buffer, p.Config.MaxAmmoSize) + } for line := 1; scanner.Scan() && (p.Limit == 0 || ammoNum < p.Limit); line++ { data := scanner.Bytes() a, err := decodeAmmo(data, p.Pool.Get().(*simple.Ammo)) if err != nil { - return errors.Wrapf(err, "failed to decode ammo at line: %v; data: %q", line, data) + if p.Config.ContinueOnError { + a.Invalidate() + } else { + return errors.Wrapf(err, "failed to decode ammo at line: %v; data: %q", line, data) + } + } + if !confutil.IsChosenCase(a.Tag(), p.Config.ChosenCases) { + continue } ammoNum++ select { @@ -59,7 +78,10 @@ func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { if p.Passes != 0 && passNum >= p.Passes { break } - ammoFile.Seek(0, 0) + _, err := ammoFile.Seek(0, 0) + if err != nil { + p.log.Info("Failed to seek ammo file", zap.Error(err)) + } } return nil } @@ -68,18 +90,18 @@ func decodeAmmo(jsonDoc []byte, am *simple.Ammo) (*simple.Ammo, error) { var data data err := data.UnmarshalJSON(jsonDoc) if err != nil { - return nil, errors.WithStack(err) + return am, errors.WithStack(err) } req, err := data.ToRequest() if err != nil { - return nil, err + return am, err } am.Reset(req, data.Tag) return am, nil } func (d *data) ToRequest() (req *http.Request, err error) { - uri := "http://" + d.Host + d.Uri + uri := "http://" + d.Host + d.URI req, err = http.NewRequest(d.Method, uri, strings.NewReader(d.Body)) if err != nil { return nil, errors.WithStack(err) diff --git a/components/phttp/ammo/simple/provider.go b/components/phttp/ammo/simple/provider.go index cd4167e5f..70ebda16f 100644 --- a/components/phttp/ammo/simple/provider.go +++ b/components/phttp/ammo/simple/provider.go @@ -11,9 +11,8 @@ import ( "github.com/pkg/errors" "github.com/spf13/afero" - "go.uber.org/atomic" - "github.com/yandex/pandora/core" + "go.uber.org/atomic" ) func NewProvider(fs afero.Fs, fileName string, start func(ctx context.Context, file afero.File) error) Provider { @@ -23,6 +22,7 @@ func NewProvider(fs afero.Fs, fileName string, start func(ctx context.Context, f start: start, Sink: make(chan *Ammo, 128), Pool: sync.Pool{New: func() interface{} { return &Ammo{} }}, + Close: func() {}, } } @@ -33,13 +33,14 @@ type Provider struct { Sink chan *Ammo Pool sync.Pool idCounter atomic.Int64 + Close func() core.ProviderDeps } func (p *Provider) Acquire() (core.Ammo, bool) { ammo, ok := <-p.Sink if ok { - ammo.SetId(int(p.idCounter.Inc() - 1)) + ammo.SetID(int(p.idCounter.Inc() - 1)) } return ammo, ok } @@ -49,6 +50,7 @@ func (p *Provider) Release(a core.Ammo) { } func (p *Provider) Run(ctx context.Context, deps core.ProviderDeps) error { + defer p.Close() p.ProviderDeps = deps defer close(p.Sink) file, err := p.fs.Open(p.fileName) diff --git a/components/phttp/ammo/simple/raw/decoder.go b/components/phttp/ammo/simple/raw/decoder.go index b6b7e63e5..e34481d68 100644 --- a/components/phttp/ammo/simple/raw/decoder.go +++ b/components/phttp/ammo/simple/raw/decoder.go @@ -6,8 +6,6 @@ import ( "net/http" "strconv" "strings" - - "github.com/pkg/errors" ) type Header struct { @@ -29,30 +27,6 @@ func decodeRequest(reqString []byte) (req *http.Request, err error) { if err != nil { return } - if req.Host != "" { - req.URL.Host = req.Host - } req.RequestURI = "" return } - -func decodeHTTPConfigHeaders(headers []string) (configHTTPHeaders []Header, err error) { - for _, header := range headers { - line := []byte(header) - if len(line) < 3 || line[0] != '[' || line[len(line)-1] != ']' { - return nil, errors.New("header line should be like '[key: value]") - } - line = line[1 : len(line)-1] - colonIdx := bytes.IndexByte(line, ':') - if colonIdx < 0 { - return nil, errors.New("missing colon") - } - configHTTPHeaders = append( - configHTTPHeaders, - Header{ - string(bytes.TrimSpace(line[:colonIdx])), - string(bytes.TrimSpace(line[colonIdx+1:])), - }) - } - return -} diff --git a/components/phttp/ammo/simple/raw/decoder_bench_test.go b/components/phttp/ammo/simple/raw/decoder_bench_test.go index f622bfa1c..adf049f4d 100644 --- a/components/phttp/ammo/simple/raw/decoder_bench_test.go +++ b/components/phttp/ammo/simple/raw/decoder_bench_test.go @@ -1,6 +1,10 @@ package raw -import "testing" +import ( + "testing" + + "github.com/yandex/pandora/components/phttp/ammo/simple" +) var ( benchTestConfigHeaders = []string{ @@ -21,22 +25,16 @@ const ( func BenchmarkRawDecoder(b *testing.B) { for i := 0; i < b.N; i++ { - decodeRequest([]byte(benchTestRequest)) + _, _ = decodeRequest([]byte(benchTestRequest)) } } func BenchmarkRawDecoderWithHeaders(b *testing.B) { b.StopTimer() - decodedHTTPConfigHeaders, _ := decodeHTTPConfigHeaders(benchTestConfigHeaders) + decodedHTTPConfigHeaders, _ := simple.DecodeHTTPConfigHeaders(benchTestConfigHeaders) b.StartTimer() for i := 0; i < b.N; i++ { req, _ := decodeRequest([]byte(benchTestRequest)) - for _, header := range decodedHTTPConfigHeaders { - if header.key == "Host" { - req.URL.Host = header.value - } else { - req.Header.Set(header.key, header.value) - } - } + simple.UpdateRequestWithHeaders(req, decodedHTTPConfigHeaders) } } diff --git a/components/phttp/ammo/simple/raw/decoder_test.go b/components/phttp/ammo/simple/raw/decoder_test.go index 9f8288306..696522f20 100644 --- a/components/phttp/ammo/simple/raw/decoder_test.go +++ b/components/phttp/ammo/simple/raw/decoder_test.go @@ -62,7 +62,7 @@ var _ = Describe("Decoder", func() { if req.Body != nil { _, err := io.Copy(&bout, req.Body) Expect(err).To(BeNil()) - req.Body.Close() + _ = req.Body.Close() } Expect(bout.String()).To(Equal("foobar")) }) @@ -83,34 +83,5 @@ var _ = Describe("Decoder", func() { req, err := decodeRequest([]byte(raw)) Expect(err).To(BeNil()) Expect(req.Host).To(Equal("hostname.tld")) - Expect(req.URL.Host).To(Equal("hostname.tld")) - }) - It("should replace header Host from config", func() { - const host = "hostname.tld" - const newhost = "newhostname.tld" - - raw := "GET / HTTP/1.1\r\n" + - "Host: " + host + "\r\n" + - "Content-Length: 0\r\n" + - "\r\n" - configHeaders := []string{ - "[Host: " + newhost + "]", - "[SomeTestKey: sometestvalue]", - } - req, err := decodeRequest([]byte(raw)) - Expect(err).To(BeNil()) - Expect(req.Host).To(Equal(host)) - Expect(req.URL.Host).To(Equal(host)) - decodedConfigHeaders, _ := decodeHTTPConfigHeaders(configHeaders) - for _, header := range decodedConfigHeaders { - // special behavior for `Host` header - if header.key == "Host" { - req.URL.Host = header.value - } else { - req.Header.Set(header.key, header.value) - } - } - Expect(req.URL.Host).To(Equal(newhost)) - Expect(req.Header.Get("SomeTestKey")).To(Equal("sometestvalue")) }) }) diff --git a/components/phttp/ammo/simple/raw/provider.go b/components/phttp/ammo/simple/raw/provider.go index e1b66c6b0..25100bb4e 100644 --- a/components/phttp/ammo/simple/raw/provider.go +++ b/components/phttp/ammo/simple/raw/provider.go @@ -7,8 +7,9 @@ import ( "github.com/pkg/errors" "github.com/spf13/afero" - "github.com/yandex/pandora/components/phttp/ammo/simple" + "github.com/yandex/pandora/lib/confutil" + "go.uber.org/zap" ) /* @@ -53,7 +54,8 @@ type Config struct { // Redefine HTTP headers Headers []string // Passes limits ammo file passes. Unlimited if zero. - Passes int `validate:"min=0"` + Passes int `validate:"min=0"` + ChosenCases []string } func NewProvider(fs afero.Fs, conf Config) *Provider { @@ -68,13 +70,14 @@ func NewProvider(fs afero.Fs, conf Config) *Provider { type Provider struct { simple.Provider Config + log *zap.Logger } func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { var passNum int var ammoNum int // parse and prepare Headers from config - decodedConfigHeaders, err := decodeHTTPConfigHeaders(p.Config.Headers) + decodedConfigHeaders, err := simple.DecodeHTTPConfigHeaders(p.Config.Headers) if err != nil { return err } @@ -95,10 +98,13 @@ func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { if len(data) == 0 { continue // skip empty lines } - reqSize, tag, err := decodeHeader(data) + reqSize, tag, _ := decodeHeader(data) if reqSize == 0 { break // start over from the beginning of file if ammo size is 0 } + if !confutil.IsChosenCase(tag, p.Config.ChosenCases) { + continue + } buff := make([]byte, reqSize) if n, err := io.ReadFull(reader, buff); err != nil { return errors.Wrapf(err, "failed to read ammo at position: %v; tried to read: %v; have read: %v", filePosition(ammoFile), reqSize, n) @@ -108,15 +114,8 @@ func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { return errors.Wrapf(err, "failed to decode ammo at position: %v; data: %q", filePosition(ammoFile), buff) } - // redefine request Headers from config - for _, header := range decodedConfigHeaders { - // special behavior for `Host` header - if header.key == "Host" { - req.URL.Host = header.value - } else { - req.Header.Set(header.key, header.value) - } - } + // add new Headers to request from config + simple.UpdateRequestWithHeaders(req, decodedConfigHeaders) sh := p.Pool.Get().(*simple.Ammo) sh.Reset(req, tag) @@ -134,7 +133,10 @@ func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { if p.Passes != 0 && passNum >= p.Passes { break } - ammoFile.Seek(0, 0) + _, err := ammoFile.Seek(0, 0) + if err != nil { + p.log.Info("Failed to seek ammo file", zap.Error(err)) + } } return nil } diff --git a/components/phttp/ammo/simple/raw/provider_test.go b/components/phttp/ammo/simple/raw/provider_test.go index 547068f39..d73931d38 100644 --- a/components/phttp/ammo/simple/raw/provider_test.go +++ b/components/phttp/ammo/simple/raw/provider_test.go @@ -13,7 +13,6 @@ import ( . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" "github.com/spf13/afero" - "github.com/yandex/pandora/components/phttp/ammo/simple" "github.com/yandex/pandora/core" ) diff --git a/components/phttp/ammo/simple/request.go b/components/phttp/ammo/simple/request.go new file mode 100644 index 000000000..08e2f1f90 --- /dev/null +++ b/components/phttp/ammo/simple/request.go @@ -0,0 +1,56 @@ +// Copyright (c) 2017 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package simple + +import ( + "bytes" + "net/http" + + "github.com/pkg/errors" +) + +type Header struct { + key string + value string +} + +func DecodeHTTPConfigHeaders(headers []string) (configHTTPHeaders []Header, err error) { + for _, header := range headers { + line := []byte(header) + if len(line) < 3 || line[0] != '[' || line[len(line)-1] != ']' { + return nil, errors.New("header line should be like '[key: value]") + } + line = line[1 : len(line)-1] + colonIdx := bytes.IndexByte(line, ':') + if colonIdx < 0 { + return nil, errors.New("missing colon") + } + configHTTPHeaders = append( + configHTTPHeaders, + Header{ + string(bytes.TrimSpace(line[:colonIdx])), + string(bytes.TrimSpace(line[colonIdx+1:])), + }) + } + return +} + +func UpdateRequestWithHeaders(req *http.Request, headers []Header) { + origHeaders := req.Header.Clone() + for _, header := range headers { + if origHeaders.Get(header.key) != "" { + continue + } + // special behavior for `Host` header + if header.key == "Host" { + if req.Host == "" { + req.Host = header.value + } + } else { + req.Header.Add(header.key, header.value) + } + } +} diff --git a/components/phttp/ammo/simple/uri/decoder.go b/components/phttp/ammo/simple/uri/decoder.go index 5980fb466..30aa6c51e 100644 --- a/components/phttp/ammo/simple/uri/decoder.go +++ b/components/phttp/ammo/simple/uri/decoder.go @@ -13,8 +13,8 @@ import ( "sync" "github.com/pkg/errors" - "github.com/yandex/pandora/components/phttp/ammo/simple" + "github.com/yandex/pandora/lib/confutil" ) type decoder struct { @@ -24,20 +24,17 @@ type decoder struct { ammoNum int header http.Header - configHeaders []ConfigHeader -} - -type ConfigHeader struct { - key string - value string + configHeaders []simple.Header + chosenCases []string } -func newDecoder(ctx context.Context, sink chan<- *simple.Ammo, pool *sync.Pool) *decoder { +func newDecoder(ctx context.Context, sink chan<- *simple.Ammo, pool *sync.Pool, chosenCases []string) *decoder { return &decoder{ - sink: sink, - header: http.Header{}, - pool: pool, - ctx: ctx, + sink: sink, + header: http.Header{}, + pool: pool, + ctx: ctx, + chosenCases: chosenCases, } } @@ -48,13 +45,11 @@ func (d *decoder) Decode(line []byte) error { return errors.New("empty line") } line = bytes.TrimSpace(line) - switch line[0] { - case '/': - return d.decodeURI(line) - case '[': + if line[0] == '[' { return d.decodeHeader(line) + } else { + return d.decodeURI(line) } - return errors.New("every line should begin with '[' or '/'") } func (d *decoder) decodeURI(line []byte) error { @@ -64,6 +59,9 @@ func (d *decoder) decodeURI(line []byte) error { if len(parts) > 1 { tag = parts[1] } + if !confutil.IsChosenCase(tag, d.chosenCases) { + return nil + } req, err := http.NewRequest("GET", string(url), nil) if err != nil { return errors.Wrap(err, "uri decode") @@ -72,20 +70,13 @@ func (d *decoder) decodeURI(line []byte) error { // http.Request.Write sends Host header based on req.URL.Host if k == "Host" { req.Host = v[0] - req.URL.Host = v[0] } else { req.Header[k] = v } } - // redefine request Headers from config - for _, configHeader := range d.configHeaders { - if configHeader.key == "Host" { - req.Host = configHeader.value - req.URL.Host = configHeader.value - } else { - req.Header.Set(configHeader.key, configHeader.value) - } - } + + // add new Headers to request from config + simple.UpdateRequestWithHeaders(req, d.configHeaders) sh := d.pool.Get().(*simple.Ammo) sh.Reset(req, tag) @@ -121,23 +112,3 @@ func (d *decoder) ResetHeader() { delete(d.header, k) } } - -func decodeHTTPConfigHeaders(headers []string) (configHTTPHeaders []ConfigHeader, err error) { - for _, header := range headers { - line := []byte(header) - if len(line) < 3 || line[0] != '[' || line[len(line)-1] != ']' { - return nil, errors.New("header line should be like '[key: value]") - } - line = line[1 : len(line)-1] - colonIdx := bytes.IndexByte(line, ':') - if colonIdx < 0 { - return nil, errors.New("missing colon") - } - preparedHeader := ConfigHeader{ - string(bytes.TrimSpace(line[:colonIdx])), - string(bytes.TrimSpace(line[colonIdx+1:])), - } - configHTTPHeaders = append(configHTTPHeaders, preparedHeader) - } - return -} diff --git a/components/phttp/ammo/simple/uri/decoder_test.go b/components/phttp/ammo/simple/uri/decoder_test.go index 37f6cf647..c3b731218 100644 --- a/components/phttp/ammo/simple/uri/decoder_test.go +++ b/components/phttp/ammo/simple/uri/decoder_test.go @@ -9,7 +9,6 @@ import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" - "github.com/yandex/pandora/components/phttp/ammo/simple" "github.com/yandex/pandora/core" ) @@ -22,7 +21,7 @@ func newAmmoPool() *sync.Pool { var _ = Describe("Decoder", func() { It("uri decode ctx cancel", func() { ctx, cancel := context.WithCancel(context.Background()) - decoder := newDecoder(ctx, make(chan *simple.Ammo), newAmmoPool()) + decoder := newDecoder(ctx, make(chan *simple.Ammo), newAmmoPool(), nil) cancel() err := decoder.Decode([]byte("/some/path")) Expect(err).To(Equal(context.Canceled)) @@ -33,7 +32,7 @@ var _ = Describe("Decoder", func() { ) BeforeEach(func() { ammoCh = make(chan *simple.Ammo, 10) - decoder = newDecoder(context.Background(), ammoCh, newAmmoPool()) + decoder = newDecoder(context.Background(), ammoCh, newAmmoPool(), nil) }) DescribeTable("invalid input", func(line string) { @@ -43,7 +42,6 @@ var _ = Describe("Decoder", func() { Expect(decoder.header).To(BeEmpty()) }, Entry("empty line", ""), - Entry("line start", "test"), Entry("empty header", "[ ]"), Entry("no closing brace", "[key: val "), Entry("no header key", "[ : val ]"), @@ -70,7 +68,6 @@ var _ = Describe("Decoder", func() { req, sample := sh.Request() Expect(*req.URL).To(MatchFields(IgnoreExtras, Fields{ "Path": Equal(line), - "Host": Equal(host), "Scheme": BeEmpty(), })) Expect(req.Host).To(Equal(host)) @@ -96,7 +93,6 @@ var _ = Describe("Decoder", func() { req, sample := sh.Request() Expect(*req.URL).To(MatchFields(IgnoreExtras, Fields{ "Path": Equal("/some/path"), - "Host": Equal(host), "Scheme": BeEmpty(), })) Expect(req.Host).To(Equal(host)) @@ -143,18 +139,6 @@ var _ = Describe("Decoder", func() { "C": []string{""}, })) }) - It("overwrite by config", func() { - decodedConfigHeaders, _ := decodeHTTPConfigHeaders([]string{ - "[Host: youhost.tld]", - "[SomeHeader: somevalue]", - }) - decoder.configHeaders = decodedConfigHeaders - cfgHeaders := []ConfigHeader{ - {"Host", "youhost.tld"}, - {"SomeHeader", "somevalue"}, - } - Expect(decoder.configHeaders).To(Equal(cfgHeaders)) - }) }) It("Reset", func() { decoder.header.Set("a", "b") diff --git a/components/phttp/ammo/simple/uri/provider.go b/components/phttp/ammo/simple/uri/provider.go index 643cfe300..a763d5c9f 100644 --- a/components/phttp/ammo/simple/uri/provider.go +++ b/components/phttp/ammo/simple/uri/provider.go @@ -8,44 +8,75 @@ package uri import ( "bufio" "context" + "fmt" "github.com/pkg/errors" "github.com/spf13/afero" - "github.com/yandex/pandora/components/phttp/ammo/simple" + "go.uber.org/zap" ) type Config struct { - File string `validate:"required"` + File string // Limit limits total num of ammo. Unlimited if zero. Limit int `validate:"min=0"` - // Redefine HTTP headers + // Additional HTTP headers Headers []string // Passes limits ammo file passes. Unlimited if zero. - Passes int `validate:"min=0"` + Passes int `validate:"min=0"` + Uris []string + ChosenCases []string } // TODO: pass logger and metricsRegistry func NewProvider(fs afero.Fs, conf Config) *Provider { + if len(conf.Uris) > 0 { + if conf.File != "" { + panic(`One should specify either 'file' or 'uris', but not both of them.`) + } + file, err := afero.TempFile(fs, "", "generated_ammo_") + if err != nil { + panic(fmt.Sprintf(`failed to create tmp ammo file: %v`, err)) + } + for _, uri := range conf.Uris { + _, err := file.WriteString(fmt.Sprintf("%s\n", uri)) + if err != nil { + panic(fmt.Sprintf(`failed to write ammo in tmp file: %v`, err)) + } + } + conf.File = file.Name() + } + if conf.File == "" { + panic(`One should specify either 'file' or 'uris'.`) + } var p Provider p = Provider{ Provider: simple.NewProvider(fs, conf.File, p.start), Config: conf, } + p.Close = func() { + if len(conf.Uris) > 0 { + err := fs.Remove(conf.File) + if err != nil { + zap.L().Error("failed to delete temp file", zap.String("file name", conf.File)) + } + } + } return &p } type Provider struct { simple.Provider Config + log *zap.Logger decoder *decoder // Initialized on start. } func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { - p.decoder = newDecoder(ctx, p.Sink, &p.Pool) + p.decoder = newDecoder(ctx, p.Sink, &p.Pool, p.Config.ChosenCases) // parse and prepare Headers from config - decodedConfigHeaders, err := decodeHTTPConfigHeaders(p.Config.Headers) + decodedConfigHeaders, err := simple.DecodeHTTPConfigHeaders(p.Config.Headers) if err != nil { return err } @@ -70,7 +101,10 @@ func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { if p.Passes != 0 && passNum >= p.Passes { break } - ammoFile.Seek(0, 0) + _, err := ammoFile.Seek(0, 0) + if err != nil { + p.log.Info("Failed to seek ammo file", zap.Error(err)) + } p.decoder.ResetHeader() } return nil diff --git a/components/phttp/ammo/simple/uri/provider_test.go b/components/phttp/ammo/simple/uri/provider_test.go index a0d05411b..ec2245ec3 100644 --- a/components/phttp/ammo/simple/uri/provider_test.go +++ b/components/phttp/ammo/simple/uri/provider_test.go @@ -9,9 +9,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" - "github.com/spf13/afero" - "github.com/pkg/errors" + "github.com/spf13/afero" "github.com/yandex/pandora/components/phttp/ammo/simple" "github.com/yandex/pandora/core" ) @@ -147,7 +146,6 @@ var _ = Describe("provider decode", func() { "Host": Equal(expectedData.host), "URL": PointTo(MatchFields(IgnoreExtras, Fields{ "Scheme": BeEmpty(), - "Host": Equal(expectedData.host), "Path": Equal(expectedData.path), })), "Header": Equal(expectedData.header), diff --git a/components/phttp/ammo/simple/uripost/decoder.go b/components/phttp/ammo/simple/uripost/decoder.go new file mode 100644 index 000000000..504eb0a21 --- /dev/null +++ b/components/phttp/ammo/simple/uripost/decoder.go @@ -0,0 +1,46 @@ +package uripost + +import ( + "bytes" + "strconv" + "strings" + + "github.com/pkg/errors" +) + +func decodeHeader(line []byte) (key string, val string, err error) { + if len(line) < 3 || line[0] != '[' || line[len(line)-1] != ']' { + return key, val, errors.New("header line should be like '[key: value]'") + } + line = line[1 : len(line)-1] + colonIdx := bytes.IndexByte(line, ':') + if colonIdx < 0 { + return key, val, errors.New("missing colon") + } + key = string(bytes.TrimSpace(line[:colonIdx])) + val = string(bytes.TrimSpace(line[colonIdx+1:])) + if key == "" { + return key, val, errors.New("missing header key") + } + return +} + +func decodeURI(uriString []byte) (bodySize int, uri string, tag string, err error) { + parts := strings.Split(string(uriString), " ") + bodySize, err = strconv.Atoi(parts[0]) + if err != nil { + err = errors.New("Wrong ammo body size, should be in bytes") + return + } + switch { + case len(parts) == 2: + uri = parts[1] + case len(parts) >= 3: + uri = parts[1] + tag = parts[2] + default: + err = errors.New("Wrong ammo format, should be like 'bodySize uri [tag]'") + } + + return +} diff --git a/components/phttp/ammo/simple/uripost/decoder_test.go b/components/phttp/ammo/simple/uripost/decoder_test.go new file mode 100644 index 000000000..10cdbb585 --- /dev/null +++ b/components/phttp/ammo/simple/uripost/decoder_test.go @@ -0,0 +1,94 @@ +package uripost + +import ( + "reflect" + "strconv" + "testing" +) + +func TestDecoderHeader(t *testing.T) { + var tests = []struct { + line []byte + key, val string + }{ + {line: []byte("[Host: some.host]"), key: "Host", val: "some.host"}, + {line: []byte("[User-agent: Tank]"), key: "User-agent", val: "Tank"}, + } + + for _, test := range tests { + if rkey, rval, _ := decodeHeader(test.line); rkey != test.key && rval != test.val { + t.Errorf("(%v) = %v %v, expected %v %v", string(test.line), rkey, rval, test.key, test.val) + } + } + +} + +func TestDecoderBadHeader(t *testing.T) { + var tests = []struct { + line []byte + err_msg string + }{ + {line: []byte("[Host some.host]"), err_msg: "missing colon"}, + {line: []byte("[User-agent: Tank"), err_msg: "header line should be like '[key: value]'"}, + {line: []byte("[: Tank]"), err_msg: "missing header key"}, + } + + for _, test := range tests { + if _, _, err := decodeHeader(test.line); err.Error() != test.err_msg { + t.Errorf("Got: %v, expected: %v", err.Error(), test.err_msg) + } + } + +} + +func TestDecodeURI(t *testing.T) { + var tests = []struct { + line []byte + size int + uri, tag string + }{ + {line: []byte("7 /test tag1"), size: 7, uri: "/test", tag: "tag1"}, + {line: []byte("10 /test"), size: 10, uri: "/test", tag: ""}, + } + + for _, test := range tests { + d_size, d_uri, d_tag, _ := decodeURI(test.line) + if d_size != test.size && d_uri != test.uri && d_tag != test.tag { + t.Errorf("Got: %v %v %v, expected: %v %v %v", strconv.Itoa(d_size), d_uri, d_tag, test.size, test.uri, test.tag) + } + } + +} + +func TestDecodeBadURI(t *testing.T) { + var tests = []struct { + line []byte + err_msg string + }{ + {line: []byte("3"), err_msg: "Wrong ammo format, should be like 'bodySize uri [tag]'"}, + {line: []byte("a"), err_msg: "Wrong ammo body size, should be in bytes"}, + } + + for _, test := range tests { + + _, _, _, err := decodeURI(test.line) + if err.Error() != test.err_msg { + t.Errorf("Got: %v, expected: %v", err.Error(), test.err_msg) + } + } + +} + +func TestDecodeHTTPConfigHeaders(t *testing.T) { + headers := []string{ + "[Host: some.host]", + "[User-Agent: Tank]", + } + + header := []Header{{key: "Host", value: "some.host"}, {key: "User-Agent", value: "Tank"}} + configHeaders, err := decodeHTTPConfigHeaders(headers) + if err == nil && !reflect.DeepEqual(configHeaders, header) { + t.Errorf("Got: %v, expected: %v", configHeaders, header) + } + +} diff --git a/components/phttp/ammo/simple/uripost/provider.go b/components/phttp/ammo/simple/uripost/provider.go new file mode 100644 index 000000000..28c5b8440 --- /dev/null +++ b/components/phttp/ammo/simple/uripost/provider.go @@ -0,0 +1,141 @@ +package uripost + +import ( + "bufio" + "bytes" + "context" + "io" + "net/http" + "strconv" + + "github.com/pkg/errors" + "github.com/spf13/afero" + "github.com/yandex/pandora/components/phttp/ammo/simple" + "github.com/yandex/pandora/lib/confutil" + "go.uber.org/zap" +) + +func filePosition(file afero.File) (position int64) { + position, _ = file.Seek(0, io.SeekCurrent) + return +} + +type Config struct { + File string `validate:"required"` + // Limit limits total num of ammo. Unlimited if zero. + Limit int `validate:"min=0"` + // Additional HTTP headers + Headers []string + // Passes limits ammo file passes. Unlimited if zero. + Passes int `validate:"min=0"` + ChosenCases []string +} + +func NewProvider(fs afero.Fs, conf Config) *Provider { + var p Provider + p = Provider{ + Provider: simple.NewProvider(fs, conf.File, p.start), + Config: conf, + } + return &p +} + +type Provider struct { + simple.Provider + Config + log *zap.Logger +} + +func (p *Provider) start(ctx context.Context, ammoFile afero.File) error { + var passNum int + var ammoNum int + // var key string + // var val string + var bodySize int + var uri string + var tag string + + header := make(http.Header) + // parse and prepare Headers from config + decodedConfigHeaders, err := simple.DecodeHTTPConfigHeaders(p.Config.Headers) + if err != nil { + return err + } + for { + passNum++ + reader := bufio.NewReader(ammoFile) + for p.Limit == 0 || ammoNum < p.Limit { + data, isPrefix, err := reader.ReadLine() + if isPrefix { + return errors.Errorf("too long header in ammo at position %v", filePosition(ammoFile)) + } + if err == io.EOF { + break // start over from the beginning + } + if err != nil { + return errors.Wrapf(err, "reading ammo failed at position: %v", filePosition(ammoFile)) + } + if len(data) == 0 { + continue // skip empty lines + } + data = bytes.TrimSpace(data) + if data[0] == '[' { + key, val, err := decodeHeader(data) + if err == nil { + header.Set(key, val) + } + continue + } + if _, err := strconv.Atoi(string(data[0])); err == nil { + bodySize, uri, tag, _ = decodeURI(data) + } + if bodySize == 0 { + break // start over from the beginning of file if ammo size is 0 + } + if !confutil.IsChosenCase(tag, p.Config.ChosenCases) { + continue + } + buff := make([]byte, bodySize) + if n, err := io.ReadFull(reader, buff); err != nil { + return errors.Wrapf(err, "failed to read ammo at position: %v; tried to read: %v; have read: %v", filePosition(ammoFile), bodySize, n) + } + req, err := http.NewRequest("POST", uri, bytes.NewReader(buff)) + if err != nil { + return errors.Wrapf(err, "failed to decode ammo at position: %v; data: %q", filePosition(ammoFile), buff) + } + + for k, v := range header { + // http.Request.Write sends Host header based on req.URL.Host + if k == "Host" { + req.Host = v[0] + } else { + req.Header[k] = v + } + } + + // add new Headers to request from config + simple.UpdateRequestWithHeaders(req, decodedConfigHeaders) + + sh := p.Pool.Get().(*simple.Ammo) + sh.Reset(req, tag) + + select { + case p.Sink <- sh: + ammoNum++ + case <-ctx.Done(): + return ctx.Err() + } + } + if ammoNum == 0 { + return errors.New("no ammo in file") + } + if p.Passes != 0 && passNum >= p.Passes { + break + } + _, err := ammoFile.Seek(0, 0) + if err != nil { + p.log.Info("Failed to seek ammo file", zap.Error(err)) + } + } + return nil +} diff --git a/components/phttp/base.go b/components/phttp/base.go index bf8eba67d..7d5756b5f 100644 --- a/components/phttp/base.go +++ b/components/phttp/base.go @@ -6,17 +6,20 @@ package phttp import ( + "bytes" "context" + "fmt" "io" "io/ioutil" "net/http" + "net/http/httptrace" + "net/http/httputil" "net/url" "github.com/pkg/errors" - "go.uber.org/zap" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/aggregator/netsample" + "go.uber.org/zap" ) const ( @@ -24,7 +27,9 @@ const ( ) type BaseGunConfig struct { - AutoTag AutoTagConfig `config:"auto-tag"` + AutoTag AutoTagConfig `config:"auto-tag"` + AnswLog AnswLogConfig `config:"answlog"` + HTTPTrace HTTPTraceConfig `config:"httptrace"` } // AutoTagConfig configure automatic tags generation based on ammo URI. First AutoTag URI path elements becomes tag. @@ -35,13 +40,34 @@ type AutoTagConfig struct { NoTagOnly bool `config:"no-tag-only"` // When true, autotagged only ammo that has no tag before. } +type AnswLogConfig struct { + Enabled bool `config:"enabled"` + Path string `config:"path"` + Filter string `config:"filter" valid:"oneof=all warning error"` +} + +type HTTPTraceConfig struct { + DumpEnabled bool `config:"dump"` + TraceEnabled bool `config:"trace"` +} + func DefaultBaseGunConfig() BaseGunConfig { return BaseGunConfig{ AutoTagConfig{ Enabled: false, URIElements: 2, NoTagOnly: true, - }} + }, + AnswLogConfig{ + Enabled: false, + Path: "answ.log", + Filter: "error", + }, + HTTPTraceConfig{ + DumpEnabled: false, + TraceEnabled: false, + }, + } } type BaseGun struct { @@ -51,6 +77,7 @@ type BaseGun struct { Connect func(ctx context.Context) error // Optional hook. OnClose func() error // Optional. Called on Close(). Aggregator netsample.Aggregator // Lazy set via BindResultTo. + AnswLog *zap.Logger core.GunDeps } @@ -78,6 +105,7 @@ func (b *BaseGun) Bind(aggregator netsample.Aggregator, deps core.GunDeps) error // Shoot is thread safe iff Do and Connect hooks are thread safe. func (b *BaseGun) Shoot(ammo Ammo) { + var bodyBytes []byte if b.Aggregator == nil { zap.L().Panic("must bind before shoot") } @@ -90,16 +118,25 @@ func (b *BaseGun) Shoot(ammo Ammo) { } req, sample := ammo.Request() + if ammo.IsInvalid() { + sample.AddTag(EmptyTag) + sample.SetProtoCode(0) + b.Aggregator.Report(sample) + b.Log.Warn("Invalid ammo", zap.Int("request", ammo.ID())) + return + } if b.DebugLog { b.Log.Debug("Prepared ammo to shoot", zap.Stringer("url", req.URL)) } - if b.Config.AutoTag.Enabled && (!b.Config.AutoTag.NoTagOnly || sample.Tags() == "") { sample.AddTag(autotag(b.Config.AutoTag.URIElements, req.URL)) } if sample.Tags() == "" { sample.AddTag(EmptyTag) } + if b.Config.AnswLog.Enabled { + bodyBytes = GetBody(req) + } var err error defer func() { @@ -110,8 +147,41 @@ func (b *BaseGun) Shoot(ammo Ammo) { err = errors.WithStack(err) }() + var timings *TraceTimings + if b.Config.HTTPTrace.TraceEnabled { + var clientTracer *httptrace.ClientTrace + clientTracer, timings = createHTTPTrace() + req = req.WithContext(httptrace.WithClientTrace(req.Context(), clientTracer)) + } + if b.Config.HTTPTrace.DumpEnabled { + requestDump, err := httputil.DumpRequest(req, true) + if err != nil { + b.Log.Error("DumpRequest error", zap.Error(err)) + } else { + sample.SetRequestBytes(len(requestDump)) + } + } var res *http.Response res, err = b.Do(req) + + if b.Config.HTTPTrace.TraceEnabled && timings != nil { + sample.SetReceiveTime(timings.GetReceiveTime()) + } + + if b.Config.HTTPTrace.DumpEnabled && res != nil { + responseDump, err := httputil.DumpResponse(res, true) + if err != nil { + b.Log.Error("DumpResponse error", zap.Error(err)) + } else { + sample.SetResponseBytes(len(responseDump)) + } + } + if b.Config.HTTPTrace.TraceEnabled && timings != nil { + sample.SetConnectTime(timings.GetConnectTime()) + sample.SetSendTime(timings.GetSendTime()) + sample.SetLatency(timings.GetLatency()) + } + if err != nil { b.Log.Warn("Request fail", zap.Error(err)) return @@ -120,6 +190,22 @@ func (b *BaseGun) Shoot(ammo Ammo) { if b.DebugLog { b.verboseLogging(res) } + if b.Config.AnswLog.Enabled { + switch b.Config.AnswLog.Filter { + case "all": + b.answLogging(req, bodyBytes, res) + + case "warning": + if res.StatusCode >= 400 { + b.answLogging(req, bodyBytes, res) + } + + case "error": + if res.StatusCode >= 500 { + b.answLogging(req, bodyBytes, res) + } + } + } sample.SetProtoCode(res.StatusCode) defer res.Body.Close() @@ -170,6 +256,27 @@ func (b *BaseGun) verboseLogging(res *http.Response) { ) } +func (b *BaseGun) answLogging(req *http.Request, bodyBytes []byte, res *http.Response) { + isBody := false + if bodyBytes != nil { + req.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + isBody = true + } + dump, err := httputil.DumpRequestOut(req, isBody) + if err != nil { + zap.L().Error("Error dumping request: %s", zap.Error(err)) + } + msg := fmt.Sprintf("REQUEST:\n%s\n\n", string(dump)) + b.AnswLog.Debug(msg) + + dump, err = httputil.DumpResponse(res, true) + if err != nil { + zap.L().Error("Error dumping response: %s", zap.Error(err)) + } + msg = fmt.Sprintf("RESPONSE:\n%s", string(dump)) + b.AnswLog.Debug(msg) +} + func autotag(depth int, URL *url.URL) string { path := URL.Path var ind int @@ -183,3 +290,14 @@ func autotag(depth int, URL *url.URL) string { } return path[:ind] } + +func GetBody(req *http.Request) []byte { + if req.Body != nil && req.Body != http.NoBody { + bodyBytes, _ := ioutil.ReadAll(req.Body) + req.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + return bodyBytes + } + + return nil + +} diff --git a/components/phttp/base_test.go b/components/phttp/base_test.go index 1b28186f8..8834f9c5b 100644 --- a/components/phttp/base_test.go +++ b/components/phttp/base_test.go @@ -18,12 +18,12 @@ import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" "github.com/stretchr/testify/mock" - ammomock "github.com/yandex/pandora/components/phttp/mocks" "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/aggregator/netsample" "github.com/yandex/pandora/core/coretest" "github.com/yandex/pandora/lib/ginkgoutil" + "go.uber.org/zap" ) func testDeps() core.GunDeps { @@ -45,15 +45,15 @@ var _ = Describe("BaseGun", func() { Context("BindResultTo", func() { It("nil panics", func() { Expect(func() { - base.Bind(nil, testDeps()) + _ = base.Bind(nil, testDeps()) }).To(Panic()) }) It("second time panics", func() { res := &netsample.TestAggregator{} - base.Bind(res, testDeps()) + _ = base.Bind(res, testDeps()) Expect(base.Aggregator).To(Equal(res)) Expect(func() { - base.Bind(&netsample.TestAggregator{}, testDeps()) + _ = base.Bind(&netsample.TestAggregator{}, testDeps()) }).To(Panic()) }) }) @@ -90,7 +90,7 @@ var _ = Describe("BaseGun", func() { req = httptest.NewRequest("GET", "/1/2/3/4", nil) tag = "" results = &netsample.TestAggregator{} - base.Bind(results, testDeps()) + _ = base.Bind(results, testDeps()) }) JustBeforeEach(func() { @@ -109,6 +109,7 @@ var _ = Describe("BaseGun", func() { Context("Do ok", func() { BeforeEach(func() { body = ioutil.NopCloser(strings.NewReader("aaaaaaa")) + base.AnswLog = zap.NewNop() base.Do = func(doReq *http.Request) (*http.Response, error) { Expect(doReq).To(Equal(req)) return res, nil diff --git a/components/phttp/client.go b/components/phttp/client.go index 0aea86e07..a131fd32c 100644 --- a/components/phttp/client.go +++ b/components/phttp/client.go @@ -9,14 +9,13 @@ import ( "crypto/tls" "net" "net/http" + "strings" "time" - "go.uber.org/zap" - "golang.org/x/net/http2" - "github.com/pkg/errors" - "github.com/yandex/pandora/core/config" "github.com/yandex/pandora/lib/netutil" + "go.uber.org/zap" + "golang.org/x/net/http2" ) //go:generate mockery -name=Client -case=underscore -inpkg -testonly @@ -64,8 +63,12 @@ func DefaultDialerConfig() DialerConfig { } func NewDialer(conf DialerConfig) netutil.Dialer { - d := &net.Dialer{} - config.Map(d, conf) + d := &net.Dialer{ + Timeout: conf.Timeout, + DualStack: conf.DualStack, + FallbackDelay: conf.FallbackDelay, + KeepAlive: conf.KeepAlive, + } if !conf.DNSCache { return d } @@ -95,19 +98,32 @@ func DefaultTransportConfig() TransportConfig { } } -func NewTransport(conf TransportConfig, dial netutil.DialerFunc) *http.Transport { - tr := &http.Transport{} +func NewTransport(conf TransportConfig, dial netutil.DialerFunc, target string) *http.Transport { + tr := &http.Transport{ + TLSHandshakeTimeout: conf.TLSHandshakeTimeout, + DisableKeepAlives: conf.DisableKeepAlives, + DisableCompression: conf.DisableCompression, + MaxIdleConns: conf.MaxIdleConns, + MaxIdleConnsPerHost: conf.MaxIdleConnsPerHost, + IdleConnTimeout: conf.IdleConnTimeout, + ResponseHeaderTimeout: conf.ResponseHeaderTimeout, + ExpectContinueTimeout: conf.ExpectContinueTimeout, + } + host, _, err := net.SplitHostPort(target) + if err != nil { + zap.L().Panic("HTTP transport configure fail", zap.Error(err)) + } tr.TLSClientConfig = &tls.Config{ InsecureSkipVerify: true, // We should not spend time for this stuff. NextProtos: []string{"http/1.1"}, // Disable HTTP/2. Use HTTP/2 transport explicitly, if needed. + ServerName: host, } - config.Map(tr, conf) tr.DialContext = dial return tr } -func NewHTTP2Transport(conf TransportConfig, dial netutil.DialerFunc) *http.Transport { - tr := NewTransport(conf, dial) +func NewHTTP2Transport(conf TransportConfig, dial netutil.DialerFunc, target string) *http.Transport { + tr := NewTransport(conf, dial, target) err := http2.ConfigureTransport(tr) if err != nil { zap.L().Panic("HTTP/2 transport configure fail", zap.Error(err)) @@ -145,6 +161,11 @@ const notHTTP2PanicMsg = "Non HTTP/2 connection established. Seems that target d func (c *panicOnHTTP1Client) Do(req *http.Request) (*http.Response, error) { res, err := c.Client.Do(req) if err != nil { + var opError *net.OpError + // Unfortunately, Go doesn't expose tls.alert (https://github.com/golang/go/issues/35234), so we make decisions based on the error message + if errors.As(err, &opError) && opError.Op == "remote error" && strings.Contains(err.Error(), "no application protocol") { + zap.L().Panic(notHTTP2PanicMsg, zap.Error(err)) + } return nil, err } err = checkHTTP2(res.TLS) @@ -166,3 +187,11 @@ func checkHTTP2(state *tls.ConnectionState) error { } return nil } + +func getHostWithoutPort(target string) string { + host, _, err := net.SplitHostPort(target) + if err != nil { + host = target + } + return host +} diff --git a/components/phttp/connect.go b/components/phttp/connect.go index bac6a8b5f..64f439b3e 100644 --- a/components/phttp/connect.go +++ b/components/phttp/connect.go @@ -16,6 +16,7 @@ import ( "github.com/pkg/errors" "github.com/yandex/pandora/lib/netutil" + "go.uber.org/zap" ) type ConnectGunConfig struct { @@ -26,7 +27,7 @@ type ConnectGunConfig struct { BaseGunConfig `config:",squash"` } -func NewConnectGun(conf ConnectGunConfig) *ConnectGun { +func NewConnectGun(conf ConnectGunConfig, answLog *zap.Logger) *ConnectGun { scheme := "http" if conf.SSL { scheme = "https" @@ -41,6 +42,7 @@ func NewConnectGun(conf ConnectGunConfig) *ConnectGun { client.CloseIdleConnections() return nil }, + AnswLog: answLog, }, scheme: scheme, client: client, @@ -75,7 +77,7 @@ func newConnectClient(conf ConnectGunConfig) Client { conf.Target, conf.ConnectSSL, NewDialer(conf.Client.Dialer), - )) + ), conf.Target) return newClient(transport, conf.Client.Redirect) } @@ -88,7 +90,7 @@ func newConnectDialFunc(target string, connectSSL bool, dialer netutil.Dialer) n } defer func() { if err != nil && conn != nil { - conn.Close() + _ = conn.Close() conn = nil } }() diff --git a/components/phttp/connect_test.go b/components/phttp/connect_test.go index 39339e6d5..cf8abc521 100644 --- a/components/phttp/connect_test.go +++ b/components/phttp/connect_test.go @@ -15,8 +15,8 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/yandex/pandora/core/aggregator/netsample" + "go.uber.org/zap" ) var _ = Describe("connect", func() { @@ -35,8 +35,8 @@ var _ = Describe("connect", func() { "Current implementation should not send requested data before got response.") _, err = io.WriteString(conn, "HTTP/1.1 200 Connection established\r\n\r\n") Expect(err).To(BeNil()) - go func() { io.Copy(toOrigin, conn) }() - go func() { io.Copy(conn, toOrigin) }() + go func() { _, _ = io.Copy(toOrigin, conn) }() + go func() { _, _ = io.Copy(conn, toOrigin) }() }) } @@ -85,12 +85,13 @@ var _ = Describe("connect", func() { proxy := httptest.NewServer(tunnelHandler(origin.URL)) defer proxy.Close() + log := zap.NewNop() conf := DefaultConnectGunConfig() conf.Target = proxy.Listener.Addr().String() - connectGun := NewConnectGun(conf) + connectGun := NewConnectGun(conf, log) results := &netsample.TestAggregator{} - connectGun.Bind(results, testDeps()) + _ = connectGun.Bind(results, testDeps()) connectGun.Shoot(newAmmoURL(origin.URL)) Expect(results.Samples[0].Err()).To(BeNil()) diff --git a/components/phttp/core.go b/components/phttp/core.go index 0615255ed..eeac62fd5 100644 --- a/components/phttp/core.go +++ b/components/phttp/core.go @@ -22,7 +22,8 @@ type Ammo interface { // TODO(skipor): instead of sample use it wrapper with httptrace and more usable interface. Request() (*http.Request, *netsample.Sample) // Id unique ammo id. Usually equals to ammo num got from provider. - Id() int + ID() int + IsInvalid() bool } type Gun interface { diff --git a/components/phttp/http.go b/components/phttp/http.go index c057b49f6..4924f9a31 100644 --- a/components/phttp/http.go +++ b/components/phttp/http.go @@ -9,6 +9,7 @@ import ( "net/http" "github.com/pkg/errors" + "go.uber.org/zap" ) type ClientGunConfig struct { @@ -27,26 +28,26 @@ type HTTP2GunConfig struct { Client ClientConfig `config:",squash"` } -func NewHTTPGun(conf HTTPGunConfig) *HTTPGun { - transport := NewTransport(conf.Client.Transport, NewDialer(conf.Client.Dialer).DialContext) +func NewHTTPGun(conf HTTPGunConfig, answLog *zap.Logger, targetResolved string) *HTTPGun { + transport := NewTransport(conf.Client.Transport, NewDialer(conf.Client.Dialer).DialContext, conf.Gun.Target) client := newClient(transport, conf.Client.Redirect) - return NewClientGun(client, conf.Gun) + return NewClientGun(client, conf.Gun, answLog, targetResolved) } // NewHTTP2Gun return simple HTTP/2 gun that can shoot sequentially through one connection. -func NewHTTP2Gun(conf HTTP2GunConfig) (*HTTPGun, error) { +func NewHTTP2Gun(conf HTTP2GunConfig, answLog *zap.Logger, targetResolved string) (*HTTPGun, error) { if !conf.Gun.SSL { // Open issue on github if you really need this feature. return nil, errors.New("HTTP/2.0 over TCP is not supported. Please leave SSL option true by default.") } - transport := NewHTTP2Transport(conf.Client.Transport, NewDialer(conf.Client.Dialer).DialContext) + transport := NewHTTP2Transport(conf.Client.Transport, NewDialer(conf.Client.Dialer).DialContext, conf.Gun.Target) client := newClient(transport, conf.Client.Redirect) // Will panic and cancel shooting whet target doesn't support HTTP/2. client = &panicOnHTTP1Client{client} - return NewClientGun(client, conf.Gun), nil + return NewClientGun(client, conf.Gun, answLog, targetResolved), nil } -func NewClientGun(client Client, conf ClientGunConfig) *HTTPGun { +func NewClientGun(client Client, conf ClientGunConfig, answLog *zap.Logger, targetResolved string) *HTTPGun { scheme := "http" if conf.SSL { scheme = "https" @@ -60,26 +61,32 @@ func NewClientGun(client Client, conf ClientGunConfig) *HTTPGun { client.CloseIdleConnections() return nil }, + AnswLog: answLog, }, - scheme: scheme, - target: conf.Target, - client: client, + scheme: scheme, + hostname: getHostWithoutPort(conf.Target), + targetResolved: targetResolved, + client: client, } return &g } type HTTPGun struct { BaseGun - scheme string - target string - client Client + scheme string + hostname string + targetResolved string + client Client } var _ Gun = (*HTTPGun)(nil) func (g *HTTPGun) Do(req *http.Request) (*http.Response, error) { - req.Host = req.URL.Host - req.URL.Host = g.target + if req.Host == "" { + req.Host = g.hostname + } + + req.URL.Host = g.targetResolved req.URL.Scheme = g.scheme return g.client.Do(req) } diff --git a/components/phttp/http_test.go b/components/phttp/http_test.go index 3ffa0db23..be5cb5c7d 100644 --- a/components/phttp/http_test.go +++ b/components/phttp/http_test.go @@ -14,13 +14,12 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" - "go.uber.org/atomic" - "go.uber.org/zap" - "golang.org/x/net/http2" - ammomock "github.com/yandex/pandora/components/phttp/mocks" "github.com/yandex/pandora/core/aggregator/netsample" "github.com/yandex/pandora/core/config" + "go.uber.org/atomic" + "go.uber.org/zap" + "golang.org/x/net/http2" ) var _ = Describe("BaseGun", func() { @@ -45,11 +44,13 @@ var _ = Describe("BaseGun", func() { actualReq = req })) defer server.Close() + log := zap.NewNop() conf := DefaultHTTPGunConfig() - conf.Gun.Target = strings.TrimPrefix(server.URL, "http://") + conf.Gun.Target = host + ":80" + targetResolved := strings.TrimPrefix(server.URL, "http://") results := &netsample.TestAggregator{} - httpGun := NewHTTPGun(conf) - httpGun.Bind(results, testDeps()) + httpGun := NewHTTPGun(conf, log, targetResolved) + _ = httpGun.Bind(results, testDeps()) am := newAmmoReq(expectedReq) httpGun.Shoot(am) @@ -58,9 +59,9 @@ var _ = Describe("BaseGun", func() { Expect(*actualReq).To(MatchFields(IgnoreExtras, Fields{ "Method": Equal("GET"), "Proto": Equal("HTTP/1.1"), - "Host": Equal(host), // Not server host, but host from ammo. + "Host": Equal(host), // Server host "URL": PointTo(MatchFields(IgnoreExtras, Fields{ - "Host": BeEmpty(), // Server request. + "Host": BeEmpty(), // Set in Do(). "Path": Equal(path), })), })) @@ -94,12 +95,13 @@ var _ = Describe("HTTP", func() { server.Start() } defer server.Close() + log := zap.NewNop() conf := DefaultHTTPGunConfig() conf.Gun.Target = server.Listener.Addr().String() conf.Gun.SSL = https - gun := NewHTTPGun(conf) + gun := NewHTTPGun(conf, log, conf.Gun.Target) var aggr netsample.TestAggregator - gun.Bind(&aggr, testDeps()) + _ = gun.Bind(&aggr, testDeps()) gun.Shoot(newAmmoURL("/")) Expect(aggr.Samples).To(HaveLen(1)) @@ -119,12 +121,13 @@ var _ = Describe("HTTP", func() { } })) defer server.Close() + log := zap.NewNop() conf := DefaultHTTPGunConfig() conf.Gun.Target = server.Listener.Addr().String() conf.Client.Redirect = redirect - gun := NewHTTPGun(conf) + gun := NewHTTPGun(conf, log, conf.Gun.Target) var aggr netsample.TestAggregator - gun.Bind(&aggr, testDeps()) + _ = gun.Bind(&aggr, testDeps()) gun.Shoot(newAmmoURL("/redirect")) Expect(aggr.Samples).To(HaveLen(1)) @@ -156,12 +159,13 @@ var _ = Describe("HTTP", func() { Expect(err).NotTo(HaveOccurred()) Expect(res.StatusCode).To(Equal(http.StatusForbidden)) + log := zap.NewNop() conf := DefaultHTTPGunConfig() conf.Gun.Target = server.Listener.Addr().String() conf.Gun.SSL = true - gun := NewHTTPGun(conf) + gun := NewHTTPGun(conf, log, conf.Gun.Target) var results netsample.TestAggregator - gun.Bind(&results, testDeps()) + _ = gun.Bind(&results, testDeps()) gun.Shoot(newAmmoURL("/")) Expect(results.Samples).To(HaveLen(1)) @@ -180,11 +184,12 @@ var _ = Describe("HTTP/2", func() { } })) defer server.Close() + log := zap.NewNop() conf := DefaultHTTP2GunConfig() conf.Gun.Target = server.Listener.Addr().String() - gun, _ := NewHTTP2Gun(conf) + gun, _ := NewHTTP2Gun(conf, log, conf.Gun.Target) var results netsample.TestAggregator - gun.Bind(&results, testDeps()) + _ = gun.Bind(&results, testDeps()) gun.Shoot(newAmmoURL("/")) Expect(results.Samples[0].ProtoCode()).To(Equal(http.StatusOK)) }) @@ -194,11 +199,12 @@ var _ = Describe("HTTP/2", func() { zap.S().Info("Served") })) defer server.Close() + log := zap.NewNop() conf := DefaultHTTP2GunConfig() conf.Gun.Target = server.Listener.Addr().String() - gun, _ := NewHTTP2Gun(conf) + gun, _ := NewHTTP2Gun(conf, log, conf.Gun.Target) var results netsample.TestAggregator - gun.Bind(&results, testDeps()) + _ = gun.Bind(&results, testDeps()) var r interface{} func() { defer func() { @@ -215,10 +221,11 @@ var _ = Describe("HTTP/2", func() { zap.S().Info("Served") })) defer server.Close() + log := zap.NewNop() conf := DefaultHTTP2GunConfig() conf.Gun.Target = server.Listener.Addr().String() conf.Gun.SSL = false - _, err := NewHTTP2Gun(conf) + _, err := NewHTTP2Gun(conf, log, conf.Gun.Target) Expect(err).To(HaveOccurred()) }) @@ -230,7 +237,7 @@ func isHTTP2Request(req *http.Request) bool { func newHTTP2TestServer(handler http.Handler) *httptest.Server { server := httptest.NewUnstartedServer(handler) - http2.ConfigureServer(server.Config, nil) + _ = http2.ConfigureServer(server.Config, nil) server.TLS = server.Config.TLSConfig // StartTLS takes TLS configuration from that field. server.StartTLS() return server diff --git a/components/phttp/import/import.go b/components/phttp/import/import.go index 3f2d5e1a2..3441b80f1 100644 --- a/components/phttp/import/import.go +++ b/components/phttp/import/import.go @@ -9,18 +9,20 @@ import ( "net" "github.com/spf13/afero" - "go.uber.org/zap" - - . "github.com/yandex/pandora/components/phttp" + "github.com/yandex/pandora/components/phttp" "github.com/yandex/pandora/components/phttp/ammo/simple/jsonline" "github.com/yandex/pandora/components/phttp/ammo/simple/raw" "github.com/yandex/pandora/components/phttp/ammo/simple/uri" + "github.com/yandex/pandora/components/phttp/ammo/simple/uripost" "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/register" + "github.com/yandex/pandora/lib/answlog" "github.com/yandex/pandora/lib/netutil" + "go.uber.org/zap" ) func Import(fs afero.Fs) { + register.Provider("http/json", func(conf jsonline.Config) core.Provider { return jsonline.NewProvider(fs, conf) }) @@ -29,29 +31,36 @@ func Import(fs afero.Fs) { return uri.NewProvider(fs, conf) }) + register.Provider("uripost", func(conf uripost.Config) core.Provider { + return uripost.NewProvider(fs, conf) + }) + register.Provider("raw", func(conf raw.Config) core.Provider { return raw.NewProvider(fs, conf) }) - register.Gun("http", func(conf HTTPGunConfig) func() core.Gun { - preResolveTargetAddr(&conf.Client, &conf.Gun.Target) - return func() core.Gun { return WrapGun(NewHTTPGun(conf)) } - }, DefaultHTTPGunConfig) + register.Gun("http", func(conf phttp.HTTPGunConfig) func() core.Gun { + targetResolved, _ := preResolveTargetAddr(&conf.Client, conf.Gun.Target) + answLog := answlog.Init(conf.Gun.Base.AnswLog.Path) + return func() core.Gun { return phttp.WrapGun(phttp.NewHTTPGun(conf, answLog, targetResolved)) } + }, phttp.DefaultHTTPGunConfig) - register.Gun("http2", func(conf HTTP2GunConfig) func() (core.Gun, error) { - preResolveTargetAddr(&conf.Client, &conf.Gun.Target) + register.Gun("http2", func(conf phttp.HTTP2GunConfig) func() (core.Gun, error) { + targetResolved, _ := preResolveTargetAddr(&conf.Client, conf.Gun.Target) + answLog := answlog.Init(conf.Gun.Base.AnswLog.Path) return func() (core.Gun, error) { - gun, err := NewHTTP2Gun(conf) - return WrapGun(gun), err + gun, err := phttp.NewHTTP2Gun(conf, answLog, targetResolved) + return phttp.WrapGun(gun), err } - }, DefaultHTTP2GunConfig) + }, phttp.DefaultHTTP2GunConfig) - register.Gun("connect", func(conf ConnectGunConfig) func() core.Gun { - preResolveTargetAddr(&conf.Client, &conf.Target) + register.Gun("connect", func(conf phttp.ConnectGunConfig) func() core.Gun { + conf.Target, _ = preResolveTargetAddr(&conf.Client, conf.Target) + answLog := answlog.Init(conf.BaseGunConfig.AnswLog.Path) return func() core.Gun { - return WrapGun(NewConnectGun(conf)) + return phttp.WrapGun(phttp.NewConnectGun(conf, answLog)) } - }, DefaultConnectGunConfig) + }, phttp.DefaultConnectGunConfig) } // DNS resolve optimisation. @@ -60,22 +69,24 @@ func Import(fs afero.Fs) { // If we can resolve accessible target addr - use it as target, not use caching. // Otherwise just use DNS cache - we should not fail shooting, we should try to // connect on every shoot. DNS cache will save resolved addr after first successful connect. -func preResolveTargetAddr(clientConf *ClientConfig, target *string) (err error) { +func preResolveTargetAddr(clientConf *phttp.ClientConfig, target string) (targetAddr string, err error) { + targetAddr = target + if !clientConf.Dialer.DNSCache { return } - if endpointIsResolved(*target) { + if endpointIsResolved(target) { clientConf.Dialer.DNSCache = false return } - resolved, err := netutil.LookupReachable(*target) + resolved, err := netutil.LookupReachable(target, clientConf.Dialer.Timeout) if err != nil { zap.L().Warn("DNS target pre resolve failed", - zap.String("target", *target), zap.Error(err)) + zap.String("target", target), zap.Error(err)) return } clientConf.Dialer.DNSCache = false - *target = resolved + targetAddr = resolved return } diff --git a/components/phttp/import/import_suite_test.go b/components/phttp/import/import_suite_test.go index 3a0b5a844..fd427c25a 100644 --- a/components/phttp/import/import_suite_test.go +++ b/components/phttp/import/import_suite_test.go @@ -8,7 +8,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/spf13/afero" - . "github.com/yandex/pandora/components/phttp" "github.com/yandex/pandora/lib/ginkgoutil" ) @@ -31,14 +30,16 @@ var _ = Describe("preResolveTargetAddr", func() { conf.Dialer.DNSCache = true listener, err := net.ListenTCP("tcp4", nil) - defer listener.Close() + if listener != nil { + defer listener.Close() + } Expect(err).NotTo(HaveOccurred()) port := strconv.Itoa(listener.Addr().(*net.TCPAddr).Port) target := "localhost:" + port expectedResolved := "127.0.0.1:" + port - err = preResolveTargetAddr(conf, &target) + target, err = preResolveTargetAddr(conf, target) Expect(err).NotTo(HaveOccurred()) Expect(conf.Dialer.DNSCache).To(BeFalse()) @@ -51,7 +52,7 @@ var _ = Describe("preResolveTargetAddr", func() { const addr = "127.0.0.1:80" target := addr - err := preResolveTargetAddr(conf, &target) + target, err := preResolveTargetAddr(conf, target) Expect(err).NotTo(HaveOccurred()) Expect(conf.Dialer.DNSCache).To(BeFalse()) Expect(target).To(Equal(addr)) @@ -63,7 +64,7 @@ var _ = Describe("preResolveTargetAddr", func() { const addr = "localhost:54321" target := addr - err := preResolveTargetAddr(conf, &target) + target, err := preResolveTargetAddr(conf, target) Expect(err).To(HaveOccurred()) Expect(conf.Dialer.DNSCache).To(BeTrue()) Expect(target).To(Equal(addr)) diff --git a/components/phttp/mock_client_test.go b/components/phttp/mock_client_test.go index a56da548a..bee5eca6a 100644 --- a/components/phttp/mock_client_test.go +++ b/components/phttp/mock_client_test.go @@ -1,8 +1,11 @@ // Code generated by mockery v1.0.0 package phttp -import "net/http" -import "github.com/stretchr/testify/mock" +import ( + "net/http" + + "github.com/stretchr/testify/mock" +) // MockClient is an autogenerated mock type for the Client type type MockClient struct { diff --git a/components/phttp/mocks/ammo.go b/components/phttp/mocks/ammo.go index 533d7fccc..cb43bff1b 100644 --- a/components/phttp/mocks/ammo.go +++ b/components/phttp/mocks/ammo.go @@ -1,17 +1,21 @@ // Code generated by mockery v1.0.0 package ammomock -import http "net/http" -import mock "github.com/stretchr/testify/mock" -import netsample "github.com/yandex/pandora/core/aggregator/netsample" +import ( + http "net/http" + + mock "github.com/stretchr/testify/mock" + netsample "github.com/yandex/pandora/core/aggregator/netsample" +) // Ammo is an autogenerated mock type for the Ammo type type Ammo struct { mock.Mock + isInvalid bool } // Id provides a mock function with given fields: -func (_m *Ammo) Id() int { +func (_m *Ammo) ID() int { ret := _m.Called() var r0 int @@ -48,3 +52,15 @@ func (_m *Ammo) Request() (*http.Request, *netsample.Sample) { return r0, r1 } + +func (_m *Ammo) Invalidate() { + _m.isInvalid = true +} + +func (_m *Ammo) IsInvalid() bool { + return _m.isInvalid +} + +func (_m *Ammo) IsValid() bool { + return !_m.isInvalid +} diff --git a/components/phttp/trace.go b/components/phttp/trace.go new file mode 100644 index 000000000..70cedaa45 --- /dev/null +++ b/components/phttp/trace.go @@ -0,0 +1,65 @@ +package phttp + +import ( + "net/http/httptrace" + "time" +) + +type TraceTimings struct { + GotConnTime time.Time + GetConnTime time.Time + DNSStartTime time.Time + DNSDoneTime time.Time + ConnectDoneTime time.Time + ConnectStartTime time.Time + WroteRequestTime time.Time + GotFirstResponseByte time.Time +} + +func (t *TraceTimings) GetReceiveTime() time.Duration { + return time.Since(t.GotFirstResponseByte) +} + +func (t *TraceTimings) GetConnectTime() time.Duration { + return t.GotConnTime.Sub(t.GetConnTime) +} + +func (t *TraceTimings) GetSendTime() time.Duration { + return t.WroteRequestTime.Sub(t.GotConnTime) +} + +func (t *TraceTimings) GetLatency() time.Duration { + return t.GotFirstResponseByte.Sub(t.WroteRequestTime) +} + +func createHTTPTrace() (*httptrace.ClientTrace, *TraceTimings) { + timings := &TraceTimings{} + tracer := &httptrace.ClientTrace{ + GetConn: func(_ string) { + timings.GetConnTime = time.Now() + }, + GotConn: func(_ httptrace.GotConnInfo) { + timings.GotConnTime = time.Now() + }, + DNSStart: func(_ httptrace.DNSStartInfo) { + timings.DNSStartTime = time.Now() + }, + DNSDone: func(info httptrace.DNSDoneInfo) { + timings.DNSDoneTime = time.Now() + }, + ConnectStart: func(network, addr string) { + timings.ConnectStartTime = time.Now() + }, + ConnectDone: func(network, addr string, err error) { + timings.ConnectDoneTime = time.Now() + }, + WroteRequest: func(wr httptrace.WroteRequestInfo) { + timings.WroteRequestTime = time.Now() + }, + GotFirstResponseByte: func() { + timings.GotFirstResponseByte = time.Now() + }, + } + + return tracer, timings +} diff --git a/core/aggregator/encoder.go b/core/aggregator/encoder.go index 0b0dca75b..a13ae9df4 100644 --- a/core/aggregator/encoder.go +++ b/core/aggregator/encoder.go @@ -11,7 +11,6 @@ import ( "time" "github.com/pkg/errors" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/coreutil" "github.com/yandex/pandora/lib/errutil" @@ -120,7 +119,7 @@ HandleLoop: if err != nil { return } - case _ = <-flushTick: + case <-flushTick: if previousFlushes == flushes { a.Log.Debug("Flushing") err = encoder.Flush() diff --git a/core/aggregator/encoder_test.go b/core/aggregator/encoder_test.go index d9175627e..302f2df6a 100644 --- a/core/aggregator/encoder_test.go +++ b/core/aggregator/encoder_test.go @@ -12,18 +12,17 @@ import ( "testing" "time" - "github.com/hashicorp/go-multierror" + multierror "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "go.uber.org/zap" - "github.com/yandex/pandora/core" aggregatemock "github.com/yandex/pandora/core/aggregator/mocks" coremock "github.com/yandex/pandora/core/mocks" iomock "github.com/yandex/pandora/lib/ioutil2/mocks" "github.com/yandex/pandora/lib/testutil" + "go.uber.org/zap" ) type EncoderAggregatorTester struct { @@ -70,7 +69,7 @@ func NewEncoderAggregatorTester(t testutil.TestingT) *EncoderAggregatorTester { ReporterConfig: ReporterConfig{100}, } tr.ctx, tr.cancel = context.WithCancel(context.Background()) - tr.deps = core.AggregatorDeps{zap.L()} + tr.deps = core.AggregatorDeps{Log: zap.L()} return tr } @@ -234,7 +233,7 @@ func TestEncoderAggregator_ManualFlush(t *testing.T) { defer writeTicker.Stop() for { select { - case _ = <-writeTicker.C: + case <-writeTicker.C: testee.Report(0) case <-tr.ctx.Done(): return diff --git a/core/aggregator/jsonlines.go b/core/aggregator/jsonlines.go index 2d4747c7d..a66e26eeb 100644 --- a/core/aggregator/jsonlines.go +++ b/core/aggregator/jsonlines.go @@ -10,11 +10,9 @@ import ( "io" jsoniter "github.com/json-iterator/go" - "github.com/yandex/pandora/lib/ioutil2" - "github.com/yandex/pandora/core" - "github.com/yandex/pandora/core/config" "github.com/yandex/pandora/core/coreutil" + "github.com/yandex/pandora/lib/ioutil2" ) type JSONLineAggregatorConfig struct { @@ -52,8 +50,11 @@ func NewJSONLinesAggregator(conf JSONLineAggregatorConfig) core.Aggregator { } func NewJSONEncoder(w io.Writer, conf JSONLineEncoderConfig) SampleEncoder { - var apiConfig jsoniter.Config - config.Map(&apiConfig, conf.JSONIterConfig) + apiConfig := jsoniter.Config{ + SortMapKeys: conf.JSONIterConfig.SortMapKeys, + MarshalFloatWith6Digits: conf.JSONIterConfig.MarshalFloatWith6Digits, + } + api := apiConfig.Froze() // NOTE(skipor): internal buffering is not working really. Don't know why // OPTIMIZE(skipor): don't wrap into buffer, if already ioutil2.ByteWriter @@ -75,6 +76,6 @@ func (e *jsonEncoder) Encode(s core.Sample) error { func (e *jsonEncoder) Flush() error { err := e.Stream.Flush() - e.buf.Flush() + _ = e.buf.Flush() return err } diff --git a/core/aggregator/jsonlines_test.go b/core/aggregator/jsonlines_test.go index ec8ae97eb..0ca60fbf9 100644 --- a/core/aggregator/jsonlines_test.go +++ b/core/aggregator/jsonlines_test.go @@ -12,10 +12,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/zap" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/datasink" + "go.uber.org/zap" ) type jsonTestData struct { @@ -38,7 +37,7 @@ func TestNewJSONLinesAggregator(t *testing.T) { runErr := make(chan error) go func() { - runErr <- testee.Run(ctx, core.AggregatorDeps{zap.L()}) + runErr <- testee.Run(ctx, core.AggregatorDeps{Log: zap.L()}) }() for _, sample := range samples { diff --git a/core/aggregator/log.go b/core/aggregator/log.go index cf40e5ca3..681dca3ee 100644 --- a/core/aggregator/log.go +++ b/core/aggregator/log.go @@ -8,9 +8,8 @@ package aggregator import ( "context" - "go.uber.org/zap" - "github.com/yandex/pandora/core" + "go.uber.org/zap" ) func NewLog() core.Aggregator { diff --git a/core/aggregator/mocks/sample_encode_closer.go b/core/aggregator/mocks/sample_encode_closer.go index 65735cf03..0e409af97 100644 --- a/core/aggregator/mocks/sample_encode_closer.go +++ b/core/aggregator/mocks/sample_encode_closer.go @@ -1,8 +1,10 @@ // Code generated by mockery v1.0.0 package aggregatemock -import core "github.com/yandex/pandora/core" -import mock "github.com/stretchr/testify/mock" +import ( + mock "github.com/stretchr/testify/mock" + core "github.com/yandex/pandora/core" +) // SampleEncodeCloser is an autogenerated mock type for the SampleEncodeCloser type type SampleEncodeCloser struct { diff --git a/core/aggregator/mocks/sample_encoder.go b/core/aggregator/mocks/sample_encoder.go index 344213eb5..8e1dd9474 100644 --- a/core/aggregator/mocks/sample_encoder.go +++ b/core/aggregator/mocks/sample_encoder.go @@ -1,8 +1,10 @@ // Code generated by mockery v1.0.0 package aggregatemock -import core "github.com/yandex/pandora/core" -import mock "github.com/stretchr/testify/mock" +import ( + mock "github.com/stretchr/testify/mock" + core "github.com/yandex/pandora/core" +) // SampleEncoder is an autogenerated mock type for the SampleEncoder type type SampleEncoder struct { diff --git a/core/aggregator/netsample/phout.go b/core/aggregator/netsample/phout.go index 6be94a31b..a72733cf6 100644 --- a/core/aggregator/netsample/phout.go +++ b/core/aggregator/netsample/phout.go @@ -11,14 +11,13 @@ import ( "github.com/c2h5oh/datasize" "github.com/pkg/errors" "github.com/spf13/afero" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/coreutil" ) type PhoutConfig struct { Destination string // Destination file name - Id bool // Print ammo ids if true. + ID bool // Print ammo ids if true. FlushTime time.Duration `config:"flush-time"` SampleQueueSize int `config:"sample-queue-size"` Buffer coreutil.BufferSizeConfig `config:",squash"` @@ -67,8 +66,8 @@ func (a *phoutAggregator) Report(s *Sample) { a.sink <- s } func (a *phoutAggregator) Run(ctx context.Context, _ core.AggregatorDeps) error { shouldFlush := time.NewTicker(1 * time.Second) defer func() { - a.writer.Flush() - a.file.Close() + _ = a.writer.Flush() + _ = a.file.Close() shouldFlush.Stop() }() loop: @@ -80,11 +79,11 @@ loop: } select { case <-shouldFlush.C: - a.writer.Flush() + _ = a.writer.Flush() default: } case <-time.After(1 * time.Second): - a.writer.Flush() + _ = a.writer.Flush() case <-ctx.Done(): // Context is done, but we should read all data from sink for { @@ -103,7 +102,7 @@ loop: } func (a *phoutAggregator) handle(s *Sample) error { - a.buf = appendPhout(s, a.buf, a.config.Id) + a.buf = appendPhout(s, a.buf, a.config.ID) a.buf = append(a.buf, '\n') _, err := a.writer.Write(a.buf) a.buf = a.buf[:0] @@ -119,7 +118,7 @@ func appendPhout(s *Sample, dst []byte, id bool) []byte { dst = append(dst, s.tags...) if id { dst = append(dst, '#') - dst = strconv.AppendInt(dst, int64(s.Id()), 10) + dst = strconv.AppendInt(dst, int64(s.ID()), 10) } for _, v := range s.fields { dst = append(dst, phoutDelimiter) diff --git a/core/aggregator/netsample/phout_test.go b/core/aggregator/netsample/phout_test.go index 67f1093e4..33f6a7bea 100644 --- a/core/aggregator/netsample/phout_test.go +++ b/core/aggregator/netsample/phout_test.go @@ -8,7 +8,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/spf13/afero" - "github.com/yandex/pandora/core" ) @@ -48,11 +47,11 @@ var _ = Describe("Phout", func() { testee.Report(newTestSample()) cancel() Expect(<-runErr).NotTo(HaveOccurred()) - Expect(getOutput()).To(Equal(strings.Repeat(testSampleNoIdPhout+"\n", 2))) + Expect(getOutput()).To(Equal(strings.Repeat(testSampleNoIDPhout+"\n", 2))) }, 1) Context("id option set", func() { BeforeEach(func() { - conf.Id = true + conf.ID = true }) It("id printed", func() { testee.Report(newTestSample()) @@ -67,13 +66,13 @@ var _ = Describe("Phout", func() { const ( testSamplePhout = "1484660999.002 tag1|tag2#42 333333 0 0 0 0 0 0 0 13 999" - testSampleNoIdPhout = "1484660999.002 tag1|tag2 333333 0 0 0 0 0 0 0 13 999" + testSampleNoIDPhout = "1484660999.002 tag1|tag2 333333 0 0 0 0 0 0 0 13 999" ) func newTestSample() *Sample { s := &Sample{} s.timeStamp = time.Unix(1484660999, 002*1000000) - s.SetId(42) + s.SetID(42) s.AddTag("tag1|tag2") s.setDuration(keyRTTMicro, time.Second/3) s.set(keyErrno, 13) diff --git a/core/aggregator/netsample/sample.go b/core/aggregator/netsample/sample.go index e74ed75f6..536e65306 100644 --- a/core/aggregator/netsample/sample.go +++ b/core/aggregator/netsample/sample.go @@ -7,6 +7,7 @@ package netsample import ( "net" + "net/url" "os" "sync" "syscall" @@ -16,7 +17,9 @@ import ( ) const ( - ProtoCodeError = 999 + ProtoCodeError = 999 + DiscardedShootCodeError = 777 + DiscardedShootTag = "discarded" ) const ( @@ -63,8 +66,8 @@ func (s *Sample) AddTag(tag string) { s.tags += "|" + tag } -func (s *Sample) Id() int { return s.id } -func (s *Sample) SetId(id int) { s.id = id } +func (s *Sample) ID() int { return s.id } +func (s *Sample) SetID(id int) { s.id = id } func (s *Sample) ProtoCode() int { return s.get(keyProtoCode) } func (s *Sample) SetProtoCode(code int) { @@ -88,11 +91,51 @@ func (s *Sample) setRTT() { } } +func (s *Sample) SetUserDuration(d time.Duration) { + s.setDuration(keyRTTMicro, d) +} + +func (s *Sample) SetUserProto(code int) { + s.set(keyProtoCode, code) +} + +func (s *Sample) SetUserNet(code int) { + s.set(keyErrno, code) +} + +func (s *Sample) SetConnectTime(d time.Duration) { + s.setDuration(keyConnectMicro, d) +} + +func (s *Sample) SetSendTime(d time.Duration) { + s.setDuration(keySendMicro, d) +} + +func (s *Sample) SetLatency(d time.Duration) { + s.setDuration(keyLatencyMicro, d) +} + +func (s *Sample) SetReceiveTime(d time.Duration) { + s.setDuration(keyReceiveMicro, d) +} + +func (s *Sample) SetRequestBytes(b int) { + s.set(keyRequestBytes, b) +} + +func (s *Sample) SetResponseBytes(b int) { + s.set(keyResponseBytes, b) +} + func (s *Sample) String() string { return string(appendPhout(s, nil, true)) } func getErrno(err error) int { + // + if e, ok := err.(net.Error); ok && e.Timeout() { + return 110 // Handle client Timeout as if it connection timeout + } // stackerr.Error and etc. type hasUnderlying interface { Underlying() error @@ -111,6 +154,8 @@ func getErrno(err error) int { err = typed.Err case *os.SyscallError: err = typed.Err + case *url.Error: + err = typed.Err case syscall.Errno: return int(typed) default: @@ -119,3 +164,13 @@ func getErrno(err error) int { } } } + +func DiscardedShootSample() *Sample { + sample := &Sample{ + timeStamp: time.Now(), + tags: DiscardedShootTag, + } + sample.SetUserNet(DiscardedShootCodeError) + + return sample +} diff --git a/core/aggregator/netsample/sample_test.go b/core/aggregator/netsample/sample_test.go index 64948fba4..94cdeef53 100644 --- a/core/aggregator/netsample/sample_test.go +++ b/core/aggregator/netsample/sample_test.go @@ -27,7 +27,7 @@ func TestSampleBehaviour(t *testing.T) { const id = 42 sample := Acquire(tag) sample.AddTag(tag2) - sample.SetId(id) + sample.SetID(id) const sleep = time.Millisecond time.Sleep(sleep) sample.SetErr(syscall.EINVAL) @@ -51,6 +51,42 @@ func TestSampleBehaviour(t *testing.T) { assert.Equal(t, expected, sample.String()) } +func TestCustomSets(t *testing.T) { + const tag = "UserDefine" + s := Acquire(tag) + + userDuration := 100 * time.Millisecond + s.SetUserDuration(userDuration) + + s.SetUserProto(0) + s.SetUserNet(110) + + latency := 200 * time.Millisecond + s.SetLatency(latency) + + reqBytes := 4 + s.SetRequestBytes(reqBytes) + + respBytes := 8 + s.SetResponseBytes(respBytes) + + expectedTimeStamp := fmt.Sprintf("%v.%3.f", + s.timeStamp.Unix(), + float32((s.timeStamp.UnixNano()/1e6)%1000)) + expectedTimeStamp = strings.Replace(expectedTimeStamp, " ", "0", -1) + expected := fmt.Sprintf("%s\t%s#0\t%v\t0\t0\t%v\t0\t0\t%v\t%v\t%v\t%v", + expectedTimeStamp, + tag, + int(userDuration.Nanoseconds()/1000), // keyRTTMicro + int(latency.Nanoseconds()/1000), // keyLatencyMicro + reqBytes, // keyRequestBytes + respBytes, // keyResponseBytes + 110, + 0, + ) + assert.Equal(t, s.String(), expected) +} + func TestGetErrno(t *testing.T) { var err error = syscall.EINVAL err = &os.SyscallError{Err: err} diff --git a/core/aggregator/reporter.go b/core/aggregator/reporter.go index 5a3cfed1b..c7809e897 100644 --- a/core/aggregator/reporter.go +++ b/core/aggregator/reporter.go @@ -8,11 +8,10 @@ package aggregator import ( "fmt" - "go.uber.org/atomic" - "go.uber.org/zap" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/coreutil" + "go.uber.org/atomic" + "go.uber.org/zap" ) type ReporterConfig struct { diff --git a/core/aggregator/reporter_test.go b/core/aggregator/reporter_test.go index 1857e7f7b..a1f950fec 100644 --- a/core/aggregator/reporter_test.go +++ b/core/aggregator/reporter_test.go @@ -10,11 +10,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/zap" - "go.uber.org/zap/zaptest/observer" - coremock "github.com/yandex/pandora/core/mocks" "github.com/yandex/pandora/lib/testutil" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" ) func TestReporter_DroppedErr(t *testing.T) { @@ -24,7 +23,7 @@ func TestReporter_DroppedErr(t *testing.T) { reporter := NewReporter(ReporterConfig{1}) reporter.Report(1) - assert.Nil(t, reporter.DroppedErr()) + assert.NoError(t, reporter.DroppedErr()) reporter.Report(2) err := reporter.DroppedErr() require.Error(t, err) diff --git a/core/config/config.go b/core/config/config.go index 429cbbaf8..37f24d2ea 100644 --- a/core/config/config.go +++ b/core/config/config.go @@ -6,7 +6,6 @@ package config import ( - "github.com/fatih/structs" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -39,18 +38,36 @@ func DecodeAndValidate(conf interface{}, result interface{}) error { // in such case you can from this subset of fields struct Single, decode config // into it, and map it on Multi. func Map(dst, src interface{}) { - conf := &mapstructure.DecoderConfig{ + // dst and src conf for compatibility with old fatih/structs. + // src map from "map:" tags -> tmp -> map to "mapstructure:" tags in dst + dstConf := &mapstructure.DecoderConfig{ ErrorUnused: true, ZeroFields: true, Result: dst, } - d, err := mapstructure.NewDecoder(conf) + d, err := mapstructure.NewDecoder(dstConf) if err != nil { panic(err) } - s := structs.New(src) - s.TagName = "map" - err = d.Decode(s.Map()) + + tmp := make(map[string]interface{}) + srcConf := &mapstructure.DecoderConfig{ + ErrorUnused: true, + ZeroFields: true, + Result: &tmp, + TagName: "map", + } + s, err := mapstructure.NewDecoder(srcConf) + if err != nil { + panic(err) + } + + err = s.Decode(src) + if err != nil { + panic(err) + } + + err = d.Decode(tmp) if err != nil { panic(err) } @@ -84,6 +101,7 @@ func AddKindHook(hook KindHook) (_ struct{}) { func DefaultHooks() []mapstructure.DecodeHookFunc { return []mapstructure.DecodeHookFunc{ + VariableInjectHook, DebugHook, TextUnmarshallerHook, mapstructure.StringToTimeDurationHookFunc(), diff --git a/core/config/config_test.go b/core/config/config_test.go index a602c6bc1..dde391b62 100644 --- a/core/config/config_test.go +++ b/core/config/config_test.go @@ -6,12 +6,14 @@ package config import ( + "net" "testing" "time" "github.com/facebookgo/stack" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/yandex/pandora/lib/confutil" ) type M map[string]interface{} @@ -184,7 +186,6 @@ func TestMapTagged(t *testing.T) { Map(n, &M{SomeOtherFieldName: MultiStrings{A: "a"}}) assert.Equal(t, &N{A: "a", MultiStrings: MultiStrings{A: "a"}}, n) } - func TestDeltaUpdate(t *testing.T) { var l2 Level2 err := Decode(M{ @@ -231,3 +232,35 @@ func TestNextSquash(t *testing.T) { require.NoError(t, err) assert.Equal(t, "baz", data.Level1.Level2.Foo) } + +func TestConfigEnvVarReplacement(t *testing.T) { + confutil.RegisterTagResolver("", confutil.EnvTagResolver) + confutil.RegisterTagResolver("ENV", confutil.EnvTagResolver) + + t.Setenv("ENV_VAR_1", "value1") + t.Setenv("VAR_2", "value2") + t.Setenv("INT_VAR_3", "15") + t.Setenv("IP_SEQ", "1.2") + t.Setenv("DURATION", "30s") + var l1 struct { + Val1 string + Val2 string + Val3 int + Val4 net.IP + Val5 time.Duration + } + + err := Decode(M{ + "val1": "aa-${ENV_VAR_1}", + "val2": "${ENV:VAR_2}", + "val3": "${INT_VAR_3}", + "val4": "1.1.${ENV:IP_SEQ}", + "val5": "${DURATION}", + }, &l1) + assert.NoError(t, err) + assert.Equal(t, "aa-value1", l1.Val1) + assert.Equal(t, "value2", l1.Val2) + assert.Equal(t, 15, l1.Val3) + assert.Equal(t, net.IPv4(1, 1, 1, 2), l1.Val4) + assert.Equal(t, 30*time.Second, l1.Val5) +} diff --git a/core/config/hooks.go b/core/config/hooks.go index 7e1238b87..0fe69096b 100644 --- a/core/config/hooks.go +++ b/core/config/hooks.go @@ -17,9 +17,9 @@ import ( "github.com/c2h5oh/datasize" "github.com/facebookgo/stack" "github.com/pkg/errors" - "go.uber.org/zap" - + "github.com/yandex/pandora/lib/confutil" "github.com/yandex/pandora/lib/tag" + "go.uber.org/zap" ) var InvalidURLError = errors.New("string is not valid URL") @@ -53,7 +53,7 @@ func StringToURLHook(f reflect.Type, t reflect.Type, data interface{}) (interfac return urlPtr, nil } -var InvalidIPError = stderrors.New("string is not valid IP") +var ErrInvalidIP = stderrors.New("string is not valid IP") // StringToIPHook converts string to net.IP func StringToIPHook(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) { @@ -66,7 +66,7 @@ func StringToIPHook(f reflect.Type, t reflect.Type, data interface{}) (interface str := data.(string) ip := net.ParseIP(str) if ip == nil { - return nil, errors.WithStack(InvalidIPError) + return nil, errors.WithStack(ErrInvalidIP) } return ip, nil } @@ -111,7 +111,7 @@ func TextUnmarshallerHook(f reflect.Type, t reflect.Type, data interface{}) (int func unmarhsallText(v reflect.Value, data interface{}) error { unmarshaller := v.Interface().(encoding.TextUnmarshaler) - unmarshaller.UnmarshalText([]byte(data.(string))) + // unmarshaller.UnmarshalText([]byte(data.(string))) err := unmarshaller.UnmarshalText([]byte(data.(string))) return err } @@ -138,3 +138,22 @@ func DebugHook(f reflect.Type, t reflect.Type, data interface{}) (p interface{}, ) return } + +// VariableInjectHook injects values into ${VAR_NAME} placeholders +func VariableInjectHook(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + + str := data.(string) + res, err := confutil.ResolveCustomTags(str, t) + if err == confutil.ErrNoTagsFound { + return data, nil + } + + if err != nil { + return data, err + } + + return res, nil +} diff --git a/core/config/validations.go b/core/config/validations.go index 18eaff0b1..6dd310584 100644 --- a/core/config/validations.go +++ b/core/config/validations.go @@ -10,7 +10,7 @@ import ( "github.com/asaskevich/govalidator" "github.com/c2h5oh/datasize" - "gopkg.in/bluesuncorp/validator.v9" + validator "gopkg.in/bluesuncorp/validator.v9" ) func MinTimeValidation(fl validator.FieldLevel) bool { diff --git a/core/config/validator.go b/core/config/validator.go index d1abf67a9..6134d3981 100644 --- a/core/config/validator.go +++ b/core/config/validator.go @@ -5,7 +5,7 @@ package config import ( "github.com/pkg/errors" - "gopkg.in/bluesuncorp/validator.v9" + validator "gopkg.in/bluesuncorp/validator.v9" ) var validations = []struct { @@ -36,10 +36,10 @@ func newValidator() *validator.Validate { validate := validator.New() validate.SetTagName("validate") for _, val := range validations { - validate.RegisterValidation(val.key, val.val) + _ = validate.RegisterValidation(val.key, val.val) } for _, val := range stringValidations { - validate.RegisterValidation(val.key, StringToAbstractValidation(val.val)) + _ = validate.RegisterValidation(val.key, StringToAbstractValidation(val.val)) } return validate } diff --git a/core/config/validator_test.go b/core/config/validator_test.go index f2a7b0396..bbfc6fd35 100644 --- a/core/config/validator_test.go +++ b/core/config/validator_test.go @@ -72,7 +72,7 @@ type D struct { func TestValidateInvalidValidatorName(t *testing.T) { require.Panics(t, func() { - Validate(&D{"test"}) + _ = Validate(&D{"test"}) }) } diff --git a/core/core.go b/core/core.go index 07164e034..51ac7a7c2 100644 --- a/core/core.go +++ b/core/core.go @@ -63,7 +63,8 @@ type Provider interface { // WARN: another fields could be added in next MINOR versions. // That is NOT considered as a breaking compatibility change. type ProviderDeps struct { - Log *zap.Logger + Log *zap.Logger + PoolID string } //go:generate mockery -name=Gun -case=underscore -outpkg=coremock @@ -104,7 +105,9 @@ type GunDeps struct { // Pool set's ids to Instances from 0, incrementing it after Instance Run. // There is a race between Instances for Ammo Acquire, so it's not guaranteed, that // Instance with lower InstanceId gets it's Ammo earlier. - InstanceId int + InstanceID int + PoolID string + // TODO(skipor): https://github.com/yandex/pandora/issues/71 // Pass parallelism value. InstanceId MUST be -1 if parallelism > 1. } diff --git a/core/coretest/schedule.go b/core/coretest/schedule.go index 8af768391..73d9511d2 100644 --- a/core/coretest/schedule.go +++ b/core/coretest/schedule.go @@ -8,17 +8,17 @@ package coretest import ( "time" - . "github.com/onsi/gomega" + "github.com/onsi/gomega" "github.com/yandex/pandora/core" ) func ExpectScheduleNextsStartAt(sched core.Schedule, startAt time.Time, nexts ...time.Duration) { beforeStartLeft := sched.Left() tokensExpected := len(nexts) - 1 // Last next is finish time. - Expect(beforeStartLeft).To(Equal(tokensExpected)) + gomega.Expect(beforeStartLeft).To(gomega.Equal(tokensExpected)) sched.Start(startAt) actualNexts := DrainScheduleDuration(sched, startAt) - Expect(actualNexts).To(Equal(nexts)) + gomega.Expect(actualNexts).To(gomega.Equal(nexts)) } func ExpectScheduleNexts(sched core.Schedule, nexts ...time.Duration) { @@ -47,11 +47,11 @@ func DrainSchedule(sched core.Schedule) []time.Time { next, ok := sched.Next() nexts = append(nexts, next) if !ok { - Expect(sched.Left()).To(Equal(0)) + gomega.Expect(sched.Left()).To(gomega.Equal(0)) return nexts } expectedLeft-- - Expect(sched.Left()).To(Equal(expectedLeft)) + gomega.Expect(sched.Left()).To(gomega.Equal(expectedLeft)) } panic("drain limit reached") } diff --git a/core/coretest/sink.go b/core/coretest/sink.go index a0158263b..191c44111 100644 --- a/core/coretest/sink.go +++ b/core/coretest/sink.go @@ -14,7 +14,6 @@ import ( "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/yandex/pandora/core" ) @@ -38,13 +37,13 @@ func AssertSinkEqualStdStream(t *testing.T, expectedPtr **os.File, getSink func( err = wc.Close() require.NoError(t, err) - temp.Seek(0, io.SeekStart) - data, err := ioutil.ReadAll(temp) + _, _ = temp.Seek(0, io.SeekStart) + data, _ := ioutil.ReadAll(temp) assert.Equal(t, testdata, string(data)) } func AssertSinkEqualFile(t *testing.T, fs afero.Fs, filename string, sink core.DataSink) { - afero.WriteFile(fs, filename, []byte("should be truncated"), 644) + _ = afero.WriteFile(fs, filename, []byte("should be truncated"), 0644) wc, err := sink.OpenSink() require.NoError(t, err) diff --git a/core/coretest/source.go b/core/coretest/source.go index 23d4acb5c..9aa94bdb7 100644 --- a/core/coretest/source.go +++ b/core/coretest/source.go @@ -14,7 +14,6 @@ import ( "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/lib/testutil" ) @@ -38,14 +37,14 @@ func AssertSourceEqualStdStream(t *testing.T, expectedPtr **os.File, getSource f err = rc.Close() require.NoError(t, err, "std stream should not be closed") - temp.Seek(0, io.SeekStart) - data, err := ioutil.ReadAll(temp) + _, _ = temp.Seek(0, io.SeekStart) + data, _ := ioutil.ReadAll(temp) assert.Equal(t, testdata, string(data)) } func AssertSourceEqualFile(t *testing.T, fs afero.Fs, filename string, source core.DataSource) { const testdata = "abcd" - afero.WriteFile(fs, filename, []byte(testdata), 644) + _ = afero.WriteFile(fs, filename, []byte(testdata), 0644) rc, err := source.OpenSource() require.NoError(t, err) diff --git a/core/coreutil/buffer_size_config_test.go b/core/coreutil/buffer_size_config_test.go index 13223be73..aeb8fbd13 100644 --- a/core/coreutil/buffer_size_config_test.go +++ b/core/coreutil/buffer_size_config_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/c2h5oh/datasize" - "github.com/magiconair/properties/assert" + "github.com/stretchr/testify/assert" ) func TestBufferSizeConfig_BufferSizeOrDefault(t *testing.T) { diff --git a/core/coreutil/schedule_test.go b/core/coreutil/schedule_test.go index 93d84f3c5..a4a3613b1 100644 --- a/core/coreutil/schedule_test.go +++ b/core/coreutil/schedule_test.go @@ -10,7 +10,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/yandex/pandora/core/schedule" ) diff --git a/core/coreutil/waiter.go b/core/coreutil/waiter.go index dbebba4b4..88d3ce3dc 100644 --- a/core/coreutil/waiter.go +++ b/core/coreutil/waiter.go @@ -14,8 +14,9 @@ import ( // Waiter goroutine unsafe wrapper for efficient waiting schedule. type Waiter struct { - sched core.Schedule - ctx context.Context + sched core.Schedule + ctx context.Context + slowDownItems int // Lazy initialized. timer *time.Timer @@ -33,23 +34,28 @@ func (w *Waiter) Wait() (ok bool) { // Check, that context is not done. Very quick: 5 ns for op, due to benchmark. select { case <-w.ctx.Done(): + w.slowDownItems = 0 return false default: } next, ok := w.sched.Next() if !ok { + w.slowDownItems = 0 return false } // Get current time lazily. // For once schedule, for example, we need to get it only once. if next.Before(w.lastNow) { + w.slowDownItems++ return true } w.lastNow = time.Now() waitFor := next.Sub(w.lastNow) if waitFor <= 0 { + w.slowDownItems++ return true } + w.slowDownItems = 0 // Lazy init. We don't need timer for unlimited and once schedule. if w.timer == nil { w.timer = time.NewTimer(waitFor) @@ -57,13 +63,23 @@ func (w *Waiter) Wait() (ok bool) { w.timer.Reset(waitFor) } select { - case _ = <-w.timer.C: + case <-w.timer.C: return true case <-w.ctx.Done(): return false } } +// IsSlowDown returns true, if schedule contains 2 elements before current time. +func (w *Waiter) IsSlowDown() (ok bool) { + select { + case <-w.ctx.Done(): + return false + default: + return w.slowDownItems >= 2 + } +} + // IsFinished is quick check, that wait context is not canceled and there are some tokens left in // schedule. func (w *Waiter) IsFinished() (ok bool) { diff --git a/core/coreutil/waiter_test.go b/core/coreutil/waiter_test.go index 4cb0309e8..d5629ecc2 100644 --- a/core/coreutil/waiter_test.go +++ b/core/coreutil/waiter_test.go @@ -11,7 +11,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/yandex/pandora/core/schedule" ) diff --git a/core/datasink/file.go b/core/datasink/file.go index 7ea859b48..610cff137 100644 --- a/core/datasink/file.go +++ b/core/datasink/file.go @@ -10,7 +10,6 @@ import ( "os" "github.com/spf13/afero" - "github.com/yandex/pandora/core" ) @@ -21,7 +20,7 @@ type FileConfig struct { } func NewFile(fs afero.Fs, conf FileConfig) core.DataSink { - return &fileSink{afero.Afero{fs}, conf} + return &fileSink{afero.Afero{Fs: fs}, conf} } type fileSink struct { diff --git a/core/datasink/file_test.go b/core/datasink/file_test.go index 018ba9720..df6ef0dd9 100644 --- a/core/datasink/file_test.go +++ b/core/datasink/file_test.go @@ -10,7 +10,6 @@ import ( "testing" "github.com/spf13/afero" - "github.com/yandex/pandora/core/coretest" ) diff --git a/core/datasource/file.go b/core/datasource/file.go index ab6f4ce1b..f5a885521 100644 --- a/core/datasource/file.go +++ b/core/datasource/file.go @@ -10,7 +10,6 @@ import ( "os" "github.com/spf13/afero" - "github.com/yandex/pandora/core" ) @@ -21,7 +20,7 @@ type FileConfig struct { } func NewFile(fs afero.Fs, conf FileConfig) core.DataSource { - return &fileSource{afero.Afero{fs}, conf} + return &fileSource{afero.Afero{Fs: fs}, conf} } type fileSource struct { diff --git a/core/engine/engine.go b/core/engine/engine.go index 2e2ef4a47..a5ac5ed6e 100644 --- a/core/engine/engine.go +++ b/core/engine/engine.go @@ -11,12 +11,12 @@ import ( "sync" "github.com/pkg/errors" - "go.uber.org/zap" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/coreutil" + "github.com/yandex/pandora/core/warmup" "github.com/yandex/pandora/lib/errutil" "github.com/yandex/pandora/lib/monitoring" + "go.uber.org/zap" ) type Config struct { @@ -24,13 +24,14 @@ type Config struct { } type InstancePoolConfig struct { - Id string + ID string Provider core.Provider `config:"ammo" validate:"required"` Aggregator core.Aggregator `config:"result" validate:"required"` NewGun func() (core.Gun, error) `config:"gun" validate:"required"` RPSPerInstance bool `config:"rps-per-instance"` NewRPSSchedule func() (core.Schedule, error) `config:"rps" validate:"required"` StartupSchedule core.Schedule `config:"startup" validate:"required"` + DiscardOverflow bool `config:"discard_overflow"` } // TODO(skipor): use something github.com/rcrowley/go-metrics based. @@ -67,18 +68,18 @@ func (e *Engine) Run(ctx context.Context) error { runRes := make(chan poolRunResult, 1) for i, conf := range e.config.Pools { - if conf.Id == "" { - conf.Id = fmt.Sprintf("pool_%v", i) + if conf.ID == "" { + conf.ID = fmt.Sprintf("pool_%v", i) } e.wait.Add(1) pool := newPool(e.log, e.metrics, e.wait.Done, conf) go func() { err := pool.Run(ctx) select { - case runRes <- poolRunResult{pool.Id, err}: + case runRes <- poolRunResult{pool.ID, err}: case <-ctx.Done(): pool.log.Info("Pool run result suppressed", - zap.String("id", pool.Id), zap.Error(err)) + zap.String("id", pool.ID), zap.Error(err)) } }() } @@ -87,14 +88,14 @@ func (e *Engine) Run(ctx context.Context) error { select { case res := <-runRes: e.log.Debug("Pool awaited", zap.Int("awaited", i), - zap.String("id", res.Id), zap.Error(res.Err)) + zap.String("id", res.ID), zap.Error(res.Err)) if res.Err != nil { select { case <-ctx.Done(): return ctx.Err() default: } - return errors.WithMessage(res.Err, fmt.Sprintf("%q pool run failed", res.Id)) + return errors.WithMessage(res.Err, fmt.Sprintf("%q pool run failed", res.ID)) } case <-ctx.Done(): e.log.Info("Engine run canceled") @@ -111,8 +112,8 @@ func (e *Engine) Wait() { } func newPool(log *zap.Logger, m Metrics, onWaitDone func(), conf InstancePoolConfig) *instancePool { - log = log.With(zap.String("pool", conf.Id)) - return &instancePool{log, m, onWaitDone, conf} + log = log.With(zap.String("pool", conf.ID)) + return &instancePool{log, m, onWaitDone, conf, nil} } type instancePool struct { @@ -120,6 +121,7 @@ type instancePool struct { metrics Metrics onWaitDone func() InstancePoolConfig + gunWarmUpResult interface{} } // Run start instance pool. Run blocks until fail happen, or all instances finish. @@ -141,6 +143,11 @@ func (p *instancePool) Run(ctx context.Context) error { cancel() }() + if err := p.warmUpGun(ctx); err != nil { + p.onWaitDone() + return err + } + rh, err := p.runAsync(ctx) if err != nil { return err @@ -162,6 +169,23 @@ func (p *instancePool) Run(ctx context.Context) error { } } +func (p *instancePool) warmUpGun(ctx context.Context) error { + dummyGun, err := p.NewGun() + if err != nil { + return fmt.Errorf("can't initiate a gun: %w", err) + } + if gunWithWarmUp, ok := dummyGun.(warmup.WarmedUp); ok { + p.gunWarmUpResult, err = gunWithWarmUp.WarmUp(&warmup.Options{ + Log: p.log, + Ctx: ctx, + }) + if err != nil { + return fmt.Errorf("gun warm up failed: %w", err) + } + } + return nil +} + type poolAsyncRunHandle struct { runCtx context.Context runCancel context.CancelFunc @@ -178,6 +202,7 @@ type poolAsyncRunHandle struct { func (p *instancePool) runAsync(runCtx context.Context) (*poolAsyncRunHandle, error) { // Canceled in case all instances finish, fail or run runCancel. runCtx, runCancel := context.WithCancel(runCtx) + _ = runCancel // Canceled also on out of ammo, and finish of shared RPS schedule. instanceStartCtx, instanceStartCancel := context.WithCancel(runCtx) newInstanceSchedule, err := p.buildNewInstanceSchedule(instanceStartCtx, instanceStartCancel) @@ -194,11 +219,11 @@ func (p *instancePool) runAsync(runCtx context.Context) (*poolAsyncRunHandle, er runRes = make(chan instanceRunResult, runResultBufSize) ) go func() { - deps := core.ProviderDeps{p.log} + deps := core.ProviderDeps{Log: p.log, PoolID: p.ID} providerErr <- p.Provider.Run(runCtx, deps) }() go func() { - deps := core.AggregatorDeps{p.log} + deps := core.AggregatorDeps{Log: p.log} aggregatorErr <- p.Aggregator.Run(runCtx, deps) }() go func() { @@ -265,10 +290,6 @@ func (ah *runAwaitHandle) awaitRun() { if errutil.IsNotCtxError(ah.runCtx, err) { ah.onErrAwaited(errors.WithMessage(err, "provider failed")) } - if err == nil && !ah.isStartFinished() { - ah.log.Debug("Canceling instance start because out of ammo") - ah.instanceStartCancel() - } case err := <-ah.aggregatorErr: ah.aggregatorErr = nil ah.toWait-- @@ -288,10 +309,16 @@ func (ah *runAwaitHandle) awaitRun() { case res := <-ah.runRes: ah.awaitedInstances++ if ent := ah.log.Check(zap.DebugLevel, "Instance run awaited"); ent != nil { - ent.Write(zap.Int("id", res.Id), zap.Int("awaited", ah.awaitedInstances), zap.Error(res.Err)) + ent.Write(zap.Int("id", res.ID), zap.Int("awaited", ah.awaitedInstances), zap.Error(res.Err)) } - if errutil.IsNotCtxError(ah.runCtx, res.Err) { - ah.onErrAwaited(errors.WithMessage(res.Err, fmt.Sprintf("instance %q run failed", res.Id))) + + if res.Err == outOfAmmoErr { + if !ah.isStartFinished() { + ah.log.Debug("Canceling instance start because out of ammo") + ah.instanceStartCancel() + } + } else if errutil.IsNotCtxError(ah.runCtx, res.Err) { + ah.onErrAwaited(errors.WithMessage(res.Err, fmt.Sprintf("instance %q run failed", res.ID))) } ah.checkAllInstancesAreFinished() } @@ -336,10 +363,9 @@ func (p *instancePool) startInstances( newInstanceSchedule func() (core.Schedule, error), runRes chan<- instanceRunResult) (started int, err error) { deps := instanceDeps{ - p.Aggregator, newInstanceSchedule, p.NewGun, - instanceSharedDeps{p.Provider, p.metrics}, + instanceSharedDeps{p.Provider, p.metrics, p.gunWarmUpResult, p.Aggregator, p.DiscardOverflow}, } waiter := coreutil.NewWaiter(p.StartupSchedule, startCtx) @@ -350,20 +376,22 @@ func (p *instancePool) startInstances( err = startCtx.Err() return } - firstInstance, err := newInstance(runCtx, p.log, 0, deps) + firstInstance, err := newInstance(runCtx, p.log, p.ID, 0, deps) if err != nil { return } started++ go func() { - defer firstInstance.Close() - runRes <- instanceRunResult{0, firstInstance.Run(runCtx)} + runRes <- instanceRunResult{0, func() error { + defer firstInstance.Close() + return firstInstance.Run(runCtx) + }()} }() for ; waiter.Wait(); started++ { id := started go func() { - runRes <- instanceRunResult{id, runNewInstance(runCtx, p.log, id, deps)} + runRes <- instanceRunResult{id, runNewInstance(runCtx, p.log, p.ID, id, deps)} }() } err = startCtx.Err() @@ -398,8 +426,8 @@ func (p *instancePool) buildNewInstanceSchedule(startCtx context.Context, cancel return } -func runNewInstance(ctx context.Context, log *zap.Logger, id int, deps instanceDeps) error { - instance, err := newInstance(ctx, log, id, deps) +func runNewInstance(ctx context.Context, log *zap.Logger, poolID string, id int, deps instanceDeps) error { + instance, err := newInstance(ctx, log, poolID, id, deps) if err != nil { return err } @@ -408,12 +436,12 @@ func runNewInstance(ctx context.Context, log *zap.Logger, id int, deps instanceD } type poolRunResult struct { - Id string + ID string Err error } type instanceRunResult struct { - Id int + ID int Err error } diff --git a/core/engine/engine_test.go b/core/engine/engine_test.go index 187434156..5729b7456 100644 --- a/core/engine/engine_test.go +++ b/core/engine/engine_test.go @@ -9,8 +9,6 @@ import ( . "github.com/onsi/gomega" "github.com/pkg/errors" "github.com/stretchr/testify/mock" - "go.uber.org/atomic" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/aggregator" "github.com/yandex/pandora/core/config" @@ -18,6 +16,7 @@ import ( "github.com/yandex/pandora/core/provider" "github.com/yandex/pandora/core/schedule" "github.com/yandex/pandora/lib/ginkgoutil" + "go.uber.org/atomic" ) var _ = Describe("config validation", func() { @@ -213,6 +212,21 @@ var _ = Describe("multiple instance", func() { Expect(pool.metrics.InstanceStart.Get()).To(BeNumerically("<=", 3)) }, 1) + It("when provider run done it does not mean out of ammo; instance start is not canceled", func() { + conf, _ := newTestPoolConf() + conf.Provider = provider.NewNumBuffered(3) + conf.NewRPSSchedule = func() (core.Schedule, error) { + return schedule.NewOnce(1), nil + } + conf.StartupSchedule = schedule.NewOnce(3) + pool := newPool(ginkgoutil.NewLogger(), newTestMetrics(), nil, conf) + ctx := context.Background() + + err := pool.Run(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(pool.metrics.InstanceStart.Get()).To(BeNumerically("==", 3)) + }, 1) + It("out of RPS - instance start is canceled", func() { conf, _ := newTestPoolConf() conf.NewRPSSchedule = func() (core.Schedule, error) { diff --git a/core/engine/instance.go b/core/engine/instance.go index fe93b6c5f..fdcd0b6a1 100644 --- a/core/engine/instance.go +++ b/core/engine/instance.go @@ -7,14 +7,16 @@ package engine import ( "context" + "fmt" "io" "github.com/pkg/errors" - "go.uber.org/zap" - "github.com/yandex/pandora/core" + "github.com/yandex/pandora/core/aggregator/netsample" "github.com/yandex/pandora/core/coreutil" + "github.com/yandex/pandora/core/warmup" "github.com/yandex/pandora/lib/tag" + "go.uber.org/zap" ) type instance struct { @@ -25,9 +27,9 @@ type instance struct { instanceSharedDeps } -func newInstance(ctx context.Context, log *zap.Logger, id int, deps instanceDeps) (*instance, error) { +func newInstance(ctx context.Context, log *zap.Logger, poolID string, id int, deps instanceDeps) (*instance, error) { log = log.With(zap.Int("instance", id)) - gunDeps := core.GunDeps{ctx, log, id} + gunDeps := core.GunDeps{Ctx: ctx, Log: log, PoolID: poolID, InstanceID: id} sched, err := deps.newSchedule() if err != nil { return nil, err @@ -36,6 +38,11 @@ func newInstance(ctx context.Context, log *zap.Logger, id int, deps instanceDeps if err != nil { return nil, err } + if warmedUp, ok := gun.(warmup.WarmedUp); ok { + if err := warmedUp.AcceptWarmUpResult(deps.gunWarmUpResult); err != nil { + return nil, fmt.Errorf("gun failed to accept warmup result: %w", err) + } + } err = gun.Bind(deps.aggregator, gunDeps) if err != nil { return nil, err @@ -45,7 +52,6 @@ func newInstance(ctx context.Context, log *zap.Logger, id int, deps instanceDeps } type instanceDeps struct { - aggregator core.Aggregator newSchedule func() (core.Schedule, error) newGun func() (core.Gun, error) instanceSharedDeps @@ -54,6 +60,9 @@ type instanceDeps struct { type instanceSharedDeps struct { provider core.Provider Metrics + gunWarmUpResult interface{} + aggregator core.Aggregator + discardOverflow bool } // Run blocks until ammo finish, error or context cancel. @@ -81,24 +90,34 @@ func (i *instance) shoot(ctx context.Context) (err error) { // Checking, that schedule is not finished, required, to not consume extra ammo, // on finish in case of per instance schedule. for !waiter.IsFinished() { - ammo, ok := i.provider.Acquire() - if !ok { - i.log.Debug("Out of ammo") - break + err := func() error { + ammo, ok := i.provider.Acquire() + if !ok { + i.log.Debug("Out of ammo") + return outOfAmmoErr + } + defer i.provider.Release(ammo) + if tag.Debug { + i.log.Debug("Ammo acquired", zap.Any("ammo", ammo)) + } + if !waiter.Wait() { + return nil + } + if !i.discardOverflow || !waiter.IsSlowDown() { + i.Metrics.Request.Add(1) + if tag.Debug { + i.log.Debug("Shooting", zap.Any("ammo", ammo)) + } + i.gun.Shoot(ammo) + i.Metrics.Response.Add(1) + } else { + i.aggregator.Report(netsample.DiscardedShootSample()) + } + return nil + }() + if err != nil { + return err } - if tag.Debug { - i.log.Debug("Ammo acquired", zap.Any("ammo", ammo)) - } - if !waiter.Wait() { - break - } - i.Metrics.Request.Add(1) - if tag.Debug { - i.log.Debug("Shooting", zap.Any("ammo", ammo)) - } - i.gun.Shoot(ammo) - i.Metrics.Response.Add(1) - i.provider.Release(ammo) } return ctx.Err() } @@ -115,3 +134,5 @@ func (i *instance) Close() error { i.log.Debug("Gun closed") return err } + +var outOfAmmoErr = errors.New("Out of ammo") diff --git a/core/engine/instance_test.go b/core/engine/instance_test.go index 23629705d..029d93aa6 100644 --- a/core/engine/instance_test.go +++ b/core/engine/instance_test.go @@ -8,7 +8,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/stretchr/testify/mock" - "github.com/yandex/pandora/core" coremock "github.com/yandex/pandora/core/mocks" "github.com/yandex/pandora/core/schedule" @@ -48,15 +47,18 @@ var _ = Describe("Instance", func() { JustBeforeEach(func() { deps := instanceDeps{ - aggregator, + newSchedule, newGun, instanceSharedDeps{ provider, metrics, + nil, + aggregator, + false, }, } - ins, insCreateErr = newInstance(ctx, ginkgoutil.NewLogger(), 0, deps) + ins, insCreateErr = newInstance(ctx, ginkgoutil.NewLogger(), "pool_0", 0, deps) }) AfterEach(func() { @@ -113,7 +115,7 @@ var _ = Describe("Instance", func() { Context("context canceled after run", func() { BeforeEach(func() { - ctx, _ = context.WithTimeout(ctx, 10*time.Millisecond) + ctx, _ = context.WithTimeout(context.Background(), 10*time.Millisecond) sched := sched.(*coremock.Schedule) sched.On("Next").Return(time.Now().Add(5*time.Second), true) sched.On("Left").Return(1) diff --git a/core/import/import.go b/core/import/import.go index 3bf142802..87f88255c 100644 --- a/core/import/import.go +++ b/core/import/import.go @@ -9,8 +9,6 @@ import ( "reflect" "github.com/spf13/afero" - "go.uber.org/zap" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/aggregator" "github.com/yandex/pandora/core/aggregator/netsample" @@ -22,7 +20,9 @@ import ( "github.com/yandex/pandora/core/provider" "github.com/yandex/pandora/core/register" "github.com/yandex/pandora/core/schedule" + "github.com/yandex/pandora/lib/confutil" "github.com/yandex/pandora/lib/tag" + "go.uber.org/zap" ) const ( @@ -30,6 +30,11 @@ const ( compositeScheduleKey = "composite" ) +// getter for fs to avoid afero dependency in custom guns +func GetFs() afero.Fs { + return afero.NewOsFs() +} + func Import(fs afero.Fs) { register.DataSink(fileDataKey, func(conf datasink.FileConfig) core.DataSink { @@ -82,11 +87,16 @@ func Import(fs afero.Fs) { register.Limiter("const", schedule.NewConstConf) register.Limiter("once", schedule.NewOnceConf) register.Limiter("unlimited", schedule.NewUnlimitedConf) + register.Limiter("step", schedule.NewStepConf) + register.Limiter("instance_step", schedule.NewInstanceStepConf) register.Limiter(compositeScheduleKey, schedule.NewCompositeConf) config.AddTypeHook(sinkStringHook) config.AddTypeHook(scheduleSliceToCompositeConfigHook) + confutil.RegisterTagResolver("", confutil.EnvTagResolver) + confutil.RegisterTagResolver("ENV", confutil.EnvTagResolver) + // Required for decoding plugins. Need to be added after Composite Schedule hacky hook. pluginconfig.AddHooks() } diff --git a/core/import/import_suite_test.go b/core/import/import_suite_test.go index a39a2163d..a83785439 100644 --- a/core/import/import_suite_test.go +++ b/core/import/import_suite_test.go @@ -10,14 +10,13 @@ import ( . "github.com/onsi/gomega" "github.com/spf13/afero" "github.com/stretchr/testify/require" - "go.uber.org/zap" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/config" "github.com/yandex/pandora/core/coretest" "github.com/yandex/pandora/core/plugin" "github.com/yandex/pandora/lib/ginkgoutil" "github.com/yandex/pandora/lib/testutil" + "go.uber.org/zap" ) func TestImport(t *testing.T) { @@ -121,7 +120,7 @@ func TestProviderJSONLine(t *testing.T) { conf.Aggregator.Report([]int{0, 1, 2}) ctx, cancel := context.WithCancel(context.Background()) cancel() - err = conf.Aggregator.Run(ctx, core.AggregatorDeps{zap.L()}) + err = conf.Aggregator.Run(ctx, core.AggregatorDeps{Log: zap.L()}) require.NoError(t, err) testutil.AssertFileEqual(t, fs, filename, "[0,1,2]\n") diff --git a/core/mocks/aggregator.go b/core/mocks/aggregator.go index ecead2740..b325dcbc9 100644 --- a/core/mocks/aggregator.go +++ b/core/mocks/aggregator.go @@ -1,9 +1,12 @@ // Code generated by mockery v1.0.0 package coremock -import "context" -import "github.com/yandex/pandora/core" -import "github.com/stretchr/testify/mock" +import ( + "context" + + "github.com/stretchr/testify/mock" + "github.com/yandex/pandora/core" +) // Aggregator is an autogenerated mock type for the Aggregator type type Aggregator struct { diff --git a/core/mocks/data_sink.go b/core/mocks/data_sink.go index ada729f56..24ed8be47 100644 --- a/core/mocks/data_sink.go +++ b/core/mocks/data_sink.go @@ -1,8 +1,11 @@ // Code generated by mockery v1.0.0 package coremock -import io "io" -import mock "github.com/stretchr/testify/mock" +import ( + io "io" + + mock "github.com/stretchr/testify/mock" +) // DataSink is an autogenerated mock type for the DataSink type type DataSink struct { diff --git a/core/mocks/data_source.go b/core/mocks/data_source.go index 854c923c0..e89eb4855 100644 --- a/core/mocks/data_source.go +++ b/core/mocks/data_source.go @@ -1,8 +1,11 @@ // Code generated by mockery v1.0.0 package coremock -import io "io" -import mock "github.com/stretchr/testify/mock" +import ( + io "io" + + mock "github.com/stretchr/testify/mock" +) // DataSource is an autogenerated mock type for the DataSource type type DataSource struct { diff --git a/core/mocks/gun.go b/core/mocks/gun.go index c93ff36b2..aa69c41d7 100644 --- a/core/mocks/gun.go +++ b/core/mocks/gun.go @@ -1,8 +1,10 @@ // Code generated by mockery v1.0.0 package coremock -import core "github.com/yandex/pandora/core" -import mock "github.com/stretchr/testify/mock" +import ( + mock "github.com/stretchr/testify/mock" + core "github.com/yandex/pandora/core" +) // Gun is an autogenerated mock type for the Gun type type Gun struct { diff --git a/core/mocks/provider.go b/core/mocks/provider.go index d94bd028a..fd3cb0b1e 100644 --- a/core/mocks/provider.go +++ b/core/mocks/provider.go @@ -1,9 +1,12 @@ // Code generated by mockery v1.0.0 package coremock -import context "context" -import core "github.com/yandex/pandora/core" -import mock "github.com/stretchr/testify/mock" +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + core "github.com/yandex/pandora/core" +) // Provider is an autogenerated mock type for the Provider type type Provider struct { diff --git a/core/mocks/schedule.go b/core/mocks/schedule.go index 291b4554c..080509065 100644 --- a/core/mocks/schedule.go +++ b/core/mocks/schedule.go @@ -1,8 +1,11 @@ // Code generated by mockery v1.0.0 package coremock -import mock "github.com/stretchr/testify/mock" -import time "time" +import ( + time "time" + + mock "github.com/stretchr/testify/mock" +) // Schedule is an autogenerated mock type for the Schedule type type Schedule struct { diff --git a/core/plugin/pluginconfig/hooks.go b/core/plugin/pluginconfig/hooks.go index fe669bb49..4c3e44106 100644 --- a/core/plugin/pluginconfig/hooks.go +++ b/core/plugin/pluginconfig/hooks.go @@ -14,11 +14,10 @@ import ( "strings" "github.com/pkg/errors" - "go.uber.org/zap" - "github.com/yandex/pandora/core/config" "github.com/yandex/pandora/core/plugin" "github.com/yandex/pandora/lib/tag" + "go.uber.org/zap" ) func AddHooks() { diff --git a/core/plugin/ptest_test.go b/core/plugin/ptest_test.go index cfae16f84..69850d285 100644 --- a/core/plugin/ptest_test.go +++ b/core/plugin/ptest_test.go @@ -10,7 +10,6 @@ import ( . "github.com/onsi/gomega" "github.com/pkg/errors" - "github.com/yandex/pandora/core/config" ) diff --git a/core/provider/chunk_decoder.go b/core/provider/chunk_decoder.go index 0904b4497..2fd851ca7 100644 --- a/core/provider/chunk_decoder.go +++ b/core/provider/chunk_decoder.go @@ -10,7 +10,6 @@ import ( "fmt" "github.com/pkg/errors" - "github.com/yandex/pandora/core" ) diff --git a/core/provider/decoder.go b/core/provider/decoder.go index f6c8735c4..01a923c38 100644 --- a/core/provider/decoder.go +++ b/core/provider/decoder.go @@ -11,11 +11,10 @@ import ( "io" "github.com/pkg/errors" - "go.uber.org/zap" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/lib/errutil" "github.com/yandex/pandora/lib/ioutil2" + "go.uber.org/zap" ) type NewAmmoDecoder func(deps core.ProviderDeps, source io.Reader) (AmmoDecoder, error) @@ -77,7 +76,7 @@ func (p *DecodeProvider) Run(ctx context.Context, deps core.ProviderDeps) (err e return errors.WithMessage(err, "data source open failed") } defer func() { - errutil.Join(err, errors.Wrap(source.Close(), "data source close failed")) + _ = errutil.Join(err, errors.Wrap(source.Close(), "data source close failed")) }() // Problem: can't use decoder after io.EOF, because decoder is invalidated. But decoder recreation diff --git a/core/provider/json.go b/core/provider/json.go index 586935c58..820a991f0 100644 --- a/core/provider/json.go +++ b/core/provider/json.go @@ -9,7 +9,6 @@ import ( "io" jsoniter "github.com/json-iterator/go" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/coreutil" "github.com/yandex/pandora/lib/ioutil2" diff --git a/core/provider/json_test.go b/core/provider/json_test.go index eca80711a..1690d0086 100644 --- a/core/provider/json_test.go +++ b/core/provider/json_test.go @@ -13,13 +13,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/datasource" ) type testJSONAmmo struct { - Id string + ID string Data string } @@ -158,12 +157,12 @@ func TestDecoderWhitespaces(t *testing.T) { func TestDecoderReset(t *testing.T) { val := testJSONAmmo{ - Id: "id", + ID: "id", } input := strings.NewReader(`{"data":"first"}`) decoder := NewJSONAmmoDecoder(input, 512) err := decoder.Decode(&val) require.NoError(t, err) assert.Equal(t, "first", val.Data) - assert.Zero(t, val.Id) + assert.Zero(t, val.ID) } diff --git a/core/provider/num.go b/core/provider/num.go index 401ca41af..772153b76 100644 --- a/core/provider/num.go +++ b/core/provider/num.go @@ -20,6 +20,13 @@ func NewNum(limit int) core.Provider { } } +func NewNumBuffered(limit int) core.Provider { + return &num{ + limit: limit, + sink: make(chan core.Ammo, limit), + } +} + type NumConfig struct { Limit int } diff --git a/core/provider/num_test.go b/core/provider/num_test.go index a7722cb67..caf7242f6 100644 --- a/core/provider/num_test.go +++ b/core/provider/num_test.go @@ -5,7 +5,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/yandex/pandora/core" ) diff --git a/core/provider/provider_suite_test.go b/core/provider/provider_suite_test.go index 9c500eede..1de38db7a 100644 --- a/core/provider/provider_suite_test.go +++ b/core/provider/provider_suite_test.go @@ -12,5 +12,5 @@ func TestProvider(t *testing.T) { } func testDeps() core.ProviderDeps { - return core.ProviderDeps{ginkgoutil.NewLogger()} + return core.ProviderDeps{Log: ginkgoutil.NewLogger()} } diff --git a/core/schedule/composite_test.go b/core/schedule/composite_test.go index e4e0dc86c..67c053837 100644 --- a/core/schedule/composite_test.go +++ b/core/schedule/composite_test.go @@ -11,7 +11,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/coretest" "go.uber.org/atomic" diff --git a/core/schedule/do_at.go b/core/schedule/do_at.go index e33b8ae23..dddcaf696 100644 --- a/core/schedule/do_at.go +++ b/core/schedule/do_at.go @@ -8,9 +8,8 @@ package schedule import ( "time" - "go.uber.org/atomic" - "github.com/yandex/pandora/core" + "go.uber.org/atomic" ) // DoAt returns when i'th operation should be performed, assuming that schedule diff --git a/core/schedule/instance_step.go b/core/schedule/instance_step.go new file mode 100644 index 000000000..9724854ef --- /dev/null +++ b/core/schedule/instance_step.go @@ -0,0 +1,35 @@ +// Copyright (c) 2017 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package schedule + +import ( + "time" + + "github.com/yandex/pandora/core" +) + +func NewInstanceStep(from, to int64, step int64, stepDuration time.Duration) core.Schedule { + var nexts []core.Schedule + nexts = append(nexts, NewOnce(from)) + + for i := from + step; i <= to; i += step { + nexts = append(nexts, NewConst(0, stepDuration)) + nexts = append(nexts, NewOnce(step)) + } + + return NewCompositeConf(CompositeConf{nexts}) +} + +type InstanceStepConfig struct { + From int64 `validate:"min=0"` + To int64 `validate:"min=0"` + Step int64 `validate:"min=1"` + StepDuration time.Duration `validate:"min-time=1ms"` +} + +func NewInstanceStepConf(conf InstanceStepConfig) core.Schedule { + return NewInstanceStep(conf.From, conf.To, conf.Step, conf.StepDuration) +} diff --git a/core/schedule/schedule_suite_test.go b/core/schedule/schedule_suite_test.go index 0658482bd..aea7fe558 100644 --- a/core/schedule/schedule_suite_test.go +++ b/core/schedule/schedule_suite_test.go @@ -7,7 +7,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/yandex/pandora/core" "github.com/yandex/pandora/core/coretest" "github.com/yandex/pandora/lib/ginkgoutil" @@ -192,6 +191,36 @@ var _ = Describe("line", func() { }) +var _ = Describe("step", func() { + It("", func() { + conf := StepConfig{ + From: 1, + To: 2, + Step: 1, + Duration: 2 * time.Second, + } + testee := NewStepConf(conf) + Expect(testee.Left()).To(Equal(6)) + + }) + +}) + +var _ = Describe("instance_step", func() { + It("", func() { + conf := InstanceStepConfig{ + From: 1, + To: 3, + Step: 1, + StepDuration: 2 * time.Second, + } + testee := NewInstanceStepConf(conf) + Expect(testee.Left()).To(Equal(3)) + + }) + +}) + func BenchmarkLineSchedule(b *testing.B) { schedule := NewLine(0, float64(b.N), 2*time.Second) benchmarkScheduleNext(b, schedule) diff --git a/core/schedule/start_sync.go b/core/schedule/start_sync.go index 4b2b5215f..43f35682b 100644 --- a/core/schedule/start_sync.go +++ b/core/schedule/start_sync.go @@ -6,9 +6,9 @@ package schedule import ( - "go.uber.org/atomic" - "sync" + + "go.uber.org/atomic" ) // StartSync is util to make schedule start goroutine safe. diff --git a/core/schedule/step.go b/core/schedule/step.go new file mode 100644 index 000000000..7f48598ca --- /dev/null +++ b/core/schedule/step.go @@ -0,0 +1,37 @@ +// Copyright (c) 2017 Yandex LLC. All rights reserved. +// Use of this source code is governed by a MPL 2.0 +// license that can be found in the LICENSE file. +// Author: Vladimir Skipor + +package schedule + +import ( + "time" + + "github.com/yandex/pandora/core" +) + +func NewStep(from, to float64, step int64, duration time.Duration) core.Schedule { + var nexts []core.Schedule + + if from == to { + return NewConst(from, duration) + } + + for i := from; i <= to; i += float64(step) { + nexts = append(nexts, NewConst(i, duration)) + } + + return NewCompositeConf(CompositeConf{nexts}) +} + +type StepConfig struct { + From float64 `validate:"min=0"` + To float64 `validate:"min=0"` + Step int64 `validate:"min=1"` + Duration time.Duration `validate:"min-time=1ms"` +} + +func NewStepConf(conf StepConfig) core.Schedule { + return NewStep(conf.From, conf.To, conf.Step, conf.Duration) +} diff --git a/core/warmup/interface.go b/core/warmup/interface.go new file mode 100644 index 000000000..6438074bf --- /dev/null +++ b/core/warmup/interface.go @@ -0,0 +1,6 @@ +package warmup + +type WarmedUp interface { + WarmUp(*Options) (interface{}, error) + AcceptWarmUpResult(interface{}) error +} diff --git a/core/warmup/options.go b/core/warmup/options.go new file mode 100644 index 000000000..a09d8224a --- /dev/null +++ b/core/warmup/options.go @@ -0,0 +1,12 @@ +package warmup + +import ( + "context" + + "go.uber.org/zap" +) + +type Options struct { + Log *zap.Logger + Ctx context.Context +} diff --git a/docs/advanced.rst b/docs/advanced.rst index 03d2d0a52..25e843242 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -18,7 +18,7 @@ Pay attention to special header `Host` defined ``outside`` of Headers dictionary Ammofile sample: :: - {"uri": "/", "method": "GET", "headers": {"Accept": "*/*", "Accept-Encoding": "gzip, deflate", "User-Agent": "Pandora"}, "host": "example.com"} + {"tag": "tag1", "uri": "/", "method": "GET", "headers": {"Accept": "*/*", "Accept-Encoding": "gzip, deflate", "User-Agent": "Pandora"}, "host": "example.com"} Config sample: @@ -70,7 +70,7 @@ Config sample: type: raw # ammo format file: ./ammofile # ammo file path -You can redefine any headers using special config option `headers`. Format: list of strings. +You can define common headers using special config option `headers`. Headers in ammo file have priority. Format: list of strings. Example: @@ -111,7 +111,7 @@ Config sample: file: ./ammofile # ammo file path -You can redefine any headers using special config option `headers`. Format: list of strings. +You can define common headers using special config option `headers`. Headers in ammo file have priority. Format: list of strings. Example: @@ -124,3 +124,32 @@ Example: headers: - "[Host: yourhost.tld]" - "[User-Agent: some user agent]" + +Ammo filters +------------ + +Each http ammo provider lets you choose specific ammo for your test from ammo file with `chosencases` setting: +.. code-block:: yaml + + pools: + - ammo: + type: uri # ammo format + chosencases: ["tag1", "tag2"] # use only "tag1" and "tag2" ammo for this test + file: ./ammofile # ammo file path + +Tags are defined in ammo files as shown below: + +http/json: +:: + {"tag": "tag1", "uri": "/", + +raw (request-style): +:: + 73 tag1 + GET / HTTP/1.0 + +uri-style: +:: + /?drg tag1 + / + /buy tag2 \ No newline at end of file diff --git a/docs/custom.rst b/docs/custom.rst index c72c1a537..31a78f8cf 100644 --- a/docs/custom.rst +++ b/docs/custom.rst @@ -361,7 +361,7 @@ Websockets } else { code = 200 } - defer func() { + func() { sample.SetProtoCode(code) g.aggr.Report(sample) }() diff --git a/docs/guns.rst b/docs/guns.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/index.rst b/docs/index.rst index ccb6a6d76..d7273bdce 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,7 +15,6 @@ Pandora is a high-performance load generator in Go language. It has built-in HTT install tutorial advanced - guns custom performance architecture diff --git a/docs/tutorial.rst b/docs/tutorial.rst index e5db5e0b0..d4aff36b8 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -17,7 +17,7 @@ Pandora supports config files in `YAML`_ format. Create a new file named ``load. type: http # gun type target: example.com:80 # gun target ammo: - type: uri # ammo format + type: uri # ammo format file: ./ammo.uri # ammo File result: type: phout # report format (phout is compatible with Yandex.Tank) @@ -77,5 +77,5 @@ References .. target-notes:: .. _`Overload`: https://overload.yandex.net -.. _`Yandex.Tank`: http://yandextank.readthedocs.org/en/latest/configuration.html#pandora -.. _`YAML`: https://en.wikipedia.org/wiki/YAML \ No newline at end of file +.. _`Yandex.Tank`: https://yandextank.readthedocs.io/en/latest/core_and_modules.html#pandora +.. _`YAML`: https://en.wikipedia.org/wiki/YAML diff --git a/examples/custom_pandora/custom_main.go b/examples/custom_pandora/custom_main.go index b5304501d..c25569f85 100644 --- a/examples/custom_pandora/custom_main.go +++ b/examples/custom_pandora/custom_main.go @@ -10,13 +10,12 @@ import ( "time" "github.com/spf13/afero" - "go.uber.org/zap" - "github.com/yandex/pandora/cli" phttp "github.com/yandex/pandora/components/phttp/import" "github.com/yandex/pandora/core" coreimport "github.com/yandex/pandora/core/import" "github.com/yandex/pandora/core/register" + "go.uber.org/zap" ) type Ammo struct { diff --git a/go.mod b/go.mod index 838b874e3..ebade8e63 100644 --- a/go.mod +++ b/go.mod @@ -1,45 +1,52 @@ module github.com/yandex/pandora +go 1.17 + require ( - github.com/BurntSushi/toml v0.3.1 // indirect github.com/asaskevich/govalidator v0.0.0-20171111151018-521b25f4b05f github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae - github.com/davecgh/go-spew v1.1.0 // indirect github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 github.com/facebookgo/stackerr v0.0.0-20150612192056-c2fcf88613f4 - github.com/fatih/structs v1.0.0 - github.com/fsnotify/fsnotify v1.4.7 // indirect github.com/ghodss/yaml v1.0.0 - github.com/go-playground/locales v0.11.2 // indirect - github.com/go-playground/universal-translator v0.16.0 // indirect - github.com/golang/protobuf v1.3.1 // indirect - github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 - github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb // indirect + github.com/jhump/protoreflect v1.10.1 github.com/json-iterator/go v0.0.0-20180214060632-e7c7f3b33712 - github.com/kr/pretty v0.1.0 // indirect - github.com/magiconair/properties v1.7.6 github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047 github.com/onsi/ginkgo v1.4.0 github.com/onsi/gomega v1.3.0 - github.com/pelletier/go-toml v1.1.0 // indirect - github.com/pkg/errors v0.8.0 - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pkg/errors v0.9.0 github.com/pquerna/ffjson v0.0.0-20171002144729-d49c2bc1aa13 github.com/spf13/afero v1.0.2 + github.com/spf13/viper v1.0.0 + github.com/stretchr/testify v1.7.0 + go.uber.org/atomic v1.3.1 + go.uber.org/zap v1.7.1 + golang.org/x/net v0.0.0-20200822124328-c89045814202 + google.golang.org/grpc v1.41.0 + gopkg.in/bluesuncorp/validator.v9 v9.10.0 +) + +require ( + github.com/davecgh/go-spew v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/go-playground/locales v0.11.2 // indirect + github.com/go-playground/universal-translator v0.16.0 // indirect + github.com/golang/protobuf v1.4.3 // indirect + github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect + github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb // indirect + github.com/magiconair/properties v1.8.6 // indirect + github.com/pelletier/go-toml v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/cast v1.2.0 // indirect github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec // indirect github.com/spf13/pflag v1.0.0 // indirect - github.com/spf13/viper v1.0.0 github.com/stretchr/objx v0.1.0 // indirect - github.com/stretchr/testify v1.2.1 - github.com/uber-go/atomic v1.3.0 - go.uber.org/atomic v1.3.1 go.uber.org/multierr v1.1.0 // indirect - go.uber.org/zap v1.7.1 - golang.org/x/net v0.0.0-20190311183353-d8887717615a - gopkg.in/bluesuncorp/validator.v9 v9.10.0 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25 // indirect + golang.org/x/text v0.3.0 // indirect + google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect + google.golang.org/protobuf v1.25.1-0.20200805231151-a709e31e5d12 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect - gopkg.in/yaml.v2 v2.0.0 // indirect + gopkg.in/yaml.v2 v2.2.3 // indirect + gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) diff --git a/go.sum b/go.sum index 6c128cc18..c1e357636 100644 --- a/go.sum +++ b/go.sum @@ -1,56 +1,98 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/asaskevich/govalidator v0.0.0-20171111151018-521b25f4b05f h1:xHxhygLkJBQaXZ7H0JUpmqK/gfKO2DZXB7gAKT6bbBs= github.com/asaskevich/govalidator v0.0.0-20171111151018-521b25f4b05f/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae h1:2Zmk+8cNvAGuY8AyvZuWpUdpQUAXwfom4ReVMe/CTIo= github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= github.com/facebookgo/stackerr v0.0.0-20150612192056-c2fcf88613f4 h1:fP04zlkPjAGpsduG7xN3rRkxjAqkJaIQnnkNYYw/pAk= github.com/facebookgo/stackerr v0.0.0-20150612192056-c2fcf88613f4/go.mod h1:SBHk9aNQtiw4R4bEuzHjVmZikkUKCnO1v3lPQ21HZGk= -github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= -github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-playground/locales v0.11.2 h1:wH6Ksuvzk0SU9M6wUeGz/EaRWnavAHCOsFre1njzgi8= github.com/go-playground/locales v0.11.2/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874 h1:em+tTnzgU7N22woTBMcSJAOW7tRHAkK597W+MD/CpK8= github.com/hashicorp/go-multierror v0.0.0-20171204182908-b7773ae21874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb h1:1OvvPvZkn/yCQ3xBcM8y4020wdkMXPHLB4+NfoGWh4U= github.com/hashicorp/hcl v0.0.0-20171017181929-23c074d0eceb/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= +github.com/jhump/protoreflect v1.10.1 h1:iH+UZfsbRE6vpyZH7asAjTPWJf7RJbpZ9j/N3lDlKs0= +github.com/jhump/protoreflect v1.10.1/go.mod h1:7GcYQDdMU/O/BBrl/cX6PNHpXh6cenjd8pneu5yW7Tg= github.com/json-iterator/go v0.0.0-20180214060632-e7c7f3b33712 h1:nANBg0vxBeNql2DGe4fxaAGskRFEWtXg5cgWJYuHQ14= github.com/json-iterator/go v0.0.0-20180214060632-e7c7f3b33712/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.7.6 h1:U+1DqNen04MdEPgFiIwdOUiqZ8qPa37xgogX/sd3+54= -github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= +github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047 h1:zCoDWFD5nrJJVjbXiDZcVhOBSzKn3o9LgRLLMRNuru8= github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/nishanths/predeclared v0.0.0-20200524104333-86fad755b4d3/go.mod h1:nt3d53pc1VYcphSCIaYAJtnPYnr3Zyn8fMq2wvPGPso= github.com/onsi/ginkgo v1.4.0 h1:n60/4GZK0Sr9O2iuGKq876Aoa0ER2ydgpMOBwzJ8e2c= github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.3.0 h1:yPHEatyQC4jN3vdfvqJXG7O9vfC6LhaAV1NEdYpP+h0= github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/pelletier/go-toml v1.1.0 h1:cmiOvKzEunMsAxyhXSzpL5Q1CRKpVv0KQsnAIcSEVYM= github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.0 h1:J8lpUdobwIeCI7OiSxHqEwJUKvJwicL5+3v1oe2Yb4k= +github.com/pkg/errors v0.9.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/ffjson v0.0.0-20171002144729-d49c2bc1aa13 h1:AUK/hm/tPsiNNASdb3J8fySVRZoI7fnK5mlOvdFD43o= github.com/pquerna/ffjson v0.0.0-20171002144729-d49c2bc1aa13/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/spf13/afero v1.0.2 h1:5bRmqmInNmNFkI9NG9O0Xc/Lgl9wOWWUUA/O8XZqTCo= github.com/spf13/afero v1.0.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg= @@ -63,10 +105,12 @@ github.com/spf13/viper v1.0.0 h1:RUA/ghS2i64rlnn4ydTfblY8Og8QzcPtCcHvgMn+w/I= github.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U= -github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/uber-go/atomic v1.3.0 h1:ylWoWcs+jXihgo3Us1Sdsatf2R6+OlBGm8fexR3oFG4= -github.com/uber-go/atomic v1.3.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.1 h1:U8WaWEmp56LGz7PReduqHRVF6zzs9GbMC2NEZ42dxSQ= go.uber.org/atomic v1.3.1/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= @@ -74,17 +118,96 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/zap v1.7.1 h1:wKPciimwkIgV4Aag/wpSDzvtO5JrfwdHKHO7blTHx7Q= go.uber.org/zap v1.7.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25 h1:nwzwVf0l2Y/lkov/+IYgMMbFyI+QypZDds9RxlSmsFQ= +golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.25.1-0.20200805231151-a709e31e5d12 h1:OwhZOOMuf7leLaSCuxtQ9FW7ui2L2L6UKOtKAUqovUQ= +google.golang.org/protobuf v1.25.1-0.20200805231151-a709e31e5d12/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/bluesuncorp/validator.v9 v9.10.0 h1:eyhz/IzFglUqngYr1p7WCfKVAAQh9E/IsqJcnwG/OWg= gopkg.in/bluesuncorp/validator.v9 v9.10.0/go.mod h1:sz1RrKEIYJCpC5S6ruDsBWo5vYV69E+bEZ86LbUsSZ8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/yaml.v2 v2.0.0 h1:uUkhRGrsEyx/laRdeS6YIQKIys8pg+lRSRdVMTYjivs= -gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= diff --git a/lib/answlog/logger.go b/lib/answlog/logger.go new file mode 100644 index 000000000..28720a1c5 --- /dev/null +++ b/lib/answlog/logger.go @@ -0,0 +1,26 @@ +package answlog + +import ( + "os" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func Init(path string) *zap.Logger { + writerSyncer := getAnswWriter(path) + encoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) + core := zapcore.NewCore(encoder, writerSyncer, zapcore.DebugLevel) + + Log := zap.New(core) + defer Log.Sync() + return Log +} + +func getAnswWriter(path string) zapcore.WriteSyncer { + if path == "" { + path = "./answ.log" + } + file, _ := os.Create(path) + return zapcore.AddSync(file) +} diff --git a/lib/confutil/chosen_cases_filter.go b/lib/confutil/chosen_cases_filter.go new file mode 100644 index 000000000..bd1619a5c --- /dev/null +++ b/lib/confutil/chosen_cases_filter.go @@ -0,0 +1,15 @@ +package confutil + +// Creates filter that returns true if ammo tag is in chosenCases. If no chosenCases provided - returns true +func IsChosenCase(checkCase string, chosenCases []string) bool { + if len(chosenCases) == 0 { + return true + } + + for _, c := range chosenCases { + if c == checkCase { + return true + } + } + return false +} diff --git a/lib/confutil/chosen_cases_filter_test.go b/lib/confutil/chosen_cases_filter_test.go new file mode 100644 index 000000000..c73a42c69 --- /dev/null +++ b/lib/confutil/chosen_cases_filter_test.go @@ -0,0 +1,25 @@ +package confutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestChosenCases(t *testing.T) { + type testCase struct { + ammoTag string + expected bool + } + + cases := []string{"tag1", "tag3"} + + tests := []testCase{ + {"tag1", true}, + {"tag2", false}, + } + + for _, tc := range tests { + assert.Equal(t, tc.expected, IsChosenCase(tc.ammoTag, cases)) + } +} diff --git a/lib/confutil/custom_tag_resolver.go b/lib/confutil/custom_tag_resolver.go new file mode 100644 index 000000000..8ab249e67 --- /dev/null +++ b/lib/confutil/custom_tag_resolver.go @@ -0,0 +1,180 @@ +package confutil + +import ( + "errors" + "fmt" + "reflect" + "regexp" + "strconv" + "strings" +) + +var ( + notoken = "" + ErrNoTagsFound = errors.New("no tags found") + ErrUnsupportedKind = errors.New("unsupported kind") + ErrCantCastVariableToTargetType = errors.New("can't cast variable") + ErrResolverNotRegistered = errors.New("unknown tag type") +) + +type TagResolver func(string) (string, error) + +type tagEntry struct { + tagType string + string string + varname string +} + +var resolvers map[string]TagResolver = make(map[string]TagResolver) + +// Register custom tag resolver for config variables +func RegisterTagResolver(tagType string, resolver TagResolver) { + tagType = strings.ToLower(tagType) + // silent overwrite existing resolver + resolvers[tagType] = resolver +} + +func getTagResolver(tagType string) (TagResolver, error) { + tagType = strings.ToLower(tagType) + r, ok := resolvers[tagType] + if !ok { + return nil, ErrResolverNotRegistered + } + return r, nil +} + +// Resolve config variables in format ${tagType:variable} +func ResolveCustomTags(s string, targetType reflect.Type) (interface{}, error) { + tokens, err := findTags(s) + if err != nil { + return nil, err + } + + if len(tokens) == 0 { + return s, ErrNoTagsFound + } + + res := s + for _, t := range tokens { + resolver, err := getTagResolver(t.tagType) + if err == ErrResolverNotRegistered { + continue + } else if err != nil { + return nil, err + } + + resolved, err := resolver(t.varname) + if err != nil { + return nil, err + } + res = strings.ReplaceAll(res, t.string, resolved) + } + + // try to cast result to target type, because mapstructure will not attempt to cast strings to bool, int and floats + // if target type is unknown, we still let other hooks process result (time.Duration, ipv4 and other hooks will do) + if len(tokens) == 1 && strings.TrimSpace(s) == tokens[0].string { + castedRes, err := cast(res, targetType) + if err == nil || !errors.Is(err, ErrCantCastVariableToTargetType) { + return castedRes, err + } + } + return res, nil +} + +func findTags(s string) ([]*tagEntry, error) { + tagRegexp := regexp.MustCompile(`\$\{(?:([^}]+?):)?([^{}]+?)\}`) + tokensFound := tagRegexp.FindAllStringSubmatch(s, -1) + result := make([]*tagEntry, 0, len(tokensFound)) + + for _, token := range tokensFound { + tag := &tagEntry{ + tagType: strings.TrimSpace(token[1]), + varname: strings.TrimSpace(token[2]), + string: token[0], + } + result = append(result, tag) + } + + return result, nil +} + +func cast(v string, t reflect.Type) (interface{}, error) { + switch t.Kind() { + case reflect.Bool: + return castBool(v) + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64, + reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64: + return castInt(v, t) + case reflect.Float32, + reflect.Float64: + return castFloat(v, t) + case reflect.String: + return v, nil + } + return nil, ErrUnsupportedKind +} + +func castBool(v string) (interface{}, error) { + res, err := strconv.ParseBool(v) + if err != nil { + return false, fmt.Errorf("'%s' cast to bool failed: %w", v, ErrCantCastVariableToTargetType) + } + + return res, nil +} + +func castInt(v string, t reflect.Type) (interface{}, error) { + intV, err := strconv.ParseInt(v, 0, t.Bits()) + if err != nil { + return nil, fmt.Errorf("'%s' cast to %s failed: %w", v, t, ErrCantCastVariableToTargetType) + } + + switch t.Kind() { + case reflect.Int: + return int(intV), nil + case reflect.Int8: + return int8(intV), nil + case reflect.Int16: + return int16(intV), nil + case reflect.Int32: + return int32(intV), nil + case reflect.Int64: + return int64(intV), nil + case reflect.Uint: + return uint(intV), nil + case reflect.Uint8: + return uint8(intV), nil + case reflect.Uint16: + return uint16(intV), nil + case reflect.Uint32: + return uint32(intV), nil + case reflect.Uint64: + return uint64(intV), nil + } + + return nil, ErrUnsupportedKind +} + +func castFloat(v string, t reflect.Type) (interface{}, error) { + intV, err := strconv.ParseFloat(v, 64) + if err != nil { + return 0.0, fmt.Errorf("'%s' cast to %s failed: %w", v, t, ErrCantCastVariableToTargetType) + } + + switch t.Kind() { + case reflect.Float32: + return float32(intV), nil + case reflect.Float64: + return float64(intV), nil + } + + return nil, ErrUnsupportedKind +} diff --git a/lib/confutil/custom_tag_resolver_test.go b/lib/confutil/custom_tag_resolver_test.go new file mode 100644 index 000000000..5a887d580 --- /dev/null +++ b/lib/confutil/custom_tag_resolver_test.go @@ -0,0 +1,122 @@ +package confutil + +import ( + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStringToExpectedCast(t *testing.T) { + type testCase struct { + val string + expected interface{} + err error + } + + tests := []testCase{ + {"True", true, nil}, + {"T", true, nil}, + {"t", true, nil}, + {"TRUE", true, nil}, + {"true", true, nil}, + {"1", true, nil}, + {"False", false, nil}, + {"false", false, nil}, + {"0", false, nil}, + {"f", false, nil}, + {"", false, ErrCantCastVariableToTargetType}, + + {"11", uint(11), nil}, + {"10", uint8(10), nil}, + {"10", uint16(10), nil}, + {"10", uint32(10), nil}, + {"10", uint64(10), nil}, + {"11", int(11), nil}, + {"10", int8(10), nil}, + {"10", int16(10), nil}, + {"10", int32(10), nil}, + {"10", int64(10), nil}, + {"", int(0), ErrCantCastVariableToTargetType}, + {"asdf", int(0), ErrCantCastVariableToTargetType}, + {" -14", int(0), ErrCantCastVariableToTargetType}, + + {"-10", float32(-10), nil}, + {"10.23", float32(10.23), nil}, + {"-10", float64(-10), nil}, + {"10.23", float64(10.23), nil}, + {"", float64(0), ErrCantCastVariableToTargetType}, + {"asdf", float64(0), ErrCantCastVariableToTargetType}, + {" -14", float64(0), ErrCantCastVariableToTargetType}, + + {"10", "10", nil}, + {"value-port", "value-port", nil}, + {"", "", nil}, + } + + for _, test := range tests { + expectedType := reflect.TypeOf(test.expected) + t.Run(fmt.Sprintf("Test string to %s cast", expectedType), func(t *testing.T) { + actual, err := cast(test.val, expectedType) + if test.err == nil { + assert.NoError(t, err) + assert.Exactly(t, test.expected, actual) + } else { + assert.ErrorIs(t, err, test.err) + } + + }) + } +} + +func TestFindTokens(t *testing.T) { + type testCase struct { + val string + expected []*tagEntry + err error + } + + tests := []testCase{ + { + "${token}", + []*tagEntry{{string: "${token}", tagType: "", varname: "token"}}, + nil, + }, + { + "${token}-${ second\t}", + []*tagEntry{ + {string: "${token}", tagType: "", varname: "token"}, + {string: "${ second\t}", tagType: "", varname: "second"}, + }, + nil, + }, + { + "asdf${ee:token}aa", + []*tagEntry{ + {string: "${ee:token}", tagType: "ee", varname: "token"}, + }, + nil, + }, + { + "asdf${ee: to:ken}aa-${ e2 :to }ken}", + []*tagEntry{ + {string: "${ee: to:ken}", tagType: "ee", varname: "to:ken"}, + {string: "${ e2 :to }", tagType: "e2", varname: "to"}, + }, + nil, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("Test findTokens in %s", test.val), func(t *testing.T) { + actual, err := findTags(test.val) + if test.err == nil { + assert.NoError(t, err) + assert.EqualValues(t, test.expected, actual) + } else { + assert.ErrorIs(t, err, test.err) + } + }) + } +} diff --git a/lib/confutil/env_var_resolver.go b/lib/confutil/env_var_resolver.go new file mode 100644 index 000000000..a3cb79072 --- /dev/null +++ b/lib/confutil/env_var_resolver.go @@ -0,0 +1,24 @@ +package confutil + +import ( + "errors" + "fmt" + "os" +) + +var ErrEnvVariableNotProvided error = errors.New("env variable not set") + +// Resolve custom tag token with env variable value +var EnvTagResolver TagResolver = envTokenResolver + +func envTokenResolver(in string) (string, error) { + // TODO: windows os is case-insensitive for env variables, + // so it may requre to load all vars and lookup for env var manually + + val, ok := os.LookupEnv(in) + if !ok { + return "", fmt.Errorf("%s: %w", in, ErrEnvVariableNotProvided) + } + + return val, nil +} diff --git a/lib/confutil/env_var_resolver_test.go b/lib/confutil/env_var_resolver_test.go new file mode 100644 index 000000000..0d452032b --- /dev/null +++ b/lib/confutil/env_var_resolver_test.go @@ -0,0 +1,37 @@ +package confutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEnvVarResolver(t *testing.T) { + type testCase struct { + varname string + val string + err error + } + + tests := []testCase{ + {"SOME_BOOL", "True", nil}, + {"INT_VALUE", "10", nil}, + {"V_NAME", "10", nil}, + } + + for _, test := range tests { + t.Setenv(test.varname, test.val) + } + + tests = append(tests, testCase{"NOT_EXISTS", "", ErrEnvVariableNotProvided}) + + for _, test := range tests { + actual, err := envTokenResolver(test.varname) + if test.err != nil { + assert.ErrorIs(t, err, test.err) + } else { + assert.NoError(t, err) + assert.Exactly(t, test.val, actual) + } + } +} diff --git a/lib/errutil/errutil.go b/lib/errutil/errutil.go index bffd469b7..ab231d04f 100644 --- a/lib/errutil/errutil.go +++ b/lib/errutil/errutil.go @@ -8,7 +8,7 @@ package errutil import ( "context" - "github.com/hashicorp/go-multierror" + multierror "github.com/hashicorp/go-multierror" "github.com/pkg/errors" ) diff --git a/lib/ginkgoutil/ginkgo.go b/lib/ginkgoutil/ginkgo.go index 51c2c731e..2d1fac0c0 100644 --- a/lib/ginkgoutil/ginkgo.go +++ b/lib/ginkgoutil/ginkgo.go @@ -9,8 +9,8 @@ import ( "strings" "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" "github.com/onsi/gomega/format" "github.com/spf13/viper" "github.com/stretchr/testify/mock" @@ -21,12 +21,12 @@ import ( func SetupSuite() { format.UseStringerRepresentation = true // Otherwise error stacks have binary format. ReplaceGlobalLogger() - RegisterFailHandler(Fail) + gomega.RegisterFailHandler(ginkgo.Fail) } func RunSuite(t *testing.T, description string) { SetupSuite() - RunSpecs(t, description) + ginkgo.RunSpecs(t, description) } func ReplaceGlobalLogger() *zap.Logger { @@ -39,7 +39,7 @@ func ReplaceGlobalLogger() *zap.Logger { func NewLogger() *zap.Logger { conf := zap.NewDevelopmentConfig() enc := zapcore.NewConsoleEncoder(conf.EncoderConfig) - core := zapcore.NewCore(enc, zapcore.AddSync(GinkgoWriter), zap.DebugLevel) + core := zapcore.NewCore(enc, zapcore.AddSync(ginkgo.GinkgoWriter), zap.DebugLevel) log := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zap.DPanicLevel)) return log } @@ -51,18 +51,18 @@ type Mock interface { func AssertExpectations(mocks ...Mock) { for _, m := range mocks { - m.AssertExpectations(GinkgoT(1)) + m.AssertExpectations(ginkgo.GinkgoT(1)) } } func AssertNotCalled(mock Mock, methodName string) { - mock.AssertNotCalled(GinkgoT(1), methodName) + mock.AssertNotCalled(ginkgo.GinkgoT(1), methodName) } func ParseYAML(data string) map[string]interface{} { v := viper.New() v.SetConfigType("yaml") err := v.ReadConfig(strings.NewReader(data)) - Expect(err).NotTo(HaveOccurred()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) return v.AllSettings() } diff --git a/lib/ginkgoutil/matchers.go b/lib/ginkgoutil/matchers.go index 5ecf59181..8049198d2 100644 --- a/lib/ginkgoutil/matchers.go +++ b/lib/ginkgoutil/matchers.go @@ -8,11 +8,11 @@ package ginkgoutil import ( "reflect" - . "github.com/onsi/gomega" + "github.com/onsi/gomega" ) func ExpectFuncsEqual(f1, f2 interface{}) { val1 := reflect.ValueOf(f1) val2 := reflect.ValueOf(f2) - Expect(val1.Pointer()).To(Equal(val2.Pointer())) + gomega.Expect(val1.Pointer()).To(gomega.Equal(val2.Pointer())) } diff --git a/lib/ioutil2/reader.go b/lib/ioutil2/reader.go index 261e93de7..c70bf25f4 100644 --- a/lib/ioutil2/reader.go +++ b/lib/ioutil2/reader.go @@ -41,9 +41,9 @@ func (r *MultiPassReader) Read(p []byte) (n int, err error) { return } -func (r *MultiPassReader) PassesLeft() int { - return r.PassesLeft() -} +// func (r *MultiPassReader) PassesLeft() int { +// return r.PassesLeft() +// } func (r *MultiPassReader) Unwrap() io.Reader { return r.rs diff --git a/lib/monitoring/counter.go b/lib/monitoring/counter.go index 405e52cb4..17d4e8972 100644 --- a/lib/monitoring/counter.go +++ b/lib/monitoring/counter.go @@ -9,7 +9,7 @@ import ( "expvar" "strconv" - "github.com/uber-go/atomic" + "go.uber.org/atomic" ) // TODO: use one rcrowley/go-metrics instead. diff --git a/lib/netutil/dial.go b/lib/netutil/dial.go index 71709873f..e03de0a9d 100644 --- a/lib/netutil/dial.go +++ b/lib/netutil/dial.go @@ -9,6 +9,7 @@ import ( "context" "net" "sync" + "time" "github.com/pkg/errors" ) @@ -42,7 +43,7 @@ func NewDNSCachingDialer(dialer Dialer, cache DNSCache) DialerFunc { remoteAddr := conn.RemoteAddr().(*net.TCPAddr) _, port, err := net.SplitHostPort(addr) if err != nil { - conn.Close() + _ = conn.Close() return nil, errors.Wrap(err, "invalid address, but successful dial - should not happen") } cache.Add(addr, net.JoinHostPort(remoteAddr.IP.String(), port)) @@ -56,8 +57,8 @@ var DefaultDNSCache = &SimpleDNSCache{} // This method has much more overhead, but get guaranteed reachable resolved addr. // Example: host is resolved to IPv4 and IPv6, but IPv4 is not working on machine. // LookupReachable will return IPv6 in that case. -func LookupReachable(addr string) (string, error) { - d := net.Dialer{DualStack: true} +func LookupReachable(addr string, timeout time.Duration) (string, error) { + d := net.Dialer{DualStack: true, Timeout: timeout} conn, err := d.Dial("tcp", addr) if err != nil { return "", err @@ -78,7 +79,7 @@ func WarmDNSCache(c DNSCache, addr string) error { if err != nil { return err } - conn.Close() + _ = conn.Close() return nil } diff --git a/lib/netutil/mocks/conn.go b/lib/netutil/mocks/conn.go index 7f10c79c0..c71a21ce7 100644 --- a/lib/netutil/mocks/conn.go +++ b/lib/netutil/mocks/conn.go @@ -1,10 +1,12 @@ // Code generated by mockery v1.0.0 package netmock -import "github.com/stretchr/testify/mock" -import "net" +import ( + "net" + "time" -import "time" + "github.com/stretchr/testify/mock" +) // Conn is an autogenerated mock type for the Conn type type Conn struct { diff --git a/lib/netutil/mocks/dialer.go b/lib/netutil/mocks/dialer.go index 27e1ea755..77007eb9b 100644 --- a/lib/netutil/mocks/dialer.go +++ b/lib/netutil/mocks/dialer.go @@ -1,9 +1,12 @@ // Code generated by mockery v1.0.0 package netmock -import "context" -import "github.com/stretchr/testify/mock" -import "net" +import ( + "context" + "net" + + "github.com/stretchr/testify/mock" +) // Dialer is an autogenerated mock type for the Dialer type type Dialer struct { diff --git a/lib/netutil/netutil_suite_test.go b/lib/netutil/netutil_suite_test.go index baee6a76a..a397d4121 100644 --- a/lib/netutil/netutil_suite_test.go +++ b/lib/netutil/netutil_suite_test.go @@ -5,10 +5,10 @@ import ( "net" "strconv" "testing" + "time" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" "github.com/pkg/errors" "github.com/yandex/pandora/lib/ginkgoutil" netmock "github.com/yandex/pandora/lib/netutil/mocks" @@ -18,20 +18,20 @@ func TestNetutil(t *testing.T) { ginkgoutil.RunSuite(t, "Netutil Suite") } -var _ = Describe("DNS", func() { +var _ = ginkgo.Describe("DNS", func() { - It("lookup reachable", func() { + ginkgo.It("lookup reachable", func() { listener, err := net.ListenTCP("tcp4", nil) - defer listener.Close() - Expect(err).NotTo(HaveOccurred()) + defer func() { _ = listener.Close() }() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) port := strconv.Itoa(listener.Addr().(*net.TCPAddr).Port) addr := "localhost:" + port expectedResolved := "127.0.0.1:" + port - resolved, err := LookupReachable(addr) - Expect(err).NotTo(HaveOccurred()) - Expect(resolved).To(Equal(expectedResolved)) + resolved, err := LookupReachable(addr, time.Second) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(resolved).To(gomega.Equal(expectedResolved)) }) const ( @@ -39,19 +39,19 @@ var _ = Describe("DNS", func() { resolved = "[::1]:8888" ) - It("cache", func() { + ginkgo.It("cache", func() { cache := &SimpleDNSCache{} got, ok := cache.Get(addr) - Expect(ok).To(BeFalse()) - Expect(got).To(BeEmpty()) + gomega.Expect(ok).To(gomega.BeFalse()) + gomega.Expect(got).To(gomega.BeEmpty()) cache.Add(addr, resolved) got, ok = cache.Get(addr) - Expect(ok).To(BeTrue()) - Expect(got).To(Equal(resolved)) + gomega.Expect(ok).To(gomega.BeTrue()) + gomega.Expect(got).To(gomega.Equal(resolved)) }) - It("Dialer cache miss", func() { + ginkgo.It("Dialer cache miss", func() { ctx := context.Background() mockConn := &netmock.Conn{} mockConn.On("RemoteAddr").Return(&net.TCPAddr{ @@ -66,13 +66,13 @@ var _ = Describe("DNS", func() { testee := NewDNSCachingDialer(dialer, cache) conn, err := testee.DialContext(ctx, "tcp", addr) - Expect(err).NotTo(HaveOccurred()) - Expect(conn).To(Equal(mockConn)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(conn).To(gomega.Equal(mockConn)) ginkgoutil.AssertExpectations(mockConn, cache, dialer) }) - It("Dialer cache hit", func() { + ginkgo.It("Dialer cache hit", func() { ctx := context.Background() mockConn := &netmock.Conn{} cache := &netmock.DNSCache{} @@ -82,13 +82,13 @@ var _ = Describe("DNS", func() { testee := NewDNSCachingDialer(dialer, cache) conn, err := testee.DialContext(ctx, "tcp", addr) - Expect(err).NotTo(HaveOccurred()) - Expect(conn).To(Equal(mockConn)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(conn).To(gomega.Equal(mockConn)) ginkgoutil.AssertExpectations(mockConn, cache, dialer) }) - It("Dialer cache miss err", func() { + ginkgo.It("Dialer cache miss err", func() { ctx := context.Background() expectedErr := errors.New("dial failed") cache := &netmock.DNSCache{} @@ -98,8 +98,8 @@ var _ = Describe("DNS", func() { testee := NewDNSCachingDialer(dialer, cache) conn, err := testee.DialContext(ctx, "tcp", addr) - Expect(err).To(Equal(expectedErr)) - Expect(conn).To(BeNil()) + gomega.Expect(err).To(gomega.Equal(expectedErr)) + gomega.Expect(conn).To(gomega.BeNil()) ginkgoutil.AssertExpectations(cache, dialer) }) diff --git a/lib/tag/debug.go b/lib/tag/debug.go index aaa7c857c..ee36c25b0 100644 --- a/lib/tag/debug.go +++ b/lib/tag/debug.go @@ -3,6 +3,7 @@ // license that can be found in the LICENSE file. // Author: Vladimir Skipor +//go:build debug // +build debug package tag diff --git a/lib/tag/no_degug.go b/lib/tag/no_degug.go index 0877c0d71..697fb8ef2 100644 --- a/lib/tag/no_degug.go +++ b/lib/tag/no_degug.go @@ -3,6 +3,7 @@ // license that can be found in the LICENSE file. // Author: Vladimir Skipor +//go:build !debug // +build !debug package tag diff --git a/lib/tag/no_race.go b/lib/tag/no_race.go index d746b261b..7ccebb392 100644 --- a/lib/tag/no_race.go +++ b/lib/tag/no_race.go @@ -3,6 +3,7 @@ // license that can be found in the LICENSE file. // Author: Vladimir Skipor +//go:build !race // +build !race package tag diff --git a/lib/tag/race.go b/lib/tag/race.go index a19547375..0cdaf0afb 100644 --- a/lib/tag/race.go +++ b/lib/tag/race.go @@ -3,6 +3,7 @@ // license that can be found in the LICENSE file. // Author: Vladimir Skipor +//go:build race // +build race package tag diff --git a/lib/testutil/matchers.go b/lib/testutil/matchers.go index cda57389c..78d6d268d 100644 --- a/lib/testutil/matchers.go +++ b/lib/testutil/matchers.go @@ -26,7 +26,7 @@ var _ TestingT = &flakyT{} func (ff *flakyT) Logf(format string, args ...interface{}) { getHelper(ff.t).Helper() - ff.Logf(format, args...) + ff.t.Logf(format, args...) } func (ff *flakyT) Errorf(format string, args ...interface{}) { diff --git a/lib/zaputil/stack_extract_core.go b/lib/zaputil/stack_extract_core.go index 4387012a6..e596912b9 100644 --- a/lib/zaputil/stack_extract_core.go +++ b/lib/zaputil/stack_extract_core.go @@ -75,7 +75,7 @@ func (c *errStackExtractCore) Check(ent zapcore.Entry, ce *zapcore.CheckedEntry) func (c *errStackExtractCore) cloneBuffer() zapBuffer { clone := getBuffer() - clone.Write(c.stacksBuff.Bytes()) + _, _ = clone.Write(c.stacksBuff.Bytes()) return clone } diff --git a/lib/zaputil/stack_extract_core_test.go b/lib/zaputil/stack_extract_core_test.go index c208c4048..1f09ee8a3 100644 --- a/lib/zaputil/stack_extract_core_test.go +++ b/lib/zaputil/stack_extract_core_test.go @@ -47,11 +47,11 @@ var _ = Describe("stack_extract_core", func() { testee = testee.With(noStackFields1()) entry := zapcore.Entry{Message: "test"} - testee.Write(entry, noStackFields2()) + _ = testee.Write(entry, noStackFields2()) Expect(logs.Len()).To(Equal(1)) Expect(logs.All()[0]).To(Equal( - observer.LoggedEntry{entry, append(noStackFields1(), noStackFields2()...)}, + observer.LoggedEntry{Entry: entry, Context: append(noStackFields1(), noStackFields2()...)}, )) }) @@ -67,15 +67,15 @@ var _ = Describe("stack_extract_core", func() { fieldsCopy := make([]zapcore.Field, len(fields)) copy(fieldsCopy, fields) entry := zapcore.Entry{Message: "test"} - testee.Write(entry, fields) + _ = testee.Write(entry, fields) expectedEntry := entry expectedEntry.Stack = "error stacktrace:" + sampleStack Expect(logs.Len()).To(Equal(1)) Expect(logs.All()[0]).To(Equal( observer.LoggedEntry{ - expectedEntry, - append(noStackFields1(), zap.String("error", sampleErrMsg)), + Entry: expectedEntry, + Context: append(noStackFields1(), zap.String("error", sampleErrMsg)), }, )) Expect(fields).To(Equal(fieldsCopy)) @@ -95,15 +95,15 @@ var _ = Describe("stack_extract_core", func() { copy(fieldsCopy, fields) entry := zapcore.Entry{Message: "test"} testee = testee.With(fields) - testee.Write(entry, nil) + _ = testee.Write(entry, nil) expectedEntry := entry expectedEntry.Stack = "error stacktrace:" + sampleStack Expect(logs.Len()).To(Equal(1)) Expect(logs.All()[0]).To(Equal( observer.LoggedEntry{ - expectedEntry, - append(noStackFields1(), zap.Error(sampleCause)), + Entry: expectedEntry, + Context: append(noStackFields1(), zap.Error(sampleCause)), }, )) Expect(fields).To(Equal(fieldsCopy)) @@ -120,15 +120,15 @@ var _ = Describe("stack_extract_core", func() { const entryStack = "entry stack" entry := zapcore.Entry{Message: "test", Stack: entryStack} const customKey = "custom-key" - testee.Write(entry, []zapcore.Field{zap.NamedError(customKey, sampleErr)}) + _ = testee.Write(entry, []zapcore.Field{zap.NamedError(customKey, sampleErr)}) expectedEntry := entry expectedEntry.Stack = entryStack + "\n" + customKey + " stacktrace:" + sampleStack Expect(logs.Len()).To(Equal(1)) Expect(logs.All()[0]).To(Equal( observer.LoggedEntry{ - expectedEntry, - []zapcore.Field{zap.String(customKey, sampleErrMsg)}, + Entry: expectedEntry, + Context: []zapcore.Field{zap.String(customKey, sampleErrMsg)}, }, )) }) diff --git a/main.go b/main.go index be760ad7b..dfd8cba40 100644 --- a/main.go +++ b/main.go @@ -7,9 +7,9 @@ package main import ( "github.com/spf13/afero" - "github.com/yandex/pandora/cli" example "github.com/yandex/pandora/components/example/import" + grpc "github.com/yandex/pandora/components/grpc/import" phttp "github.com/yandex/pandora/components/phttp/import" coreimport "github.com/yandex/pandora/core/import" ) @@ -21,6 +21,7 @@ func main() { coreimport.Import(fs) phttp.Import(fs) example.Import() + grpc.Import(fs) cli.Run() } diff --git a/test-sync.txt b/test-sync.txt new file mode 100644 index 000000000..7c9415122 --- /dev/null +++ b/test-sync.txt @@ -0,0 +1 @@ +test-sync-with-arcadia