diff --git a/cmd/avalanche.go b/cmd/avalanche.go index 5a0925a..6bcd996 100644 --- a/cmd/avalanche.go +++ b/cmd/avalanche.go @@ -30,25 +30,28 @@ import ( ) var ( - metricCount = kingpin.Flag("metric-count", "Number of metrics to serve.").Default("500").Int() - labelCount = kingpin.Flag("label-count", "Number of labels per-metric.").Default("10").Int() - seriesCount = kingpin.Flag("series-count", "Number of series per-metric.").Default("10").Int() - metricLength = kingpin.Flag("metricname-length", "Modify length of metric names.").Default("5").Int() - labelLength = kingpin.Flag("labelname-length", "Modify length of label names.").Default("5").Int() - constLabels = kingpin.Flag("const-label", "Constant label to add to every metric. Format is labelName=labelValue. Flag can be specified multiple times.").Strings() - valueInterval = kingpin.Flag("value-interval", "Change series values every {interval} seconds.").Default("30").Int() - labelInterval = kingpin.Flag("series-interval", "Change series_id label values every {interval} seconds.").Default("60").Int() - metricInterval = kingpin.Flag("metric-interval", "Change __name__ label values every {interval} seconds.").Default("120").Int() - port = kingpin.Flag("port", "Port to serve at").Default("9001").Int() - remoteURL = kingpin.Flag("remote-url", "URL to send samples via remote_write API.").URL() - remotePprofURLs = kingpin.Flag("remote-pprof-urls", "a list of urls to download pprofs during the remote write: --remote-pprof-urls=http://127.0.0.1:10902/debug/pprof/heap --remote-pprof-urls=http://127.0.0.1:10902/debug/pprof/profile").URLList() - remotePprofInterval = kingpin.Flag("remote-pprof-interval", "how often to download pprof profiles.When not provided it will download a profile once before the end of the test.").Duration() - remoteBatchSize = kingpin.Flag("remote-batch-size", "how many samples to send with each remote_write API request.").Default("2000").Int() - remoteRequestCount = kingpin.Flag("remote-requests-count", "how many requests to send in total to the remote_write API.").Default("100").Int() - remoteReqsInterval = kingpin.Flag("remote-write-interval", "delay between each remote write request.").Default("100ms").Duration() - remoteTenant = kingpin.Flag("remote-tenant", "Tenant ID to include in remote_write send").Default("0").String() - tlsClientInsecure = kingpin.Flag("tls-client-insecure", "Skip certificate check on tls connection").Default("false").Bool() - remoteTenantHeader = kingpin.Flag("remote-tenant-header", "Tenant ID to include in remote_write send. The default, is the default tenant header expected by Cortex.").Default("X-Scope-OrgID").String() + metricCount = kingpin.Flag("metric-count", "Number of metrics to serve.").Default("500").Int() + labelCount = kingpin.Flag("label-count", "Number of labels per-metric.").Default("10").Int() + seriesCount = kingpin.Flag("series-count", "Number of series per-metric.").Default("10").Int() + seriesChangeRate = kingpin.Flag("series-change-rate", "The rate at which the number of active series changes over time. Positive value increases the series, negative value decreases the series.").Default("10").Int() + metricLength = kingpin.Flag("metricname-length", "Modify length of metric names.").Default("5").Int() + labelLength = kingpin.Flag("labelname-length", "Modify length of label names.").Default("5").Int() + constLabels = kingpin.Flag("const-label", "Constant label to add to every metric. Format is labelName=labelValue. Flag can be specified multiple times.").Strings() + valueInterval = kingpin.Flag("value-interval", "Change series values every {interval} seconds.").Default("30").Int() + labelInterval = kingpin.Flag("series-interval", "Change series_id label values every {interval} seconds.").Default("60").Int() + metricInterval = kingpin.Flag("metric-interval", "Change __name__ label values every {interval} seconds.").Default("120").Int() + seriesChangeInterval = kingpin.Flag("series-change-interval", "Change the number of series every {interval} seconds.").Default("10").Int() + operationMode = kingpin.Flag("operation-mode", "Mode of operation: 'gradual-change', 'series-spike', 'double-halve'").Default("gradual-change").String() + port = kingpin.Flag("port", "Port to serve at").Default("9001").Int() + remoteURL = kingpin.Flag("remote-url", "URL to send samples via remote_write API.").URL() + remotePprofURLs = kingpin.Flag("remote-pprof-urls", "a list of urls to download pprofs during the remote write: --remote-pprof-urls=http://127.0.0.1:10902/debug/pprof/heap --remote-pprof-urls=http://127.0.0.1:10902/debug/pprof/profile").URLList() + remotePprofInterval = kingpin.Flag("remote-pprof-interval", "how often to download pprof profiles.When not provided it will download a profile once before the end of the test.").Duration() + remoteBatchSize = kingpin.Flag("remote-batch-size", "how many samples to send with each remote_write API request.").Default("2000").Int() + remoteRequestCount = kingpin.Flag("remote-requests-count", "how many requests to send in total to the remote_write API.").Default("100").Int() + remoteReqsInterval = kingpin.Flag("remote-write-interval", "delay between each remote write request.").Default("100ms").Duration() + remoteTenant = kingpin.Flag("remote-tenant", "Tenant ID to include in remote_write send").Default("0").String() + tlsClientInsecure = kingpin.Flag("tls-client-insecure", "Skip certificate check on tls connection").Default("false").Bool() + remoteTenantHeader = kingpin.Flag("remote-tenant-header", "Tenant ID to include in remote_write send. The default, is the default tenant header expected by Cortex.").Default("X-Scope-OrgID").String() ) func main() { @@ -59,7 +62,7 @@ func main() { stop := make(chan struct{}) defer close(stop) - updateNotify, err := metrics.RunMetrics(*metricCount, *labelCount, *seriesCount, *metricLength, *labelLength, *valueInterval, *labelInterval, *metricInterval, *constLabels, stop) + updateNotify, err := metrics.RunMetrics(*metricCount, *labelCount, *seriesCount, *seriesChangeRate, *metricLength, *labelLength, *valueInterval, *labelInterval, *metricInterval, *seriesChangeInterval, *operationMode, *constLabels, stop) if err != nil { log.Fatal(err) } diff --git a/go.mod b/go.mod index ad629d9..58fe1b8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/prometheus-community/avalanche -go 1.14 +go 1.17 require ( github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4 // indirect @@ -11,6 +11,21 @@ require ( github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.15.0 github.com/prometheus/prometheus v1.8.2-0.20201119181812-c8f810083d3f - golang.org/x/sys v0.1.0 // indirect + github.com/stretchr/testify v1.9.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 ) + +require ( + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/protobuf v1.4.3 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/nelkinda/http-go v0.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/procfs v0.2.0 // indirect + golang.org/x/sys v0.1.0 // indirect + google.golang.org/protobuf v1.25.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index 4ec4086..6b578e5 100644 --- a/go.sum +++ b/go.sum @@ -144,12 +144,9 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cucumber/gherkin-go/v11 v11.0.0 h1:cwVwN1Qn2VRSfHZNLEh5x00tPBmZcjATBWDpxsR5Xug= github.com/cucumber/gherkin-go/v11 v11.0.0/go.mod h1:CX33k2XU2qog4e+TFjOValoq6mIUq0DmVccZs238R9w= -github.com/cucumber/godog v0.9.0 h1:QOb8wyC7f+FVFXzY3RdgowwJUb4WeJfqbnQqaH4jp+A= github.com/cucumber/godog v0.9.0/go.mod h1:roWCHkpeK6UTOyIRRl7IR+fgfBeZ4vZR7OSq2J/NbM4= github.com/cucumber/messages-go/v10 v10.0.1/go.mod h1:kA5T38CBlBbYLU12TIrJ4fk4wSkVVOgyh7Enyy8WnSg= -github.com/cucumber/messages-go/v10 v10.0.3 h1:m/9SD/K/A15WP7i1aemIv7cwvUw+viS51Ui5HBw1cdE= github.com/cucumber/messages-go/v10 v10.0.3/go.mod h1:9jMZ2Y8ZxjLY6TG2+x344nt5rXstVVDYSdS5ySfI1WY= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -308,7 +305,6 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -689,13 +685,20 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -1116,8 +1119,9 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/metrics/serve.go b/metrics/serve.go index f4a495f..6895d2e 100644 --- a/metrics/serve.go +++ b/metrics/serve.go @@ -83,7 +83,7 @@ func cycleValues(labelKeys, labelValues []string, seriesCount, seriesCycle int) } // RunMetrics creates a set of Prometheus test series that update over time -func RunMetrics(metricCount, labelCount, seriesCount, metricLength, labelLength, valueInterval, seriesInterval, metricInterval int, constLabels []string, stop chan struct{}) (chan struct{}, error) { +func RunMetrics(metricCount, labelCount, seriesCount, seriesChangeRate, metricLength, labelLength, valueInterval, seriesInterval, metricInterval, seriesChangeInterval int, operationMode string, constLabels []string, stop chan struct{}) (chan struct{}, error) { labelKeys := make([]string, labelCount) for idx := 0; idx < labelCount; idx++ { labelKeys[idx] = fmt.Sprintf("label_key_%s_%v", strings.Repeat("k", labelLength), idx) @@ -108,6 +108,7 @@ func RunMetrics(metricCount, labelCount, seriesCount, metricLength, labelLength, valueTick := time.NewTicker(time.Duration(valueInterval) * time.Second) seriesTick := time.NewTicker(time.Duration(seriesInterval) * time.Second) metricTick := time.NewTicker(time.Duration(metricInterval) * time.Second) + changeSeriesTick := time.NewTicker(time.Duration(seriesChangeInterval) * time.Second) updateNotify := make(chan struct{}, 1) go func() { @@ -153,6 +154,37 @@ func RunMetrics(metricCount, labelCount, seriesCount, metricLength, labelLength, } } }() + switch operationMode { + case "double-halve": + seriesIncrease := true + go func() { + for tick := range changeSeriesTick.C { + fmt.Printf("%v: Adjusting series count. New count: %d\n", tick, seriesCount) + + metricsMux.Lock() + + if seriesIncrease { + seriesCount *= 2 + } else { + seriesCount /= 2 + if seriesCount < 1 { + seriesCount = 1 + } + } + + unregisterMetrics() + registerMetrics(metricCount, metricLength, metricCycle, labelKeys) + cycleValues(labelKeys, labelValues, seriesCount, seriesCycle) + metricsMux.Unlock() + + seriesIncrease = !seriesIncrease + select { + case updateNotify <- struct{}{}: + default: + } + } + }() + } go func() { <-stop diff --git a/metrics/serve_test.go b/metrics/serve_test.go new file mode 100644 index 0000000..f63aef4 --- /dev/null +++ b/metrics/serve_test.go @@ -0,0 +1,77 @@ +package metrics + +import ( + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" +) + +// Helper function to count the series in the registry +func countSeries(t *testing.T, registry *prometheus.Registry) int { + metricsFamilies, err := registry.Gather() + assert.NoError(t, err) + + seriesCount := 0 + for _, mf := range metricsFamilies { + for range mf.Metric { + seriesCount++ + } + } + + return seriesCount +} + +func TestRunMetricsSeriesCountChangeDoubleHalve(t *testing.T) { + const ( + initialSeriesCount = 5 + metricCount = 1 + labelCount = 1 + seriesChangeRate = 1 + metricLength = 1 + labelLength = 1 + valueInterval = 1 + seriesInterval = 1 + metricInterval = 1 + seriesChangeInterval = 3 + operationMode = "double-halve" + constLabel = "constLabel=test" + updateNotifyTimeout = 3 * time.Second + waitTimeBetweenChecks = 3 * time.Second + ) + + stop := make(chan struct{}) + defer close(stop) + + promRegistry = prometheus.NewRegistry() + + updateNotify, err := RunMetrics(metricCount, labelCount, initialSeriesCount, seriesChangeRate, metricLength, labelLength, valueInterval, seriesInterval, metricInterval, seriesChangeInterval, operationMode, []string{constLabel}, stop) + assert.NoError(t, err) + + initialCount := countSeries(t, promRegistry) + expectedInitialCount := initialSeriesCount + assert.Equal(t, expectedInitialCount, initialCount, "Initial series count should be %d but got %d", expectedInitialCount, initialCount) + + // Test for doubling the series count + select { + case <-updateNotify: + time.Sleep(waitTimeBetweenChecks) + doubledCount := countSeries(t, promRegistry) + expectedDoubledCount := initialSeriesCount * 2 + assert.Equal(t, expectedDoubledCount, doubledCount, "Doubled series count should be %d but got %d", expectedDoubledCount, doubledCount) + case <-time.After(updateNotifyTimeout): + t.Fatal("Did not receive update notification for series count doubling in time") + } + + // Test for halving the series count + select { + case <-updateNotify: + time.Sleep(waitTimeBetweenChecks) + halvedCount := countSeries(t, promRegistry) + expectedHalvedCount := initialSeriesCount + assert.Equal(t, expectedHalvedCount, halvedCount, "Halved series count should be %d but got %d", expectedHalvedCount, halvedCount) + case <-time.After(updateNotifyTimeout): + t.Fatal("Did not receive update notification for series count halving in time") + } +}