Updated prombenchy to support multiple benchmarks on one cluster; adjusted avalanche, removed some experiments for now.

.github/workflows/go.yml

@@ -0,0 +1,21 @@
+name: Go
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Unit tests on Go ${{ matrix.go }} ${{ matrix.platform }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Setup Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.22.x'
+    - uses: actions/cache@v3
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+    - name: Run unit tests.
+      env:
+        GOBIN: /tmp/.bin
+      run: make test

.github/workflows/golangci-lint.yml

@@ -1,31 +1,30 @@
 ---
-# This action is synced from https://github.com/prometheus/prometheus
 name: golangci-lint
 on:
   push:
     paths:
-      - "go.sum"
-      - "go.mod"
-      - "**.go"
-      - ".github/workflows/golangci-lint.yml"
-      - ".golangci.yml"
-  pull_request:
+    - "**.go"
+    - "scripts/errcheck_excludes.txt"
+    - ".github/workflows/golangci-lint.yml"
+    - ".golangci.yml"
+
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
+env:
+  GO_VERSION: stable
+  GOLANGCI_LINT_VERSION: v1.60
 
 jobs:
-  golangci:
-    name: lint
+  golangci-lint:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
-      - name: install Go
-        uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
-        with:
-          go-version: 1.20.x
-      - name: Install snmp_exporter/generator dependencies
-        run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
-        if: github.repository == 'prometheus/snmp_exporter'
-      - name: Lint
-        uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc # v3.7.0
-        with:
-          version: v1.54.2
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v5
+      with:
+        go-version: ${{ env.GO_VERSION }}
+    - name: golangci-lint mtypes
+      uses: golangci/golangci-lint-action@v6
+      with:
+        version: ${{ env.GOLANGCI_LINT_VERSION }}
+        working-directory: ./tools/mtypes

.golangci.yml

@@ -1,8 +1,135 @@
+# Generally copied from prometheus/prometheus.
+# TODO(bwplotka): Add golangcilint, right now not
 run:
-  deadline: 5m
-  skip-files:
-    # Skip autogenerated files.
-    - ^.*\.(pb|y)\.go$
+  timeout: 15m
 
 output:
   sort-results: true
+
+linters:
+  enable:
+  - depguard
+  - errorlint
+  - gocritic
+  - godot
+  - gofumpt
+  - goimports
+  - misspell
+  - nolintlint
+  - perfsprint
+  - predeclared
+  - revive
+  - testifylint
+  - unconvert
+  - unused
+  - usestdlibvars
+  - whitespace
+  - loggercheck
+
+issues:
+  max-issues-per-linter: 0
+  max-same-issues: 0
+  # The default exclusions are too aggressive. For one, they
+  # essentially disable any linting on doc comments. We disable
+  # default exclusions here and add exclusions fitting our codebase
+  # further down.
+  exclude-use-default: false
+  exclude-files:
+  # Skip autogenerated files.
+  - ^.*\.(pb|y)\.go$
+  exclude-rules:
+  - linters:
+    - errcheck
+    # Taken from the default exclusions (that are otherwise disabled above).
+    text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked
+  - linters:
+    - govet
+    # We use many Seek methods that do not follow the usual pattern.
+    text: "stdmethods: method Seek.* should have signature Seek"
+  - path: _test.go
+    linters:
+    - errcheck
+  - linters:
+    - godot
+    source: "^// ==="
+  - linters:
+    - perfsprint
+    text: "fmt.Sprintf can be replaced with string concatenation"
+linters-settings:
+  depguard:
+    rules:
+      main:
+        deny:
+        - pkg: "sync/atomic"
+          desc: "Use go.uber.org/atomic instead of sync/atomic"
+        - pkg: "github.com/stretchr/testify/assert"
+          desc: "Use github.com/stretchr/testify/require instead of github.com/stretchr/testify/assert"
+        - pkg: "github.com/go-kit/kit/log"
+          desc: "Use github.com/go-kit/log instead of github.com/go-kit/kit/log"
+        - pkg: "io/ioutil"
+          desc: "Use corresponding 'os' or 'io' functions instead."
+        - pkg: "regexp"
+          desc: "Use github.com/grafana/regexp instead of regexp"
+        - pkg: "github.com/pkg/errors"
+          desc: "Use 'errors' or 'fmt' instead of github.com/pkg/errors"
+        - pkg: "gzip"
+          desc: "Use github.com/klauspost/compress instead of gzip"
+        - pkg: "zlib"
+          desc: "Use github.com/klauspost/compress instead of zlib"
+        - pkg: "golang.org/x/exp/slices"
+          desc: "Use 'slices' instead."
+  errcheck:
+    exclude-functions:
+    # Don't flag lines such as "io.Copy(io.Discard, resp.Body)".
+    - io.Copy
+    # The next two are used in HTTP handlers, any error is handled by the server itself.
+    - io.WriteString
+    - (net/http.ResponseWriter).Write
+    # No need to check for errors on server's shutdown.
+    - (*net/http.Server).Shutdown
+    # Never check for logger errors.
+    - (github.com/go-kit/log.Logger).Log
+  goimports:
+    local-prefixes: github.com/prometheus/prometheus
+  gofumpt:
+    extra-rules: true
+  perfsprint:
+    # Optimizes `fmt.Errorf`.
+    errorf: false
+  revive:
+    # By default, revive will enable only the linting rules that are named in the configuration file.
+    # So, it's needed to explicitly enable all required rules here.
+    rules:
+    # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md
+    - name: blank-imports
+    - name: comment-spacings
+    - name: context-as-argument
+      arguments:
+      # Allow functions with test or bench signatures.
+      - allowTypesBefore: "*testing.T,testing.TB"
+    - name: context-keys-type
+    - name: dot-imports
+    - name: empty-block
+    - name: error-naming
+    - name: error-return
+    - name: error-strings
+    - name: errorf
+    - name: exported
+    - name: increment-decrement
+    - name: indent-error-flow
+    - name: package-comments
+    - name: range
+    - name: receiver-naming
+    - name: redefines-builtin-id
+    - name: superfluous-else
+    - name: time-naming
+    - name: unexported-return
+    - name: unreachable-code
+    - name: var-declaration
+    - name: var-naming
+      disabled: true # Consistency with dto type.
+  testifylint:
+    disable:
+    - float-compare
+    - go-require
+    enable-all: true

Makefile

@@ -8,13 +8,13 @@ help:
 	@awk 'BEGIN {FS = ": ##"; printf "Usage:\n  make <target>\n\nTargets:\n"} /^[a-zA-Z0-9_\.\-\/%]+: ##/ { printf "  %-45s %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
 
 .PHONY: check-deps
-check-deps: ## Check local dependencies.
+check-deps: $(GOMPLATE) ## Check local dependencies.
 	@command -v gcloud >/dev/null 2>&1 || { echo 'Please install gcloud (https://cloud.google.com/sdk/gcloud#download_and_install_the)'; exit 1; }
 	@command -v go >/dev/null 2>&1 || { echo 'Please install go (https://go.dev/doc/install)'; exit 1; }
 	@command -v kubectl >/dev/null 2>&1 || { echo 'Please install kubectl'; exit 1; }
 
 .PHONY: start
-start: check-deps ## Start benchmark on the current cluster.
+start: check-deps ## Start a new benchmark on the current cluster.
 	@test -n "$(BENCH_NAME)" || (echo "BENCH_NAME variable is not set, what name for this benchmark you want to use?" ; exit 1)
 	@# TODO(bwplotka): Check against cluster mismatches.
 	@# TODO(bwplotka): Check if this benchmark is already running.
@@ -41,3 +41,23 @@ cluster-destroy: check-deps ## Tear down the benchmarking GKE cluster.
 .PHONY: lint
 lint: ## Lint resources.
 	bash ./scripts/shellcheck.sh
+
+GOMODS := $(shell find . -name "go.mod" | grep -v .bingo | xargs dirname)
+.PHONY: test
+test:
+	@for gomod in $(GOMODS); do \
+		cd $$gomod && go test -v ./...; \
+    done
+
+GOFUMPT = gofumpt
+$(GOFUMPT):
+	@go install mvdan.cc/gofumpt@latest
+
+GO_FILES = $(shell find . -path ./vendor -prune -o -name '*.go' -print)
+
+.PHONY: format
+format: $(GOFUMPT) $(GOIMPORTS)
+	@echo ">> formating imports"
+	@$(GOIMPORTS) -w $(GO_FILES)
+	@echo ">> gofumpt-ing the code; golangci-lint requires this"
+	@$(GOFUMPT) -extra -w $(GO_FILES)

README.md

@@ -1,16 +1,34 @@
 # prombenchy
 
 This repo contains very simplistic, experimental and opinionated [prombench](https://github.com/prometheus/test-infra/tree/master/prombench) alternative
-that focuses on benchmarking (and testing) the collection mode for Prometheus metrics (discovery + scrape + basic processing + remote write) on GKE.
+that focuses on benchmarking (and testing) the agents, so collection modes for Prometheus metrics (discovery + scrape + basic processing + remote write/alternative protocols) on GKE.
 
-This may or may not end up as the another mode/feature/version of prombench itself, but feel free to use it as you wish. 
+Feel free to use it as you wish.
 
 ### Usage
 
-TBD (:
+Check `make help` on what's possible. Then if anything is failing check `scripts/` bash scripts and adjust according to your setup. Those are shell script, will be always flaky for edge cases or races, but it's better than nothing (: 
+
+The general flow looks as follows:
+
+* You setup your GKE cluster once: `make cluster-setup CLUSTER_NAME=my-prombenchy`
+* Then to set up any benchmark run you do `make start CLUSTER_NAME=my-prombenchy BENCH_NAME=<name of benchmark, also k8s namespace> SCENARIO=./manifests/scenarios/gmp`. This will setup node-pool and your collector (e.g. as daemon set or separate pod - up to you, as long as you do correct node section!)
+
+You can start as many scenarios as you want on the single cluster (make sure to use unique `BENCH_NAME` though!)
+
+The scenario is a path to the "collector" manifest, so anything that will scrape `./manifests/load/avalanche.exampletarget.yaml`. Feel free to adjust anything in `./manifests/scenarios/` or add your own.
+
+This setup uses separate Prometheus for gathering metrics about core resources and collectors (available locally and in GCM). Make sure your pod has `app=collector` label and relevant port name has `-ins` suffix, to be scraped by this core Prometheus. There is also a dashboard you can apply to GCM in `./dashboards/`. 
+
+* `make stop CLUSTER_NAME=my-prombenchy BENCH_NAME=<name of benchmark, also k8s namespace> SCENARIO=./manifests/scenarios/gmp` kill the node-pool and experiment.
+
+### TODOs
+
+* [ ] All scenarios are GMP aware, so they send data to GCM. In the future, we plan to also benchmark remote-write or OTLP, but proper test reivers would need to be added. Help welcome!
+* [ ] Probably Go code for scripts instead of bash, for reliability.
+* [ ] Cleanup svc account permissions on stopped scenarios.
+* [ ] Make config-reloader work with otel-collector (annoying to delete pod after config changes).
 
 ### Credits
 
-This repo was started by sharing a lot of design and resources from https://github.com/prometheus/test-infra repo, which we maintain in the
-Prometheus team mostly for [prombench](https://github.com/prometheus/test-infra/tree/master/prombench) functionality. Kudos to prombench
-project for the hard work so far!
+This repo was started by sharing a lot of design and resources from https://github.com/prometheus/test-infra repo, which we maintain in the Prometheus team mostly for [prombench](https://github.com/prometheus/test-infra/tree/master/prombench) functionality. Kudos to prombench project for the hard work so far! Since then, it was completely redesigned and simplified.