diff --git a/.github/badges/.gitkeep b/.github/badges/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.github/badges/coverage.svg b/.github/badges/coverage.svg new file mode 100644 index 0000000..8c89c29 --- /dev/null +++ b/.github/badges/coverage.svg @@ -0,0 +1 @@ +coverage: 33.7%coverage33.7% \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..9cf5649 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,52 @@ +name: Analysis +on: + push: + branches: + - main + pull_request: + +permissions: + contents: read + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.20' + cache: false + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + # Require: The version of golangci-lint to use. + # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version. + # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit. + version: v1.54 + + # Optional: working directory, useful for monorepos + # working-directory: somedir + + # Optional: golangci-lint command line arguments. + # + # Note: By default, the `.golangci.yml` file should be at the root of the repository. + # The location of the configuration file can be changed by using `--config=` + # args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0 + + # Optional: show only new issues if it's a pull request. The default value is `false`. + # only-new-issues: true + + # Optional: if set to true, then all caching functionality will be completely disabled, + # takes precedence over all other caching options. + # skip-cache: true + + # Optional: if set to true, then the action won't cache or restore ~/go/pkg. + # skip-pkg-cache: true + + # Optional: if set to true, then the action won't cache or restore ~/.cache/go-build. + # skip-build-cache: true + + # Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'. + # install-mode: "goinstall" \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..0b98ea2 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,52 @@ +name: Test + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + + test: + name: Test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + - uses: actions/setup-go@v4 + with: + go-version: '^1.20' + - run: go version + + + - name: Build + run: go build -v ./... + + - name: Test + run: | + CVPKG=$(go list ./... | grep -v mocks | tr '\n' ',') + go test -p 1 -coverpkg=${CVPKG} -coverprofile=coverage.out -covermode=count ./... + + - name: Code Coverage Badge + run: | + set -x + total=`go tool cover -func=coverage.out | grep total | grep -Eo '[0-9]+\.[0-9]+'` + if (( $(echo "$total <= 50" | bc -l) )) ; then + COLOR=red + elif (( $(echo "$total > 80" | bc -l) )); then + COLOR=green + else + COLOR=orange + fi + curl "https://img.shields.io/badge/coverage-$total%25-$COLOR" > .github/badges/coverage.svg + + - name: Commit and push the badge (if it changed) + uses: EndBug/add-and-commit@v9 + with: + default_author: github_actor + message: 'commit badge' + add: '*.svg' \ No newline at end of file diff --git a/.gitignore b/.gitignore index bc8a670..ca95085 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -.idea/* \ No newline at end of file +.idea/* + +cache/*.json \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..295837a --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +run: + export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:4317 + go run main.go + +test: + go test + golangci-lint run \ No newline at end of file diff --git a/cache/.gitkeep b/cache/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml index 68e1d27..704c92d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,5 +42,7 @@ services: dockerfile: Dockerfile environment: - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://otel-collector:4317 + volumes: + - ./cache:/app/cache depends_on: - otel-collector \ No newline at end of file diff --git a/file_cache.go b/file_cache.go new file mode 100644 index 0000000..6afb163 --- /dev/null +++ b/file_cache.go @@ -0,0 +1,86 @@ +package main + +import ( + "encoding/json" + "os" +) + +type SingleFileCache struct { + location string +} + +func NewSingleFileCache(location string) *SingleFileCache { + if _, err := os.Stat(location); os.IsNotExist(err) { + // Create an empty JSON file if it doesn't exist + data := []byte("{}") + err := os.WriteFile(location, data, 0644) + if err != nil { + panic(err) + } + } + + return &SingleFileCache{ + location: location, + } +} + +func (c *SingleFileCache) AddToCache(key string, value string) error { + // Read the existing cache + data, err := os.ReadFile(c.location) + if err != nil { + return err + } + + // Unmarshal the JSON data into a map + cache := map[string]string{} + err = json.Unmarshal(data, &cache) + if err != nil { + return err + } + + // Add or update the key-value pair + cache[key] = value + + // Marshal the updated cache and write it back to the file + updatedData, err := json.Marshal(cache) + if err != nil { + return err + } + + err = os.WriteFile(c.location, updatedData, 0644) + if err != nil { + return err + } + + return nil +} + +func (c *SingleFileCache) RetrieveValue(key string) (string, error) { + // Read the existing cache + data, err := os.ReadFile(c.location) + if err != nil { + return "", err + } + + // Unmarshal the JSON data into a map + cache := map[string]string{} + err = json.Unmarshal(data, &cache) + if err != nil { + return "", err + } + + return cache[key], nil +} + +func (c *SingleFileCache) DeleteCache() error { + if _, err := os.Stat(c.location); os.IsNotExist(err) { + return nil // Cache file doesn't exist, nothing to delete + } + + err := os.Remove(c.location) + if err != nil { + return err + } + + return nil +} diff --git a/file_cache_test.go b/file_cache_test.go new file mode 100644 index 0000000..9ecb55a --- /dev/null +++ b/file_cache_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCacheAdd(t *testing.T) { + cache := NewSingleFileCache("test-cache") + err := cache.AddToCache("test", "value") + if err != nil { + t.Failed() + } + + value, err := cache.RetrieveValue("test") + if err != nil { + t.Failed() + } + + assert.Equal(t, "value", value) + empty, _ := cache.RetrieveValue("not existent") + assert.Empty(t, empty) + err = cache.DeleteCache() + if err != nil { + t.Failed() + } +} diff --git a/github_client.go b/github_client.go index 97b87ee..50bbf6d 100644 --- a/github_client.go +++ b/github_client.go @@ -4,7 +4,7 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" ) @@ -44,7 +44,7 @@ func (c *GitHubClient) GetMostRecentCommit(repo, timestamp, branch string) (stri if response.StatusCode == http.StatusOK { defer response.Body.Close() - body, err := ioutil.ReadAll(response.Body) + body, err := io.ReadAll(response.Body) if err != nil { return "", err } @@ -80,7 +80,7 @@ func (c *GitHubClient) GetFileAtCommit(repository, filepath, commitSHA string) ( if response.StatusCode == http.StatusOK { defer response.Body.Close() - body, err := ioutil.ReadAll(response.Body) + body, err := io.ReadAll(response.Body) if err != nil { return "", err } diff --git a/grafana/dashboards/instrumentation-benchmarks.json b/grafana/dashboards/instrumentation-benchmarks.json index 113c147..acdc8b2 100644 --- a/grafana/dashboards/instrumentation-benchmarks.json +++ b/grafana/dashboards/instrumentation-benchmarks.json @@ -82,10 +82,10 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, "y": 0 - }, + }, "id": 1, "options": { "legend": { @@ -114,12 +114,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Thread switch rate'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Max heap used (MB)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Thread switch rate", + "title": "Max heap used (MB)", "type": "timeseries" },{ "datasource": { @@ -181,10 +181,10 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, + "w": 8, + "x": 8, "y": 0 - }, + }, "id": 1, "options": { "legend": { @@ -213,12 +213,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Req. p95 (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Max. CPU (user) %'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Req. p95 (ms)", + "title": "Max. CPU (user) %", "type": "timeseries" },{ "datasource": { @@ -280,10 +280,10 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, + "w": 8, + "x": 16, "y": 0 - }, + }, "id": 1, "options": { "legend": { @@ -312,12 +312,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Iter. p95 (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'GC pause time (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Iter. p95 (ms)", + "title": "GC pause time (ms)", "type": "timeseries" },{ "datasource": { @@ -379,10 +379,10 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 0 - }, + "y": 8 + }, "id": 1, "options": { "legend": { @@ -411,12 +411,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Net write avg (bps)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Req. p95 (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Net write avg (bps)", + "title": "Req. p95 (ms)", "type": "timeseries" },{ "datasource": { @@ -478,10 +478,10 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, + "w": 8, + "x": 8, + "y": 8 + }, "id": 1, "options": { "legend": { @@ -510,12 +510,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Avg. mch tot cpu %'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Iter. p95 (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Avg. mch tot cpu %", + "title": "Iter. p95 (ms)", "type": "timeseries" },{ "datasource": { @@ -577,10 +577,10 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, + "w": 8, + "x": 16, + "y": 8 + }, "id": 1, "options": { "legend": { @@ -609,12 +609,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Iter. mean (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Peak threads'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Iter. mean (ms)", + "title": "Peak threads", "type": "timeseries" },{ "datasource": { @@ -676,10 +676,10 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 0 - }, + "y": 16 + }, "id": 1, "options": { "legend": { @@ -708,12 +708,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Net read avg (bps)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Avg. mch tot cpu %'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Net read avg (bps)", + "title": "Avg. mch tot cpu %", "type": "timeseries" },{ "datasource": { @@ -775,10 +775,10 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, + "w": 8, + "x": 8, + "y": 16 + }, "id": 1, "options": { "legend": { @@ -874,10 +874,10 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, + "w": 8, + "x": 16, + "y": 16 + }, "id": 1, "options": { "legend": { @@ -906,12 +906,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Min heap used (MB)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Startup time (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Min heap used (MB)", + "title": "Startup time (ms)", "type": "timeseries" },{ "datasource": { @@ -973,10 +973,10 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 0 - }, + "y": 24 + }, "id": 1, "options": { "legend": { @@ -1005,12 +1005,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Max heap used (MB)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Avg. CPU (user) %'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Max heap used (MB)", + "title": "Avg. CPU (user) %", "type": "timeseries" },{ "datasource": { @@ -1072,10 +1072,10 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, + "w": 8, + "x": 8, + "y": 24 + }, "id": 1, "options": { "legend": { @@ -1104,12 +1104,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Peak threads'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Thread switch rate'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Peak threads", + "title": "Thread switch rate", "type": "timeseries" },{ "datasource": { @@ -1171,10 +1171,10 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, + "w": 8, + "x": 16, + "y": 24 + }, "id": 1, "options": { "legend": { @@ -1203,12 +1203,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Max. CPU (user) %'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Total allocated MB'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Max. CPU (user) %", + "title": "Total allocated MB", "type": "timeseries" },{ "datasource": { @@ -1270,10 +1270,10 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 0 - }, + "y": 32 + }, "id": 1, "options": { "legend": { @@ -1302,12 +1302,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Startup time (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Net write avg (bps)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Startup time (ms)", + "title": "Net write avg (bps)", "type": "timeseries" },{ "datasource": { @@ -1369,10 +1369,10 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, + "w": 8, + "x": 8, + "y": 32 + }, "id": 1, "options": { "legend": { @@ -1401,12 +1401,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'GC time (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Iter. mean (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "GC time (ms)", + "title": "Iter. mean (ms)", "type": "timeseries" },{ "datasource": { @@ -1468,10 +1468,10 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, + "w": 8, + "x": 16, + "y": 32 + }, "id": 1, "options": { "legend": { @@ -1500,12 +1500,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'GC pause time (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Min heap used (MB)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "GC pause time (ms)", + "title": "Min heap used (MB)", "type": "timeseries" },{ "datasource": { @@ -1567,10 +1567,10 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 0 - }, + "y": 40 + }, "id": 1, "options": { "legend": { @@ -1599,12 +1599,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Avg. CPU (user) %'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'GC time (ms)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Avg. CPU (user) %", + "title": "GC time (ms)", "type": "timeseries" },{ "datasource": { @@ -1666,10 +1666,10 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, + "w": 8, + "x": 8, + "y": 40 + }, "id": 1, "options": { "legend": { @@ -1698,12 +1698,12 @@ } }, "queryType": "sql", - "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Total allocated MB'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", + "rawSql": "SELECT\n MetricName,\n StartTimeUnix,\n MAX(IF(Attributes['entity'] = 'none', Value, NULL)) AS none,\n MAX(IF(Attributes['entity'] = 'snapshot', Value, NULL)) AS snapshot,\n MAX(IF(Attributes['entity'] = 'latest', Value, NULL)) AS latest\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'Net read avg (bps)'\nGROUP BY MetricName, StartTimeUnix\nORDER BY StartTimeUnix;", "refId": "A", "selectedFormat": 4 } ], - "title": "Total allocated MB", + "title": "Net read avg (bps)", "type": "timeseries" } ], @@ -1724,4 +1724,4 @@ "uid": "dfc2be2e-f435-4ebf-956a-782d7d16c6b0", "version": 2, "weekStart": "" -} +} \ No newline at end of file diff --git a/grafana_dashboard.go b/grafana_dashboard.go index b85f1d1..a19296e 100644 --- a/grafana_dashboard.go +++ b/grafana_dashboard.go @@ -1,8 +1,38 @@ package main -import "fmt" +import ( + "fmt" + "os" + "strings" +) -func generateDashboard(panels string) string { +func generateDashboard(metrics []string) { + var panels []string + var currentX = 0 + var currentY = 0 + panelWidth := 8 + panelHeight := 8 + panelsPerRow := 3 + + for _, metric := range metrics { + panels = append(panels, generatePanel(metric, metric, panelHeight, panelWidth, currentX, currentY)) + // Update the current position for the next panel + currentX += panelWidth + if currentX >= panelsPerRow*panelWidth { + currentX = 0 + currentY += panelHeight + } + } + + // Update Dashboard based on metrics + dashboard := generateDashboardJson(strings.Join(panels, ",")) + err := os.WriteFile("grafana/dashboards/instrumentation-benchmarks.json", []byte(dashboard), 0644) + if err != nil { + panic(err) + } +} + +func generateDashboardJson(panels string) string { return fmt.Sprintf(`{ "annotations": { "list": [ @@ -49,7 +79,15 @@ func generateDashboard(panels string) string { }`, panels) } -func generatePanel(metricName, friendlyName string) string { +func generatePanel(metricName, friendlyName string, panelHeight, panelWidth, currentX, currentY int) string { + + gridPos := fmt.Sprintf(`{ + "h": %d, + "w": %d, + "x": %d, + "y": %d + }`, panelHeight, panelWidth, currentX, currentY) + return fmt.Sprintf(`{ "datasource": { "type": "grafana-clickhouse-datasource", @@ -108,12 +146,7 @@ func generatePanel(metricName, friendlyName string) string { }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, + "gridPos": %s, "id": 1, "options": { "legend": { @@ -149,5 +182,5 @@ func generatePanel(metricName, friendlyName string) string { ], "title": "%s", "type": "timeseries" - }`, metricName, friendlyName) + }`, gridPos, metricName, friendlyName) } diff --git a/main.go b/main.go index 6083d88..13ee579 100644 --- a/main.go +++ b/main.go @@ -8,7 +8,6 @@ import ( "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" "os" - "strings" ) func main() { @@ -16,15 +15,41 @@ func main() { repo := "open-telemetry/opentelemetry-java-instrumentation" filePath := "benchmark-overhead/results/release/summary.txt" + // Cache API calls to github to prevent repeated calls when testing + commitCache := NewSingleFileCache("cache/commit-cache.json") + reportCache := NewSingleFileCache("cache/report-cache.json") + client := NewGitHubClient(token) - timeframe, _ := generateTimeframeToToday("2023-04-01", 30) + timeframe, _ := generateTimeframeToToday("2022-02-14", 7) dataPoints := map[string][]metricdata.DataPoint[float64]{} for _, timestamp := range timeframe { - commit, _ := client.GetMostRecentCommit(repo, timestamp, "gh-pages") - contents, _ := client.GetFileAtCommit(repo, filePath, commit) + var commit string + cached, _ := commitCache.RetrieveValue(timestamp) + if cached == "" { + commit, _ = client.GetMostRecentCommit(repo, timestamp, "gh-pages") + err := commitCache.AddToCache(timestamp, commit) + if err != nil { + fmt.Println("Error adding to cache") + } + } else { + commit = cached + } + + var contents string + cached, _ = reportCache.RetrieveValue(timestamp) + if cached == "" { + contents, _ = client.GetFileAtCommit(repo, filePath, commit) + err := reportCache.AddToCache(timestamp, contents) + if err != nil { + fmt.Println("Error adding to cache") + } + } else { + contents = cached + } + report := ParseReport(contents) for entity, metrics := range report.Metrics { @@ -39,17 +64,14 @@ func main() { } } - panels := []string{} + var metricNames []string var metrics []metricdata.Metrics for metric, metricData := range dataPoints { metrics = append(metrics, *generateMetrics(metric, metricData)) - panels = append(panels, generatePanel(metric, metric)) + metricNames = append(metricNames, metric) } - dashboard := generateDashboard(strings.Join(panels, ",")) - fmt.Print(dashboard) - resourceMetrics := generateResourceMetrics(metrics) ctx := context.Background() @@ -67,7 +89,9 @@ func main() { otel.SetMeterProvider(meterProvider) // export to collector - fmt.Sprintf("Exporting metrics") + fmt.Println("Exporting metrics") _ = exp.Export(ctx, resourceMetrics) + // Update Dashboard based on metrics + generateDashboard(metricNames) } diff --git a/media/benchmarks.png b/media/benchmarks.png new file mode 100644 index 0000000..00a3cca Binary files /dev/null and b/media/benchmarks.png differ diff --git a/readme.md b/readme.md index 25808cc..d23d1b1 100644 --- a/readme.md +++ b/readme.md @@ -1,13 +1,25 @@ # Benchmark Metrics +![Coverage](../.github/badges/coverage.svg) + +## Context + Given a repository with a branch that includes reports of benchmark test runs, convert this data into timeseries metrics that can be visualized in grafana. -- [x] Generate time frames -- [x] Github client - - [x] get commits - - [x] get contents - - [ ] cache -- [ ] Convert to metrics -- [ ] Grafana dashboard -- [ ] Tag commit meta data? \ No newline at end of file +The motivation behind this project was to analyze the historical benchmark data from the +https://github.com/open-telemetry/opentelemetry-java-instrumentation project. + +## Setup + +It helps to have a github API key set via a `GITHUB_TOKEN` env variable, but not needed (although you might get rate limited) + + +You can run everything via docker + +`docker compose up -d` + +Access via [Grafana](http://localhost:3001/): + +![grafana](./media/benchmarks.png) + diff --git a/report_parser.go b/report_parser.go index 41e10d9..0029f82 100644 --- a/report_parser.go +++ b/report_parser.go @@ -13,7 +13,6 @@ type ReportMetrics struct { } func ParseReport(report string) ReportMetrics { - split := strings.Split(report, "----------------------------------------------------------\n") date := strings.Split(strings.Split(split[1], "Run at ")[1], "\n")[0] @@ -32,11 +31,21 @@ func ParseReport(report string) ReportMetrics { if strings.HasPrefix(line, "Agent") || strings.HasPrefix(line, "Run duration") || line == "" { continue } + + // Bad data in some of the reports + if strings.Contains(line, "8796093022208.00") { + continue + } + metricList := strings.Split(line, ":") metricName := strings.TrimSpace(metricList[0]) + for index, value := range entities { - thisMetric, _ := strconv.ParseFloat(splitByMultipleSpaces(metricList[1])[index], 32) - metrics[value][metricName] = math.Round(thisMetric*100) / 100 + silo := splitByMultipleSpaces(metricList[1]) + thisMetric, err := strconv.ParseFloat(silo[index], 32) + if err == nil { + metrics[value][metricName] = math.Round(thisMetric*100) / 100 + } } } diff --git a/report_parser_test.go b/report_parser_test.go index 79a253c..28bcf95 100644 --- a/report_parser_test.go +++ b/report_parser_test.go @@ -1,8 +1,7 @@ package main import ( - "fmt" - "reflect" + "github.com/stretchr/testify/assert" "testing" ) @@ -36,29 +35,27 @@ Peak threads : 42 53 55` func TestParseDateFromSummary(t *testing.T) { expected := "2023-09-01" result := ParseReport(report) - if expected != result.Date.Format("2006-01-02") { - t.Errorf(fmt.Sprintf("should be equal %s %s", expected, result.Date)) - } + assert.Equal(t, expected, result.Date.Format("2006-01-02")) } func TestParseConfigsFromReport(t *testing.T) { - expected := []string{"none", "latest", "snapshot"} + expected := []string{"latest", "snapshot", "none"} result := ParseReport(report) - if reflect.DeepEqual(expected, result) { - t.Errorf(fmt.Sprintf("should be equal")) + + var entities []string + for entity := range result.Metrics { + entities = append(entities, entity) } + + assert.ElementsMatch(t, expected, entities) } func TestParseMetricsFromReport(t *testing.T) { expected := 92.64 result := ParseReport(report).Metrics["none"]["Min heap used (MB)"] - if expected != result { - t.Errorf(fmt.Sprintf("should be equal")) - } + assert.Equal(t, expected, result) expected = 53 result = ParseReport(report).Metrics["latest"]["Peak threads"] - if expected != result { - t.Errorf(fmt.Sprintf("should be equal")) - } + assert.Equal(t, expected, result) } diff --git a/utilities.go b/utilities.go index a40d52f..e030d10 100644 --- a/utilities.go +++ b/utilities.go @@ -7,18 +7,22 @@ import ( "time" ) +var ( + layout = "2006-01-02" +) + func generateTimeframeToToday(start string, interval int) ([]string, error) { currentTime := time.Now() - return generateTimeframeSlice(start, currentTime.Format("2006-01-02"), interval) + return generateTimeframeSlice(start, currentTime.Format(layout), interval) } func generateTimeframeSlice(start, end string, interval int) ([]string, error) { - startDate, err := time.Parse("2006-01-02", start) + startDate, err := time.Parse(layout, start) if err != nil { return nil, fmt.Errorf("failed to parse start date: %v", err) } - endDate, err := time.Parse("2006-01-02", end) + endDate, err := time.Parse(layout, end) if err != nil { return nil, fmt.Errorf("failed to parse end date: %v", err) } @@ -28,7 +32,7 @@ func generateTimeframeSlice(start, end string, interval int) ([]string, error) { // Increment the start date by the interval until it reaches or exceeds the end date currentDate := startDate for currentDate.Before(endDate) || currentDate.Equal(endDate) { - dateList = append(dateList, currentDate.Format("2006-01-02")) + dateList = append(dateList, currentDate.Format(layout)) currentDate = currentDate.AddDate(0, 0, interval) } @@ -58,6 +62,5 @@ func splitByMultipleSpaces(input string) []string { cleanedValues = append(cleanedValues, trimmedValue) } } - return cleanedValues } diff --git a/utilities_test.go b/utilities_test.go index 65bf94b..7b41a38 100644 --- a/utilities_test.go +++ b/utilities_test.go @@ -1,8 +1,7 @@ package main import ( - "fmt" - "reflect" + "github.com/stretchr/testify/assert" "testing" ) @@ -12,25 +11,19 @@ func TestGenerateTimeframeSlice(t *testing.T) { interval := 1 expected := []string{"2023-09-20", "2023-09-21", "2023-09-22", "2023-09-23"} result, _ := generateTimeframeSlice(start, end, interval) - if !reflect.DeepEqual(expected, result) { - t.Errorf(fmt.Sprintf("should be equal %s %s", expected, result)) - } + assert.Equal(t, expected, result) } func TestConvertDateFormat(t *testing.T) { start := "Fri Sep 01 05:16:59 UTC 2023" expected := "2023-09-01" result := convertDateFormat(start) - if !reflect.DeepEqual(expected, result) { - t.Errorf(fmt.Sprintf("should be equal %s %s", expected, result)) - } + assert.Equal(t, expected, result.Format("2006-01-02")) } func TestSplitByMultipleSpaces(t *testing.T) { start := ` 111.87 129.19 129.52` expected := []string{"111.87", "129.19", "129.52"} result := splitByMultipleSpaces(start) - if !reflect.DeepEqual(expected, result) { - t.Errorf(fmt.Sprintf("should be equal %s %s", expected, result)) - } + assert.Equal(t, expected, result) }