Refactor and add docker scripts

Benchmarks/.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+/.build
+/Packages
+xcuserdata/
+DerivedData/
+.swiftpm/configuration/registries.json
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
+.netrc
+.benchmarkBaselines/

Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift

@@ -16,82 +16,49 @@
 import Benchmark
 import Prometheus
 
+let registry = PrometheusCollectorRegistry()
+
+public func makeLabels(_ idx: Int) -> [(String, String)] {
+    [
+        ("job", "api_server_\(idx)"),
+        ("handler", "/api/handler_\(idx)"),
+        ("status_code", "200"),
+        ("version", "\(idx).0.0"),
+    ]
+}
+
 let benchmarks = {
     Benchmark.defaultConfiguration.maxDuration = .seconds(5)
     Benchmark.defaultConfiguration.scalingFactor = .kilo
+    // Benchmark.defaultConfiguration.metrics = [.wallClock, .throughput, .mallocCountTotal]
+    Benchmark.defaultConfiguration.metrics = [.mallocCountTotal]
 
-    let registry = PrometheusCollectorRegistry()
-
-    func metricsDimensions(_ idx: Int) -> [(String, String)] {
-        [
-            ("job", "api_server_\(idx)"),
-            ("handler", "/api/handler_\(idx)"),
-            ("status_code", "200"),
-            ("version", "\(idx).0.0"),
-        ]
+    Benchmark("Counter #1") { benchmark in
+        runCounterBench(benchmark.scaledIterations)
     }
 
-    Benchmark("1 - Metrics: Counter benchmark") { benchmark in
-        let ctr = registry.makeCounter(name: "counter_1", labels: metricsDimensions(1))
-        benchmark.startMeasurement()
+    Benchmark("Counter #2") { benchmark, run in
         for _ in benchmark.scaledIterations {
-            blackHole(ctr.increment())
+            run()
         }
+    } setup: {
+        setupCounterBench()
     }
 
-    Benchmark("2 - Metrics: Gauge benchmark") { benchmark in
-        let gauge = registry.makeGauge(name: "gauge_1", labels: metricsDimensions(2))
-        benchmark.startMeasurement()
-        for _ in benchmark.scaledIterations {
-            blackHole(gauge.increment())
-        }
+    Benchmark("Gauge") { benchmark in
+        runGaugeBench(benchmark.scaledIterations)
     }
 
-    Benchmark("3 - Metrics: Histogram benchmark") { benchmark in
-        let histogram = registry.makeDurationHistogram(name: "histogram_1", labels: metricsDimensions(3),
-                                                       buckets: [
-                                                           .milliseconds(100),
-                                                           .milliseconds(250),
-                                                           .milliseconds(500),
-                                                           .seconds(1),
-                                                       ])
-        benchmark.startMeasurement()
-        for _ in benchmark.scaledIterations {
-            histogram.record(Duration.milliseconds(400))
-        }
+    Benchmark("DurationHistogram") { benchmark in
+        runDurationHistogramBench(benchmark.scaledIterations)
     }
 
-    Benchmark("4 - Metrics: export 5000 metrics", 
-              configuration: .init(scalingFactor: .one)) { benchmark in
-        let metricsCount = 5000
-
-        let registryExport = PrometheusCollectorRegistry()
-
-        var counterArray = [Counter]()
-        var gaugeArray = [Gauge]()
-        var buffer = [UInt8]()
-
-        let counterExportSize = 620_000
-        counterArray.reserveCapacity(metricsCount)
-        gaugeArray.reserveCapacity(metricsCount)
-        buffer.reserveCapacity(counterExportSize)
-
-        for i in 0..<(metricsCount / 2) {
-            let counter = registryExport.makeCounter(name: "http_requests_total", labels: metricsDimensions(i))
-            counter.increment()
-            counterArray.append(counter)
-
-            let gauge = registryExport.makeGauge(name: "export_gauge_\(i)", labels: metricsDimensions(i))
-            gauge.increment()
-            gaugeArray.append(gauge)
-        }
-
-        benchmark.startMeasurement()
+    Benchmark("RegistryEmit - 5000 metrics",
+              configuration: .init(scalingFactor: .one)) { benchmark, run in
         for _ in benchmark.scaledIterations {
-            blackHole(registryExport.emit(into: &buffer))
-
+            run()
         }
-        benchmark.stopMeasurement()
-        buffer.removeAll(keepingCapacity: true)
+    } setup: {
+        setupRegistryExport(numberOfMetrics: 5000)
     }
 }

Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift

@@ -0,0 +1,31 @@
+// swift-tools-version:5.7
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftPrometheus open source project
+//
+// Copyright (c) 2018-2023 SwiftPrometheus project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Prometheus
+
+public func runCounterBench(_ iterations: Range<Int>) {
+    let ctr = registry.makeCounter(name: "counter_1", labels: makeLabels(1))
+    for _ in iterations {
+        blackHole(ctr.increment())
+    }
+}
+
+public func setupCounterBench() -> () -> Void {
+    let ctr = registry.makeCounter(name: "counter_2", labels: makeLabels(2))
+    return {
+        blackHole(ctr.increment())
+    }
+}

Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift

@@ -0,0 +1,30 @@
+// swift-tools-version:5.7
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftPrometheus open source project
+//
+// Copyright (c) 2018-2023 SwiftPrometheus project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Prometheus
+
+public func runDurationHistogramBench(_ iterations: Range<Int>) {
+    let histogram = registry.makeDurationHistogram(name: "histogram_1", labels: makeLabels(3),
+                                                   buckets: [
+                                                       .milliseconds(100),
+                                                       .milliseconds(250),
+                                                       .milliseconds(500),
+                                                       .seconds(1),
+                                                   ])
+    for _ in iterations {
+        blackHole(histogram.record(Duration.milliseconds(400)))
+    }
+}

Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift

@@ -0,0 +1,31 @@
+// swift-tools-version:5.7
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftPrometheus open source project
+//
+// Copyright (c) 2018-2023 SwiftPrometheus project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Prometheus
+
+public func runGaugeBench(_ iterations: Range<Int>) {
+    let gauge = registry.makeGauge(name: "gauge_1", labels: makeLabels(2))
+    for _ in iterations {
+        blackHole(gauge.increment())
+    }
+}
+
+public func setupGaugeBench() -> () -> Void {
+    let gauge = registry.makeGauge(name: "gauge_1", labels: makeLabels(2))
+    return {
+        blackHole(gauge.increment())
+    }
+}

Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift

@@ -0,0 +1,43 @@
+// swift-tools-version:5.7
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the SwiftPrometheus open source project
+//
+// Copyright (c) 2018-2023 SwiftPrometheus project authors
+// Licensed under Apache License v2.0
+//
+// See LICENSE.txt for license information
+// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+//===----------------------------------------------------------------------===//
+
+import Benchmark
+import Prometheus
+
+public func setupRegistryExport(numberOfMetrics: Int) -> () -> Void {
+    let registryExport = PrometheusCollectorRegistry()
+
+    var counterArray = [Counter]()
+    var gaugeArray = [Gauge]()
+    var buffer = [UInt8]()
+
+    let counterExportSize = 620_000
+    counterArray.reserveCapacity(numberOfMetrics)
+    gaugeArray.reserveCapacity(numberOfMetrics)
+    buffer.reserveCapacity(counterExportSize)
+
+    for i in 0..<(numberOfMetrics / 2) {
+        let counter = registryExport.makeCounter(name: "http_requests_total", labels: makeLabels(i))
+        counter.increment()
+        counterArray.append(counter)
+
+        let gauge = registryExport.makeGauge(name: "export_gauge_\(i)", labels: makeLabels(i))
+        gauge.increment()
+        gaugeArray.append(gauge)
+    }
+    return {
+        blackHole(registryExport.emit(into: &buffer))
+    }
+}

Benchmarks/Package.swift

@@ -18,7 +18,7 @@ import PackageDescription
 let package = Package(
     name: "benchmarks",
     platforms: [
-        .macOS("13"),
+        .macOS("14"),
     ],
     dependencies: [
         .package(path: "../"),
@@ -29,10 +29,12 @@ let package = Package(
             name: "PrometheusBenchmarks",
             dependencies: [
                 .product(name: "Benchmark", package: "package-benchmark"),
-                .product(name: "BenchmarkPlugin", package: "package-benchmark"),
                 .product(name: "Prometheus", package: "swift-prometheus"),
             ],
-            path: "Benchmarks/PrometheusBenchmarks"
+            path: "Benchmarks/PrometheusBenchmarks",
+            plugins: [
+                .plugin(name: "BenchmarkPlugin", package: "package-benchmark"),
+            ]
         ),
     ]
 )

Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#1.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#2.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 0
+}

Benchmarks/Thresholds/5.9/PrometheusBenchmarks.DurationHistogram.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 2
+}

Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Gauge.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 1
+}

Benchmarks/Thresholds/5.9/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json

@@ -0,0 +1,3 @@
+{
+  "mallocCountTotal" : 0
+}

README.md

@@ -22,3 +22,17 @@ If you find a bug or have issues, please [create an issue](https://github.com/sw
 
 [Documentation]: https://swiftpackageindex.com/swift-server/swift-prometheus/documentation/prometheus
 [SSWG-Incubation]: https://www.swift.org/sswg/incubation-process.html
+
+
+## Benchmarks
+
+Benchmarks for `swift-prometheus` are in a separate Swift Package in the `Benchmarks` subfolder of this repository.
+They use the [`package-benchmark`](https://github.com/ordo-one/package-benchmark) plugin.
+Benchmarks depends on the [`jemalloc`](https://jemalloc.net) memory allocation library, which is used by `package-benchmark` to capture memory allocation statistics.
+An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`.
+Afterwards you can run the benchmarks from CLI by going to the `Benchmarks` subfolder (e.g. `cd Benchmarks`) and invoking:
+```
+swift package benchmark
+```
+
+For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark). 

dev/update-benchmark-tresholds.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the SwiftNIO open source project
+##
+## Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of SwiftNIO project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+##===----------------------------------------------------------------------===##
+##
+## This source file is part of the SwiftCertificates open source project
+##
+## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors
+## Licensed under Apache License v2.0
+##
+## See LICENSE.txt for license information
+## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors
+##
+## SPDX-License-Identifier: Apache-2.0
+##
+##===----------------------------------------------------------------------===##
+
+set -eu
+set -o pipefail
+
+here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+target_repo=${2-"$here/.."}
+
+for f in 57 58 59 510 -nightly; do
+    echo "swift$f"
+
+    docker_file=$(if [[ "$f" == "-nightly" ]]; then f=main; fi && ls "$target_repo/docker/docker-compose."*"$f"*".yaml")
+
+    docker-compose -f docker/docker-compose.yaml -f $docker_file run update-benchmark-baseline
+done

docker/docker-compose.2204.57.yaml

@@ -17,5 +17,10 @@ services:
     environment: []
       #- SANITIZER_ARG=--sanitize=thread
 
+  update-benchmark-baseline:
+    image: prometheus:22.04-5.7
+    environment:
+      - SWIFT_VERSION=5.7
+
   shell:
     image: prometheus:22.04-5.7

docker/docker-compose.2204.58.yaml

@@ -18,5 +18,10 @@ services:
       - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error
       #- SANITIZER_ARG=--sanitize=thread
 
+  update-benchmark-baseline:
+    image: prometheus:22.04-5.8
+    environment:
+      - SWIFT_VERSION=5.8
+
   shell:
     image: prometheus:22.04-5.8