diff --git a/Benchmarks/.gitignore b/Benchmarks/.gitignore new file mode 100644 index 0000000..785db78 --- /dev/null +++ b/Benchmarks/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +/.build +/Packages +xcuserdata/ +DerivedData/ +.swiftpm/configuration/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc +.benchmarkBaselines/ diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift index 10ac212..29d9d0b 100644 --- a/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift @@ -16,82 +16,49 @@ import Benchmark import Prometheus +let registry = PrometheusCollectorRegistry() + +public func makeLabels(_ idx: Int) -> [(String, String)] { + [ + ("job", "api_server_\(idx)"), + ("handler", "/api/handler_\(idx)"), + ("status_code", "200"), + ("version", "\(idx).0.0"), + ] +} + let benchmarks = { Benchmark.defaultConfiguration.maxDuration = .seconds(5) Benchmark.defaultConfiguration.scalingFactor = .kilo + // Benchmark.defaultConfiguration.metrics = [.wallClock, .throughput, .mallocCountTotal] + Benchmark.defaultConfiguration.metrics = [.mallocCountTotal] - let registry = PrometheusCollectorRegistry() - - func metricsDimensions(_ idx: Int) -> [(String, String)] { - [ - ("job", "api_server_\(idx)"), - ("handler", "/api/handler_\(idx)"), - ("status_code", "200"), - ("version", "\(idx).0.0"), - ] + Benchmark("Counter #1") { benchmark in + runCounterBench(benchmark.scaledIterations) } - Benchmark("1 - Metrics: Counter benchmark") { benchmark in - let ctr = registry.makeCounter(name: "counter_1", labels: metricsDimensions(1)) - benchmark.startMeasurement() + Benchmark("Counter #2") { benchmark, run in for _ in benchmark.scaledIterations { - blackHole(ctr.increment()) + run() } + } setup: { + setupCounterBench() } - Benchmark("2 - Metrics: Gauge benchmark") { benchmark in - let gauge = registry.makeGauge(name: "gauge_1", labels: metricsDimensions(2)) - benchmark.startMeasurement() - for _ in benchmark.scaledIterations { - blackHole(gauge.increment()) - } + Benchmark("Gauge") { benchmark in + runGaugeBench(benchmark.scaledIterations) } - Benchmark("3 - Metrics: Histogram benchmark") { benchmark in - let histogram = registry.makeDurationHistogram(name: "histogram_1", labels: metricsDimensions(3), - buckets: [ - .milliseconds(100), - .milliseconds(250), - .milliseconds(500), - .seconds(1), - ]) - benchmark.startMeasurement() - for _ in benchmark.scaledIterations { - histogram.record(Duration.milliseconds(400)) - } + Benchmark("DurationHistogram") { benchmark in + runDurationHistogramBench(benchmark.scaledIterations) } - Benchmark("4 - Metrics: export 5000 metrics", - configuration: .init(scalingFactor: .one)) { benchmark in - let metricsCount = 5000 - - let registryExport = PrometheusCollectorRegistry() - - var counterArray = [Counter]() - var gaugeArray = [Gauge]() - var buffer = [UInt8]() - - let counterExportSize = 620_000 - counterArray.reserveCapacity(metricsCount) - gaugeArray.reserveCapacity(metricsCount) - buffer.reserveCapacity(counterExportSize) - - for i in 0..<(metricsCount / 2) { - let counter = registryExport.makeCounter(name: "http_requests_total", labels: metricsDimensions(i)) - counter.increment() - counterArray.append(counter) - - let gauge = registryExport.makeGauge(name: "export_gauge_\(i)", labels: metricsDimensions(i)) - gauge.increment() - gaugeArray.append(gauge) - } - - benchmark.startMeasurement() + Benchmark("RegistryEmit - 5000 metrics", + configuration: .init(scalingFactor: .one)) { benchmark, run in for _ in benchmark.scaledIterations { - blackHole(registryExport.emit(into: &buffer)) - + run() } - benchmark.stopMeasurement() - buffer.removeAll(keepingCapacity: true) + } setup: { + setupRegistryExport(numberOfMetrics: 5000) } } diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift new file mode 100644 index 0000000..a3f76a0 --- /dev/null +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift @@ -0,0 +1,31 @@ +// swift-tools-version:5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftPrometheus open source project +// +// Copyright (c) 2018-2023 SwiftPrometheus project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Prometheus + +public func runCounterBench(_ iterations: Range) { + let ctr = registry.makeCounter(name: "counter_1", labels: makeLabels(1)) + for _ in iterations { + blackHole(ctr.increment()) + } +} + +public func setupCounterBench() -> () -> Void { + let ctr = registry.makeCounter(name: "counter_2", labels: makeLabels(2)) + return { + blackHole(ctr.increment()) + } +} diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift new file mode 100644 index 0000000..9e6c0f3 --- /dev/null +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift @@ -0,0 +1,30 @@ +// swift-tools-version:5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftPrometheus open source project +// +// Copyright (c) 2018-2023 SwiftPrometheus project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Prometheus + +public func runDurationHistogramBench(_ iterations: Range) { + let histogram = registry.makeDurationHistogram(name: "histogram_1", labels: makeLabels(3), + buckets: [ + .milliseconds(100), + .milliseconds(250), + .milliseconds(500), + .seconds(1), + ]) + for _ in iterations { + blackHole(histogram.record(Duration.milliseconds(400))) + } +} diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift new file mode 100644 index 0000000..2cf34eb --- /dev/null +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift @@ -0,0 +1,31 @@ +// swift-tools-version:5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftPrometheus open source project +// +// Copyright (c) 2018-2023 SwiftPrometheus project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Prometheus + +public func runGaugeBench(_ iterations: Range) { + let gauge = registry.makeGauge(name: "gauge_1", labels: makeLabels(2)) + for _ in iterations { + blackHole(gauge.increment()) + } +} + +public func setupGaugeBench() -> () -> Void { + let gauge = registry.makeGauge(name: "gauge_1", labels: makeLabels(2)) + return { + blackHole(gauge.increment()) + } +} diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift new file mode 100644 index 0000000..8b92add --- /dev/null +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift @@ -0,0 +1,43 @@ +// swift-tools-version:5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftPrometheus open source project +// +// Copyright (c) 2018-2023 SwiftPrometheus project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Prometheus + +public func setupRegistryExport(numberOfMetrics: Int) -> () -> Void { + let registryExport = PrometheusCollectorRegistry() + + var counterArray = [Counter]() + var gaugeArray = [Gauge]() + var buffer = [UInt8]() + + let counterExportSize = 620_000 + counterArray.reserveCapacity(numberOfMetrics) + gaugeArray.reserveCapacity(numberOfMetrics) + buffer.reserveCapacity(counterExportSize) + + for i in 0..<(numberOfMetrics / 2) { + let counter = registryExport.makeCounter(name: "http_requests_total", labels: makeLabels(i)) + counter.increment() + counterArray.append(counter) + + let gauge = registryExport.makeGauge(name: "export_gauge_\(i)", labels: makeLabels(i)) + gauge.increment() + gaugeArray.append(gauge) + } + return { + blackHole(registryExport.emit(into: &buffer)) + } +} diff --git a/Benchmarks/Package.resolved b/Benchmarks/Package.resolved deleted file mode 100644 index eb89cb0..0000000 --- a/Benchmarks/Package.resolved +++ /dev/null @@ -1,113 +0,0 @@ -{ - "pins" : [ - { - "identity" : "package-benchmark", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/package-benchmark.git", - "state" : { - "revision" : "070dee37abe72c0a3d7e0feff5dca3effd66b541", - "version" : "1.11.2" - } - }, - { - "identity" : "package-datetime", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/package-datetime", - "state" : { - "revision" : "d1242188c9f48aad297e6ca9b717776f8660bc31", - "version" : "1.0.2" - } - }, - { - "identity" : "package-histogram", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/package-histogram", - "state" : { - "revision" : "a69fa24d7b70421870cafa86340ece900489e17e", - "version" : "0.1.2" - } - }, - { - "identity" : "package-jemalloc", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/package-jemalloc", - "state" : { - "revision" : "e8a5db026963f5bfeac842d9d3f2cc8cde323b49", - "version" : "1.0.0" - } - }, - { - "identity" : "progress.swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/Progress.swift", - "state" : { - "revision" : "29dc5dc29d8408f42878b832c7aae38a35ff26ee", - "version" : "1.0.3" - } - }, - { - "identity" : "swift-argument-parser", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-argument-parser", - "state" : { - "revision" : "8f4d2753f0e4778c76d5f05ad16c74f707390531", - "version" : "1.2.3" - } - }, - { - "identity" : "swift-atomics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-atomics.git", - "state" : { - "revision" : "cd142fd2f64be2100422d658e7411e39489da985", - "version" : "1.2.0" - } - }, - { - "identity" : "swift-extras-json", - "kind" : "remoteSourceControl", - "location" : "https://github.com/swift-extras/swift-extras-json", - "state" : { - "revision" : "122b9454ef01bf89a4c190b8fd3717ddd0a2fbd0", - "version" : "0.6.0" - } - }, - { - "identity" : "swift-metrics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-metrics.git", - "state" : { - "revision" : "971ba26378ab69c43737ee7ba967a896cb74c0d1", - "version" : "2.4.1" - } - }, - { - "identity" : "swift-numerics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-numerics", - "state" : { - "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b", - "version" : "1.0.2" - } - }, - { - "identity" : "swift-system", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-system", - "state" : { - "revision" : "025bcb1165deab2e20d4eaba79967ce73013f496", - "version" : "1.2.1" - } - }, - { - "identity" : "texttable", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/TextTable", - "state" : { - "revision" : "a27a07300cf4ae322e0079ca0a475c5583dd575f", - "version" : "0.0.2" - } - } - ], - "version" : 2 -} diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift index 4cee4b4..9674c1b 100644 --- a/Benchmarks/Package.swift +++ b/Benchmarks/Package.swift @@ -18,7 +18,7 @@ import PackageDescription let package = Package( name: "benchmarks", platforms: [ - .macOS("13"), + .macOS("14"), ], dependencies: [ .package(path: "../"), @@ -29,10 +29,12 @@ let package = Package( name: "PrometheusBenchmarks", dependencies: [ .product(name: "Benchmark", package: "package-benchmark"), - .product(name: "BenchmarkPlugin", package: "package-benchmark"), .product(name: "Prometheus", package: "swift-prometheus"), ], - path: "Benchmarks/PrometheusBenchmarks" + path: "Benchmarks/PrometheusBenchmarks", + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark"), + ] ), ] ) diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#1.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#1.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#1.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#2.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#2.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#2.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.DurationHistogram.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.DurationHistogram.p90.json new file mode 100644 index 0000000..282dc64 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.DurationHistogram.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 2 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Gauge.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Gauge.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Gauge.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/README.md b/README.md index 7aab78a..195418b 100644 --- a/README.md +++ b/README.md @@ -22,3 +22,17 @@ If you find a bug or have issues, please [create an issue](https://github.com/sw [Documentation]: https://swiftpackageindex.com/swift-server/swift-prometheus/documentation/prometheus [SSWG-Incubation]: https://www.swift.org/sswg/incubation-process.html + + +## Benchmarks + +Benchmarks for `swift-prometheus` are in a separate Swift Package in the `Benchmarks` subfolder of this repository. +They use the [`package-benchmark`](https://github.com/ordo-one/package-benchmark) plugin. +Benchmarks depends on the [`jemalloc`](https://jemalloc.net) memory allocation library, which is used by `package-benchmark` to capture memory allocation statistics. +An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`. +Afterwards you can run the benchmarks from CLI by going to the `Benchmarks` subfolder (e.g. `cd Benchmarks`) and invoking: +``` +swift package benchmark +``` + +For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark). diff --git a/dev/update-benchmark-tresholds.sh b/dev/update-benchmark-tresholds.sh new file mode 100755 index 0000000..9a37298 --- /dev/null +++ b/dev/update-benchmark-tresholds.sh @@ -0,0 +1,41 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftCertificates open source project +## +## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +set -eu +set -o pipefail + +here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +target_repo=${2-"$here/.."} + +for f in 57 58 59 510 -nightly; do + echo "swift$f" + + docker_file=$(if [[ "$f" == "-nightly" ]]; then f=main; fi && ls "$target_repo/docker/docker-compose."*"$f"*".yaml") + + docker-compose -f docker/docker-compose.yaml -f $docker_file run update-benchmark-baseline +done diff --git a/docker/docker-compose.2204.57.yaml b/docker/docker-compose.2204.57.yaml index fa96acd..5f1524f 100644 --- a/docker/docker-compose.2204.57.yaml +++ b/docker/docker-compose.2204.57.yaml @@ -17,5 +17,10 @@ services: environment: [] #- SANITIZER_ARG=--sanitize=thread + update-benchmark-baseline: + image: prometheus:22.04-5.7 + environment: + - SWIFT_VERSION=5.7 + shell: image: prometheus:22.04-5.7 diff --git a/docker/docker-compose.2204.58.yaml b/docker/docker-compose.2204.58.yaml index 1d27d8e..5d653b6 100644 --- a/docker/docker-compose.2204.58.yaml +++ b/docker/docker-compose.2204.58.yaml @@ -18,5 +18,10 @@ services: - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error #- SANITIZER_ARG=--sanitize=thread + update-benchmark-baseline: + image: prometheus:22.04-5.8 + environment: + - SWIFT_VERSION=5.8 + shell: image: prometheus:22.04-5.8 diff --git a/docker/docker-compose.2204.59.yaml b/docker/docker-compose.2204.59.yaml index 1ae4771..e893b44 100644 --- a/docker/docker-compose.2204.59.yaml +++ b/docker/docker-compose.2204.59.yaml @@ -6,7 +6,8 @@ services: image: prometheus:22.04-5.9 build: args: - base_image: "swiftlang/swift:nightly-5.9-jammy" + ubuntu_version: "jammy" + swift_version: "5.9" documentation-check: image: prometheus:22.04-5.9 @@ -14,8 +15,14 @@ services: test: image: prometheus:22.04-5.9 environment: + - SWIFT_VERSION=main - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error #- SANITIZER_ARG=--sanitize=thread + update-benchmark-baseline: + image: prometheus:22.04-5.9 + environment: + - SWIFT_VERSION=5.9 + shell: image: prometheus:22.04-5.9 diff --git a/docker/docker-compose.2204.main.yaml b/docker/docker-compose.2204.main.yaml index 461eefc..707b242 100644 --- a/docker/docker-compose.2204.main.yaml +++ b/docker/docker-compose.2204.main.yaml @@ -17,5 +17,8 @@ services: - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error #- SANITIZER_ARG=--sanitize=thread + update-benchmark-baseline: + image: prometheus:22.04-main + shell: image: prometheus:22.04-main diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index a53b7e5..5affd44 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -33,7 +33,11 @@ services: test: <<: *common depends_on: [runtime-setup] - command: /bin/bash -xcl "swift test -Xswiftc -warnings-as-errors --enable-test-discovery $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-}" + command: /bin/bash -xcl "swift test -Xswiftc -warnings-as-errors --enable-test-discovery $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} && cd Benchmarks && swift package benchmark baseline check --check-absolute-path Thresholds/$${SWIFT_VERSION-}/" + + update-benchmark-baseline: + <<: *common + command: /bin/bash -xcl "cd Benchmarks && swift package --disable-sandbox --scratch-path .build/$${SWIFT_VERSION-}/ --allow-writing-to-package-directory benchmark --format metricP90AbsoluteThresholds --path Thresholds/$${SWIFT_VERSION-}/" # util