Migrate benchmarks to k6

.github/workflows/bench.yml

@@ -35,15 +35,19 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
 
-      - name: Build devcontainer and run setup and benchmark
+      - name: Build devcontainer
         uses: devcontainers/[email protected]
         with:
           imageName: graphql-benchmarks
           push: never
-          runCmd: |
-            bash ./graphql/${{ matrix.service }}/setup.sh
-            bash run_benchmarks.sh ${{ matrix.service }}
 
+      - name: Setup k6
+        uses: grafana/setup-k6-action@v1
+
+      - name: Run benchmarks
+        run: |
+          bash ./graphql/${{ matrix.service }}/setup.sh
+          bash run_benchmarks.sh ${{ matrix.service }}
       - name: List benchmark files
         run: |
           ls -la bench*.txt || echo "No matching files found"
@@ -57,10 +61,28 @@ jobs:
   analyze:
     needs: build
     runs-on: benchmarking-runner
+    env:
+      GRAFANA_API_KEY: ${{ secrets.GRAFANA_API_KEY }}
+      INFLUXDB_TOKEN: ${{ secrets.INFLUXDB_TOKEN }}
+      INFLUXDB_ORG: ${{ secrets.INFLUXDB_ORG }}
+      INFLUXDB_URL: ${{ secrets.INFLUXDB_URL }}
     steps:
       - name: Checkout (GitHub)
         uses: actions/checkout@v4
 
+      - name: Setup influxdb-cli
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: |
+          echo "UPLOAD_TO_CLOUD=true" >> $GITHUB_ENV
+          mkdir -p influxdb
+          cd influxdb
+          wget https://download.influxdata.com/influxdb/releases/influxdb2-client-2.7.5-linux-amd64.tar.gz
+          tar xvzf influxdb2-client-2.7.5-linux-amd64.tar.gz
+          cp influx /usr/local/bin/
+          influx config create --config-name benchmark --host-url $INFLUXDB_URL --org $INFLUXDB_ORG --token $INFLUXDB_TOKEN --active
+          cd ..
+          rm -rf influxdb
+
       - name: Download all benchmark results
         uses: actions/download-artifact@v3
         with:
@@ -73,7 +95,6 @@ jobs:
       - name: Analyze results
         run: |
           bash run_analyze_script.sh
-
       - name: Print benchmark results
         run: cat ./results.md
 

README.md

@@ -7,10 +7,8 @@ Explore and compare the performance of the fastest GraphQL frameworks through ou
 - [Introduction](#introduction)
 - [Quick Start](#quick-start)
 - [Benchmark Results](#benchmark-results)
-  - [Throughput (Higher is better)](#throughput-higher-is-better)
-  - [Latency (Lower is better)](#latency-lower-is-better)
 - [Architecture](#architecture)
-  - [WRK](#wrk)
+  - [K6](#k6)
   - [GraphQL](#graphql)
   - [Nginx](#nginx)
   - [Jsonplaceholder](#jsonplaceholder)
@@ -46,6 +44,15 @@ Get started with the benchmarks:
 
 ## Benchmark Results
 
+| Throughput (Higher is better) | Latency (Lower is better) | 
+|-------:|--------:|
+|  `{{ posts { id userId title user { id name email }}}}` |
+| ![](assets/posts_users_req.png) | ![](assets/posts_users_latency.png) |
+|  `{ posts { title }}` |
+| ![](assets/posts_req.png) | ![](assets/posts_latency.png) |
+|  `{greet}` |
+| ![](assets/greet_req.png) | ![](assets/greet_latency.png) |
+
 <!-- PERFORMANCE_RESULTS_START -->
 
 | Query | Server | Requests/sec | Latency (ms) | Relative |
@@ -80,44 +87,15 @@ Get started with the benchmarks:
 
 <!-- PERFORMANCE_RESULTS_END -->
 
-
-
-### 1. `{posts {title body user {name}}}`
-#### Throughput (Higher is better)
-
-![Throughput Histogram](assets/req_sec_histogram1.png)
-
-#### Latency (Lower is better)
-
-![Latency Histogram](assets/latency_histogram1.png)
-
-### 2. `{posts {title body}}`
-#### Throughput (Higher is better)
-
-![Throughput Histogram](assets/req_sec_histogram2.png)
-
-#### Latency (Lower is better)
-
-![Latency Histogram](assets/latency_histogram2.png)
-
-### 3. `{greet}`
-#### Throughput (Higher is better)
-
-![Throughput Histogram](assets/req_sec_histogram3.png)
-
-#### Latency (Lower is better)
-
-![Latency Histogram](assets/latency_histogram3.png)
-
 ## Architecture
 
 ![Architecture Diagram](assets/architecture.png)
 
-A client (`wrk`) sends requests to a GraphQL server to fetch post titles. The GraphQL server, in turn, retrieves data from an external source, `jsonplaceholder.typicode.com`, routed through the `nginx` reverse proxy.
+A client (`k6`) sends requests to a GraphQL server to fetch post titles. The GraphQL server, in turn, retrieves data from an external source, `jsonplaceholder.typicode.com`, routed through the `nginx` reverse proxy.
 
-### WRK
+### K6
 
-`wrk` serves as our test client, sending GraphQL requests at a high rate.
+`k6` serves as our test client, sending GraphQL requests at a high rate.
 
 ### GraphQL
 

analyze.sh

@@ -44,57 +44,13 @@ for idx in "${!servers[@]}"; do
   avgLatencies[${servers[$idx]}]=$(average "${latencyVals[@]}")
 done
 
-# Generating data files for gnuplot
-reqSecData="/tmp/reqSec.dat"
-latencyData="/tmp/latency.dat"
-
-echo "Server Value" >"$reqSecData"
-for server in "${servers[@]}"; do
-  echo "$server ${avgReqSecs[$server]}" >>"$reqSecData"
-done
-
-echo "Server Value" >"$latencyData"
-for server in "${servers[@]}"; do
-  echo "$server ${avgLatencies[$server]}" >>"$latencyData"
-done
-
 whichBench=1
 if [[ $1 == bench2* ]]; then
     whichBench=2
 elif [[ $1 == bench3* ]]; then
     whichBench=3
 fi
 
-reqSecHistogramFile="req_sec_histogram${whichBench}.png"
-latencyHistogramFile="latency_histogram${whichBench}.png"
-
-# Plotting using gnuplot
-gnuplot <<-EOF
-    set term pngcairo size 1280,720 enhanced font "Courier,12"
-    set output "$reqSecHistogramFile"
-    set style data histograms
-    set style histogram cluster gap 1
-    set style fill solid border -1
-    set xtics rotate by -45
-    set boxwidth 0.9
-    set title "Requests/Sec"
-    stats "$reqSecData" using 2 nooutput
-    set yrange [0:STATS_max*1.2]
-    set key outside right top
-    plot "$reqSecData" using 2:xtic(1) title "Req/Sec"
-
-    set output "$latencyHistogramFile"
-    set title "Latency (in ms)"
-    stats "$latencyData" using 2 nooutput
-    set yrange [0:STATS_max*1.2]
-    plot "$latencyData" using 2:xtic(1) title "Latency"
-EOF
-
-# Move PNGs to assets
-mkdir -p assets
-mv $reqSecHistogramFile assets/
-mv $latencyHistogramFile assets/
-
 # Declare an associative array for server RPS
 declare -A serverRPS
 
@@ -123,6 +79,11 @@ fi
 for server in "${sortedServers[@]}"; do
     formattedReqSecs=$(printf "%.2f" ${avgReqSecs[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g')
     formattedLatencies=$(printf "%.2f" ${avgLatencies[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g')
+    echo "Writing to influx for $server and benchmark $whichBench with ${avgReqSecs[$server]} and ${avgLatencies[$server]}"
+    influx write -b bench "
+    http_reqs,test_name=$server,benchmark=$whichBench value=${avgReqSecs[$server]}
+    latency,test_name=$server,benchmark=$whichBench value=${avgLatencies[$server]}
+    "
     # Calculate the relative performance
     relativePerformance=$(echo "${avgReqSecs[$server]} $lastServerReqSecs" | awk '{printf "%.2f", $1 / $2}')
 
@@ -157,10 +118,6 @@ if [[ $whichBench == 3 ]]; then
     fi
 fi
 
-# Move the generated images to the assets folder
-mv $reqSecHistogramFile assets/
-mv $latencyHistogramFile assets/
-
 # Delete the result TXT files
 for file in "${resultFiles[@]}"; do
   rm "$file"