diff --git a/analyze.js b/analyze.js new file mode 100644 index 00000000..fc69a597 --- /dev/null +++ b/analyze.js @@ -0,0 +1,209 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const { execSync } = require('child_process'); + +const extractMetric = (file, metric) => { + const data = fs.readFileSync(file, 'utf8'); + const line = data.split('\n').find((line) => line.includes(metric)); + if (!line) return null; + return parseFloat(line.split(' ')[1].replace('ms', '')); +}; + +const average = (arr) => arr.reduce((a, b) => a + b, 0) / arr.length; + +const servers = [ + 'apollo', + 'caliban', + 'netflixdgs', + 'gqlgen', + 'tailcall', + 'async_graphql', + 'hasura', + 'graphql_jit', +]; + +// Associative array for formatted server names +const formattedServerNames = { + tailcall: 'Tailcall', + gqlgen: 'Gqlgen', + apollo: 'Apollo GraphQL', + netflixdgs: 'Netflix DGZ', + caliban: 'Caliban', + async_graphql: 'async-graphql', + hasura: 'Hasura', + graphql_jit: 'GraphQL JIT', +}; + +const resultFiles = process.argv.slice(2).filter((arg) => fs.existsSync(arg)); + +// Extract metrics and calculate averages +const calculateAverages = () => { + const avgReqSecs = {}; + const avgLatencies = {}; + + servers.forEach((server, idx) => { + const startIdx = idx * 3; + const reqSecVals = []; + const latencyVals = []; + + for (let j = 0; j < 3; j++) { + const fileIdx = startIdx + j; + reqSecVals.push(extractMetric(resultFiles[fileIdx], 'Requests/sec')); + latencyVals.push(extractMetric(resultFiles[fileIdx], 'Latency')); + } + + avgReqSecs[server] = average(reqSecVals); + avgLatencies[server] = average(latencyVals); + }); + + return { avgReqSecs, avgLatencies }; +}; + +const writeDataFile = (filename, data) => { + fs.writeFileSync(filename, 'Server Value\n'); + Object.entries(data).forEach(([server, value]) => { + fs.appendFileSync(filename, `${server} ${value}\n`); + }); +}; + +// Plotting using gnuplot +const generateGnuplotScript = ( + reqSecData, + latencyData, + reqSecHistogramFile, + latencyHistogramFile, +) => ` +set term pngcairo size 1280,720 enhanced font "Courier,12" +set style data histograms +set style histogram cluster gap 1 +set style fill solid border -1 +set xtics rotate by -45 +set boxwidth 0.9 +set key outside right top +set output "${reqSecHistogramFile}" +set title "Requests/Sec" +stats "${reqSecData}" using 2 nooutput +set yrange [0:STATS_max*1.2] +plot "${reqSecData}" using 2:xtic(1) title "Req/Sec" + +set output "${latencyHistogramFile}" +set title "Latency (in ms)" +stats "${latencyData}" using 2 nooutput +set yrange [0:STATS_max*1.2] +plot "${latencyData}" using 2:xtic(1) title "Latency" +`; + +// Generate the results table for README +const generateResultsTable = (avgReqSecs, avgLatencies, whichBench) => { + const sortedServers = Object.keys(avgReqSecs).sort( + (a, b) => avgReqSecs[b] - avgReqSecs[a], + ); + const lastServer = sortedServers[sortedServers.length - 1]; + const lastServerReqSecs = avgReqSecs[lastServer]; + + console.log(`Sorted servers: ${sortedServers.join(', ')}`); + + // Start building the resultsTable + let resultsTable; + if (whichBench === 1) { + resultsTable = + '\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|\n| ' + + whichBench + + ' | `{ posts { id userId title user { id name email }}}` |'; + } else if (whichBench === 2) { + resultsTable = '| ' + whichBench + ' | `{ posts { title }}` |'; + } else if (whichBench === 3) { + resultsTable = '| ' + whichBench + ' | `{ greet }` |'; + } + + // Build the resultsTable with sorted servers and formatted numbers + sortedServers.forEach((server) => { + const formattedReqSecs = avgReqSecs[server].toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const formattedLatencies = avgLatencies[server].toLocaleString(undefined, { + minimumFractionDigits: 2, + maximumFractionDigits: 2, + }); + const relativePerformance = ( + avgReqSecs[server] / lastServerReqSecs + ).toFixed(2); + + resultsTable += `\n| [${formattedServerNames[server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |`; + }); + + if (whichBench === 3) { + resultsTable += '\n\n'; + } + + return resultsTable; +}; + +// Print the results table in a new file +const updateReadme = (resultsTable) => { + const readmePath = 'README.md'; + let content = fs.readFileSync(readmePath, 'utf8'); + const regex = + /[\s\S]*/; + content = content.includes('PERFORMANCE_RESULTS_START') + ? content.replace(regex, resultsTable) + : content + '\n\n' + resultsTable; + fs.writeFileSync(readmePath, content); +}; + +const main = () => { + const { avgReqSecs, avgLatencies } = calculateAverages(); + + // Generating data files for gnuplot + const reqSecData = '/tmp/reqSec.dat'; + const latencyData = '/tmp/latency.dat'; + + writeDataFile(reqSecData, avgReqSecs); + writeDataFile(latencyData, avgLatencies); + + let whichBench = 1; + if (resultFiles[0].startsWith('bench2')) { + whichBench = 2; + } else if (resultFiles[0].startsWith('bench3')) { + whichBench = 3; + } + + // Move the generated images to the assets folder + const reqSecHistogramFile = `req_sec_histogram${whichBench}.png`; + const latencyHistogramFile = `latency_histogram${whichBench}.png`; + + const gnuplotScript = generateGnuplotScript( + reqSecData, + latencyData, + reqSecHistogramFile, + latencyHistogramFile, + ); + fs.writeFileSync('/tmp/gnuplot_script', gnuplotScript); + execSync('gnuplot /tmp/gnuplot_script'); + + // Move PNGs to assets + if (!fs.existsSync('assets')) fs.mkdirSync('assets'); + fs.renameSync(reqSecHistogramFile, `assets/${reqSecHistogramFile}`); + fs.renameSync(latencyHistogramFile, `assets/${latencyHistogramFile}`); + + const resultsTable = generateResultsTable( + avgReqSecs, + avgLatencies, + whichBench, + ); + const resultsFile = 'results.md'; + fs.writeFileSync(resultsFile, resultsTable); + + if (whichBench === 3) { + const finalResults = resultsTable.replace(/\n?/g, ''); + console.log(finalResults); + updateReadme(resultsTable); + } + + // Delete the result TXT files + resultFiles.forEach((file) => fs.unlinkSync(file)); +}; + +main(); diff --git a/analyze.sh b/analyze.sh deleted file mode 100755 index 17bbbce1..00000000 --- a/analyze.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash - -function extractMetric() { - local file="$1" - local metric="$2" - grep "$metric" "$file" | awk '{print $2}' | sed 's/ms//' -} - -function average() { - echo "$@" | awk '{for(i=1;i<=NF;i++) s+=$i; print s/NF}' -} - -declare -A formattedServerNames -formattedServerNames=( - ["tailcall"]="Tailcall" - ["gqlgen"]="Gqlgen" - ["apollo"]="Apollo GraphQL" - ["netflixdgs"]="Netflix DGS" - ["caliban"]="Caliban" - ["async_graphql"]="async-graphql" - ["hasura"]="Hasura" - ["graphql_jit"]="GraphQL JIT" -) - -servers=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") -resultFiles=("$@") -declare -A avgReqSecs -declare -A avgLatencies - -# Extract metrics and calculate averages -for idx in "${!servers[@]}"; do - startIdx=$((idx * 3)) - reqSecVals=() - latencyVals=() - for j in 0 1 2; do - fileIdx=$((startIdx + j)) - reqSecVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Requests/sec")) - latencyVals+=($(extractMetric "${resultFiles[$fileIdx]}" "Latency")) - done - avgReqSecs[${servers[$idx]}]=$(average "${reqSecVals[@]}") - avgLatencies[${servers[$idx]}]=$(average "${latencyVals[@]}") -done - -# Generating data files for gnuplot -reqSecData="/tmp/reqSec.dat" -latencyData="/tmp/latency.dat" - -echo "Server Value" >"$reqSecData" -for server in "${servers[@]}"; do - echo "$server ${avgReqSecs[$server]}" >>"$reqSecData" -done - -echo "Server Value" >"$latencyData" -for server in "${servers[@]}"; do - echo "$server ${avgLatencies[$server]}" >>"$latencyData" -done - -whichBench=1 -if [[ $1 == bench2* ]]; then - whichBench=2 -elif [[ $1 == bench3* ]]; then - whichBench=3 -fi - -reqSecHistogramFile="req_sec_histogram${whichBench}.png" -latencyHistogramFile="latency_histogram${whichBench}.png" - -# Plotting using gnuplot -gnuplot <<-EOF - set term pngcairo size 1280,720 enhanced font "Courier,12" - set output "$reqSecHistogramFile" - set style data histograms - set style histogram cluster gap 1 - set style fill solid border -1 - set xtics rotate by -45 - set boxwidth 0.9 - set title "Requests/Sec" - stats "$reqSecData" using 2 nooutput - set yrange [0:STATS_max*1.2] - set key outside right top - plot "$reqSecData" using 2:xtic(1) title "Req/Sec" - - set output "$latencyHistogramFile" - set title "Latency (in ms)" - stats "$latencyData" using 2 nooutput - set yrange [0:STATS_max*1.2] - plot "$latencyData" using 2:xtic(1) title "Latency" -EOF - -# Move PNGs to assets -mkdir -p assets -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Declare an associative array for server RPS -declare -A serverRPS - -# Populate the serverRPS array -for server in "${servers[@]}"; do - serverRPS[$server]=${avgReqSecs[$server]} -done - -# Get the servers sorted by RPS in descending order -IFS=$'\n' sortedServers=($(for server in "${!serverRPS[@]}"; do echo "$server ${serverRPS[$server]}"; done | sort -rn -k2 | cut -d' ' -f1)) - -echo "Sorted servers: ${sortedServers[@]}" -lastServer="${sortedServers[-1]}" -lastServerReqSecs=${avgReqSecs[$lastServer]} - -# Start building the resultsTable -if [[ $whichBench == 1 ]]; then - resultsTable="\n\n| Query | Server | Requests/sec | Latency (ms) | Relative |\n|-------:|--------:|--------------:|--------------:|---------:|\n| $whichBench | \`{ posts { id userId title user { id name email }}}\` |" -elif [[ $whichBench == 2 ]]; then - resultsTable="| $whichBench | \`{ posts { title }}\` |" -elif [[ $whichBench == 3 ]]; then - resultsTable="| $whichBench | \`{ greet }\` |" -fi - -# Build the resultsTable with sorted servers and formatted numbers -for server in "${sortedServers[@]}"; do - formattedReqSecs=$(printf "%.2f" ${avgReqSecs[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - formattedLatencies=$(printf "%.2f" ${avgLatencies[$server]} | perl -pe 's/(?<=\d)(?=(\d{3})+(\.\d*)?$)/,/g') - # Calculate the relative performance - relativePerformance=$(echo "${avgReqSecs[$server]} $lastServerReqSecs" | awk '{printf "%.2f", $1 / $2}') - - resultsTable+="\n|| [${formattedServerNames[$server]}] | \`${formattedReqSecs}\` | \`${formattedLatencies}\` | \`${relativePerformance}x\` |" -done - -if [[ $whichBench == 3 ]]; then - resultsTable+="\n\n" -fi - -echo "resultsTable: $resultsTable" - -# Print the results table in a new file -resultsFile="results.md" -echo -e $resultsTable >> $resultsFile - - -if [[ $whichBench == 3 ]]; then - finalResults=$(printf '%s\n' "$(cat $resultsFile)" | sed 's/$/\\n/'| tr -d '\n') - # Remove the last newline character - finalResults=${finalResults::-2} - - # Print the results as a table in the terminal - echo -e $finalResults | sed "s///;s///" - # Check if the markers are present - if grep -q "PERFORMANCE_RESULTS_START" README.md; then - # Replace the old results with the new results - sed -i "/PERFORMANCE_RESULTS_START/,/PERFORMANCE_RESULTS_END/c\\$finalResults" README.md - else - # Append the results at the end of the README.md file - echo -e "\n$finalResults" >> README.md - fi -fi - -# Move the generated images to the assets folder -mv $reqSecHistogramFile assets/ -mv $latencyHistogramFile assets/ - -# Delete the result TXT files -for file in "${resultFiles[@]}"; do - rm "$file" -done diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 263e6f69..511bfebd 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -83,6 +83,7 @@ for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "asyn fi done -bash analyze.sh "${bench1Results[@]}" -bash analyze.sh "${bench2Results[@]}" -bash analyze.sh "${bench3Results[@]}" +# Run the analysis using analyze.js +node analyze.js "${bench1Results[@]}" +node analyze.js "${bench2Results[@]}" +node analyze.js "${bench3Results[@]}"