Add SolidBench performance benchmark to CI

.github/workflows/ci.yml

@@ -123,6 +123,95 @@ jobs:
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
 
+  performance:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        benchmark:
+          - solidbench
+    needs:
+      - test
+      - lint
+    if: startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/feature') || startsWith(github.ref, 'refs/heads/next')
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18.x
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile --ignore-engines
+      - name: Run benchmarks
+        run: cd performance/benchmark-${{ matrix.benchmark }}/ && yarn run performance:ci
+      - uses: actions/upload-artifact@v4
+        with:
+          name: performance-benchmark-${{ matrix.benchmark }}
+          path: performance/benchmark-${{ matrix.benchmark }}/combinations/combination_0/output/
+      - name: Print raw output
+        run: cat performance/benchmark-${{ matrix.benchmark }}/combinations/combination_0/output/query-times.csv
+
+  performance-consolidate:
+    needs:
+      - performance
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/feature') || startsWith(github.ref, 'refs/heads/next')
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18.x
+      - uses: actions/download-artifact@v4
+        with:
+          name: performance-benchmark-solidbench
+          path: performance/benchmark-solidbench/combinations/combination_0/output/
+      - name: Process benchmark detailed results
+        run: npx -p @rubensworks/process-sparql-benchmark-results psbr csv ghbench performance/benchmark-solidbench/combinations/combination_0/output/ --overrideCombinationLabels SolidBench --total false --detailed true --name ghbench-detail.json
+      - name: Process benchmark total results
+        run: npx -p @rubensworks/process-sparql-benchmark-results psbr csv ghbench performance/benchmark-solidbench/combinations/combination_0/output/ --overrideCombinationLabels SolidBench --total true --detailed false --name ghbench-total.json
+      - name: Determine benchmark detailed results target directory name
+        run: echo "BENCHMARK_DATA_DIR_PATH_DETAIL=comunica-feature-link-traversal/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}/benchmarks-detail" >> $GITHUB_ENV
+      - name: Determine benchmarks total results target directory name
+        run: echo "BENCHMARK_DATA_DIR_PATH_TOTAL=comunica-feature-link-traversal/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}/benchmarks-total" >> $GITHUB_ENV
+      - name: Finalize benchmarks detailed results
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Benchmarks detailed results
+          tool: 'customSmallerIsBetter'
+          output-file-path: ghbench-detail.json
+          github-token: ${{ secrets.PAT }}
+          auto-push: true
+          alert-comment-cc-users: '@rubensworks'
+          gh-repository: 'github.com/comunica/comunica-performance-results'
+          gh-pages-branch: 'master'
+          benchmark-data-dir-path: ${{ env.BENCHMARK_DATA_DIR_PATH_DETAIL }}
+      - name: Cleanup benchmark-action data
+        run: rm -rf ./benchmark-data-repository
+      - name: Finalize benchmarks total results
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Benchmarks total results
+          tool: 'customSmallerIsBetter'
+          output-file-path: ghbench-total.json
+          github-token: ${{ secrets.PAT }}
+          auto-push: true
+          alert-threshold: '150%'
+          comment-always: true
+          comment-on-alert: true
+          alert-comment-cc-users: '@rubensworks'
+          summary-always: true
+          gh-repository: 'github.com/comunica/comunica-performance-results'
+          gh-pages-branch: 'master'
+          benchmark-data-dir-path: ${{ env.BENCHMARK_DATA_DIR_PATH_TOTAL }}
+      - name: Prepare comment on commit with link to performance results
+        run: echo -e "Performance benchmarks succeeded! 🚀\n\n\[[Summarized results](https://comunica.github.io/comunica-performance-results/comunica-feature-link-traversal/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}/benchmarks-total/)\] \[[Detailed results](https://comunica.github.io/comunica-performance-results/comunica-feature-link-traversal/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}/benchmarks-detail/)\]" > ./commit-comment-body.txt
+      - name: Comment on commit with link to performance results
+        uses: peter-evans/commit-comment@v3
+        with:
+          body-path: ./commit-comment-body.txt
+
   docs:
     runs-on: ubuntu-latest
     steps:

eslint.config.js

@@ -82,6 +82,7 @@ module.exports = config([
       '**/web-clients/**',
       '**/bintest/**',
       'lerna.json',
+      'performance/**',
     ],
   },
   {

lerna.json

@@ -18,4 +18,4 @@
   "loglevel": "success",
   "registry": "https://registry.npmjs.org/",
   "npmClient": "yarn"
-}
+}

package.json

@@ -43,6 +43,7 @@
     "@pollyjs/core": "^6.0.6",
     "@pollyjs/persister-fs": "^6.0.6",
     "@rubensworks/eslint-config": "^3.0.0",
+    "@rubensworks/process-sparql-benchmark-results": "^1.3.0",
     "@types/jest": "^29.5.5",
     "@types/node": "^20.0.0",
     "@types/setup-polly-jest": "^0.5.2",

performance/benchmark-solidbench/.gitignore

@@ -0,0 +1,11 @@
+/componentsjs-error-state.json
+/node_modules
+/generated/*
+!/generated/.keep
+/output/*
+!/output/.keep
+/combinations/*/generated/*
+!/combinations/*/generated/.keep
+/combinations/*/output/*
+!/combinations/*/output/.keep
+plot_queries_data.*

performance/benchmark-solidbench/README.md

@@ -0,0 +1,12 @@
+# Benchmark SolidBench
+
+This internal package benchmarks Comunica Link Traversal Solid using the [SolidBench](https://github.com/SolidBench/SolidBench.js) benchmark.
+
+Compare your current version of Comunica locally with the latest published release by running `npm run performance` from within this package.
+This will output a file called `plot_queries_data.svg` that visualizes the performance differences.
+
+If you only want to check the performance of your current version of Comunica,
+you can run `npm run performance:ci` instead,
+which is what the CI will run as well for continuous performance measurements.
+
+Continuous performance results are tracked on https://github.com/comunica/comunica-performance-results.

performance/benchmark-solidbench/combinations/combination_0/input/config-client.json

@@ -0,0 +1,8 @@
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/config-query-sparql-link-traversal/^0.0.0/components/context.jsonld"
+  ],
+  "import": [
+    "ccqslt:config/config-solid-default.json"
+  ]
+}

performance/benchmark-solidbench/combinations/combination_0/input/config-enhancer.json

@@ -0,0 +1,29 @@
+{
+  "@context": "https://linkedsoftwaredependencies.org/bundles/npm/ldbc-snb-enhancer/^2.0.0/components/context.jsonld",
+  "@id": "urn:ldbc-snb-enhancer:default",
+  "@type": "Enhancer",
+  "personsPath": "out-snb/social_network/social_network_person_0_0.ttl",
+  "activitiesPath": "out-snb/social_network/social_network_activity_0_0.ttl",
+  "staticPath": "out-snb/social_network/social_network_static_0_0.ttl",
+  "destinationPathData": "out-enhanced/social_network_auxiliary.ttl",
+  "logger": {
+    "@type": "LoggerStdout"
+  },
+  "dataSelector": {
+    "@type": "DataSelectorRandom",
+    "seed": 12345
+  },
+  "parameterEmitterPosts": {
+    "@type": "ParameterEmitterCsv",
+    "destinationPath": "out-enhanced/parameters-posts.csv"
+  },
+  "handlers": [
+    {
+      "@type": "EnhancementHandlerPersonNoise",
+      "chance": 10
+    },
+    {
+      "@type": "EnhancementHandlerVocabulary"
+    }
+  ]
+}

performance/benchmark-solidbench/combinations/combination_0/input/config-fragmenter-auxiliary.json

@@ -0,0 +1,61 @@
+{
+  "@context": "https://linkedsoftwaredependencies.org/bundles/npm/rdf-dataset-fragmenter/^2.0.0/components/context.jsonld",
+  "@id": "urn:rdf-dataset-fragmenter:default",
+  "@type": "Fragmenter",
+  "quadSource": {
+    "@id": "urn:rdf-dataset-fragmenter:source:default",
+    "@type": "QuadSourceComposite",
+    "sources": [
+	  {
+	    "@type": "QuadSourceFile",
+	    "filePath": "out-enhanced/social_network_auxiliary.ttl"
+	  }
+    ]
+  },
+  "transformers": [
+    {
+      "@type": "QuadTransformerReplaceIri",
+      "searchRegex": "^http://www.ldbc.eu",
+      "replacementString": "http://solidbench-server:3000/www.ldbc.eu"
+    },
+    {
+      "@type": "QuadTransformerReplaceIri",
+      "searchRegex": "^http://dbpedia.org",
+      "replacementString": "http://solidbench-server:3000/dbpedia.org"
+    },
+    {
+      "@type": "QuadTransformerReplaceIri",
+      "searchRegex": "^http://www.w3.org/2002/07/owl",
+      "replacementString": "http://solidbench-server:3000/www.w3.org/2002/07/owl"
+    },
+    {
+      "@type": "QuadTransformerReplaceIri",
+      "searchRegex": "^http://solidbench-server:3000/www.ldbc.eu/ldbc_socialnet/1.0/data/pers([0-9]*)$",
+      "replacementString": "http://solidbench-server:3000/pods/$1/profile/card#me"
+    },
+    {
+      "@type": "QuadTransformerRemapResourceIdentifier",
+      "newIdentifierSeparator": "../noise/",
+      "typeRegex": "vocabulary/Noise$",
+      "identifierPredicateRegex": "vocabulary/noise$",
+      "targetPredicateRegex": "vocabulary/hasCreator$"
+    }
+  ],
+  "fragmentationStrategy": {
+    "@type": "FragmentationStrategyComposite",
+    "strategies": [
+      { "@type": "FragmentationStrategySubject" }
+    ]
+  },
+  "quadSink": {
+    "@id": "urn:rdf-dataset-fragmenter:sink:default",
+    "@type": "QuadSinkFile",
+    "log": true,
+    "outputFormat": "application/n-quads",
+    "fileExtension": ".nq",
+    "iriToPath": {
+      "http://": "out-fragments/http/",
+      "https://": "out-fragments/https/"
+    }
+  }
+}