Add BSBM benchmark

.github/workflows/ci.yml

@@ -106,25 +106,54 @@ jobs:
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
 
   performance:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        benchmark:
+          - watdiv-hdt
+          - bsbm-hdt
     needs:
       - test
       - lint
-    runs-on: ubuntu-latest
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Use Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 18.x
       - name: Install dependencies
         run: yarn install
       - name: Run benchmarks
-        run: npx lerna run performance:ci --concurrency 1
+        run: cd performance/benchmark-${{ matrix.benchmark }}/ && yarn run performance:ci
+      - uses: actions/upload-artifact@v4
+        with:
+          name: performance-benchmark-${{ matrix.benchmark }}
+          path: performance/benchmark-${{ matrix.benchmark }}/combinations/combination_0/output/
+
+  performance-consolidate:
+    needs:
+      - performance
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18.x
+      - uses: actions/download-artifact@v4
+        with:
+          name: performance-benchmark-watdiv-hdt
+          path: performance/benchmark-watdiv-hdt/combinations/combination_0/output/
+      - uses: actions/download-artifact@v4
+        with:
+          name: performance-benchmark-bsbm-hdt
+          path: performance/benchmark-bsbm-hdt/combinations/combination_0/output/
       - name: Process benchmark detailed results
-        run: npx psbr csv ghbench performance/benchmark-watdiv-hdt/combinations/combination_0/output/ --overrideCombinationLabels WatDiv-HDT --total false --detailed true --name ghbench-detail.json
+        run: npx -p @rubensworks/process-sparql-benchmark-results psbr csv ghbench performance/benchmark-watdiv-hdt/combinations/combination_0/output/ performance/benchmark-bsbm-hdt/combinations/combination_0/output/ --overrideCombinationLabels WatDiv-HDT,BSBM-HDT --total false --detailed true --name ghbench-detail.json
       - name: Process benchmark total results
-        run: npx psbr csv ghbench performance/benchmark-watdiv-hdt/combinations/combination_0/output/ --overrideCombinationLabels WatDiv-HDT --total true --detailed false --name ghbench-total.json
+        run: npx -p @rubensworks/process-sparql-benchmark-results psbr csv ghbench performance/benchmark-watdiv-hdt/combinations/combination_0/output/ performance/benchmark-bsbm-hdt/combinations/combination_0/output/ --overrideCombinationLabels WatDiv-HDT,BSBM-HDT --total true --detailed false --name ghbench-total.json
       - name: Determine benchmark detailed results target directory name
         run: echo "BENCHMARK_DATA_DIR_PATH_DETAIL=comunica-feature-hdt/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}/benchmarks-detail" >> $GITHUB_ENV
       - name: Determine benchmarks total results target directory name

package.json

@@ -32,7 +32,7 @@
     "@babel/preset-env": "^7.16.4",
     "@comunica/jest": "^3.1.0",
     "@rubensworks/eslint-config": "^3.0.0",
-    "@rubensworks/process-sparql-benchmark-results": "^1.1.4",
+    "@rubensworks/process-sparql-benchmark-results": "^1.2.1",
     "@types/jest": "^29.0.0",
     "@types/node": "^20.0.0",
     "asynciterator": "^3.2.1",

performance/benchmark-bsbm-hdt/.gitignore

@@ -0,0 +1,11 @@
+/componentsjs-error-state.json
+/node_modules
+/generated/*
+!/generated/.keep
+/output/*
+!/output/.keep
+/combinations/*/generated/*
+!/combinations/*/generated/.keep
+/combinations/*/output/*
+!/combinations/*/output/.keep
+plot_queries_data.*

performance/benchmark-bsbm-hdt/README.md

@@ -0,0 +1,12 @@
+# Benchmark BSBM HDT
+
+This internal package benchmarks Comunica HDT using the [BSBM](http://wbsg.informatik.uni-mannheim.de/bizer/berlinsparqlbenchmark/) benchmark.
+
+Compare your current version of Comunica locally with the latest published release by running `npm run performance` from within this package.
+This will output a file called `plot_queries_data.svg` that visualizes the performance differences.
+
+If you only want to check the performance of your current version of Comunica,
+you can run `npm run performance:ci` instead,
+which is what the CI will run as well for continuous performance measurements.
+
+Continuous performance results are tracked on https://github.com/comunica/comunica-performance-results.

performance/benchmark-bsbm-hdt/combinations/combination_0/input/config-client.json

@@ -0,0 +1,8 @@
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/config-query-sparql-hdt/^3.0.0/components/context.jsonld"
+  ],
+  "import": [
+    "ccqsh:config/config-default.json"
+  ]
+}

performance/benchmark-bsbm-hdt/combinations/combination_0/input/context-client.json

@@ -0,0 +1,3 @@
+{
+  "sources": [{ "type": "hdt", "value": "/tmp/dataset.hdt" }]
+}

performance/benchmark-bsbm-hdt/combinations/combination_0/input/dockerfiles/Dockerfile-client

@@ -0,0 +1,19 @@
+FROM comunica/query-sparql-hdt:latest
+
+ARG CONFIG_CLIENT
+ARG QUERY_TIMEOUT
+ARG MAX_MEMORY
+ARG LOG_LEVEL
+
+ADD $CONFIG_CLIENT /tmp/config.json
+
+ENV COMUNICA_CONFIG=/tmp/config.json
+ENV NODE_ENV=production
+ENV MAX_MEMORY=$MAX_MEMORY
+ENV QUERY_TIMEOUT=$QUERY_TIMEOUT
+ENV LOG_LEVEL=$LOG_LEVEL
+
+EXPOSE 3000
+
+ENTRYPOINT []
+CMD [ "/bin/sh", "-c", "node --max-old-space-size=$MAX_MEMORY ./bin/http.js -c /tmp/context.json -p 3000 -t $QUERY_TIMEOUT -l $LOG_LEVEL -i" ]

performance/benchmark-bsbm-hdt/combinations/combination_0/jbr-experiment.json

@@ -0,0 +1,17 @@
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/jbr/^5.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@jbr-experiment/bsbm/^5.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@jbr-hook/sparql-endpoint-comunica/^5.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@jbr-hook/cli/^5.0.0/components/context.jsonld"
+  ],
+  "@id": "urn:jbr:benchmark-bsbm-hdt:combination_0",
+  "@type": "ExperimentBsbm",
+  "productCount": 1000,
+  "generateHdt": true,
+  "endpointUrl": "http://host.docker.internal:3001/sparql",
+  "endpointUrlExternal": "http://localhost:3001/sparql",
+  "warmupRuns": 5,
+  "runs": 10,
+  "hookSparqlEndpoint": { "@id": "urn:jbr:benchmark-bsbm-hdt:hookSparqlEndpoint", "@type": "HookCli", "entrypoint": ["node", "../../engines/query-sparql-hdt/bin/http.js", "hdt@generated/dataset.hdt", "-p", "3001"], "statsFilePath": "output/stats.csv" }
+}

performance/benchmark-bsbm-hdt/combinations/combination_1/input/config-client.json

@@ -0,0 +1,8 @@
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/config-query-sparql-hdt/^3.0.0/components/context.jsonld"
+  ],
+  "import": [
+    "ccqsh:config/config-default.json"
+  ]
+}

performance/benchmark-bsbm-hdt/combinations/combination_1/input/context-client.json

@@ -0,0 +1,3 @@
+{
+  "sources": [{ "type": "hdt", "value": "/tmp/dataset.hdt" }]
+}

performance/benchmark-bsbm-hdt/combinations/combination_1/input/dockerfiles/Dockerfile-client

@@ -0,0 +1,19 @@
+FROM comunica/query-sparql-hdt:latest
+
+ARG CONFIG_CLIENT
+ARG QUERY_TIMEOUT
+ARG MAX_MEMORY
+ARG LOG_LEVEL
+
+ADD $CONFIG_CLIENT /tmp/config.json
+
+ENV COMUNICA_CONFIG=/tmp/config.json
+ENV NODE_ENV=production
+ENV MAX_MEMORY=$MAX_MEMORY
+ENV QUERY_TIMEOUT=$QUERY_TIMEOUT
+ENV LOG_LEVEL=$LOG_LEVEL
+
+EXPOSE 3000
+
+ENTRYPOINT []
+CMD [ "/bin/sh", "-c", "node --max-old-space-size=$MAX_MEMORY ./bin/http.js -c /tmp/context.json -p 3000 -t $QUERY_TIMEOUT -l $LOG_LEVEL -i" ]

performance/benchmark-bsbm-hdt/combinations/combination_1/jbr-experiment.json

@@ -0,0 +1,17 @@
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/jbr/^5.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@jbr-experiment/bsbm/^5.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@jbr-hook/sparql-endpoint-comunica/^5.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@jbr-hook/cli/^5.0.0/components/context.jsonld"
+  ],
+  "@id": "urn:jbr:benchmark-bsbm-hdt:combination_1",
+  "@type": "ExperimentBsbm",
+  "productCount": 1000,
+  "generateHdt": true,
+  "endpointUrl": "http://host.docker.internal:3001/sparql",
+  "endpointUrlExternal": "http://localhost:3001/sparql",
+  "warmupRuns": 5,
+  "runs": 10,
+  "hookSparqlEndpoint": { "@id": "urn:jbr:benchmark-bsbm-hdt:hookSparqlEndpoint", "@type": "HookSparqlEndpointComunica", "dockerfileClient": "input/dockerfiles/Dockerfile-client", "resourceConstraints": { "@type": "StaticDockerResourceConstraints", "cpu_percentage": 100 }, "configClient": "input/config-client.json", "contextClient": "input/context-client.json", "additionalBinds": ["/generated/dataset.nt:/tmp/dataset.nt"], "clientPort": 3001, "clientLogLevel": "info", "maxMemory": 8192 }
+}

performance/benchmark-bsbm-hdt/input/config-client.json

@@ -0,0 +1,8 @@
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/@comunica/config-query-sparql-hdt/^3.0.0/components/context.jsonld"
+  ],
+  "import": [
+    "ccqsh:config/config-default.json"
+  ]
+}

performance/benchmark-bsbm-hdt/input/context-client.json

@@ -0,0 +1,3 @@
+{
+  "sources": [{ "type": "hdt", "value": "/tmp/dataset.hdt" }]
+}

performance/benchmark-bsbm-hdt/input/dockerfiles/Dockerfile-client

@@ -0,0 +1,19 @@
+FROM comunica/query-sparql-hdt:latest
+
+ARG CONFIG_CLIENT
+ARG QUERY_TIMEOUT
+ARG MAX_MEMORY
+ARG LOG_LEVEL
+
+ADD $CONFIG_CLIENT /tmp/config.json
+
+ENV COMUNICA_CONFIG=/tmp/config.json
+ENV NODE_ENV=production
+ENV MAX_MEMORY=$MAX_MEMORY
+ENV QUERY_TIMEOUT=$QUERY_TIMEOUT
+ENV LOG_LEVEL=$LOG_LEVEL
+
+EXPOSE 3000
+
+ENTRYPOINT []
+CMD [ "/bin/sh", "-c", "node --max-old-space-size=$MAX_MEMORY ./bin/http.js -c /tmp/context.json -p 3000 -t $QUERY_TIMEOUT -l $LOG_LEVEL -i" ]

performance/benchmark-bsbm-hdt/jbr-combinations.json

@@ -0,0 +1,14 @@
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/jbr/^5.0.0/components/context.jsonld"
+  ],
+  "@id": "urn:jbr:benchmark-bsbm-hdt-combinations",
+  "@type": "FullFactorialCombinationProvider",
+  "commonGenerated": true,
+  "factors": {
+    "hookSparqlEndpoint": [
+      "{\"@id\": \"urn:jbr:benchmark-bsbm-hdt:hookSparqlEndpoint\",\"@type\": \"HookCli\",\"entrypoint\": [  \"node\",  \"../../engines/query-sparql-hdt/bin/http.js\",  \"hdt@generated/dataset.hdt\",  \"-p\",  \"3001\"],\"statsFilePath\": \"output/stats.csv\"  }",
+      "{\"@id\": \"urn:jbr:benchmark-bsbm-hdt:hookSparqlEndpoint\",  \"@type\": \"HookSparqlEndpointComunica\",  \"dockerfileClient\": \"input/dockerfiles/Dockerfile-client\",  \"resourceConstraints\": {\"@type\": \"StaticDockerResourceConstraints\",\"cpu_percentage\": 100  },  \"configClient\": \"input/config-client.json\",  \"contextClient\": \"input/context-client.json\",  \"additionalBinds\": [\"/generated/dataset.nt:/tmp/dataset.nt\"],  \"clientPort\": 3001,  \"clientLogLevel\": \"info\",  \"maxMemory\": 8192}"
+    ]
+  }
+}

performance/benchmark-bsbm-hdt/jbr-experiment.json.template

@@ -0,0 +1,17 @@
+{
+  "@context": [
+    "https://linkedsoftwaredependencies.org/bundles/npm/jbr/^5.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@jbr-experiment/bsbm/^5.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@jbr-hook/sparql-endpoint-comunica/^5.0.0/components/context.jsonld",
+    "https://linkedsoftwaredependencies.org/bundles/npm/@jbr-hook/cli/^5.0.0/components/context.jsonld"
+  ],
+  "@id": "urn:jbr:benchmark-bsbm-hdt",
+  "@type": "ExperimentBsbm",
+  "productCount": 1000,
+  "generateHdt": true,
+  "endpointUrl": "http://host.docker.internal:3001/sparql",
+  "endpointUrlExternal": "http://localhost:3001/sparql",
+  "warmupRuns": 5,
+  "runs": 10,
+  "hookSparqlEndpoint": %FACTOR-hookSparqlEndpoint%
+}

performance/benchmark-bsbm-hdt/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "benchmark-bsbm-hdt",
+  "version": "3.1.0",
+  "private": true,
+  "scripts": {
+    "jbr": "cross-env NODE_OPTIONS=--max-old-space-size=8192 jbr",
+    "validate": "jbr validate",
+    "performance": "docker pull comunica/query-sparql-hdt:latest && jbr prepare && jbr run && psbr csv bsbm combinations/combination_*/output && psbr tex query combinations/combination_*/output --svg --logY --overrideCombinationLabels Current,Previous",
+    "performance:ci": "jbr prepare -c 0 && jbr run -c 0 && psbr csv bsbm combinations/combination_0/output"
+  },
+  "dependencies": {
+    "@jbr-experiment/bsbm": "^5.2.3",
+    "@jbr-hook/cli": "^5.2.0",
+    "@jbr-hook/sparql-endpoint-comunica": "^5.2.0",
+    "cross-env": "^7.0.3",
+    "jbr": "^5.2.0"
+  }
+}

performance/benchmark-watdiv-hdt/package.json

@@ -6,14 +6,14 @@
     "jbr": "cross-env NODE_OPTIONS=--max-old-space-size=8192 jbr",
     "validate": "jbr validate",
     "fetch-assets": "curl -L https://github.com/comunica/comunica-performance-assets/raw/master/watdiv-10.zip?download= > watdiv-10.zip && unzip watdiv-10.zip -d generated/ && rm watdiv-10.zip",
-    "performance": "docker pull comunica/query-sparql-hdt:latest && jbr prepare && jbr run && psbr tex query output/combination_* --svg --logY --overrideCombinationLabels Current,Previous",
+    "performance": "docker pull comunica/query-sparql-hdt:latest && jbr prepare && jbr run && psbr tex query combinations/combination_*/output --svg --logY --overrideCombinationLabels Current,Previous",
     "performance:ci": "npm run fetch-assets && jbr prepare -c 0 && jbr run -c 0"
   },
   "dependencies": {
-    "@jbr-experiment/watdiv": "^5.1.1",
-    "@jbr-hook/cli": "^5.1.1",
-    "@jbr-hook/sparql-endpoint-comunica": "^5.1.1",
+    "@jbr-experiment/watdiv": "^5.2.0",
+    "@jbr-hook/cli": "^5.2.0",
+    "@jbr-hook/sparql-endpoint-comunica": "^5.2.0",
     "cross-env": "^7.0.3",
-    "jbr": "^5.1.1"
+    "jbr": "^5.2.0"
   }
 }