Merge pull request #79 from samansmink/merge-benchmarking-code-in-fea…

…ture

Add automated performance regression tests to CI

.github/regression/tpcds_sf1_local.csv

@@ -0,0 +1,100 @@
+benchmark/tpcds/sf1/local/delta/q01.benchmark
+benchmark/tpcds/sf1/local/delta/q02.benchmark
+benchmark/tpcds/sf1/local/delta/q03.benchmark
+benchmark/tpcds/sf1/local/delta/q04.benchmark
+benchmark/tpcds/sf1/local/delta/q05.benchmark
+benchmark/tpcds/sf1/local/delta/q06.benchmark
+benchmark/tpcds/sf1/local/delta/q07.benchmark
+benchmark/tpcds/sf1/local/delta/q08.benchmark
+benchmark/tpcds/sf1/local/delta/q09.benchmark
+benchmark/tpcds/sf1/local/delta/q10.benchmark
+benchmark/tpcds/sf1/local/delta/q11.benchmark
+benchmark/tpcds/sf1/local/delta/q12.benchmark
+benchmark/tpcds/sf1/local/delta/q13.benchmark
+benchmark/tpcds/sf1/local/delta/q14.benchmark
+benchmark/tpcds/sf1/local/delta/q15.benchmark
+benchmark/tpcds/sf1/local/delta/q16.benchmark
+benchmark/tpcds/sf1/local/delta/q17.benchmark
+benchmark/tpcds/sf1/local/delta/q18.benchmark
+benchmark/tpcds/sf1/local/delta/q19.benchmark
+benchmark/tpcds/sf1/local/delta/q20.benchmark
+benchmark/tpcds/sf1/local/delta/q21.benchmark
+benchmark/tpcds/sf1/local/delta/q22.benchmark
+benchmark/tpcds/sf1/local/delta/q23.benchmark
+benchmark/tpcds/sf1/local/delta/q24.benchmark
+benchmark/tpcds/sf1/local/delta/q25.benchmark
+benchmark/tpcds/sf1/local/delta/q26.benchmark
+benchmark/tpcds/sf1/local/delta/q27.benchmark
+benchmark/tpcds/sf1/local/delta/q28.benchmark
+
+benchmark/tpcds/sf1/local/delta/q29.benchmark
+benchmark/tpcds/sf1/local/delta/q30.benchmark
+benchmark/tpcds/sf1/local/delta/q31.benchmark
+benchmark/tpcds/sf1/local/delta/q32.benchmark
+benchmark/tpcds/sf1/local/delta/q33.benchmark
+benchmark/tpcds/sf1/local/delta/q34.benchmark
+benchmark/tpcds/sf1/local/delta/q35.benchmark
+benchmark/tpcds/sf1/local/delta/q36.benchmark
+benchmark/tpcds/sf1/local/delta/q37.benchmark
+benchmark/tpcds/sf1/local/delta/q38.benchmark
+benchmark/tpcds/sf1/local/delta/q39.benchmark
+benchmark/tpcds/sf1/local/delta/q40.benchmark
+benchmark/tpcds/sf1/local/delta/q41.benchmark
+benchmark/tpcds/sf1/local/delta/q42.benchmark
+benchmark/tpcds/sf1/local/delta/q43.benchmark
+benchmark/tpcds/sf1/local/delta/q44.benchmark
+benchmark/tpcds/sf1/local/delta/q45.benchmark
+benchmark/tpcds/sf1/local/delta/q46.benchmark
+benchmark/tpcds/sf1/local/delta/q47.benchmark
+benchmark/tpcds/sf1/local/delta/q48.benchmark
+benchmark/tpcds/sf1/local/delta/q49.benchmark
+benchmark/tpcds/sf1/local/delta/q50.benchmark
+benchmark/tpcds/sf1/local/delta/q51.benchmark
+benchmark/tpcds/sf1/local/delta/q52.benchmark
+benchmark/tpcds/sf1/local/delta/q53.benchmark
+benchmark/tpcds/sf1/local/delta/q54.benchmark
+benchmark/tpcds/sf1/local/delta/q55.benchmark
+benchmark/tpcds/sf1/local/delta/q56.benchmark
+benchmark/tpcds/sf1/local/delta/q57.benchmark
+benchmark/tpcds/sf1/local/delta/q58.benchmark
+benchmark/tpcds/sf1/local/delta/q59.benchmark
+benchmark/tpcds/sf1/local/delta/q60.benchmark
+benchmark/tpcds/sf1/local/delta/q61.benchmark
+benchmark/tpcds/sf1/local/delta/q62.benchmark
+benchmark/tpcds/sf1/local/delta/q63.benchmark
+benchmark/tpcds/sf1/local/delta/q64.benchmark
+benchmark/tpcds/sf1/local/delta/q65.benchmark
+benchmark/tpcds/sf1/local/delta/q66.benchmark
+benchmark/tpcds/sf1/local/delta/q67.benchmark
+benchmark/tpcds/sf1/local/delta/q68.benchmark
+benchmark/tpcds/sf1/local/delta/q69.benchmark
+benchmark/tpcds/sf1/local/delta/q70.benchmark
+benchmark/tpcds/sf1/local/delta/q71.benchmark
+benchmark/tpcds/sf1/local/delta/q72.benchmark
+benchmark/tpcds/sf1/local/delta/q73.benchmark
+benchmark/tpcds/sf1/local/delta/q74.benchmark
+benchmark/tpcds/sf1/local/delta/q75.benchmark
+benchmark/tpcds/sf1/local/delta/q76.benchmark
+benchmark/tpcds/sf1/local/delta/q77.benchmark
+benchmark/tpcds/sf1/local/delta/q78.benchmark
+benchmark/tpcds/sf1/local/delta/q79.benchmark
+benchmark/tpcds/sf1/local/delta/q80.benchmark
+benchmark/tpcds/sf1/local/delta/q81.benchmark
+benchmark/tpcds/sf1/local/delta/q82.benchmark
+benchmark/tpcds/sf1/local/delta/q83.benchmark
+benchmark/tpcds/sf1/local/delta/q84.benchmark
+benchmark/tpcds/sf1/local/delta/q85.benchmark
+benchmark/tpcds/sf1/local/delta/q86.benchmark
+benchmark/tpcds/sf1/local/delta/q87.benchmark
+benchmark/tpcds/sf1/local/delta/q88.benchmark
+benchmark/tpcds/sf1/local/delta/q89.benchmark
+benchmark/tpcds/sf1/local/delta/q90.benchmark
+benchmark/tpcds/sf1/local/delta/q91.benchmark
+benchmark/tpcds/sf1/local/delta/q92.benchmark
+benchmark/tpcds/sf1/local/delta/q93.benchmark
+benchmark/tpcds/sf1/local/delta/q94.benchmark
+benchmark/tpcds/sf1/local/delta/q95.benchmark
+benchmark/tpcds/sf1/local/delta/q96.benchmark
+benchmark/tpcds/sf1/local/delta/q97.benchmark
+benchmark/tpcds/sf1/local/delta/q98.benchmark
+benchmark/tpcds/sf1/local/delta/q99.benchmark

.github/regression/tpch_sf1_local.csv

@@ -0,0 +1,22 @@
+benchmark/tpch/sf1/local/delta/q01.benchmark
+benchmark/tpch/sf1/local/delta/q02.benchmark
+benchmark/tpch/sf1/local/delta/q03.benchmark
+benchmark/tpch/sf1/local/delta/q04.benchmark
+benchmark/tpch/sf1/local/delta/q05.benchmark
+benchmark/tpch/sf1/local/delta/q06.benchmark
+benchmark/tpch/sf1/local/delta/q07.benchmark
+benchmark/tpch/sf1/local/delta/q08.benchmark
+benchmark/tpch/sf1/local/delta/q09.benchmark
+benchmark/tpch/sf1/local/delta/q10.benchmark
+benchmark/tpch/sf1/local/delta/q11.benchmark
+benchmark/tpch/sf1/local/delta/q12.benchmark
+benchmark/tpch/sf1/local/delta/q13.benchmark
+benchmark/tpch/sf1/local/delta/q14.benchmark
+benchmark/tpch/sf1/local/delta/q15.benchmark
+benchmark/tpch/sf1/local/delta/q16.benchmark
+benchmark/tpch/sf1/local/delta/q17.benchmark
+benchmark/tpch/sf1/local/delta/q18.benchmark
+benchmark/tpch/sf1/local/delta/q19.benchmark
+benchmark/tpch/sf1/local/delta/q20.benchmark
+benchmark/tpch/sf1/local/delta/q21.benchmark
+benchmark/tpch/sf1/local/delta/q22.benchmark

.github/workflows/LocalTesting.yml

@@ -7,6 +7,9 @@ defaults:
   run:
     shell: bash
 
+env:
+  BASE_BRANCH: ${{ github.base_ref || (endsWith(github.ref, '_feature') && 'feature' || 'main') }}
+
 jobs:
   azurite-tests-linux:
     name: Azurite (local azure test server) tests (Linux)
@@ -213,4 +216,81 @@ jobs:
       - name: Test
         shell: bash
         run: |
-          GENERATED_DATA_AVAILABLE=1 make test
+          GENERATED_DATA_AVAILABLE=1 make test
+
+  regression-test-benchmark-runner:
+    name: Performance Regression Tests
+    runs-on: ubuntu-latest
+    env:
+      GEN: ninja
+      BUILD_BENCHMARK: 1
+      VCPKG_TARGET_TRIPLET: x64-linux
+      VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install
+        shell: bash
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+        with:
+          key: ${{ github.job }}
+
+      - name: Setup vcpkg
+        uses: lukka/[email protected]
+        with:
+          vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6
+
+      - name: Configure OpenSSL for Rust
+        run: |
+          echo "OPENSSL_ROOT_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_DIR=`pwd`/build/release/vcpkg_installed/x64-linux" >> $GITHUB_ENV
+          echo "OPENSSL_USE_STATIC_LIBS=true" >> $GITHUB_ENV
+
+      - name: Build current release
+        shell: bash
+        run: |
+          make
+          rm -rf build/release/rust
+
+      - name: Build ${{ env.BASE_BRANCH }} branch
+        shell: bash
+        run: | 
+          git clone --branch ${{ env.BASE_BRANCH }} https://github.com/duckdb/duckdb_delta.git --depth=1 
+          cd duckdb_delta
+          git submodule init
+          git submodule update
+          make
+          rm -rf build/release/rust
+
+      - name: Generate test data
+        shell: bash
+        run: make generate-data
+
+      - name: Regression Test TPC-H
+        if: always()
+        shell: bash
+        run: |
+          python3 ./scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpch_sf1_local.csv --verbose --threads=2 --root-dir=.
+
+      - name: Regression Test TPC-DS
+        if: always()
+        shell: bash
+        run: |
+          python ./scripts/regression_test_runner.py --old=duckdb_delta/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpcds_sf1_local.csv --verbose --threads=2 --root-dir=.
+
+      - name: Test benchmark makefile
+        shell: bash
+        run: |
+          make bench-run-tpch-sf1
+          make bench-run-tpcds-sf1

Makefile

@@ -27,5 +27,5 @@ include benchmark/benchmark.Makefile
 
 # Generate some test data to test with
 generate-data:
-	python3 -m pip install delta-spark duckdb pandas deltalake pyspark delta
+	python3 -m pip install delta-spark duckdb pandas deltalake pyspark
 	python3 scripts/generate_test_data.py

benchmark/README.md

@@ -29,13 +29,4 @@ Specific benchmarks can be run from a suite using the `BENCHMARK_PATTERN` variab
 only Q01 from TPCH SF1, run:
 ```shell
 BENCHMARK_PATTERN=q01.benchmark make bench-run-tpch-sf1
-```
-
-Also, we can run all local benchmarks using:
-```shell
-make  bench-run-all-local
-```
-Or all remote benchmarks using
-```shell
-make bench-run-all-remote
 ```

benchmark/benchmark.Makefile

@@ -4,6 +4,10 @@ ifeq ("$(BENCHMARK_PATTERN)a", "a")
     BENCHMARK_PATTERN:=.*
 endif
 
+ifeq ("$(IO_MODE)a", "a")
+    IO_MODE:=local
+endif
+
 bench-output-dir:
 	mkdir -p benchmark_results
 
@@ -13,51 +17,37 @@ clean_benchmark:
 plot:
 	python3 scripts/plot.py
 
-
 ############### BENCHMARK TARGETS ###############
 
 ###
-# TPCH LOCAL
+# TPCH
 ###
 
 # TPCH SF1 on delta table
 bench-run-tpch-sf1-delta: bench-output-dir
-	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpch/sf1-delta/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-delta.csv
+	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpch/sf1/local/delta/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-delta.csv
 # TPCH SF1 on parquet files
 bench-run-tpch-sf1-parquet: bench-output-dir
 	./build/release/benchmark/benchmark_runner 'benchmark/tpch/sf1-parquet/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-parquet.csv
-# COMPARES TPCH SF1 on parquet file vs on delta files
+# TPCH SF1 on duckdb file
+bench-run-tpch-sf1-duckdb: bench-output-dir
+	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpch/sf1/local/duckdb/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-duckdb.csv
+# COMPARES TPCH SF1 on parquet file vs on delta files vs on duckdb files
 bench-run-tpch-sf1: bench-run-tpch-sf1-delta bench-run-tpch-sf1-parquet
 
 ###
-# TPCH REMOTE
-###
-
-# TPCH on remote delta table (set BENCHMARK_DATA_S3_LINEITEM_SF1)
-bench-run-tpch-sf1-remote-delta: bench-output-dir
-	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpch/sf1-delta-remote/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpch-sf1-remote-delta.csv
-# TPCH on remote parquet table (set BENCHMARK_DATA_S3_LINEITEM_SF1)
-bench-run-tpch-sf1-remote-parquet: bench-output-dir
-	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpch/sf1-parquet-remote/$(BENCHMARK_PATTERN)'  2>&1 | tee benchmark_results/tpch-sf1-remote-parquet.csv
-# COMPARES TPCH SF1 on parquet file vs on delta files
-bench-run-tpch-sf1-remote: bench-run-tpch-sf1-remote-parquet bench-run-tpch-sf1-remote-delta
-
-###
-# TPCDS LOCAL
+# TPCDS
 ###
 
 # TPCDS SF1 on delta table
 bench-run-tpcds-sf1-delta: bench-output-dir
-	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpcds/sf1-delta/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpcds-sf1-delta.csv
+	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpcds/sf1/$(IO_MODE)/delta/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpcds-sf1-delta-$(IO_MODE).csv
 # TPCDS SF1 on parquet files
 bench-run-tpcds-sf1-parquet: bench-output-dir
-	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpcds/sf1-parquet/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpcds-sf1-parquet.csv
-# COMPARES TPCDS SF1 on parquet file vs on delta files
-bench-run-tpcds-sf1: bench-run-tpcds-sf1-delta bench-run-tpcds-sf1-parquet
-
-###
-# ALL
-###
-bench-run-all-local: bench-run-tpcds-sf1 bench-run-tpch-sf1
+	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpcds/sf1/$(IO_MODE)/parquet/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpcds-sf1-parquet-$(IO_MODE).csv
+# TPCDS SF1 on duckdb files
+bench-run-tpcds-sf1-duckdb: bench-output-dir
+	./build/release/benchmark/benchmark_runner --root-dir './' 'benchmark/tpcds/sf1/$(IO_MODE)/duckdb/$(BENCHMARK_PATTERN)' 2>&1 | tee benchmark_results/tpcds-sf1-duckdb-$(IO_MODE).csv
 
-bench-run-all-remote: bench-run-tpch-sf1-remote
+# COMPARES TPCDS SF1 on parquet file vs on delta files
+bench-run-tpcds-sf1: bench-run-tpcds-sf1-delta bench-run-tpcds-sf1-parquet bench-run-tpcds-sf1-duckdb