Skip to content

Commit

Permalink
enable all benchmarks
Browse files Browse the repository at this point in the history
  • Loading branch information
@azimafroozeh
azimafroozeh committed Dec 1, 2024
1 parent 08e5e32 commit b9aaeae
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 86 deletions.
95 changes: 47 additions & 48 deletions publication/script/master_script.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,13 @@ if [ $? -ne 0 ]; then
fi

green_echo "Running compression ratio benchmarks..."
#"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_alp_compression_ratio"
#"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_alp32_compression_ratio"
#"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_zstd_compression_ratio"
#"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_chimp_compression_ratio"
#"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_chimp128_compression_ratio"
#"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_gorillas_compression_ratio"
#"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_patas_compression_ratio"
"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_alp_compression_ratio"
"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_alp32_compression_ratio"
"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_zstd_compression_ratio"
"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_chimp_compression_ratio"
"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_chimp128_compression_ratio"
"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_gorillas_compression_ratio"
"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_patas_compression_ratio"

# Generate compression ratio tables
green_echo "Generating compression ratio tables..."
Expand All @@ -124,33 +124,33 @@ if [ "$ARCH" == "arm64" ]; then
"$CLONED_DIR/build/publication/source_code/generated/arm64v8/neon_intrinsic_uf1/arm64v8_neon_intrinsic_1024_uf1_falp_bench"
else

# # 4xLarge benchmarks
# green_echo "Running I4I_4XLarge speed benchmarks..."
# "$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx512bw_intrinsic_1024_uf1_falp_bench"
# "$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx512bw_intrinsic_1024_uf1_falp_bench_bw"
# "$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx2_intrinsic_1024_uf1_falp_bench"
# "$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/fallback_scalar_aav_1024_uf1_falp_bench"
# "$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/fallback_scalar_nav_1024_uf1_falp_bench"
# "$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_cutter_decode"
# "$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_cutter_encode"
# "$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_encode"
# "$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_without_sampling"
# "$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_chimp"
# "$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_chimp128"
# "$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_gorillas"
# "$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_patas"
# "$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_zstd"
#
# # End-to-end benchmark
# SCRIPT_DIR=$(dirname "$(realpath "$0")")
# OUTPUT_FILE="$SCRIPT_DIR/publication/end_to_end_bench/result"
# HEADER="dataset,repetition,warmup_repetition,scheme,thread_n,query,time(s),result(tpc),corrected_result(tpc),validity,compression_cycles,cycles"
# green_echo "Running end-to-end benchmark and saving results to $OUTPUT_FILE ..."
# export CLONED_DIR="$CLONED_DIR"
# bash "$CLONED_DIR/publication/script/run_end_to_end.sh" >"$OUTPUT_FILE" 2>&1
# # Ensure the header line is at the start of the file
# echo -e "$HEADER\n$(cat "$OUTPUT_FILE")" >"$OUTPUT_FILE"
# green_echo "Benchmark completed. Results are saved in $OUTPUT_FILE."
# 4xLarge benchmarks
green_echo "Running I4I_4XLarge speed benchmarks..."
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx512bw_intrinsic_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx512bw_intrinsic_1024_uf1_falp_bench_bw"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx2_intrinsic_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/fallback_scalar_aav_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/fallback_scalar_nav_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_cutter_decode"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_cutter_encode"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_encode"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_without_sampling"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_chimp"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_chimp128"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_gorillas"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_patas"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_zstd"

# End-to-end benchmark
SCRIPT_DIR=$(dirname "$(realpath "$0")")
OUTPUT_FILE="$SCRIPT_DIR/publication/end_to_end_bench/result"
HEADER="dataset,repetition,warmup_repetition,scheme,thread_n,query,time(s),result(tpc),corrected_result(tpc),validity,compression_cycles,cycles"
green_echo "Running end-to-end benchmark and saving results to $OUTPUT_FILE ..."
export CLONED_DIR="$CLONED_DIR"
bash "$CLONED_DIR/publication/script/run_end_to_end.sh" >"$OUTPUT_FILE" 2>&1
# Ensure the header line is at the start of the file
echo -e "$HEADER\n$(cat "$OUTPUT_FILE")" >"$OUTPUT_FILE"
green_echo "Benchmark completed. Results are saved in $OUTPUT_FILE."

# Clone and build the BENCH_PDE repository
green_echo "Cloning the BENCH_PDE repository..."
Expand Down Expand Up @@ -183,24 +183,23 @@ else
fi

# Navigate to PDE directory and build
# green_echo "Building the BENCH_PDE repository..."
# cd "$PDE_DIR"
# mkdir -p build
# cmake -DCMAKE_TOOLCHAIN_FILE="$TOOLCHAIN_FILE" -S "$PDE_DIR" -B "$PDE_DIR/build" -DCMAKE_BUILD_TYPE=Release -DCXX=clang++
# if [ $? -ne 0 ]; then
# red_echo "CMake configuration failed for BENCH_PDE."
# exit 1
# fi
# cmake --build "$PDE_DIR/build" -j 16
# if [ $? -ne 0 ]; then
# red_echo "Build failed for BENCH_PDE."
# exit 1
# fi
green_echo "Building the BENCH_PDE repository..."
cd "$PDE_DIR"
mkdir -p build
cmake -DCMAKE_TOOLCHAIN_FILE="$TOOLCHAIN_FILE" -S "$PDE_DIR" -B "$PDE_DIR/build" -DCMAKE_BUILD_TYPE=Release -DCXX=clang++
if [ $? -ne 0 ]; then
red_echo "CMake configuration failed for BENCH_PDE."
exit 1
fi
cmake --build "$PDE_DIR/build" -j 16
if [ $? -ne 0 ]; then
red_echo "Build failed for BENCH_PDE."
exit 1
fi

"$PDE_DIR/build/test_pde"
"$PDE_DIR/build/bench_pde"


fi

# Set up Python and install dependencies
Expand Down
66 changes: 33 additions & 33 deletions publication/tables/table_4.md
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,35 @@
| Dataset | Gor | Ch | Ch128 | Patas | PDE | Elf | Alp | LWC+Alp | Zstd |
|---|---|---|---|---|---|---|---|---|---|
| Air-Pressure | 24.48 | 22.95 | 19.24 | 27.87 | 64.0 | 64.0 | 16.43 | 64.0 | 9.39 |
| Basel-Temp | 60.47 | 54.12 | 31.14 | 36.41 | 64.0 | 64.0 | 30.72 | 64.0 | 18.44 |
| Basel-Wind | 62.2 | 54.73 | 38.35 | 48.86 | 64.0 | 64.0 | 29.81 | 64.0 | 14.66 |
| Bird-Mig | 47.82 | 42.06 | 26.43 | 36.08 | 64.0 | 64.0 | 20.14 | 64.0 | 21.02 |
| Btc-Price | 54.67 | 48.07 | 44.96 | 56.8 | 64.0 | 64.0 | 26.37 | 64.0 | 42.08 |
| City-Temp | 58.82 | 46.25 | 22.96 | 24.16 | 64.0 | 64.0 | 10.74 | 64.0 | 16.77 |
| Dew-Temp | 54.86 | 51.81 | 32.63 | 39.01 | 64.0 | 64.0 | 13.4 | 64.0 | 25.07 |
| Bio-Temp | 50.69 | 46.34 | 18.86 | 22.88 | 64.0 | 64.0 | 10.75 | 64.0 | 17.46 |
| PM10-dust | 27.5 | 24.44 | 13.66 | 19.9 | 64.0 | 64.0 | 8.56 | 64.0 | 7.78 |
| Stocks-DE | 46.18 | 42.89 | 13.63 | 20.77 | 64.0 | 64.0 | 11.01 | 64.0 | 10.54 |
| Stocks-UK | 34.75 | 31.33 | 16.76 | 21.49 | 64.0 | 64.0 | 12.59 | 64.0 | 10.28 |
| Stocks-USA | 37.2 | 34.98 | 12.19 | 19.21 | 64.0 | 64.0 | 7.9 | 64.0 | 8.56 |
| Wind-dir | 58.14 | 53.89 | 27.8 | 28.14 | 64.0 | 64.0 | 15.89 | 64.0 | 25.53 |
| TS AVG. | 47.5 | 42.6 | 24.5 | 30.9 | 64.0 | 64.0 | 16.5 | 64.0 | 17.5 |
| Arade/4 | 58.0 | 55.56 | 48.95 | 59.1 | 64.0 | 64.0 | 24.94 | 64.0 | 33.9 |
| Blockchain | 62.89 | 58.29 | 53.19 | 62.58 | 64.0 | 64.0 | 36.49 | 64.0 | 43.97 |
| CMS/1 | 37.23 | 34.77 | 28.14 | 36.78 | 64.0 | 64.0 | 35.65 | 64.0 | 26.56 |
| CMS/25 | 64.35 | 59.53 | 57.2 | 70.09 | 64.0 | 64.0 | 41.11 | 64.0 | 58.27 |
| CMS/9 | 16.01 | 18.71 | 25.65 | 25.97 | 64.0 | 64.0 | 11.67 | 64.0 | 14.73 |
| Food-prices | 38.31 | 27.98 | 24.64 | 28.29 | 64.0 | 64.0 | 23.65 | 64.0 | 18.32 |
| Gov/10 | 57.45 | 45.75 | 34.15 | 35.84 | 64.0 | 64.0 | 30.99 | 64.0 | 28.09 |
| Gov/26 | 2.37 | 2.34 | 9.26 | 16.21 | 64.0 | 64.0 | 0.41 | 64.0 | 0.23 |
| Gov/30 | 10.22 | 8.86 | 12.92 | 19.28 | 64.0 | 64.0 | 7.48 | 64.0 | 4.48 |
| Gov/31 | 5.61 | 5.0 | 10.43 | 17.07 | 64.0 | 64.0 | 3.05 | 64.0 | 1.63 |
| Gov/40 | 2.71 | 2.63 | 9.41 | 16.34 | 64.0 | 64.0 | 0.83 | 64.0 | 0.46 |
| Medicare/1 | 45.49 | 42.72 | 32.32 | 39.91 | 64.0 | 64.0 | 39.35 | 64.0 | 31.18 |
| Medicare/9 | 16.97 | 19.08 | 26.02 | 26.27 | 64.0 | 64.0 | 12.26 | 64.0 | 15.03 |
| NYC/29 | 30.56 | 29.57 | 28.71 | 38.78 | 64.0 | 64.0 | 40.38 | 64.0 | 27.5 |
| POI-lat | 65.95 | 57.7 | 57.49 | 71.72 | 64.0 | 64.0 | 55.74 | 64.0 | 59.34 |
| POI-lon | 66.1 | 63.36 | 63.19 | 75.88 | 64.0 | 64.0 | 56.56 | 64.0 | 60.98 |
| SD-bench | 50.7 | 45.29 | 18.78 | 22.8 | 64.0 | 64.0 | 16.21 | 64.0 | 11.34 |
| NON-TS | 37.1 | 33.9 | 31.8 | 39.0 | 64.0 | 64.0 | 25.7 | 64.0 | 25.6 |
| ALL AVG. | 41.6 | 37.7 | 28.6 | 35.5 | 64.0 | 64.0 | 21.7 | 64.0 | 22.1 |
| Air-Pressure | 24.48 | 22.95 | 19.24 | 27.87 | 30.2277 | 64.0 | 16.43 | 64.0 | 9.39 |
| Basel-Temp | 60.47 | 54.12 | 31.14 | 36.41 | 39.3405 | 64.0 | 30.72 | 64.0 | 18.44 |
| Basel-Wind | 62.2 | 54.73 | 38.35 | 48.86 | 35.0561 | 64.0 | 29.81 | 64.0 | 14.66 |
| Bird-Mig | 47.82 | 42.06 | 26.43 | 36.08 | 35.2224 | 64.0 | 20.14 | 64.0 | 21.02 |
| Btc-Price | 54.67 | 48.07 | 44.96 | 56.8 | 44.0952 | 64.0 | 26.37 | 64.0 | 42.08 |
| City-Temp | 58.82 | 46.25 | 22.96 | 24.16 | 31.4961 | 64.0 | 10.74 | 64.0 | 16.77 |
| Dew-Temp | 54.86 | 51.81 | 32.63 | 39.01 | 29.5099 | 64.0 | 13.4 | 64.0 | 25.07 |
| Bio-Temp | 50.69 | 46.34 | 18.86 | 22.88 | 23.3673 | 64.0 | 10.75 | 64.0 | 17.46 |
| PM10-dust | 27.5 | 24.44 | 13.66 | 19.9 | 12.9093 | 64.0 | 8.56 | 64.0 | 7.78 |
| Stocks-DE | 46.18 | 42.89 | 13.63 | 20.77 | 27.7995 | 64.0 | 11.01 | 64.0 | 10.54 |
| Stocks-UK | 34.75 | 31.33 | 16.76 | 21.49 | 26.3537 | 64.0 | 12.59 | 64.0 | 10.28 |
| Stocks-USA | 37.2 | 34.98 | 12.19 | 19.21 | 26.1213 | 64.0 | 7.9 | 64.0 | 8.56 |
| Wind-dir | 58.14 | 53.89 | 27.8 | 28.14 | 31.5006 | 64.0 | 15.89 | 64.0 | 25.53 |
| TS AVG. | 47.5 | 42.6 | 24.5 | 30.9 | 30.2 | 64.0 | 16.5 | 64.0 | 17.5 |
| Arade/4 | 58.0 | 55.56 | 48.95 | 59.1 | 33.6725 | 64.0 | 24.94 | 64.0 | 33.9 |
| Blockchain | 62.89 | 58.29 | 53.19 | 62.58 | 39.1187 | 64.0 | 36.49 | 64.0 | 43.97 |
| CMS/1 | 37.23 | 34.77 | 28.14 | 36.78 | 40.717 | 64.0 | 35.65 | 64.0 | 26.56 |
| CMS/25 | 64.35 | 59.53 | 57.2 | 70.09 | 63.9107 | 64.0 | 41.11 | 64.0 | 58.27 |
| CMS/9 | 16.01 | 18.71 | 25.65 | 25.97 | 9.7 | 64.0 | 11.67 | 64.0 | 14.73 |
| Food-prices | 38.31 | 27.98 | 24.64 | 28.29 | 25.4469 | 64.0 | 23.65 | 64.0 | 18.32 |
| Gov/10 | 57.45 | 45.75 | 34.15 | 35.84 | 35.5563 | 64.0 | 30.99 | 64.0 | 28.09 |
| Gov/26 | 2.37 | 2.34 | 9.26 | 16.21 | 0.864203 | 64.0 | 0.41 | 64.0 | 0.23 |
| Gov/30 | 10.22 | 8.86 | 12.92 | 19.28 | 8.2306 | 64.0 | 7.48 | 64.0 | 4.48 |
| Gov/31 | 5.61 | 5.0 | 10.43 | 17.07 | 2.82752 | 64.0 | 3.05 | 64.0 | 1.63 |
| Gov/40 | 2.71 | 2.63 | 9.41 | 16.34 | 1.18695 | 64.0 | 0.83 | 64.0 | 0.46 |
| Medicare/1 | 45.49 | 42.72 | 32.32 | 39.91 | 42.7979 | 64.0 | 39.35 | 64.0 | 31.18 |
| Medicare/9 | 16.97 | 19.08 | 26.02 | 26.27 | 10.2 | 64.0 | 12.26 | 64.0 | 15.03 |
| NYC/29 | 30.56 | 29.57 | 28.71 | 38.78 | 64.0004 | 64.0 | 40.38 | 64.0 | 27.5 |
| POI-lat | 65.95 | 57.7 | 57.49 | 71.72 | 69.2503 | 64.0 | 55.74 | 64.0 | 59.34 |
| POI-lon | 66.1 | 63.36 | 63.19 | 75.88 | 69.2462 | 64.0 | 56.56 | 64.0 | 60.98 |
| SD-bench | 50.7 | 45.29 | 18.78 | 22.8 | 30.6101 | 64.0 | 16.21 | 64.0 | 11.34 |
| NON-TS | 37.1 | 33.9 | 31.8 | 39.0 | 32.2 | 64.0 | 25.7 | 64.0 | 25.6 |
| ALL AVG. | 41.6 | 37.7 | 28.6 | 35.5 | 31.3 | 64.0 | 21.7 | 64.0 | 22.1 |
9 changes: 4 additions & 5 deletions publication/tables/table_7.md
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
| Dataset | Gor | Ch | Ch128 | Patas | Alp | Zstd |
|---|---|---|---|---|---|---|
| Dino-Vitb16 | 32.89 | 33.42 | 33.43 | 35.25 | 28.78 | 39.57 |
| GPT2 | 32.24 | 33.46 | 33.48 | 35.24 | 28.01 | 39.76 |
| Grammarly-lg | 32.19 | 33.42 | 33.43 | 35.23 | 29.16 | 39.37 |
| W2V Tweets | 64.0 | 64.0 | 64.0 | 64.0 | 64.0 | 64.0 |
| nan | 32.25 | 27.42 | 34.39 | 26.28 | 28.01 | 43.15 |
| Dino-Vitb16 | 34.11 | 33.42 | 33.43 | 45.81 | 28.78 | 29.74 |
| GPT2 | 34.11 | 33.46 | 33.48 | 45.63 | 28.01 | 29.69 |
| Grammarly-lg | 34.11 | 33.42 | 33.43 | 45.51 | 29.16 | 29.65 |
| W2V Tweets | 32.32 | 33.5 | 33.51 | 45.6 | 28.86 | 29.65 |

0 comments on commit b9aaeae

Please sign in to comment.