Skip to content

Commit

Permalink
check if mvn and java is installed,
Browse files Browse the repository at this point in the history
  • Loading branch information
@azimafroozeh
azimafroozeh committed Dec 4, 2024
1 parent 9cc12f4 commit df2f8fc
Show file tree
Hide file tree
Showing 12 changed files with 181 additions and 134 deletions.
1 change: 1 addition & 0 deletions data/include/data.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include "float/float_dataset.hpp"
#include "generated_columns.hpp"
#include <fstream>
#include <vector>

namespace alp_data {

Expand Down
20 changes: 19 additions & 1 deletion publication/script/install_dependencies.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,23 @@ sudo apt-get install -y python3-pip
pip3 install --upgrade pip --break-system-packages
pip3 install pandas seaborn --break-system-packages

# Install Java version 8
echo "Installing Java 8..."
sudo apt-get install -y openjdk-8-jdk

# Set Java 8 as default
echo "Setting Java 8 as the default Java version..."
sudo update-alternatives --config java <<<'1'

# Verify Java installation
java -version

# Install Maven
echo "Installing Maven..."
sudo apt-get install -y maven

# Verify Maven installation
mvn -v

# Final check
echo "All dependencies installed successfully! Python dependencies are installed system-wide."
echo "All dependencies installed successfully: Clang++, CMake, Boost, BZIP2, OpenSSL, libcurl, g++, Python (pip, pandas, seaborn), Java 8, and Maven."
94 changes: 58 additions & 36 deletions publication/script/master_script.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,23 @@ green_echo() {
}

brown_echo() {
echo -e "\033[33m $1\033[0m" >/dev/tty
echo -e "\033[33m-- $1\033[0m" >/dev/tty
}

black_echo() {
echo -e "\033[30m $1\033[0m" >/dev/tty
echo -e "\033[30m--$1\033[0m" >/dev/tty
}

red_echo() {
echo -e "\033[31mError: $1\033[0m" >/dev/tty
echo -e "\033[31m-- Error: $1\033[0m" >/dev/tty
}

# Set up workspace variables
green_echo "Setting up workspace variables..."
WORKSPACE=$(pwd)
REPO_URL="https://github.com/cwida/ALP.git"
CLONED_DIR="$WORKSPACE/ALP"
BRANCH="repro"
BRANCH="elf"

# Clone or update the repository
green_echo "Cloning or updating repository..."
Expand Down Expand Up @@ -57,6 +57,33 @@ fi
green_echo "Navigating to target directory..."
cd "$CLONED_DIR"

# Check if Java is installed
java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}')
brown_echo "Detected Java version: $java_version"

if [[ "$java_version" == 1.8* || "$java_version" == 8* ]]; then
green_echo "Java 8 is installed and set as the default version."
else
red_echo "Error: Java 8 is not installed or not set as the default version."
exit 1
fi

# Check if Maven is installed
if command -v mvn >/dev/null 2>&1; then
maven_version=$(mvn -v | awk '/Apache Maven/ {print $3}')
java_version_in_maven=$(mvn -v | awk -F '"' '/Java version/ {print $2}')

green_echo "Maven is installed. Detected Maven version: $maven_version"
brown_echo "Maven is using Java version: $java_version_in_maven"
else
red_echo "Error: Maven is not installed."
exit 1
fi

green_echo "Running ELF benchmarks..."
mvn -f "$CLONED_DIR/publication/source_code/extern/elf/pom.xml" clean
mvn -f "$CLONED_DIR/publication/source_code/extern/elf/pom.xml" test

# Create build directory
green_echo "Creating build directory..."
mkdir -p "$CLONED_DIR/build"
Expand Down Expand Up @@ -103,11 +130,6 @@ if [ $? -ne 0 ]; then
exit 1
fi

# ELF
green_echo "Running ELF benchmarks..."
mvn -f "$CLONED_DIR/publication/source_code/extern/elf/pom.xml" clean
mvn -f "$CLONED_DIR/publication/source_code/extern/elf/pom.xml" test

green_echo "Running compression ratio benchmarks..."
"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_alp_compression_ratio"
"$CLONED_DIR/build/publication/source_code/bench_compression_ratio/publication_bench_alp32_compression_ratio"
Expand All @@ -129,33 +151,33 @@ if [ "$ARCH" == "arm64" ]; then
"$CLONED_DIR/build/publication/source_code/generated/arm64v8/neon_intrinsic_uf1/arm64v8_neon_intrinsic_1024_uf1_falp_bench"
else

# 4xLarge benchmarks
green_echo "Running I4I_4XLarge speed benchmarks..."
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx512bw_intrinsic_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx512bw_intrinsic_1024_uf1_falp_bench_bw"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx2_intrinsic_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/fallback_scalar_aav_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/fallback_scalar_nav_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_cutter_decode"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_cutter_encode"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_encode"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_without_sampling"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_chimp"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_chimp128"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_gorillas"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_patas"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_zstd"

# End-to-end benchmark
SCRIPT_DIR=$(dirname "$(realpath "$0")")
OUTPUT_FILE="$SCRIPT_DIR/publication/end_to_end_bench/result"
HEADER="dataset,repetition,warmup_repetition,scheme,thread_n,query,time(s),result(tpc),corrected_result(tpc),validity,compression_cycles,cycles"
green_echo "Running end-to-end benchmark and saving results to $OUTPUT_FILE ..."
export CLONED_DIR="$CLONED_DIR"
bash "$CLONED_DIR/publication/script/run_end_to_end.sh" >"$OUTPUT_FILE" 2>&1
# Ensure the header line is at the start of the file
echo -e "$HEADER\n$(cat "$OUTPUT_FILE")" >"$OUTPUT_FILE"
green_echo "Benchmark completed. Results are saved in $OUTPUT_FILE."
# 4xLarge benchmarks
green_echo "Running I4I_4XLarge speed benchmarks..."
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx512bw_intrinsic_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx512bw_intrinsic_1024_uf1_falp_bench_bw"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/x86_64_avx2_intrinsic_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/fallback_scalar_aav_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/generated/x86_64/avx512bw_intrinsic_uf1/fallback_scalar_nav_1024_uf1_falp_bench"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_cutter_decode"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_cutter_encode"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_encode"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_alp_without_sampling"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_chimp"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_chimp128"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_gorillas"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_patas"
"$CLONED_DIR/build/publication/source_code/bench_speed/publication_bench_zstd"

# End-to-end benchmark
SCRIPT_DIR=$(dirname "$(realpath "$0")")
OUTPUT_FILE="$SCRIPT_DIR/publication/end_to_end_bench/result"
HEADER="dataset,repetition,warmup_repetition,scheme,thread_n,query,time(s),result(tpc),corrected_result(tpc),validity,compression_cycles,cycles"
green_echo "Running end-to-end benchmark and saving results to $OUTPUT_FILE ..."
export CLONED_DIR="$CLONED_DIR"
bash "$CLONED_DIR/publication/script/run_end_to_end.sh" >"$OUTPUT_FILE" 2>&1
# Ensure the header line is at the start of the file
echo -e "$HEADER\n$(cat "$OUTPUT_FILE")" >"$OUTPUT_FILE"
green_echo "Benchmark completed. Results are saved in $OUTPUT_FILE."

# Clone and build the BENCH_PDE repository
green_echo "Cloning the BENCH_PDE repository..."
Expand Down
10 changes: 5 additions & 5 deletions publication/source_code/bench_end_to_end/include/common/runtime/Import.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ void importTPCH(std::string dir, Database& db);
void importSSB(std::string dir, Database& db);

/// imports alp benchmark from CSVs in dir into db
double import_alp(alp_bench::Column& col, Database& db, encoding::scheme& encoding);
double import_alp(alp_bench::ALPColumnDescriptor& col, Database& db, encoding::scheme& encoding);
} // namespace runtime

namespace experiment {
void remove_binary_file(alp_bench::Column& col);
void expand_binary_x_times(alp_bench::Column& col, size_t x);
bool is_expanded(alp_bench::Column& col);
void clean_compressed_data(alp_bench::Column& col, encoding::scheme& scheme);
void remove_binary_file(alp_bench::ALPColumnDescriptor& col);
void expand_binary_x_times(alp_bench::ALPColumnDescriptor& col, size_t x);
bool is_expanded(alp_bench::ALPColumnDescriptor& col);
void clean_compressed_data(alp_bench::ALPColumnDescriptor& col, encoding::scheme& scheme);
double sum(double* in, size_t c);

} // namespace experiment
6 changes: 3 additions & 3 deletions publication/source_code/bench_end_to_end/src/benchmarks/alp/run_query.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,8 +209,8 @@ double alp_profile(std::function<void()> fn, uint64_t rep_c, uint64_t warmup_rep
return end - start;
}

alp_bench::Column& get_dataset(const std::string& col_name) {
static std::unordered_map<std::string, alp_bench::Column> ALP_END_TO_END_DATASET = {
alp_bench::ALPColumnDescriptor& get_dataset(const std::string& col_name) {
static std::unordered_map<std::string, alp_bench::ALPColumnDescriptor> ALP_END_TO_END_DATASET = {
{"food_prices", {0, "food_prices_tw", "", "", 16, 12, 46, 20}},
{"city_temperature_f", {1, "city_temperature_f_tw", "", "", 14, 13, 0, 11}},
{"bitcoin_transactions_f", {2, "bitcoin_transactions_f_tw", "", "", 14, 10, 10, 25}},
Expand Down Expand Up @@ -273,7 +273,7 @@ int main(int argc, char* argv[]) {
throw std::runtime_error("Wrong Scheme Type.");
}

alp_bench::Column& col = get_dataset(dataset_string);
alp_bench::ALPColumnDescriptor& col = get_dataset(dataset_string);

runtime::cur_q_mtd.repetition = cfg::rep_c;
runtime::cur_q_mtd.warm_up_repetition = cfg::warmup_rep_c;
Expand Down
10 changes: 5 additions & 5 deletions publication/source_code/bench_end_to_end/src/common/runtime/Import.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1094,7 +1094,7 @@ void importSSB(std::string dir, Database& db) {
// NO SUPPORT FOR SSB
}

double import_alp(alp_bench::Column& col, Database& db, encoding::scheme& scheme) {
double import_alp(alp_bench::ALPColumnDescriptor& col, Database& db, encoding::scheme& scheme) {
auto& rel = db[cfg::tbl_name];
rel.name = col.name;
auto columns = configX({{cfg::schema, col.name, make_unique<algebra::Double>(), scheme}});
Expand All @@ -1116,7 +1116,7 @@ double sum(double* in, size_t c) {
return result;
}

void expand_binary_x_times(alp_bench::Column& col, size_t x) {
void expand_binary_x_times(alp_bench::ALPColumnDescriptor& col, size_t x) {
/**/

std::string col_file_path = cfg::data_dir + col.name + ".bin";
Expand All @@ -1136,7 +1136,7 @@ void expand_binary_x_times(alp_bench::Column& col, size_t x) {
// std::cout << __FUNCTION__ << " : " << col_file_path << " has been extended " << x << " times." << std::endl;
}

bool is_expanded(alp_bench::Column& col) {
bool is_expanded(alp_bench::ALPColumnDescriptor& col) {
/**/
bool file_exist;
bool has_correct_size;
Expand Down Expand Up @@ -1165,13 +1165,13 @@ static void remove_file(std::string_view file_path) {
// std::cout << __FUNCTION__ << " : " << file_path << " has been removed successfully." << std::endl;
}

void remove_binary_file(alp_bench::Column& col) {
void remove_binary_file(alp_bench::ALPColumnDescriptor& col) {
/**/
std::string col_extended_file_path = cfg::data_ext_dir + col.name + +".bin";
remove_file(col_extended_file_path);
}

void clean_compressed_data(alp_bench::Column& col, encoding::scheme& scheme) {
void clean_compressed_data(alp_bench::ALPColumnDescriptor& col, encoding::scheme& scheme) {
/**/
std::vector<std::string> path_vec = ff::file::get_col_paths(col.name, scheme);
for (auto& file_path : path_vec) {
Expand Down
34 changes: 17 additions & 17 deletions publication/source_code/bench_speed/bench_alp_cutter_decode.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,23 @@
using namespace alp::config;

/* Bench ALP encode. */
static __attribute__((noinline)) benchmark::BenchmarkReporter::Run b_a_e(const double* dbl_arr,
uint16_t* exc_arr,
uint16_t* pos_arr,
uint16_t* exc_c_arr,
int64_t* encoded_arr,
uint8_t& bw,
int64_t* ffor_arr,
int64_t* base_arr,
alp::state<double>& stt,
alp_bench::ALPColumnDescriptor& dataset,
uint64_t* ffor_right_arr,
uint16_t* ffor_left_arr,
uint64_t* right_arr,
uint16_t* left_arr,
uint64_t* unffor_right_arr,
uint16_t* unffor_left_arr,
double* glue_arr) {
static __attribute__((noinline)) benchmark::BenchmarkReporter::Run b_a_e(const double* dbl_arr,
uint16_t* exc_arr,
uint16_t* pos_arr,
uint16_t* exc_c_arr,
int64_t* encoded_arr,
uint8_t& bw,
int64_t* ffor_arr,
int64_t* base_arr,
alp::state<double>& stt,
alp_bench::ALPColumnDescriptor& dataset,
uint64_t* ffor_right_arr,
uint16_t* ffor_left_arr,
uint64_t* right_arr,
uint16_t* left_arr,
uint64_t* unffor_right_arr,
uint16_t* unffor_left_arr,
double* glue_arr) {

int benchmark_number = dataset.id;

Expand Down
4 changes: 2 additions & 2 deletions publication/source_code/bench_speed/bench_chimp128.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

static __attribute__((noinline)) benchmark::BenchmarkReporter::Run
bench_decode_chimp128(const double* dbl_arr,
alp_bench::ALPColumnDescriptor& dataset,
alp_bench::ALPColumnDescriptor& dataset,
uint8_t leading_zero_block_count,
alp_bench::Chimp128CompressionState<uint64_t, false> com_stt,
idx_t leading_zero_block_size,
Expand Down Expand Up @@ -93,7 +93,7 @@ bench_decode_chimp128(const double* dbl_a
}

static __attribute__((noinline)) benchmark::BenchmarkReporter::Run
bench_encode_chimp128(alp_bench::ALPColumnDescriptor& dataset,
bench_encode_chimp128(alp_bench::ALPColumnDescriptor& dataset,
alp_bench::Chimp128CompressionState<uint64_t, false> com_stt,
uint8_t leading_zero_block_count,
idx_t leading_zero_block_size,
Expand Down
4 changes: 2 additions & 2 deletions publication/source_code/bench_speed/bench_patas.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
#include "patas/patas.hpp"

static __attribute__((noinline)) benchmark::BenchmarkReporter::Run
bench_decoding_patas(alp_bench::ALPColumnDescriptor& dataset,
bench_decoding_patas(alp_bench::ALPColumnDescriptor& dataset,
uint16_t* packed_metadata,
uint8_t* data_arr,
uint64_t* dec_arr,
Expand Down Expand Up @@ -46,7 +46,7 @@ bench_decoding_patas(alp_bench::ALPColumnDescriptor& dat
}

static __attribute__((noinline)) benchmark::BenchmarkReporter::Run
bench_encoding_patas(alp_bench::ALPColumnDescriptor& dataset,
bench_encoding_patas(alp_bench::ALPColumnDescriptor& dataset,
alp_bench::patas::PatasCompressionState<uint64_t, false> patas_state,
uint8_t* data_arr,
uint16_t* packed_metadata,
Expand Down
Loading

0 comments on commit df2f8fc

Please sign in to comment.