Skip to content

Commit

Permalink
[ci](perf) 1. add perf check of tpcds, 2. adjust clickbench and tpch …
Browse files Browse the repository at this point in the history
…check (apache#28431)
  • Loading branch information
hello-stephen authored Dec 29, 2023
1 parent 269c1b1 commit 61677d1
Show file tree
Hide file tree
Showing 20 changed files with 2,103 additions and 22 deletions.
1 change: 1 addition & 0 deletions .licenserc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ header:
- "conf/mysql_ssl_default_certificate/client_certificate/client-cert.pem"
- "conf/mysql_ssl_default_certificate/client_certificate/client-key.pem"
- "regression-test/ssl_default_certificate/*"
- "regression-test/pipeline/performance/conf/session_variables"
- "extension/beats/go.mod"
- "extension/beats/go.sum"
- "pytest/hdfs"
Expand Down
154 changes: 140 additions & 14 deletions regression-test/pipeline/common/doris-utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,15 @@ function set_doris_conf_value() {

function start_doris_fe() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
if ! java -version >/dev/null; then sudo apt install openjdk-8-jdk -y >/dev/null; fi
if ! java -version >/dev/null ||
[[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*')" ]]; then
sudo apt update && sudo apt install openjdk-8-jdk -y >/dev/null
fi
JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
export JAVA_HOME
"${DORIS_HOME}"/fe/bin/start_fe.sh --daemon

if ! mysql --version >/dev/null; then sudo apt install -y mysql-client; fi
if ! mysql --version >/dev/null; then sudo apt update && sudo apt install -y mysql-client; fi
query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
cl="mysql -h127.0.0.1 -P${query_port} -uroot "
local i=1
Expand All @@ -71,12 +74,14 @@ function start_doris_fe() {
fi
done
if [[ ${i} -ge 60 ]]; then echo "ERROR: Start Doris Frontend Failed after 2 mins wait..." && return 1; fi

}

function start_doris_be() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
if ! java -version >/dev/null; then sudo apt install openjdk-8-jdk -y >/dev/null; fi
if ! java -version >/dev/null ||
[[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*')" ]]; then
sudo apt update && sudo apt install openjdk-8-jdk -y >/dev/null
fi
JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
export JAVA_HOME
sysctl -w vm.max_map_count=2000000 &&
Expand All @@ -101,7 +106,7 @@ function start_doris_be() {

function add_doris_be_to_fe() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
if ! mysql --version >/dev/null; then sudo apt install -y mysql-client; fi
if ! mysql --version >/dev/null; then sudo sudo apt update && apt install -y mysql-client; fi
query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
heartbeat_service_port=$(get_doris_conf_value "${DORIS_HOME}"/be/conf/be.conf heartbeat_service_port)
cl="mysql -h127.0.0.1 -P${query_port} -uroot "
Expand All @@ -116,7 +121,7 @@ function add_doris_be_to_fe() {
echo 'Wait for Backends ready, sleep 2 seconds ...' && sleep 2
fi
done
if [[ ${i} -eq 60 ]]; then echo "ERROR: Add Doris Backend Failed after 2 mins wait..." && return 1; fi
if [[ ${i} -ge 60 ]]; then echo "ERROR: Add Doris Backend Failed after 2 mins wait..." && return 1; fi
}

function stop_doris() {
Expand All @@ -129,6 +134,15 @@ function stop_doris() {
fi
}

function restart_doris() {
if stop_doris; then echo; fi
if ! start_doris_fe; then return 1; fi
if ! start_doris_be; then return 1; fi
# wait 10s for doris totally started, otherwize may encounter the error below,
# ERROR 1105 (HY000) at line 102: errCode = 2, detailMessage = Failed to find enough backend, please check the replication num,replication tag and storage medium.
sleep 10s
}

function check_tpch_table_rows() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
db_name="$1"
Expand All @@ -138,20 +152,133 @@ function check_tpch_table_rows() {
query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
cl="mysql -h127.0.0.1 -P${query_port} -uroot "
declare -A table_rows
if [[ "${scale_factor}" == "100" ]]; then
if [[ "${scale_factor}" == "1" ]]; then
table_rows=(['region']=5 ['nation']=25 ['supplier']=10000 ['customer']=150000 ['part']=200000 ['partsupp']=800000 ['orders']=1500000 ['lineitem']=6001215)
elif [[ "${scale_factor}" == "100" ]]; then
table_rows=(['region']=5 ['nation']=25 ['supplier']=1000000 ['customer']=15000000 ['part']=20000000 ['partsupp']=80000000 ['orders']=150000000 ['lineitem']=600037902)
else
table_rows=(['region']=5 ['nation']=25 ['supplier']=10000 ['customer']=150000 ['part']=200000 ['partsupp']=800000 ['orders']=1500000 ['lineitem']=6001215)
echo "ERROR: unsupported scale_factor ${scale_factor} for tpch" && return 1
fi
for table in ${!table_rows[*]}; do
rows_actual=$(${cl} -D"${db_name}" -e"SELECT count(*) FROM ${table}" | sed -n '2p')
rows_expect=${table_rows[${table}]}
if [[ ${rows_actual} -ne ${rows_expect} ]]; then
echo "ERROR: ${table} actual rows: ${rows_actual}, expect rows: ${rows_expect}" && return 1
fi
done
}

function check_tpcds_table_rows() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
db_name="$1"
scale_factor="$2"
if [[ -z "${scale_factor}" ]]; then return 1; fi

query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
cl="mysql -h127.0.0.1 -P${query_port} -uroot "
declare -A table_rows
if [[ "${scale_factor}" == "1" ]]; then
table_rows=(['income_band']=20 ['ship_mode']=20 ['warehouse']=5 ['reason']=35 ['web_site']=30 ['call_center']=6 ['store']=12 ['promotion']=300 ['household_demographics']=7200 ['web_page']=60 ['catalog_page']=11718 ['time_dim']=86400 ['date_dim']=73049 ['item']=18000 ['customer_demographics']=1920800 ['customer_address']=50000 ['customer']=100000 ['web_returns']=71763 ['catalog_returns']=144067 ['store_returns']=287514 ['inventory']=11745000 ['web_sales']=719384 ['catalog_sales']=1441548 ['store_sales']=2880404)
elif [[ "${scale_factor}" == "100" ]]; then
table_rows=(['income_band']=20 ['ship_mode']=20 ['warehouse']=15 ['reason']=55 ['web_site']=24 ['call_center']=30 ['store']=402 ['promotion']=1000 ['household_demographics']=7200 ['web_page']=2040 ['catalog_page']=20400 ['time_dim']=86400 ['date_dim']=73049 ['item']=204000 ['customer_demographics']=1920800 ['customer_address']=1000000 ['customer']=2000000 ['web_returns']=7197670 ['catalog_returns']=14404374 ['store_returns']=28795080 ['inventory']=399330000 ['web_sales']=72001237 ['catalog_sales']=143997065 ['store_sales']=287997024)
elif [[ "${scale_factor}" == "1000" ]]; then
table_rows=(['income_band']=20 ['ship_mode']=20 ['warehouse']=20 ['reason']=65 ['web_site']=54 ['call_center']=42 ['store']=1002 ['promotion']=1500 ['household_demographics']=7200 ['web_page']=3000 ['catalog_page']=30000 ['time_dim']=86400 ['date_dim']=73049 ['item']=300000 ['customer_demographics']=1920800 ['customer_address']=6000000 ['customer']=12000000 ['web_returns']=71997522 ['catalog_returns']=143996756 ['store_returns']=287999764 ['inventory']=783000000 ['web_sales']=720000376 ['catalog_sales']=1439980416 ['store_sales']=2879987999)
elif [[ "${scale_factor}" == "3000" ]]; then
table_rows=(['income_band']=20 ['ship_mode']=20 ['warehouse']=22 ['reason']=67 ['web_site']=66 ['call_center']=48 ['store']=1350 ['promotion']=1800 ['household_demographics']=7200 ['web_page']=3600 ['catalog_page']=36000 ['time_dim']=86400 ['date_dim']=73049 ['item']=360000 ['customer_demographics']=1920800 ['customer_address']=15000000 ['customer']=30000000 ['web_returns']=216003761 ['catalog_returns']=432018033 ['store_returns']=863989652 ['inventory']=1033560000 ['web_sales']=2159968881 ['catalog_sales']=4320078880 ['store_sales']=8639936081)
else
echo "ERROR: unsupported scale_factor ${scale_factor} for tpcds" && return 1
fi
for table in ${!table_rows[*]}; do
rows_actual=$(${cl} -D"${db_name}" -e"SELECT count(*) FROM ${table}" | sed -n '2p')
rows_expect=${table_rows[${table}]}
if [[ ${rows_actual} -ne ${rows_expect} ]]; then
echo "WARNING: ${table} actual rows: ${rows_actual}, expect rows: ${rows_expect}" && return 1
echo "ERROR: ${table} actual rows: ${rows_actual}, expect rows: ${rows_expect}" && return 1
fi
done
}

function check_clickbench_table_rows() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
db_name="$1"
if [[ -z "${db_name}" ]]; then return 1; fi
query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
cl="mysql -h127.0.0.1 -P${query_port} -uroot "
declare -A table_rows
table_rows=(['hits']=99997497)
if ${DEBUG:-false}; then table_rows=(['hits']=10000); fi
for table in ${!table_rows[*]}; do
rows_actual=$(${cl} -D"${db_name}" -e"SELECT count(*) FROM ${table}" | sed -n '2p')
rows_expect=${table_rows[${table}]}
if [[ ${rows_actual} -ne ${rows_expect} ]]; then
echo "ERROR: ${table} actual rows: ${rows_actual}, expect rows: ${rows_expect}" && return 1
fi
done
}

function check_tpch_result() {
log_file="$1"
if [[ -z "${log_file}" ]]; then return 1; fi
if ! grep '^Total cold run time' "${log_file}" || ! grep '^Total hot run time' "${log_file}"; then
echo "ERROR: can not find 'Total hot run time' in '${log_file}'"
return 1
else
cold_run_time=$(grep '^Total cold run time' "${log_file}" | awk '{print $5}')
hot_run_time=$(grep '^Total hot run time' "${log_file}" | awk '{print $5}')
fi
# 单位是毫秒
cold_run_time_threshold=${cold_run_time_threshold:-50000}
hot_run_time_threshold=${hot_run_time_threshold:-42000}
if [[ ${cold_run_time} -gt ${cold_run_time_threshold} || ${hot_run_time} -gt ${hot_run_time_threshold} ]]; then
echo "ERROR:
cold_run_time ${cold_run_time} is great than the threshold ${cold_run_time_threshold},
or, hot_run_time ${hot_run_time} is great than the threshold ${hot_run_time_threshold}"
return 1
else
echo "INFO:
cold_run_time ${cold_run_time} is less than the threshold ${cold_run_time_threshold},
hot_run_time ${hot_run_time} is less than the threshold ${hot_run_time_threshold}"
fi
}

function check_tpcds_result() {
check_tpch_result "$1"
}

function check_clickbench_query_result() {
echo "TODO"
}

function check_clickbench_performance_result() {
result_file="$1"
if [[ -z "${result_file}" ]]; then return 1; fi

empty_query_time="$(awk -F ',' '{if( ($2=="") || ($3=="") || ($4=="") ){print $1}}' "${result_file}")"
if [[ -n ${empty_query_time} ]]; then
echo -e "ERROR: find empty query time of:\n${empty_query_time}" && return 1
fi

# 单位是秒
cold_run_time_threshold=${cold_run_time_threshold:-200}
hot_run_time_threshold=${hot_run_time_threshold:-55}
cold_run_sum=$(awk -F ',' '{sum+=$2} END {print sum}' result.csv)
hot_run_time=$(awk -F ',' '{if($3<$4){sum+=$3}else{sum+=$4}} END {print sum}' "${result_file}")
if [[ $(echo "${hot_run_time} > ${hot_run_time_threshold}" | bc) -eq 1 ]] ||
[[ $(echo "${cold_run_sum} > ${cold_run_time_threshold}" | bc) -eq 1 ]]; then
echo "ERROR:
cold_run_time ${cold_run_time} is great than the threshold ${cold_run_time_threshold},
or, hot_run_time ${hot_run_time} is great than the threshold ${hot_run_time_threshold}"
return 1
else
echo "INFO:
cold_run_time ${cold_run_time} is less than the threshold ${cold_run_time_threshold},
hot_run_time ${hot_run_time} is less than the threshold ${hot_run_time_threshold}"
fi
}

function check_load_performance() {
echo "TODO"
}

get_session_variable() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
usage="
Expand Down Expand Up @@ -228,12 +355,11 @@ archive_doris_logs() {
if [[ -z ${archive_name} ]]; then echo "ERROR: archive file name required" && return 1; fi
if tar -I pigz \
--directory "${DORIS_HOME}" \
--absolute-names \
-cf "${DORIS_HOME}/${archive_name}" \
"${DORIS_HOME}"/fe/conf \
"${DORIS_HOME}"/fe/log \
"${DORIS_HOME}"/be/conf \
"${DORIS_HOME}"/be/log; then
fe/conf \
fe/log \
be/conf \
be/log; then
echo "${DORIS_HOME}/${archive_name}"
else
return 1
Expand Down
38 changes: 38 additions & 0 deletions regression-test/pipeline/common/get-or-set-tmp-env.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

tmp_env_file_path="${PWD}/.my_tmp_env"

usage() {
echo -e "
Usage:
$0 'get'
$0 'set' \"export skip_pipeline='true'\"
note: 'get' will return env file path; 'set' will add your new item into env file"
exit 1
}

if [[ $1 == 'get' ]]; then
if [[ ! -f "${tmp_env_file_path}" ]]; then touch "${tmp_env_file_path}"; fi
echo "${tmp_env_file_path}"
elif [[ $1 == 'set' ]]; then
if [[ -z $2 ]]; then usage; fi
echo "$2" >>"${tmp_env_file_path}"
else
usage
fi
55 changes: 54 additions & 1 deletion regression-test/pipeline/common/github-utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,33 @@ function create_an_issue_comment_tpch() {
local COMMENT_BODY="$2"
local machine='aliyun_ecs.c7a.8xlarge_32C64G'
COMMENT_BODY="
TPC-H test result on machine: '${machine}'
TPC-H test result on machine: '${machine}', run with scripts in https://github.com/apache/doris/tree/master/tools/tpch-tools
\`\`\`
${COMMENT_BODY}
\`\`\`
"
create_an_issue_comment "${ISSUE_NUMBER}" "${COMMENT_BODY}"
}

function create_an_issue_comment_tpcds() {
local ISSUE_NUMBER="$1"
local COMMENT_BODY="$2"
local machine='aliyun_ecs.c7a.8xlarge_32C64G'
COMMENT_BODY="
TPC-DS test result on machine: '${machine}', run with scripts in https://github.com/apache/doris/tree/master/tools/tpcds-tools
\`\`\`
${COMMENT_BODY}
\`\`\`
"
create_an_issue_comment "${ISSUE_NUMBER}" "${COMMENT_BODY}"
}

function create_an_issue_comment_clickbench() {
local ISSUE_NUMBER="$1"
local COMMENT_BODY="$2"
local machine='aliyun_ecs.c7a.8xlarge_32C64G'
COMMENT_BODY="
ClickBench test result on machine: '${machine}', run with scripts in https://github.com/apache/doris/tree/master/tools/clickbench-tools
\`\`\`
${COMMENT_BODY}
\`\`\`
Expand Down Expand Up @@ -271,3 +297,30 @@ file_changed_ckb() {
done
echo "return no need" && return 1
}

file_changed_perf() {
local all_files
all_files=$(cat all_files)
if _only_modified_regression_conf; then echo "return no need" && return 1; fi
if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi
for af in ${all_files}; do
if [[ "${af}" == 'be'* ]] ||
[[ "${af}" == 'bin'* ]] ||
[[ "${af}" == 'conf'* ]] ||
[[ "${af}" == 'fe'* ]] ||
[[ "${af}" == 'gensrc'* ]] ||
[[ "${af}" == 'thirdparty'* ]] ||
[[ "${af}" == 'build.sh' ]] ||
[[ "${af}" == 'env.sh' ]] ||
[[ "${af}" == 'regression-test/pipeline/common/github-utils.sh' ]] ||
[[ "${af}" == 'regression-test/pipeline/common/doris-utils.sh' ]] ||
[[ "${af}" == 'regression-test/pipeline/common/oss-utils.sh' ]] ||
[[ "${af}" == 'regression-test/pipeline/performance/'* ]] ||
[[ "${af}" == 'tools/tpch-tools/bin/run-tpch-queries.sh' ]] ||
[[ "${af}" == 'tools/tpcds-tools/bin/run-tpcds-queries.sh' ]] ||
[[ "${af}" == 'regression-test/pipeline/tpch/tpch-sf100/'* ]]; then
echo "performance related file changed, return need" && return 0
fi
done
echo "return no need" && return 1
}
47 changes: 47 additions & 0 deletions regression-test/pipeline/performance/clean.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# Build Step: Command Line
: <<EOF
#!/bin/bash
# Execute step even if some of the previous steps failed
if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/performance/clean.sh ]]; then
cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance
bash -x clean.sh
else
echo "Build Step file missing: regression-test/pipeline/performance/clean.sh" && exit 1
fi
EOF

#####################################################################################
## clean.sh content ##

# shellcheck source=/dev/null
source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi

# shellcheck source=/dev/null
# stop_doris
source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh

DORIS_HOME="${teamcity_build_checkoutDir}/output"
export DORIS_HOME
stop_doris
Loading

0 comments on commit 61677d1

Please sign in to comment.