Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Bug] Fix automation script path issue #4030

Merged
merged 19 commits into from
Dec 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 99 additions & 8 deletions script/bin/auto.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# debug mode
#set -x


export RED='\033[31m'
export GREEN='\033[32m'
export YELLOW='\033[33m'
Expand All @@ -11,27 +12,49 @@ export CYAN='\033[36m'
export RESET='\033[0m'


ENV_FILE="/etc/profile.d/dinky_env"
if [ -f "${ENV_FILE}" ]; then
source "${ENV_FILE}"
fi

if [ -z "${DINKY_HOME}" ]; then
echo -e "${RED}DINKY_HOME environment variable is not set. Attempting to determine the correct path...${RESET}"
DB_ENV_FILE="/etc/profile.d/dinky_db"
if [ -f "${DB_ENV_FILE}" ]; then
source "${DB_ENV_FILE}"
fi

source /etc/profile

RETURN_HOME_PATH=""
function get_home_path() {
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
export DINKY_HOME="$(dirname "$DIR")"
RETURN_HOME_PATH=$(dirname "$DIR")
}


if [ -z "${DINKY_HOME}" ]; then
echo -e "${RED}DINKY_HOME environment variable is not set. Attempting to determine the correct path...${RESET}"
get_home_path
export DINKY_HOME=${RETURN_HOME_PATH}
echo -e "${GREEN}DINKY_HOME is set to: ${DINKY_HOME}${RESET}"
else
echo -e "${GREEN}DINKY_HOME is already set to: ${DINKY_HOME}${RESET}"
get_home_path
if [ "${DINKY_HOME}" != "${RETURN_HOME_PATH}" ]; then
export DINKY_HOME=${RETURN_HOME_PATH}
echo -e "${YELLOW}DINKY_HOME is not equal to the current path, reset DINKY_HOME to: ${RETURN_HOME_PATH}${RESET}"
else
echo -e "${GREEN}DINKY_HOME is already set to: ${DINKY_HOME}${RESET}"
fi
fi


FLINK_VERSION=${2}

DINKY_HOME=${DINKY_HOME:-$(cd "$(dirname "$0")"; cd ..; pwd)}
JAVA_VERSION=$(java -version 2>&1 | sed '1!d' | sed -e 's/"//g' | awk '{print $3}' | awk -F'.' '{print $1"."$2}')

APP_HOME="${DINKY_HOME}"
Expand Down Expand Up @@ -71,6 +94,70 @@ assertIsInputVersion() {
fi
}


if [ -f "${APP_HOME}/config/application.yml" ]; then
result=$("${APP_HOME}"/bin/parse_yml.sh "${APP_HOME}/config/application.yml" "server.port")
APP_PORT=$result
fi

echo -e "${GREEN}From ${APP_HOME}/config/application.yml server.port: ${APP_PORT}${RESET}"

if [ -z "$APP_PORT" ]; then
echo -e "${RED}Could not find server.port in configuration files, using default port 8888 ${RESET}"
APP_PORT=8888
fi

# Function: Check the status of the health check endpoint
check_health() {
curl --silent --max-time 2 --output /dev/null --write-out "%{http_code}" "http://localhost:$APP_PORT/actuator/health"
}



format_time() {
local seconds=$1
local hours=$((seconds / 3600))
local minutes=$(( (seconds % 3600) / 60 ))
local remaining_seconds=$((seconds % 60))
printf "%02d:%02d:%02d" $hours $minutes $remaining_seconds
}

function wait_start_process() {
echo -e "${GREEN}>>>>>>>>>>>>>>>>>>>>> Starting application... <<<<<<<<<<<<<<<<<<<<<<<${RESET}"
local max_attempts=100
local attempt=0
local delay=0.25
local health_status=""
local success_status_codes=("200")
local start_time=$(date +%s)

while [ $attempt -lt $max_attempts ]; do
attempt=$((attempt + 1))
local current_time=$(date +%s)
local elapsed_time=$((current_time - start_time))
local formatted_time=$(format_time $elapsed_time)
health_status=$(check_health)
for code in "${success_status_codes[@]}"; do
if [ "$health_status" == "$code" ]; then
echo -ne "\r[==================================================] 100%\n"
echo -e "${GREEN}Application started completed.${RESET}"
return 0
fi
done
local progress=$((attempt * 100 / max_attempts))
local bar_length=50
local filled_length=$((progress * bar_length / 100))
local empty_length=$((bar_length - filled_length))
local bar=$(printf '>%.0s' $(seq 1 $filled_length))$(printf ' %.0s' $(seq 1 $empty_length))
echo -ne "\r[${bar}] ${progress}% (time consuming: ${formatted_time})"
sleep $delay
done
echo -ne "\r[==================================================] 100% (time consuming: ${formatted_time})\n"
echo -e "${RED}Application start failed. Please check the log for details.${RESET}"
return 1
}


# Use FLINK_HOME:
CLASS_PATH="${APP_HOME}:${APP_HOME}/lib/*:${APP_HOME}/config:${EXTENDS_HOME}/*:${CUSTOMER_JAR_PATH}/*:${EXTENDS_HOME}/flink${FLINK_VERSION}/dinky/*:${EXTENDS_HOME}/flink${FLINK_VERSION}/flink/*:${EXTENDS_HOME}/flink${FLINK_VERSION}/*"
PID_FILE="dinky.pid"
Expand Down Expand Up @@ -129,7 +216,9 @@ start() {
updatePid
if [ -z "$pid" ]; then
nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none -cp "${CLASS_PATH}" org.dinky.Dinky ${JAR_PARAMS_OPT} > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 &
echo $! >"${PID_PATH}"/${PID_FILE}
PID=$!
echo "${PID}" >"${PID_PATH}"/${PID_FILE}
wait_start_process
echo -e "${GREEN}........................................Start Dinky Successfully........................................${RESET}"
echo -e "${GREEN}current log path : ${DINKY_LOG_PATH}/dinky-start.log , you can execute tail -fn1000 ${DINKY_LOG_PATH}/dinky-start.log to watch the log${RESET}"
else
Expand All @@ -153,9 +242,11 @@ startWithJmx() {
updatePid
if [ -z "$pid" ]; then
nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none "${JMX}" -cp "${CLASS_PATH}" org.dinky.Dinky ${JAR_PARAMS_OPT} > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 &
# echo $! >"${PID_PATH}"/${PID_FILE}
updatePid
PID=$!
wait_start_process
echo -e "$GREEN........................................Start Dinky with Jmx Successfully........................................$RESET"
updatePid

else
echo -e "$YELLOW Dinky pid $pid is in ${PID_PATH}/${PID_FILE}, Please stop first !!!$RESET"
fi
Expand Down
1 change: 0 additions & 1 deletion script/bin/init_check_network.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/bin/bash
set -x

EXTERNAL_CONNECTIVITY_CHECK_URL="www.baidu.com"

Expand Down
36 changes: 36 additions & 0 deletions script/bin/init_cleanup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/bin/bash

export RED='\033[31m'
export GREEN='\033[32m'
export YELLOW='\033[33m'
export BLUE='\033[34m'
export MAGENTA='\033[35m'
export CYAN='\033[36m'
export RESET='\033[0m'

if [ -f /etc/profile.d/dinky_env ]; then
source /etc/profile.d/dinky_env
fi

if [ -z "$DINKY_HOME" ]; then
echo -e "${RED}DINKY_HOME is not set, please check the environment variable...${RESET}"
exit 1
else
EXTENDS_HOME="$DINKY_HOME"/extends
FLINK_VERSION_SCAN=$(ls -n "${EXTENDS_HOME}" | grep '^d' | grep flink | awk -F 'flink' '{print $2}')

echo -e "${GREEN}>>>>>>>>>>>>>>>>>>>. Start cleaning up the environment... <<<<<<<<<<<<<<<<<<<<${RESET}"
echo -e "${GREEN}Cleaning up the environment variables...${RESET}"
rm -rf /etc/profile.d/dinky_*
echo -e "${GREEN}Cleaning up the flink jar dependencies...${RESET}"
rm -rf "${EXTENDS_HOME}"/flink"${FLINK_VERSION_SCAN}"/flink-*
echo -e "${GREEN}Cleaning up the flink shaded hadoop jar dependencies...${RESET}"
rm -rf "${EXTENDS_HOME}"/flink-shaded-hadoop-*
echo -e "${GREEN}Cleaning up the mysql jar dependencies...${RESET}"
rm -rf "$DINKY_HOME"/lib/mysql-connector-*
echo -e "${GREEN}Cleaning up the environment variables of DINKY_HOME ...${RESET}"
unset DINKY_HOME
echo -e "${GREEN}Refresh environment variables...${RESET}"
source /etc/profile
echo -e "${GREEN}Environment cleanup completed...${RESET}"
fi
1 change: 0 additions & 1 deletion script/bin/init_db.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/bin/bash
set -x

DINKY_HOME_PARAMS=$1
DB_ENV_FILE=$2
Expand Down
5 changes: 2 additions & 3 deletions script/bin/init_env.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/bin/bash
set -x

DINKY_HOME_PARAMS=$1
ENV_FILE=$2
Expand All @@ -18,7 +17,7 @@ while true; do
echo -e "${GREEN}Use the automatically obtained DINKY_HOME environment variable${RESET}"
echo -e "${GREEN} The currently obtained path is: $DINKY_HOME_PARAMS to perform automatic configuration${RESET}"
add_to_env "DINKY_HOME" "$DINKY_HOME_PARAMS" "$ENV_FILE"
add_to_env "PATH" "\$DINKY_HOME/bin" "$ENV_FILE"
add_to_env "PATH" "\$DINKY_HOME/bin:\$PATH" "$ENV_FILE"
sleep 2
source $ENV_FILE
echo -e "${GREEN}DINKY_HOME environment variable configuration completed. Please confirm whether the following configuration is correct:${RESET}"
Expand All @@ -34,7 +33,7 @@ while true; do
else
echo -e "${GREEN}The path you entered is: $dinky_home_path${RESET}"
add_to_env "DINKY_HOME" "$dinky_home_path" "$ENV_FILE"
add_to_env "PATH" "\$DINKY_HOME/bin" "$ENV_FILE"
add_to_env "PATH" "\$DINKY_HOME/bin:\$PATH" "$ENV_FILE"
sleep 2
source $ENV_FILE
echo -e "${GREEN}DINKY_HOME environment variable configuration completed. Please confirm whether the following configuration is correct:${RESET}"
Expand Down
31 changes: 18 additions & 13 deletions script/bin/init_flink_dependences.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/bin/bash
set -x

CURRENT_FLINK_FULL_VERSION=$1
FLINK_VERSION_SCAN=$2
Expand All @@ -16,26 +15,30 @@ if [ -z "$CURRENT_FLINK_FULL_VERSION" ] || [ -z "$FLINK_VERSION_SCAN" ] || [ -z
exit 1
fi

if [ -f "$DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" ]; then
echo -e "${YELLOW}$DINKY_TMP_DIR ALREADY EXISTS flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz file,To ensure completeness, delete first ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz Download the file again${RESET}"
rm -rf ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz
if [ -d "$DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}" ]; then
echo -e "${YELLOW}The flink directory already exists, delete it $DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}"
rm -rf $DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}
FLINK_STORE_DIR=${DINKY_TMP_DIR}/flink-download

mkdir -p ${FLINK_STORE_DIR}

if [ -f "${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" ]; then
echo -e "${YELLOW}${FLINK_STORE_DIR} ALREADY EXISTS flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz file,To ensure completeness, delete first ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz Download the file again${RESET}"
rm -rf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz
if [ -d "${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}" ]; then
echo -e "${YELLOW}The flink directory already exists, delete it ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}"
rm -rf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}
fi
fi

try_tsinghua_mirror() {
local tsinghua_url="https://mirrors.tuna.tsinghua.edu.cn/apache/flink/flink-${CURRENT_FLINK_FULL_VERSION}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz"
local apache_url="https://archive.apache.org/dist/flink/flink-${CURRENT_FLINK_FULL_VERSION}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz"

echo -e "${GREEN}Start downloading the Flink-${FLINK_VERSION_SCAN} installation package... Store it in the ${DINKY_TMP_DIR} directory${RESET}"
if download_file "$tsinghua_url" "$DINKY_TMP_DIR"; then
echo -e "${GREEN}Start downloading the Flink-${FLINK_VERSION_SCAN} installation package... Store it in the ${FLINK_STORE_DIR} directory${RESET}"
if download_file "$tsinghua_url" "${FLINK_STORE_DIR}"; then
echo -e "${BLUE}The address of the currently downloaded Flink installation package is:${tsinghua_url}${RESET}"
return 0
else
echo -e "${YELLOW}File not found in Tsinghua University mirror, try downloading from Apache official source...${RESET}"
if download_file "$apache_url" "$DINKY_TMP_DIR"; then
if download_file "$apache_url" "${FLINK_STORE_DIR}"; then
echo -e "${BLUE}The address of the currently downloaded Flink installation package is:${apache_url}${RESET}"
return 0
else
Expand All @@ -53,7 +56,7 @@ fi
echo -e "${GREEN}Flink installation package download completed。${RESET}"
echo -e "\n${GREEN}===============================================================${RESET}\n"
echo -e "${GREEN}Start decompressing the Flink installation package...${RESET}"
tar -zxvf ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz -C ${DINKY_TMP_DIR}/
tar -zxvf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz -C ${FLINK_STORE_DIR}/
if [ $? -eq 0 ]; then
echo -e "${GREEN}Flink installation package decompression completed。${RESET}"
else
Expand All @@ -63,8 +66,8 @@ fi

echo -e "\n${GREEN}===============================================================${RESET}\n"

flink_dir_tmp=$(ls -n ${DINKY_TMP_DIR} | grep '^d' | grep flink | awk '{print $9}')
full_flink_dir_tmp="${DINKY_TMP_DIR}/${flink_dir_tmp}"
flink_dir_tmp=$(ls -n ${FLINK_STORE_DIR} | grep '^d' | grep flink | awk '{print $9}')
full_flink_dir_tmp="${FLINK_STORE_DIR}/${flink_dir_tmp}"
echo -e "${BLUE}Unzipped directory name:${full_flink_dir_tmp}${RESET}"


Expand Down Expand Up @@ -102,4 +105,6 @@ ls -l ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/

echo -e "${YELLOW}Please check the above dependent files。${RESET}"

rm -rf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}*

echo -e "${GREEN}The basic dependency processing is completed, please perform subsequent operations according to the actual situation.${RESET}"
1 change: 0 additions & 1 deletion script/bin/init_hadoop_dependences.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/bin/bash
set -x

EXTENDS_HOME=$1

Expand Down
3 changes: 1 addition & 2 deletions script/bin/init_jdbc_driver.sh
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
#!/bin/bash
set -x

DINKY_LIB_DIR=$1

echo -e "${GREEN}Start downloading the mysql driver package...${RESET}"
if [ -f "${DINKY_LIB_DIR}/mysql-connector-j-8.4.0.jar" ]; then
echo -e "${YELLOW}mysql The driver package already exists, no need to download it again. Skip this step。${RESET}"
else
download_file https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.4.0/mysql-connector-j-8.4.0.jar "${DINKY_LIB}"
download_file https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.4.0/mysql-connector-j-8.4.0.jar "${DINKY_LIB_DIR}"
echo -e "${GREEN}Download is complete, please verify. The downloaded file storage address is: ${DINKY_LIB_DIR}/mysql-connector-j-8.4.0.jar${RESET}"
if [ -f "${DINKY_LIB_DIR}/mysql-connector-j-8.4.0.jar" ]; then
echo -e "${GREEN}mysql driver package downloaded successfully。${RESET}"
Expand Down
Loading
Loading