From cce2e993d92347267a4c2bab3458fb5e1e5b7902 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Sun, 8 Dec 2024 23:24:45 +0800 Subject: [PATCH 01/19] Fix automation script path issue --- script/bin/init_check_network.sh | 1 - script/bin/init_db.sh | 1 - script/bin/init_env.sh | 1 - script/bin/init_flink_dependences.sh | 1 - script/bin/init_hadoop_dependences.sh | 1 - script/bin/init_jdbc_driver.sh | 3 +- script/bin/init_tools_main.sh | 59 ++++++++++++++++++++------- 7 files changed, 45 insertions(+), 22 deletions(-) diff --git a/script/bin/init_check_network.sh b/script/bin/init_check_network.sh index 66b5cd105b..92335059a9 100644 --- a/script/bin/init_check_network.sh +++ b/script/bin/init_check_network.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x EXTERNAL_CONNECTIVITY_CHECK_URL="www.baidu.com" diff --git a/script/bin/init_db.sh b/script/bin/init_db.sh index 22b7372ab2..ae8d2e4f43 100644 --- a/script/bin/init_db.sh +++ b/script/bin/init_db.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x DINKY_HOME_PARAMS=$1 DB_ENV_FILE=$2 diff --git a/script/bin/init_env.sh b/script/bin/init_env.sh index 94242d380c..964bd97cd8 100644 --- a/script/bin/init_env.sh +++ b/script/bin/init_env.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x DINKY_HOME_PARAMS=$1 ENV_FILE=$2 diff --git a/script/bin/init_flink_dependences.sh b/script/bin/init_flink_dependences.sh index 451d3b214d..4b7092e93f 100644 --- a/script/bin/init_flink_dependences.sh +++ b/script/bin/init_flink_dependences.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x CURRENT_FLINK_FULL_VERSION=$1 FLINK_VERSION_SCAN=$2 diff --git a/script/bin/init_hadoop_dependences.sh b/script/bin/init_hadoop_dependences.sh index c080a44188..0d9dc20916 100644 --- a/script/bin/init_hadoop_dependences.sh +++ b/script/bin/init_hadoop_dependences.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x EXTENDS_HOME=$1 diff --git a/script/bin/init_jdbc_driver.sh b/script/bin/init_jdbc_driver.sh index 63631af604..ca9fd9c952 100644 --- a/script/bin/init_jdbc_driver.sh +++ b/script/bin/init_jdbc_driver.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x DINKY_LIB_DIR=$1 @@ -7,7 +6,7 @@ echo -e "${GREEN}Start downloading the mysql driver package...${RESET}" if [ -f "${DINKY_LIB_DIR}/mysql-connector-j-8.4.0.jar" ]; then echo -e "${YELLOW}mysql The driver package already exists, no need to download it again. Skip this step。${RESET}" else - download_file https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.4.0/mysql-connector-j-8.4.0.jar "${DINKY_LIB}" + download_file https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.4.0/mysql-connector-j-8.4.0.jar "${DINKY_LIB_DIR}" echo -e "${GREEN}Download is complete, please verify. The downloaded file storage address is: ${DINKY_LIB_DIR}/mysql-connector-j-8.4.0.jar${RESET}" if [ -f "${DINKY_LIB_DIR}/mysql-connector-j-8.4.0.jar" ]; then echo -e "${GREEN}mysql driver package downloaded successfully。${RESET}" diff --git a/script/bin/init_tools_main.sh b/script/bin/init_tools_main.sh index 1b2aae6821..239af03643 100644 --- a/script/bin/init_tools_main.sh +++ b/script/bin/init_tools_main.sh @@ -1,5 +1,22 @@ #!/bin/bash -set -x + +export RED='\033[31m' +export GREEN='\033[32m' +export YELLOW='\033[33m' +export BLUE='\033[34m' +export MAGENTA='\033[35m' +export CYAN='\033[36m' +export RESET='\033[0m' + +# debug mode: true or false 如果为空,则默认为false +USE_DEBUG=$1 + +if [ -z "${USE_DEBUG}" ]; then + USE_DEBUG="false" + echo "${YELLOW}Debug mode is not enabled, if you need to enable debug mode, please add the first parameter 'true' when executing the script. example: init_tools_main.sh true $RESET" +elif [ "${USE_DEBUG}" = "true" ]; then + set -x +fi ENV_FILE="/etc/profile.d/dinky_env" if [ -f "${ENV_FILE}" ]; then @@ -22,13 +39,6 @@ chmod 755 $DB_ENV_FILE source /etc/profile -export RED='\033[31m' -export GREEN='\033[32m' -export YELLOW='\033[33m' -export BLUE='\033[34m' -export MAGENTA='\033[35m' -export CYAN='\033[36m' -export RESET='\033[0m' echo -e "${GREEN}=====================================================================${RESET}" echo -e "${GREEN}=====================================================================${RESET}" @@ -36,7 +46,20 @@ echo -e "${GREEN}============ Welcome to the Dinky initialization script ======= echo -e "${GREEN}======================================================================${RESET}" echo -e "${GREEN}======================================================================${RESET}" -APP_HOME=${DINKY_HOME:-$(cd "$(dirname "$0")"; cd ..; pwd)} +if [ -z "${DINKY_HOME}" ]; then + SOURCE="${BASH_SOURCE[0]}" + while [ -h "$SOURCE" ]; do + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + done + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + APP_HOME="$(dirname "$DIR")" +else + APP_HOME="${DINKY_HOME}" +fi + +echo -e "${GREEN}Dinky root path: ${APP_HOME} ${RESET}" sudo chmod +x "${APP_HOME}"/bin/init_*.sh @@ -87,7 +110,8 @@ check_command() { echo -e "${GREEN}========== Command $cmd check completed. OK, continue executing the script. ==========${RESET}" } -sh "${APP_HOME}/bin/init_check_network.sh" +# Use source transparent transmission in debug mode +source "${APP_HOME}/bin/init_check_network.sh" check_command "wget" @@ -132,7 +156,8 @@ if [ -z "$DINKY_HOME_TMP" ]; then is_init_dinky_home=$(echo "$is_init_dinky_home" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') case $is_init_dinky_home in yes | y) - sh "${APP_HOME}"/bin/init_env.sh ${APP_HOME} ${ENV_FILE} + # Use source transparent transmission in debug mode + source "${APP_HOME}"/bin/init_env.sh ${APP_HOME} ${ENV_FILE} echo -e "${GREEN}DINKY_HOME environment variable configuration completed. the configuration file is:${ENV_FILE} ${RESET}" break ;; @@ -168,7 +193,8 @@ while true; do read -p "Please enter your database type:" db_type case $db_type in 1) - sh "${APP_HOME}"/bin/init_jdbc_driver.sh "${DINKY_LIB}" + # Use source transparent transmission in debug mode + source "${APP_HOME}"/bin/init_jdbc_driver.sh "${DINKY_LIB}" break ;; 2) @@ -220,7 +246,8 @@ while true; do case $is_init_flink in yes | y ) - sh "${APP_HOME}"/bin/init_flink_dependences.sh "${CURRENT_FLINK_FULL_VERSION}" "${FLINK_VERSION_SCAN}" "${DINKY_TMP_DIR}" "${EXTENDS_HOME}" "${APP_HOME}" + # Use source transparent transmission in debug mode + source "${APP_HOME}"/bin/init_flink_dependences.sh "${CURRENT_FLINK_FULL_VERSION}" "${FLINK_VERSION_SCAN}" "${DINKY_TMP_DIR}" "${EXTENDS_HOME}" "${APP_HOME}" break ;; no | n ) @@ -248,7 +275,8 @@ while true; do is_hadoop=$(echo "$is_hadoop" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') case $is_hadoop in yes | y ) - sh "${APP_HOME}/bin/init_hadoop_dependences.sh" "${EXTENDS_HOME}" + # Use source transparent transmission in debug mode + source "${APP_HOME}/bin/init_hadoop_dependences.sh" "${EXTENDS_HOME}" break ;; no | n ) @@ -277,7 +305,8 @@ while true; do is_init_db=$(echo "$is_init_db" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') case $is_init_db in yes | y ) - sh "${APP_HOME}/bin/init_db.sh" "${DINKY_HOME}" "${DB_ENV_FILE}" + # Use source transparent transmission in debug mode + source "${APP_HOME}/bin/init_db.sh" "${DINKY_HOME}" "${DB_ENV_FILE}" echo -e "${GREEN}The database configuration file initialization script has been executed successfully the configuration file is:${DB_ENV_FILE} ${RESET}" break ;; From c2bbd6247a3beaec504d3b956a1da5c3be97cf7b Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Sun, 8 Dec 2024 23:26:45 +0800 Subject: [PATCH 02/19] Fix automation script path issue --- script/bin/init_tools_main.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/bin/init_tools_main.sh b/script/bin/init_tools_main.sh index 239af03643..fb35daea5e 100644 --- a/script/bin/init_tools_main.sh +++ b/script/bin/init_tools_main.sh @@ -8,7 +8,7 @@ export MAGENTA='\033[35m' export CYAN='\033[36m' export RESET='\033[0m' -# debug mode: true or false 如果为空,则默认为false +# debug mode: true or false If empty, defaults to false USE_DEBUG=$1 if [ -z "${USE_DEBUG}" ]; then From 32fce0adb6f182ccdf71f6dc7519aec335303414 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Sun, 8 Dec 2024 23:34:23 +0800 Subject: [PATCH 03/19] Fix automation script path issue --- script/bin/auto.sh | 13 +++++++++++++ script/bin/init_tools_main.sh | 35 +++++++++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/script/bin/auto.sh b/script/bin/auto.sh index 37b10141f3..306eaee08c 100644 --- a/script/bin/auto.sh +++ b/script/bin/auto.sh @@ -11,6 +11,19 @@ export CYAN='\033[36m' export RESET='\033[0m' +ENV_FILE="/etc/profile.d/dinky_env" +if [ -f "${ENV_FILE}" ]; then + source "${ENV_FILE}" +fi + +DB_ENV_FILE="/etc/profile.d/dinky_db" +if [ -f "${DB_ENV_FILE}" ]; then + source "${DB_ENV_FILE}" +fi + +source /etc/profile + + if [ -z "${DINKY_HOME}" ]; then echo -e "${RED}DINKY_HOME environment variable is not set. Attempting to determine the correct path...${RESET}" diff --git a/script/bin/init_tools_main.sh b/script/bin/init_tools_main.sh index fb35daea5e..7c5b1a35ca 100644 --- a/script/bin/init_tools_main.sh +++ b/script/bin/init_tools_main.sh @@ -324,9 +324,32 @@ while true; do esac done echo -e "${GREEN} ====================== Database configuration file initialization script -> End ====================== ${RESET}" -echo -echo -echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" -echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" -echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" -echo + +while true; do + read -p "Do you need to start the Dinky service?(yes/no/exit)" is_start + is_start=$(echo "$is_start" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') + case $is_start in + yes | y ) + # Use source transparent transmission in debug mode + source "${APP_HOME}/bin/auto.sh restart" + break + ;; + no | n ) + echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" + echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" + echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" + echo + echo -e "${GREEN}The Dinky service startup script has been skipped, 请先执行上述命令,然后手动启动服务 -> ${APP_HOME}/bin/auto.sh restart | start。${RESET}" + break + ;; + exit | e ) + echo -e "${GREEN}If you choose exit, the program will exit。${RESET}" + exit 0 + ;; + *) + echo -e "${RED}Invalid input, please re-enter yes/no/exit。${RESET}" + ;; + esac +done + +echo -e "${GREEN} ====================== Dinky initialization script execution completed ====================== ${RESET}" From a7d05eb62aa8f6c187b32ec64390b760753d11ad Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Sun, 8 Dec 2024 23:36:01 +0800 Subject: [PATCH 04/19] Fix automation script path issue --- script/bin/init_tools_main.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/script/bin/init_tools_main.sh b/script/bin/init_tools_main.sh index 7c5b1a35ca..954b3107b8 100644 --- a/script/bin/init_tools_main.sh +++ b/script/bin/init_tools_main.sh @@ -325,6 +325,14 @@ while true; do done echo -e "${GREEN} ====================== Database configuration file initialization script -> End ====================== ${RESET}" +function echo_warning_msg() { + echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" + echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" + echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" +} + +echo -e "${GREEN} ====================== Dinky service startup script -> Start ====================== ${RESET}" + while true; do read -p "Do you need to start the Dinky service?(yes/no/exit)" is_start is_start=$(echo "$is_start" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') @@ -335,14 +343,13 @@ while true; do break ;; no | n ) - echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" - echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" - echo -e "${RED}Note: To make these changes permanent, you may need to restart your terminal or run 'source $DB_ENV_FILE && source $ENV_FILE' ${RESET}" + echo_warning_msg echo echo -e "${GREEN}The Dinky service startup script has been skipped, 请先执行上述命令,然后手动启动服务 -> ${APP_HOME}/bin/auto.sh restart | start。${RESET}" break ;; exit | e ) + echo_warning_msg echo -e "${GREEN}If you choose exit, the program will exit。${RESET}" exit 0 ;; @@ -351,5 +358,7 @@ while true; do ;; esac done - +echo -e "${GREEN} ====================== Dinky service startup script -> End ====================== ${RESET}" +echo +echo echo -e "${GREEN} ====================== Dinky initialization script execution completed ====================== ${RESET}" From e94db08ec9298fe2ce8661a07edd1bf9b410379d Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Sun, 8 Dec 2024 23:38:12 +0800 Subject: [PATCH 05/19] Fix automation script path issue --- script/bin/init_tools_main.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/bin/init_tools_main.sh b/script/bin/init_tools_main.sh index 954b3107b8..3a01b0ed26 100644 --- a/script/bin/init_tools_main.sh +++ b/script/bin/init_tools_main.sh @@ -13,7 +13,7 @@ USE_DEBUG=$1 if [ -z "${USE_DEBUG}" ]; then USE_DEBUG="false" - echo "${YELLOW}Debug mode is not enabled, if you need to enable debug mode, please add the first parameter 'true' when executing the script. example: init_tools_main.sh true $RESET" + echo -e "${YELLOW}Debug mode is not enabled, if you need to enable debug mode, please add the first parameter 'true' when executing the script. example: init_tools_main.sh true $RESET" elif [ "${USE_DEBUG}" = "true" ]; then set -x fi From d4d4436c81dd58c99e554970cccc9e8f1125420c Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Sun, 8 Dec 2024 23:47:00 +0800 Subject: [PATCH 06/19] Fix automation script path issue --- script/bin/init_tools_main.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/bin/init_tools_main.sh b/script/bin/init_tools_main.sh index 3a01b0ed26..97d0723b35 100644 --- a/script/bin/init_tools_main.sh +++ b/script/bin/init_tools_main.sh @@ -339,7 +339,7 @@ while true; do case $is_start in yes | y ) # Use source transparent transmission in debug mode - source "${APP_HOME}/bin/auto.sh restart" + source "${APP_HOME}"/bin/auto.sh restart break ;; no | n ) From 5a777bd7c4c2ec36477bd41a039a78efb8f4e5da Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 10:52:30 +0800 Subject: [PATCH 07/19] Fix automation script path issue --- script/bin/auto.sh | 81 ++++++++++++++++++++++---- script/bin/init_env.sh | 4 +- script/bin/init_flink_dependences.sh | 2 + script/bin/init_tools_main.sh | 85 ++++++++++++++++++---------- 4 files changed, 130 insertions(+), 42 deletions(-) diff --git a/script/bin/auto.sh b/script/bin/auto.sh index 306eaee08c..e7990095a8 100644 --- a/script/bin/auto.sh +++ b/script/bin/auto.sh @@ -23,11 +23,8 @@ fi source /etc/profile - - -if [ -z "${DINKY_HOME}" ]; then - echo -e "${RED}DINKY_HOME environment variable is not set. Attempting to determine the correct path...${RESET}" - +RETURN_HOME_PATH="" +function get_home_path() { SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" @@ -35,16 +32,28 @@ if [ -z "${DINKY_HOME}" ]; then [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" done DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - export DINKY_HOME="$(dirname "$DIR")" + RETURN_HOME_PATH=$(dirname "$DIR") +} + + +if [ -z "${DINKY_HOME}" ]; then + echo -e "${RED}DINKY_HOME environment variable is not set. Attempting to determine the correct path...${RESET}" + get_home_path + export DINKY_HOME=${RETURN_HOME_PATH} echo -e "${GREEN}DINKY_HOME is set to: ${DINKY_HOME}${RESET}" else - echo -e "${GREEN}DINKY_HOME is already set to: ${DINKY_HOME}${RESET}" + get_home_path + if [ "${DINKY_HOME}" != "${RETURN_HOME_PATH}" ]; then + export DINKY_HOME=${RETURN_HOME_PATH} + echo -e "${YELLOW}DINKY_HOME is not equal to the current path, reset DINKY_HOME to: ${RETURN_HOME_PATH}${RESET}" + else + echo -e "${GREEN}DINKY_HOME is already set to: ${DINKY_HOME}${RESET}" + fi fi FLINK_VERSION=${2} -DINKY_HOME=${DINKY_HOME:-$(cd "$(dirname "$0")"; cd ..; pwd)} JAVA_VERSION=$(java -version 2>&1 | sed '1!d' | sed -e 's/"//g' | awk '{print $3}' | awk -F'.' '{print $1"."$2}') APP_HOME="${DINKY_HOME}" @@ -84,6 +93,55 @@ assertIsInputVersion() { fi } +# 从 application.yml 中读取端口号 +read_yaml() { + local key=$1 + local file=$2 + local value=$(awk -F': ' '/^'"$key"'=: /{print $2}' $file) + echo "$value" +} + +# 尝试从 application.yml 中读取端口号 +if [ -f "${APP_HOME}/config/application.yml" ]; then + APP_PORT=$(read_yaml "server.port" "${APP_HOME}/config/application.yml") +fi + +# 如果仍然没有找到,则使用默认端口 +if [ -z "$APP_PORT" ]; then + echo -e "${RED}Could not find server.port in configuration files, using default port 8888 ${RESET}" + APP_PORT=8888 +fi + +# 函数:检查健康检查端点的状态 +check_health() { + curl --silent --max-time 2 --output /dev/null --write-out "%{http_code}" "http://localhost:$APP_PORT/actuator/health" +} + + +function wait_start_process() { + echo "Starting application..." + for i in {1..100}; do + # 检查应用是否已经启动完成 + if [ "$(check_health)" == "200" ]; then + echo -ne "\r[==================================================] 100%\n" + echo "Application started successfully." + break + else + # 打印进度条 + echo -ne "\r[=$(printf '=%.0s' $(seq 1 $((i-1))))>$(printf ' %.0s' $(seq 1 $((100-i))))] ${i}%" + sleep 0.5 # 调整等待时间 + fi + + # 防止无限循环,设置最大尝试次数 + if [ $i -eq 100 ]; then + echo -ne "\r[==================================================] 100%\n" + echo "Application startup timed out." + exit 1 + fi + done +} + + # Use FLINK_HOME: CLASS_PATH="${APP_HOME}:${APP_HOME}/lib/*:${APP_HOME}/config:${EXTENDS_HOME}/*:${CUSTOMER_JAR_PATH}/*:${EXTENDS_HOME}/flink${FLINK_VERSION}/dinky/*:${EXTENDS_HOME}/flink${FLINK_VERSION}/flink/*:${EXTENDS_HOME}/flink${FLINK_VERSION}/*" PID_FILE="dinky.pid" @@ -142,7 +200,9 @@ start() { updatePid if [ -z "$pid" ]; then nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none -cp "${CLASS_PATH}" org.dinky.Dinky ${JAR_PARAMS_OPT} > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 & - echo $! >"${PID_PATH}"/${PID_FILE} + PID=$! + echo "${PID}" >"${PID_PATH}"/${PID_FILE} + wait_start_process echo -e "${GREEN}........................................Start Dinky Successfully........................................${RESET}" echo -e "${GREEN}current log path : ${DINKY_LOG_PATH}/dinky-start.log , you can execute tail -fn1000 ${DINKY_LOG_PATH}/dinky-start.log to watch the log${RESET}" else @@ -166,7 +226,8 @@ startWithJmx() { updatePid if [ -z "$pid" ]; then nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none "${JMX}" -cp "${CLASS_PATH}" org.dinky.Dinky ${JAR_PARAMS_OPT} > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 & -# echo $! >"${PID_PATH}"/${PID_FILE} + PID=$! + wait_start_process updatePid echo -e "$GREEN........................................Start Dinky with Jmx Successfully........................................$RESET" else diff --git a/script/bin/init_env.sh b/script/bin/init_env.sh index 964bd97cd8..57587a14c0 100644 --- a/script/bin/init_env.sh +++ b/script/bin/init_env.sh @@ -17,7 +17,7 @@ while true; do echo -e "${GREEN}Use the automatically obtained DINKY_HOME environment variable${RESET}" echo -e "${GREEN} The currently obtained path is: $DINKY_HOME_PARAMS to perform automatic configuration${RESET}" add_to_env "DINKY_HOME" "$DINKY_HOME_PARAMS" "$ENV_FILE" - add_to_env "PATH" "\$DINKY_HOME/bin" "$ENV_FILE" + add_to_env "PATH" "\$DINKY_HOME/bin:\$PATH" "$ENV_FILE" sleep 2 source $ENV_FILE echo -e "${GREEN}DINKY_HOME environment variable configuration completed. Please confirm whether the following configuration is correct:${RESET}" @@ -33,7 +33,7 @@ while true; do else echo -e "${GREEN}The path you entered is: $dinky_home_path${RESET}" add_to_env "DINKY_HOME" "$dinky_home_path" "$ENV_FILE" - add_to_env "PATH" "\$DINKY_HOME/bin" "$ENV_FILE" + add_to_env "PATH" "\$DINKY_HOME/bin:\$PATH" "$ENV_FILE" sleep 2 source $ENV_FILE echo -e "${GREEN}DINKY_HOME environment variable configuration completed. Please confirm whether the following configuration is correct:${RESET}" diff --git a/script/bin/init_flink_dependences.sh b/script/bin/init_flink_dependences.sh index 4b7092e93f..84d70e6823 100644 --- a/script/bin/init_flink_dependences.sh +++ b/script/bin/init_flink_dependences.sh @@ -101,4 +101,6 @@ ls -l ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ echo -e "${YELLOW}Please check the above dependent files。${RESET}" +rm -rf ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}* + echo -e "${GREEN}The basic dependency processing is completed, please perform subsequent operations according to the actual situation.${RESET}" \ No newline at end of file diff --git a/script/bin/init_tools_main.sh b/script/bin/init_tools_main.sh index 97d0723b35..36a9685bec 100644 --- a/script/bin/init_tools_main.sh +++ b/script/bin/init_tools_main.sh @@ -22,23 +22,18 @@ ENV_FILE="/etc/profile.d/dinky_env" if [ -f "${ENV_FILE}" ]; then source "${ENV_FILE}" else - echo "" > "${ENV_FILE}" - source "${ENV_FILE}" + echo "export PATH=/bin:/usr/bin:\$PATH" > "${ENV_FILE}" && source "${ENV_FILE}" fi DB_ENV_FILE="/etc/profile.d/dinky_db" if [ -f "${DB_ENV_FILE}" ]; then source "${DB_ENV_FILE}" else - echo "" > "${DB_ENV_FILE}" - source "${DB_ENV_FILE}" + echo "export PATH=/bin:/usr/bin:\$PATH" > "${DB_ENV_FILE}" && source "${DB_ENV_FILE}" fi chmod 755 $ENV_FILE chmod 755 $DB_ENV_FILE -source /etc/profile - - echo -e "${GREEN}=====================================================================${RESET}" echo -e "${GREEN}=====================================================================${RESET}" @@ -46,7 +41,9 @@ echo -e "${GREEN}============ Welcome to the Dinky initialization script ======= echo -e "${GREEN}======================================================================${RESET}" echo -e "${GREEN}======================================================================${RESET}" -if [ -z "${DINKY_HOME}" ]; then + +RETURN_HOME_PATH="" +function get_home_path() { SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" @@ -54,9 +51,23 @@ if [ -z "${DINKY_HOME}" ]; then [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" done DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - APP_HOME="$(dirname "$DIR")" + RETURN_HOME_PATH=$(dirname "$DIR") +} + + + +if [ -z "${DINKY_HOME}" ]; then + echo -e "${RED}DINKY_HOME environment variable is not set. Attempting to determine the correct path...${RESET}" + get_home_path + APP_HOME="${RETURN_HOME_PATH}" else - APP_HOME="${DINKY_HOME}" + get_home_path + if [ "${DINKY_HOME}" != "${RETURN_HOME_PATH}" ]; then + echo -e "${YELLOW}DINKY_HOME is not equal to the current path, use new path to init: ${RETURN_HOME_PATH}${RESET}" + APP_HOME="${RETURN_HOME_PATH}" + else + echo -e "${GREEN}DINKY_HOME is already set to: ${DINKY_HOME}${RESET}" + fi fi echo -e "${GREEN}Dinky root path: ${APP_HOME} ${RESET}" @@ -148,30 +159,41 @@ export -f add_to_env echo echo +function init_env() { + while true; do + read -p "Do you need to configure the DINKY_HOME environment variable? (yes/no):" is_init_dinky_home + is_init_dinky_home=$(echo "$is_init_dinky_home" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') + case $is_init_dinky_home in + yes | y) + # Use source transparent transmission in debug mode + source "${APP_HOME}"/bin/init_env.sh ${APP_HOME} ${ENV_FILE} + echo -e "${GREEN}DINKY_HOME environment variable configuration completed. the configuration file is:${ENV_FILE} ${RESET}" + break + ;; + no | n) + echo -e "${GREEN}Skip DINKY_HOME environment variable configuration.${RESET}" + break + ;; + *) + echo -e "${RED}The entered value is incorrect, please rerun the script to select the correct value.${RESET}" + ;; + esac + done +} + + + echo -e "${GREEN} ====================== Environment variable initialization script -> Start ====================== ${RESET}" DINKY_HOME_TMP=$(echo $DINKY_HOME) if [ -z "$DINKY_HOME_TMP" ]; then - while true; do - read -p "Do you need to configure the DINKY_HOME environment variable? (yes/no):" is_init_dinky_home - is_init_dinky_home=$(echo "$is_init_dinky_home" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') - case $is_init_dinky_home in - yes | y) - # Use source transparent transmission in debug mode - source "${APP_HOME}"/bin/init_env.sh ${APP_HOME} ${ENV_FILE} - echo -e "${GREEN}DINKY_HOME environment variable configuration completed. the configuration file is:${ENV_FILE} ${RESET}" - break - ;; - no | n) - echo -e "${GREEN}Skip DINKY_HOME environment variable configuration.${RESET}" - break - ;; - *) - echo -e "${RED}The entered value is incorrect, please rerun the script to select the correct value.${RESET}" - ;; - esac - done + init_env else - echo -e "${GREEN}DINKY_HOME environment variable has been configured at ${DINKY_HOME_TMP},Skip configuration.${RESET}" + if [ "$APP_HOME" != "$DINKY_HOME_TMP" ]; then + echo -e "${RED}DINKY_HOME is not equal to the current path, The previous one was: ${DINKY_HOME_TMP}. The current one is: ${APP_HOME}, which needs to be reconfigured.${RESET}" + init_env + else + echo -e "${GREEN}DINKY_HOME environment variable has been configured at ${DINKY_HOME_TMP},Skip configuration.${RESET}" + fi fi @@ -362,3 +384,6 @@ echo -e "${GREEN} ====================== Dinky service startup script -> End === echo echo echo -e "${GREEN} ====================== Dinky initialization script execution completed ====================== ${RESET}" + + +set +x \ No newline at end of file From 30e2bc5540d108e938b9a977a606890d3d3a44db Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 13:58:22 +0800 Subject: [PATCH 08/19] Fix automation script path issue --- script/bin/auto.sh | 64 ++++++++++++++++++++-------------------- script/bin/parse_yml.sh | 65 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 31 deletions(-) create mode 100644 script/bin/parse_yml.sh diff --git a/script/bin/auto.sh b/script/bin/auto.sh index e7990095a8..550324a8f7 100644 --- a/script/bin/auto.sh +++ b/script/bin/auto.sh @@ -2,6 +2,7 @@ # debug mode #set -x + export RED='\033[31m' export GREEN='\033[32m' export YELLOW='\033[33m' @@ -93,20 +94,14 @@ assertIsInputVersion() { fi } -# 从 application.yml 中读取端口号 -read_yaml() { - local key=$1 - local file=$2 - local value=$(awk -F': ' '/^'"$key"'=: /{print $2}' $file) - echo "$value" -} -# 尝试从 application.yml 中读取端口号 if [ -f "${APP_HOME}/config/application.yml" ]; then - APP_PORT=$(read_yaml "server.port" "${APP_HOME}/config/application.yml") + result=$("${APP_HOME}"/bin/parse_yml.sh "${APP_HOME}/config/application.yml" "server.port") + APP_PORT=$result fi -# 如果仍然没有找到,则使用默认端口 +echo -e "${GREEN}From ${APP_HOME}/config/application.yml server.port: ${APP_PORT}${RESET}" + if [ -z "$APP_PORT" ]; then echo -e "${RED}Could not find server.port in configuration files, using default port 8888 ${RESET}" APP_PORT=8888 @@ -117,28 +112,35 @@ check_health() { curl --silent --max-time 2 --output /dev/null --write-out "%{http_code}" "http://localhost:$APP_PORT/actuator/health" } - function wait_start_process() { - echo "Starting application..." - for i in {1..100}; do - # 检查应用是否已经启动完成 - if [ "$(check_health)" == "200" ]; then - echo -ne "\r[==================================================] 100%\n" - echo "Application started successfully." - break - else - # 打印进度条 - echo -ne "\r[=$(printf '=%.0s' $(seq 1 $((i-1))))>$(printf ' %.0s' $(seq 1 $((100-i))))] ${i}%" - sleep 0.5 # 调整等待时间 - fi - - # 防止无限循环,设置最大尝试次数 - if [ $i -eq 100 ]; then - echo -ne "\r[==================================================] 100%\n" - echo "Application startup timed out." - exit 1 - fi - done + echo "Starting application..." + local max_attempts=100 + local attempt=0 + local delay=0.25 + local health_status="" + local success_status_codes=("200" "201" "202" "203" "204") # 可以根据实际情况调整认为成功的状态码范围 + + while [ $attempt -lt $max_attempts ]; do + attempt=$((attempt + 1)) + health_status=$(check_health) + for code in "${success_status_codes[@]}"; do + if [ "$health_status" == "$code" ]; then + echo -ne "\r[==================================================] 100%\n" + echo -e "${GREEN}Application started completed.${RESET}" + return 0 + fi + done + local progress=$((attempt * 100 / max_attempts)) + local bar_length=50 + local filled_length=$((progress * bar_length / 100)) + local empty_length=$((bar_length - filled_length)) + local bar=$(printf '=%.0s' $(seq 1 $filled_length))$(printf ' %.0s' $(seq 1 $empty_length)) + echo -ne "\r[${bar}] ${progress}%" + sleep $delay + done + echo -ne "\r[==================================================] 100%\n" + echo "Application startup timed out." + return 1 } diff --git a/script/bin/parse_yml.sh b/script/bin/parse_yml.sh new file mode 100644 index 0000000000..db7c700ec6 --- /dev/null +++ b/script/bin/parse_yml.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# 检查参数个数是否正确 +if [ $# -ne 2 ]; then + echo "请按正确格式输入参数,格式为: $0 yaml_file_path key" + exit 1 +fi + +#yaml_file_path="/Users/zhumingye/idea_works/dinky/dinky-admin/src/main/resources/application.yml" +#key="spring.port" + +yaml_file_path="$1" +key="$2" +# 将键按照.分割为数组(用于处理层级结构的键) +IFS='.' read -ra key_parts <<< "$key" + +# 用于存储最终找到的值 +value="" + +# 先处理文件,去除注释行(假设注释以#开头)和空白行 +temp_file=$(mktemp) +grep -Ev '^(#|$)' "$yaml_file_path" > "$temp_file" + +# 开始按层级查找值 +current_data=$(cat "$temp_file") +for part in "${key_parts[@]}"; do + found=false + while IFS= read -r line; do + + if [[ $line =~ ^$part: ]]; then + # 如果是最后一个键,提取值 + if [ "$part" == "${key_parts[${#key_parts[@]}-1]}" ]; then + value=$(echo "$line" | sed 's/.*: //') + found=true + break + else + # 如果不是最后一个键,获取下一层级的数据范围 + start_line_num=$(grep -n "$line" "$temp_file" | cut -d: -f1) + end_line_num=$(awk -v start="$start_line_num" '$0 ~ /^[a-zA-Z]/ && NR > start {print NR - 1; exit}' "$temp_file") + if [ -z "$end_line_num" ]; then + end_line_num=$(wc -l < "$temp_file") + fi + + current_data=$(sed -n "$((start_line_num + 1)),$((end_line_num))p" "$temp_file") + current_data=$(echo "$current_data" | sed 's/^[[:space:]]*//') + found=true + break + fi + fi + done <<< "$current_data" + if [ "$found" = false ]; then + value="" + break + fi +done + +# 删除临时文件 +rm -f "$temp_file" + +# 将找到的值输出到标准输出,方便外部获取 +if [ -n "$value" ]; then + echo "$value" +else + echo "" +fi \ No newline at end of file From e3c83be534c03458ffeb439aa1965b8f3793ede6 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 14:01:57 +0800 Subject: [PATCH 09/19] Fix automation script path issue --- script/bin/auto.sh | 4 ++-- script/bin/parse_yml.sh | 18 ++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/script/bin/auto.sh b/script/bin/auto.sh index 550324a8f7..be899b9d9e 100644 --- a/script/bin/auto.sh +++ b/script/bin/auto.sh @@ -113,7 +113,7 @@ check_health() { } function wait_start_process() { - echo "Starting application..." + echo ">>>>>>>>>>>>>>>>>>>>> Starting application... <<<<<<<<<<<<<<<<<<<<<<<" local max_attempts=100 local attempt=0 local delay=0.25 @@ -134,7 +134,7 @@ function wait_start_process() { local bar_length=50 local filled_length=$((progress * bar_length / 100)) local empty_length=$((bar_length - filled_length)) - local bar=$(printf '=%.0s' $(seq 1 $filled_length))$(printf ' %.0s' $(seq 1 $empty_length)) + local bar=$(printf '>%.0s' $(seq 1 $filled_length))$(printf ' %.0s' $(seq 1 $empty_length)) echo -ne "\r[${bar}] ${progress}%" sleep $delay done diff --git a/script/bin/parse_yml.sh b/script/bin/parse_yml.sh index db7c700ec6..bc974669e9 100644 --- a/script/bin/parse_yml.sh +++ b/script/bin/parse_yml.sh @@ -2,22 +2,20 @@ # 检查参数个数是否正确 if [ $# -ne 2 ]; then - echo "请按正确格式输入参数,格式为: $0 yaml_file_path key" + echo "Please enter the parameters in the correct format, the format is: $0 yaml_file_path key" exit 1 fi -#yaml_file_path="/Users/zhumingye/idea_works/dinky/dinky-admin/src/main/resources/application.yml" -#key="spring.port" yaml_file_path="$1" key="$2" -# 将键按照.分割为数组(用于处理层级结构的键) +# Split keys into arrays according to . (used to handle hierarchical keys) IFS='.' read -ra key_parts <<< "$key" -# 用于存储最终找到的值 +# Used to store the finally found value value="" -# 先处理文件,去除注释行(假设注释以#开头)和空白行 +# Process the file first and remove comment lines (assuming the comment starts with #) and blank lines temp_file=$(mktemp) grep -Ev '^(#|$)' "$yaml_file_path" > "$temp_file" @@ -28,13 +26,13 @@ for part in "${key_parts[@]}"; do while IFS= read -r line; do if [[ $line =~ ^$part: ]]; then - # 如果是最后一个键,提取值 + # If it is the last key, extract the value if [ "$part" == "${key_parts[${#key_parts[@]}-1]}" ]; then value=$(echo "$line" | sed 's/.*: //') found=true break else - # 如果不是最后一个键,获取下一层级的数据范围 + # If it is not the last key, get the data range of the next level start_line_num=$(grep -n "$line" "$temp_file" | cut -d: -f1) end_line_num=$(awk -v start="$start_line_num" '$0 ~ /^[a-zA-Z]/ && NR > start {print NR - 1; exit}' "$temp_file") if [ -z "$end_line_num" ]; then @@ -54,10 +52,10 @@ for part in "${key_parts[@]}"; do fi done -# 删除临时文件 +# Delete temporary files rm -f "$temp_file" -# 将找到的值输出到标准输出,方便外部获取 +# Output the found value to standard output for easy external acquisition if [ -n "$value" ]; then echo "$value" else From 495db25e195f20621f7dc5eee3d4ee082eafe392 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 14:31:16 +0800 Subject: [PATCH 10/19] Fix automation script path issue --- script/bin/auto.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/script/bin/auto.sh b/script/bin/auto.sh index be899b9d9e..67055c71cf 100644 --- a/script/bin/auto.sh +++ b/script/bin/auto.sh @@ -139,7 +139,7 @@ function wait_start_process() { sleep $delay done echo -ne "\r[==================================================] 100%\n" - echo "Application startup timed out." + echo -e "${RED}Application start failed. Please check the log for details.${RESET}" return 1 } @@ -204,8 +204,10 @@ start() { nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none -cp "${CLASS_PATH}" org.dinky.Dinky ${JAR_PARAMS_OPT} > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 & PID=$! echo "${PID}" >"${PID_PATH}"/${PID_FILE} - wait_start_process - echo -e "${GREEN}........................................Start Dinky Successfully........................................${RESET}" + run_result=$(wait_start_process) + if [ "$run_result" -eq 0 ]; then + echo -e "${GREEN}........................................Start Dinky Successfully........................................${RESET}" + fi echo -e "${GREEN}current log path : ${DINKY_LOG_PATH}/dinky-start.log , you can execute tail -fn1000 ${DINKY_LOG_PATH}/dinky-start.log to watch the log${RESET}" else echo -e "$YELLOW Dinky pid $pid is in ${PID_PATH}/${PID_FILE}, Please stop first !!!$RESET" @@ -229,9 +231,12 @@ startWithJmx() { if [ -z "$pid" ]; then nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none "${JMX}" -cp "${CLASS_PATH}" org.dinky.Dinky ${JAR_PARAMS_OPT} > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 & PID=$! - wait_start_process - updatePid - echo -e "$GREEN........................................Start Dinky with Jmx Successfully........................................$RESET" + run_result=$(wait_start_process) + if [ "$run_result" -eq 0 ]; then + echo -e "$GREEN........................................Start Dinky with Jmx Successfully........................................$RESET" + updatePid + fi + else echo -e "$YELLOW Dinky pid $pid is in ${PID_PATH}/${PID_FILE}, Please stop first !!!$RESET" fi From a4026925832727ec4cc6cd73c1bb07d3859e884a Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 15:15:45 +0800 Subject: [PATCH 11/19] Fix automation script path issue --- script/bin/init_cleanup.sh | 36 ++++++++++++++++++++++++++++ script/bin/init_flink_dependences.sh | 30 +++++++++++++---------- script/bin/init_tools_main.sh | 7 +++--- 3 files changed, 57 insertions(+), 16 deletions(-) create mode 100644 script/bin/init_cleanup.sh diff --git a/script/bin/init_cleanup.sh b/script/bin/init_cleanup.sh new file mode 100644 index 0000000000..6bcdf048de --- /dev/null +++ b/script/bin/init_cleanup.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +export RED='\033[31m' +export GREEN='\033[32m' +export YELLOW='\033[33m' +export BLUE='\033[34m' +export MAGENTA='\033[35m' +export CYAN='\033[36m' +export RESET='\033[0m' + +if [ -f /etc/profile.d/dinky_env ]; then + source /etc/profile.d/dinky_env +fi + +if [ -z "$DINKY_HOME" ]; then + echo -e "${RED}DINKY_HOME is not set, please check the environment variable...${RESET}" + exit 1 +else + EXTENDS_HOME="$DINKY_HOME"/extends + FLINK_VERSION_SCAN=$(ls -n "${EXTENDS_HOME}" | grep '^d' | grep flink | awk -F 'flink' '{print $2}') + + echo -e "${GREEN}>>>>>>>>>>>>>>>>>>>. Start cleaning up the environment... <<<<<<<<<<<<<<<<<<<<${RESET}" + echo -e "${GREEN}Cleaning up the environment variables...${RESET}" + rm -rf /etc/profile.d/dinky_* + echo -e "${GREEN}Cleaning up the flink jar dependencies...${RESET}" + rm -rf "${EXTENDS_HOME}"/flink"${FLINK_VERSION_SCAN}"/flink-* + echo -e "${GREEN}Cleaning up the flink shaded hadoop jar dependencies...${RESET}" + rm -rf "${EXTENDS_HOME}"/flink-shaded-hadoop-* + echo -e "${GREEN}Cleaning up the mysql jar dependencies...${RESET}" + rm -rf "$DINKY_HOME"/lib/mysql-connector-* + echo -e "${GREEN}Cleaning up the environment variables of DINKY_HOME ...${RESET}" + unset DINKY_HOME + echo -e "${GREEN}Refresh environment variables...${RESET}" + source /etc/profile + echo -e "${GREEN}Environment cleanup completed...${RESET}" +fi diff --git a/script/bin/init_flink_dependences.sh b/script/bin/init_flink_dependences.sh index 84d70e6823..b17be3aff0 100644 --- a/script/bin/init_flink_dependences.sh +++ b/script/bin/init_flink_dependences.sh @@ -15,12 +15,16 @@ if [ -z "$CURRENT_FLINK_FULL_VERSION" ] || [ -z "$FLINK_VERSION_SCAN" ] || [ -z exit 1 fi -if [ -f "$DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" ]; then - echo -e "${YELLOW}$DINKY_TMP_DIR ALREADY EXISTS flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz file,To ensure completeness, delete first ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz Download the file again${RESET}" - rm -rf ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz - if [ -d "$DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}" ]; then - echo -e "${YELLOW}The flink directory already exists, delete it $DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}" - rm -rf $DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION} +FLINK_STORE_DIR=${DINKY_TMP_DIR}/flink-download + +mkdir -p ${FLINK_STORE_DIR} + +if [ -f "${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" ]; then + echo -e "${YELLOW}${FLINK_STORE_DIR} ALREADY EXISTS flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz file,To ensure completeness, delete first ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz Download the file again${RESET}" + rm -rf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz + if [ -d "${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}" ]; then + echo -e "${YELLOW}The flink directory already exists, delete it ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}" + rm -rf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION} fi fi @@ -28,13 +32,13 @@ try_tsinghua_mirror() { local tsinghua_url="https://mirrors.tuna.tsinghua.edu.cn/apache/flink/flink-${CURRENT_FLINK_FULL_VERSION}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" local apache_url="https://archive.apache.org/dist/flink/flink-${CURRENT_FLINK_FULL_VERSION}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" - echo -e "${GREEN}Start downloading the Flink-${FLINK_VERSION_SCAN} installation package... Store it in the ${DINKY_TMP_DIR} directory${RESET}" - if download_file "$tsinghua_url" "$DINKY_TMP_DIR"; then + echo -e "${GREEN}Start downloading the Flink-${FLINK_VERSION_SCAN} installation package... Store it in the ${FLINK_STORE_DIR} directory${RESET}" + if download_file "$tsinghua_url" "${FLINK_STORE_DIR}"; then echo -e "${BLUE}The address of the currently downloaded Flink installation package is:${tsinghua_url}${RESET}" return 0 else echo -e "${YELLOW}File not found in Tsinghua University mirror, try downloading from Apache official source...${RESET}" - if download_file "$apache_url" "$DINKY_TMP_DIR"; then + if download_file "$apache_url" "${FLINK_STORE_DIR}"; then echo -e "${BLUE}The address of the currently downloaded Flink installation package is:${apache_url}${RESET}" return 0 else @@ -52,7 +56,7 @@ fi echo -e "${GREEN}Flink installation package download completed。${RESET}" echo -e "\n${GREEN}===============================================================${RESET}\n" echo -e "${GREEN}Start decompressing the Flink installation package...${RESET}" -tar -zxvf ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz -C ${DINKY_TMP_DIR}/ +tar -zxvf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz -C ${FLINK_STORE_DIR}/ if [ $? -eq 0 ]; then echo -e "${GREEN}Flink installation package decompression completed。${RESET}" else @@ -62,8 +66,8 @@ fi echo -e "\n${GREEN}===============================================================${RESET}\n" -flink_dir_tmp=$(ls -n ${DINKY_TMP_DIR} | grep '^d' | grep flink | awk '{print $9}') -full_flink_dir_tmp="${DINKY_TMP_DIR}/${flink_dir_tmp}" +flink_dir_tmp=$(ls -n ${FLINK_STORE_DIR} | grep '^d' | grep flink | awk '{print $9}') +full_flink_dir_tmp="${FLINK_STORE_DIR}/${flink_dir_tmp}" echo -e "${BLUE}Unzipped directory name:${full_flink_dir_tmp}${RESET}" @@ -101,6 +105,6 @@ ls -l ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ echo -e "${YELLOW}Please check the above dependent files。${RESET}" -rm -rf ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}* +rm -rf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}* echo -e "${GREEN}The basic dependency processing is completed, please perform subsequent operations according to the actual situation.${RESET}" \ No newline at end of file diff --git a/script/bin/init_tools_main.sh b/script/bin/init_tools_main.sh index 36a9685bec..da14e5c25f 100644 --- a/script/bin/init_tools_main.sh +++ b/script/bin/init_tools_main.sh @@ -74,7 +74,8 @@ echo -e "${GREEN}Dinky root path: ${APP_HOME} ${RESET}" sudo chmod +x "${APP_HOME}"/bin/init_*.sh - +/init_cleanup.sh +./init_tools_main.sh EXTENDS_HOME="${APP_HOME}/extends" if [ ! -d "${EXTENDS_HOME}" ]; then @@ -209,7 +210,7 @@ while true; do echo -e "${BLUE} ======== (h2 comes with it by default and does not need to perform this step)=========== ${RESET}" echo -e "${BLUE} ============================== Please select 1, 2, 3 ====================================== ${RESET}" echo -e "${BLUE} ==================================== 1. mysql ============================================= ${RESET}" - echo -e "${BLUE} ==================================== 2. pgsql ========================================= ${RESET}" + echo -e "${BLUE} ==================================== 2. postgresql ========================================= ${RESET}" echo -e "${BLUE} ================================ 3. Skip this step ========================================== ${RESET}" echo -e "${BLUE} ================================ Enter number selection ================================== ${RESET}" read -p "Please enter your database type:" db_type @@ -220,7 +221,7 @@ while true; do break ;; 2) - echo -e "${GREEN}It seems that pgsql has been integrated by default, so there is no need to perform this step. Please perform subsequent installation and configuration operations as needed.${RESET}" + echo -e "${GREEN}It seems that postgresql has been integrated by default, so there is no need to perform this step. Please perform subsequent installation and configuration operations as needed.${RESET}" break ;; 3) From fe713f1524550604eccec14b49056242ed216fd8 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 15:37:43 +0800 Subject: [PATCH 12/19] Fix automation script path issue --- script/bin/auto.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/script/bin/auto.sh b/script/bin/auto.sh index 67055c71cf..ac439e67a7 100644 --- a/script/bin/auto.sh +++ b/script/bin/auto.sh @@ -204,10 +204,8 @@ start() { nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none -cp "${CLASS_PATH}" org.dinky.Dinky ${JAR_PARAMS_OPT} > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 & PID=$! echo "${PID}" >"${PID_PATH}"/${PID_FILE} - run_result=$(wait_start_process) - if [ "$run_result" -eq 0 ]; then - echo -e "${GREEN}........................................Start Dinky Successfully........................................${RESET}" - fi + wait_start_process + echo -e "${GREEN}........................................Start Dinky Successfully........................................${RESET}" echo -e "${GREEN}current log path : ${DINKY_LOG_PATH}/dinky-start.log , you can execute tail -fn1000 ${DINKY_LOG_PATH}/dinky-start.log to watch the log${RESET}" else echo -e "$YELLOW Dinky pid $pid is in ${PID_PATH}/${PID_FILE}, Please stop first !!!$RESET" @@ -231,11 +229,9 @@ startWithJmx() { if [ -z "$pid" ]; then nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none "${JMX}" -cp "${CLASS_PATH}" org.dinky.Dinky ${JAR_PARAMS_OPT} > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 & PID=$! - run_result=$(wait_start_process) - if [ "$run_result" -eq 0 ]; then - echo -e "$GREEN........................................Start Dinky with Jmx Successfully........................................$RESET" - updatePid - fi + wait_start_process + echo -e "$GREEN........................................Start Dinky with Jmx Successfully........................................$RESET" + updatePid else echo -e "$YELLOW Dinky pid $pid is in ${PID_PATH}/${PID_FILE}, Please stop first !!!$RESET" From d30bf6201c4aaa9a1a85efd8f842849f4642b639 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 15:42:47 +0800 Subject: [PATCH 13/19] Fix automation script path issue --- script/bin/auto.sh | 68 ++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/script/bin/auto.sh b/script/bin/auto.sh index ac439e67a7..2d805d5156 100644 --- a/script/bin/auto.sh +++ b/script/bin/auto.sh @@ -112,35 +112,49 @@ check_health() { curl --silent --max-time 2 --output /dev/null --write-out "%{http_code}" "http://localhost:$APP_PORT/actuator/health" } + + +format_time() { + local seconds=$1 + local hours=$((seconds / 3600)) + local minutes=$(( (seconds % 3600) / 60 )) + local remaining_seconds=$((seconds % 60)) + printf "%02d:%02d:%02d" $hours $minutes $remaining_seconds +} + function wait_start_process() { echo ">>>>>>>>>>>>>>>>>>>>> Starting application... <<<<<<<<<<<<<<<<<<<<<<<" - local max_attempts=100 - local attempt=0 - local delay=0.25 - local health_status="" - local success_status_codes=("200" "201" "202" "203" "204") # 可以根据实际情况调整认为成功的状态码范围 - - while [ $attempt -lt $max_attempts ]; do - attempt=$((attempt + 1)) - health_status=$(check_health) - for code in "${success_status_codes[@]}"; do - if [ "$health_status" == "$code" ]; then - echo -ne "\r[==================================================] 100%\n" - echo -e "${GREEN}Application started completed.${RESET}" - return 0 - fi - done - local progress=$((attempt * 100 / max_attempts)) - local bar_length=50 - local filled_length=$((progress * bar_length / 100)) - local empty_length=$((bar_length - filled_length)) - local bar=$(printf '>%.0s' $(seq 1 $filled_length))$(printf ' %.0s' $(seq 1 $empty_length)) - echo -ne "\r[${bar}] ${progress}%" - sleep $delay - done - echo -ne "\r[==================================================] 100%\n" - echo -e "${RED}Application start failed. Please check the log for details.${RESET}" - return 1 + local max_attempts=100 + local attempt=0 + local delay=0.25 + local health_status="" + local success_status_codes=("200") + local start_time=$(date +%s) + + while [ $attempt -lt $max_attempts ]; do + attempt=$((attempt + 1)) + local current_time=$(date +%s) + local elapsed_time=$((current_time - start_time)) + local formatted_time=$(format_time $elapsed_time) + health_status=$(check_health) + for code in "${success_status_codes[@]}"; do + if [ "$health_status" == "$code" ]; then + echo -ne "\r[==================================================] 100%\n" + echo -e "${GREEN}Application started completed.${RESET}" + return 0 + fi + done + local progress=$((attempt * 100 / max_attempts)) + local bar_length=50 + local filled_length=$((progress * bar_length / 100)) + local empty_length=$((bar_length - filled_length)) + local bar=$(printf '>%.0s' $(seq 1 $filled_length))$(printf ' %.0s' $(seq 1 $empty_length)) + echo -ne "\r[${bar}] ${progress}% (耗时: ${formatted_time})" + sleep $delay + done + echo -ne "\r[==================================================] 100% (耗时: ${formatted_time})\n" + echo -e "${RED}Application start failed. Please check the log for details.${RESET}" + return 1 } From 55e3e1b2c3c1dcbe846a3d6a31d0f6f6efb18136 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 15:43:23 +0800 Subject: [PATCH 14/19] Fix automation script path issue --- script/bin/auto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/bin/auto.sh b/script/bin/auto.sh index 2d805d5156..4789c275f7 100644 --- a/script/bin/auto.sh +++ b/script/bin/auto.sh @@ -123,7 +123,7 @@ format_time() { } function wait_start_process() { - echo ">>>>>>>>>>>>>>>>>>>>> Starting application... <<<<<<<<<<<<<<<<<<<<<<<" + echo -e "${GREEN}>>>>>>>>>>>>>>>>>>>>> Starting application... <<<<<<<<<<<<<<<<<<<<<<<${RESET}" local max_attempts=100 local attempt=0 local delay=0.25 From ca05385a289459295d9c0f11e3a8bb25462c072d Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 16:39:54 +0800 Subject: [PATCH 15/19] Fix automation script path issue --- docs/docs/developer_guide/local_debug.md | 4 ++-- script/bin/auto.sh | 2 +- script/bin/init_tools_main.sh | 7 +------ 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/docs/docs/developer_guide/local_debug.md b/docs/docs/developer_guide/local_debug.md index ca4831c394..a21562056e 100644 --- a/docs/docs/developer_guide/local_debug.md +++ b/docs/docs/developer_guide/local_debug.md @@ -29,14 +29,14 @@ title: 本地调试 ### 环境要求 -| 环境 | 版本 | +**| 环境 | 版本 | |:----------------:|:------------------------:| | npm | 10+ | | node.js | 18+ | | jdk | Java8 或者 Java11 | | maven | 3.8+ | | lombok | IDEA插件安装 | -| MySQL/PostgreSQL | MySQL5.7+ / PostgreSQL15 | +| MySQL/PostgreSQL | MySQL5.7+ / PostgreSQL15 |** ### 代码克隆 diff --git a/script/bin/auto.sh b/script/bin/auto.sh index 4789c275f7..654abf08a3 100644 --- a/script/bin/auto.sh +++ b/script/bin/auto.sh @@ -107,7 +107,7 @@ if [ -z "$APP_PORT" ]; then APP_PORT=8888 fi -# 函数:检查健康检查端点的状态 +# Function: Check the status of the health check endpoint check_health() { curl --silent --max-time 2 --output /dev/null --write-out "%{http_code}" "http://localhost:$APP_PORT/actuator/health" } diff --git a/script/bin/init_tools_main.sh b/script/bin/init_tools_main.sh index da14e5c25f..c15177d846 100644 --- a/script/bin/init_tools_main.sh +++ b/script/bin/init_tools_main.sh @@ -74,9 +74,6 @@ echo -e "${GREEN}Dinky root path: ${APP_HOME} ${RESET}" sudo chmod +x "${APP_HOME}"/bin/init_*.sh -/init_cleanup.sh -./init_tools_main.sh - EXTENDS_HOME="${APP_HOME}/extends" if [ ! -d "${EXTENDS_HOME}" ]; then echo -e "${RED} ${EXTENDS_HOME} Directory does not exist, please check${RESET}" @@ -103,7 +100,7 @@ if [ ! -d "${DINKY_LIB}" ]; then exit 1 fi -# 函数:检查命令是否存在,不存在则尝试安装 +# Function: Check whether the command exists, if not, try to install it check_command() { local cmd="$1" echo -e "${BLUE}Check if command: $cmd exists...${RESET}" @@ -257,12 +254,10 @@ else echo -e "${GREEN}The current Flink version number deployed by Dinky:${FLINK_VERSION_SCAN}${RESET}" fi -# 根据 Dinky 部署的Flink对应的版本号,获取对应的 Flink 版本 CURRENT_FLINK_FULL_VERSION=${version_map[$FLINK_VERSION_SCAN]} echo -e "${GREEN}Obtain the version number corresponding to the deployed Flink (full version number) based on the scanned current Flink version number: flink-${CURRENT_FLINK_FULL_VERSION}${RESET}" -# 步骤2:获取Dinky部署的Flink对应的版本号,然后下载Flink安装包 while true; do read -p "It is detected that the Flink version number deployed by Dinky is: ${FLINK_VERSION_SCAN}, and the Flink installation package version number that needs to be downloaded is: flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz. Please choose whether to initialize Flink related dependencies?(yes/no/exit)" is_init_flink is_init_flink=$(echo "$is_init_flink" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') From 1991d2160e84f3f3a056a522a8a0194a2035a4a5 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 16:40:52 +0800 Subject: [PATCH 16/19] Fix automation script path issue --- docs/docs/developer_guide/local_debug.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/developer_guide/local_debug.md b/docs/docs/developer_guide/local_debug.md index a21562056e..8af6691678 100644 --- a/docs/docs/developer_guide/local_debug.md +++ b/docs/docs/developer_guide/local_debug.md @@ -29,7 +29,7 @@ title: 本地调试 ### 环境要求 -**| 环境 | 版本 | +| 环境 | 版本 | |:----------------:|:------------------------:| | npm | 10+ | | node.js | 18+ | From b9ee5937ff3b4f3a5fcb95380d238586bdff41f3 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 16:41:12 +0800 Subject: [PATCH 17/19] Fix automation script path issue --- docs/docs/developer_guide/local_debug.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/developer_guide/local_debug.md b/docs/docs/developer_guide/local_debug.md index 8af6691678..ca4831c394 100644 --- a/docs/docs/developer_guide/local_debug.md +++ b/docs/docs/developer_guide/local_debug.md @@ -36,7 +36,7 @@ title: 本地调试 | jdk | Java8 或者 Java11 | | maven | 3.8+ | | lombok | IDEA插件安装 | -| MySQL/PostgreSQL | MySQL5.7+ / PostgreSQL15 |** +| MySQL/PostgreSQL | MySQL5.7+ / PostgreSQL15 | ### 代码克隆 From 38929f3c8c8d0979eab894e568cdd4cb1b3391c1 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 16:44:06 +0800 Subject: [PATCH 18/19] Fix automation script path issue --- script/bin/auto.sh | 4 ++-- script/bin/init_tools_main.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/script/bin/auto.sh b/script/bin/auto.sh index 654abf08a3..0397a85c3d 100644 --- a/script/bin/auto.sh +++ b/script/bin/auto.sh @@ -149,10 +149,10 @@ function wait_start_process() { local filled_length=$((progress * bar_length / 100)) local empty_length=$((bar_length - filled_length)) local bar=$(printf '>%.0s' $(seq 1 $filled_length))$(printf ' %.0s' $(seq 1 $empty_length)) - echo -ne "\r[${bar}] ${progress}% (耗时: ${formatted_time})" + echo -ne "\r[${bar}] ${progress}% (time consuming: ${formatted_time})" sleep $delay done - echo -ne "\r[==================================================] 100% (耗时: ${formatted_time})\n" + echo -ne "\r[==================================================] 100% (time consuming: ${formatted_time})\n" echo -e "${RED}Application start failed. Please check the log for details.${RESET}" return 1 } diff --git a/script/bin/init_tools_main.sh b/script/bin/init_tools_main.sh index c15177d846..d8757652f6 100644 --- a/script/bin/init_tools_main.sh +++ b/script/bin/init_tools_main.sh @@ -363,7 +363,7 @@ while true; do no | n ) echo_warning_msg echo - echo -e "${GREEN}The Dinky service startup script has been skipped, 请先执行上述命令,然后手动启动服务 -> ${APP_HOME}/bin/auto.sh restart | start。${RESET}" + echo -e "${GREEN}The Dinky service startup script has been skipped, Please execute the above command first, and then start the service manually -> ${APP_HOME}/bin/auto.sh restart | start。${RESET}" break ;; exit | e ) From fed46fe516066b1e4c9976d0c835fe0b2df132b8 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Mon, 9 Dec 2024 16:44:58 +0800 Subject: [PATCH 19/19] Fix automation script path issue --- script/bin/parse_yml.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/bin/parse_yml.sh b/script/bin/parse_yml.sh index bc974669e9..63df089c35 100644 --- a/script/bin/parse_yml.sh +++ b/script/bin/parse_yml.sh @@ -1,6 +1,6 @@ #!/bin/bash -# 检查参数个数是否正确 +# Check whether the number of parameters is correct if [ $# -ne 2 ]; then echo "Please enter the parameters in the correct format, the format is: $0 yaml_file_path key" exit 1 @@ -19,7 +19,7 @@ value="" temp_file=$(mktemp) grep -Ev '^(#|$)' "$yaml_file_path" > "$temp_file" -# 开始按层级查找值 +# Start looking for values by levels current_data=$(cat "$temp_file") for part in "${key_parts[@]}"; do found=false