[Bug] Fix automation script path issue (#4030)

script/bin/auto.sh

@@ -2,6 +2,7 @@
 # debug mode
 #set -x
 
+
 export RED='\033[31m'
 export GREEN='\033[32m'
 export YELLOW='\033[33m'
@@ -11,27 +12,49 @@ export CYAN='\033[36m'
 export RESET='\033[0m'
 
 
+ENV_FILE="/etc/profile.d/dinky_env"
+if [ -f "${ENV_FILE}" ]; then
+    source "${ENV_FILE}"
+fi
 
-if [ -z "${DINKY_HOME}" ]; then
-    echo -e "${RED}DINKY_HOME environment variable is not set. Attempting to determine the correct path...${RESET}"
+DB_ENV_FILE="/etc/profile.d/dinky_db"
+if [ -f "${DB_ENV_FILE}" ]; then
+    source "${DB_ENV_FILE}"
+fi
+
+source /etc/profile
 
+RETURN_HOME_PATH=""
+function get_home_path() {
     SOURCE="${BASH_SOURCE[0]}"
     while [ -h "$SOURCE" ]; do
         DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
         SOURCE="$(readlink "$SOURCE")"
         [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
     done
     DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-    export DINKY_HOME="$(dirname "$DIR")"
+    RETURN_HOME_PATH=$(dirname "$DIR")
+}
+
+
+if [ -z "${DINKY_HOME}" ]; then
+    echo -e "${RED}DINKY_HOME environment variable is not set. Attempting to determine the correct path...${RESET}"
+    get_home_path
+    export DINKY_HOME=${RETURN_HOME_PATH}
     echo -e "${GREEN}DINKY_HOME is set to: ${DINKY_HOME}${RESET}"
 else
-    echo -e "${GREEN}DINKY_HOME is already set to: ${DINKY_HOME}${RESET}"
+    get_home_path
+    if [ "${DINKY_HOME}" != "${RETURN_HOME_PATH}" ]; then
+        export DINKY_HOME=${RETURN_HOME_PATH}
+        echo -e "${YELLOW}DINKY_HOME is not equal to the current path, reset DINKY_HOME to: ${RETURN_HOME_PATH}${RESET}"
+    else
+        echo -e "${GREEN}DINKY_HOME is already set to: ${DINKY_HOME}${RESET}"
+    fi
 fi
 
 
 FLINK_VERSION=${2}
 
-DINKY_HOME=${DINKY_HOME:-$(cd "$(dirname "$0")"; cd ..; pwd)}
 JAVA_VERSION=$(java -version 2>&1 | sed '1!d' | sed -e 's/"//g' | awk '{print $3}' | awk -F'.' '{print $1"."$2}')
 
 APP_HOME="${DINKY_HOME}"
@@ -71,6 +94,70 @@ assertIsInputVersion() {
   fi
 }
 
+
+if [ -f "${APP_HOME}/config/application.yml" ]; then
+  result=$("${APP_HOME}"/bin/parse_yml.sh "${APP_HOME}/config/application.yml" "server.port")
+  APP_PORT=$result
+fi
+
+echo -e "${GREEN}From ${APP_HOME}/config/application.yml server.port: ${APP_PORT}${RESET}"
+
+if [ -z "$APP_PORT" ]; then
+    echo -e "${RED}Could not find server.port in configuration files, using default port 8888 ${RESET}"
+    APP_PORT=8888
+fi
+
+# Function: Check the status of the health check endpoint
+check_health() {
+    curl --silent --max-time 2 --output /dev/null --write-out "%{http_code}" "http://localhost:$APP_PORT/actuator/health"
+}
+
+
+
+format_time() {
+    local seconds=$1
+    local hours=$((seconds / 3600))
+    local minutes=$(( (seconds % 3600) / 60 ))
+    local remaining_seconds=$((seconds % 60))
+    printf "%02d:%02d:%02d" $hours $minutes $remaining_seconds
+}
+
+function wait_start_process() {
+    echo -e "${GREEN}>>>>>>>>>>>>>>>>>>>>> Starting application... <<<<<<<<<<<<<<<<<<<<<<<${RESET}"
+     local max_attempts=100
+     local attempt=0
+     local delay=0.25
+     local health_status=""
+     local success_status_codes=("200")
+     local start_time=$(date +%s)
+
+     while [ $attempt -lt $max_attempts ]; do
+         attempt=$((attempt + 1))
+         local current_time=$(date +%s)
+         local elapsed_time=$((current_time - start_time))
+         local formatted_time=$(format_time $elapsed_time)
+         health_status=$(check_health)
+         for code in "${success_status_codes[@]}"; do
+             if [ "$health_status" == "$code" ]; then
+                 echo -ne "\r[==================================================] 100%\n"
+                 echo -e "${GREEN}Application started completed.${RESET}"
+                 return 0
+             fi
+         done
+         local progress=$((attempt * 100 / max_attempts))
+         local bar_length=50
+         local filled_length=$((progress * bar_length / 100))
+         local empty_length=$((bar_length - filled_length))
+         local bar=$(printf '>%.0s' $(seq 1 $filled_length))$(printf ' %.0s' $(seq 1 $empty_length))
+         echo -ne "\r[${bar}] ${progress}% (time consuming: ${formatted_time})"
+         sleep $delay
+     done
+     echo -ne "\r[==================================================] 100% (time consuming: ${formatted_time})\n"
+     echo -e "${RED}Application start failed. Please check the log for details.${RESET}"
+     return 1
+}
+
+
 # Use FLINK_HOME:
 CLASS_PATH="${APP_HOME}:${APP_HOME}/lib/*:${APP_HOME}/config:${EXTENDS_HOME}/*:${CUSTOMER_JAR_PATH}/*:${EXTENDS_HOME}/flink${FLINK_VERSION}/dinky/*:${EXTENDS_HOME}/flink${FLINK_VERSION}/flink/*:${EXTENDS_HOME}/flink${FLINK_VERSION}/*"
 PID_FILE="dinky.pid"
@@ -129,7 +216,9 @@ start() {
   updatePid
   if [ -z "$pid" ]; then
     nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none -cp "${CLASS_PATH}" org.dinky.Dinky ${JAR_PARAMS_OPT}  > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 &
-    echo $! >"${PID_PATH}"/${PID_FILE}
+    PID=$!
+    echo "${PID}" >"${PID_PATH}"/${PID_FILE}
+    wait_start_process
     echo -e "${GREEN}........................................Start Dinky Successfully........................................${RESET}"
     echo -e "${GREEN}current log path : ${DINKY_LOG_PATH}/dinky-start.log , you can execute tail -fn1000 ${DINKY_LOG_PATH}/dinky-start.log to watch the log${RESET}"
   else
@@ -153,9 +242,11 @@ startWithJmx() {
   updatePid
   if [ -z "$pid" ]; then
     nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none "${JMX}" -cp "${CLASS_PATH}" org.dinky.Dinky  ${JAR_PARAMS_OPT}  > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 &
-#    echo $! >"${PID_PATH}"/${PID_FILE}
-    updatePid
+    PID=$!
+    wait_start_process
     echo -e "$GREEN........................................Start Dinky with Jmx Successfully........................................$RESET"
+    updatePid
+
   else
     echo -e "$YELLOW Dinky pid $pid is in ${PID_PATH}/${PID_FILE}, Please stop first !!!$RESET"
   fi

script/bin/init_check_network.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -x
 
 EXTERNAL_CONNECTIVITY_CHECK_URL="www.baidu.com"
 

script/bin/init_cleanup.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+export RED='\033[31m'
+export GREEN='\033[32m'
+export YELLOW='\033[33m'
+export BLUE='\033[34m'
+export MAGENTA='\033[35m'
+export CYAN='\033[36m'
+export RESET='\033[0m'
+
+if [ -f /etc/profile.d/dinky_env ]; then
+    source /etc/profile.d/dinky_env
+fi
+
+if [ -z "$DINKY_HOME" ]; then
+  echo -e "${RED}DINKY_HOME is not set, please check the environment variable...${RESET}"
+  exit 1
+else
+  EXTENDS_HOME="$DINKY_HOME"/extends
+  FLINK_VERSION_SCAN=$(ls -n "${EXTENDS_HOME}" | grep '^d' | grep flink | awk -F 'flink' '{print $2}')
+
+  echo -e "${GREEN}>>>>>>>>>>>>>>>>>>>. Start cleaning up the environment... <<<<<<<<<<<<<<<<<<<<${RESET}"
+  echo -e "${GREEN}Cleaning up the environment variables...${RESET}"
+  rm -rf /etc/profile.d/dinky_*
+  echo -e "${GREEN}Cleaning up the flink jar dependencies...${RESET}"
+  rm -rf  "${EXTENDS_HOME}"/flink"${FLINK_VERSION_SCAN}"/flink-*
+  echo -e "${GREEN}Cleaning up the flink shaded hadoop jar dependencies...${RESET}"
+  rm -rf  "${EXTENDS_HOME}"/flink-shaded-hadoop-*
+  echo -e "${GREEN}Cleaning up the mysql jar dependencies...${RESET}"
+  rm -rf "$DINKY_HOME"/lib/mysql-connector-*
+  echo -e "${GREEN}Cleaning up the environment variables of DINKY_HOME ...${RESET}"
+  unset DINKY_HOME
+  echo -e "${GREEN}Refresh environment variables...${RESET}"
+  source /etc/profile
+  echo -e "${GREEN}Environment cleanup completed...${RESET}"
+fi

script/bin/init_db.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -x
 
 DINKY_HOME_PARAMS=$1
 DB_ENV_FILE=$2

script/bin/init_env.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -x
 
 DINKY_HOME_PARAMS=$1
 ENV_FILE=$2
@@ -18,7 +17,7 @@ while true; do
         echo -e "${GREEN}Use the automatically obtained DINKY_HOME environment variable${RESET}"
         echo -e "${GREEN} The currently obtained path is: $DINKY_HOME_PARAMS to perform automatic configuration${RESET}"
         add_to_env "DINKY_HOME" "$DINKY_HOME_PARAMS" "$ENV_FILE"
-        add_to_env "PATH" "\$DINKY_HOME/bin" "$ENV_FILE"
+        add_to_env "PATH" "\$DINKY_HOME/bin:\$PATH" "$ENV_FILE"
         sleep 2
         source $ENV_FILE
         echo -e "${GREEN}DINKY_HOME environment variable configuration completed. Please confirm whether the following configuration is correct：${RESET}"
@@ -34,7 +33,7 @@ while true; do
         else
             echo -e "${GREEN}The path you entered is: $dinky_home_path${RESET}"
             add_to_env "DINKY_HOME" "$dinky_home_path" "$ENV_FILE"
-            add_to_env "PATH" "\$DINKY_HOME/bin" "$ENV_FILE"
+            add_to_env "PATH" "\$DINKY_HOME/bin:\$PATH" "$ENV_FILE"
             sleep 2
             source $ENV_FILE
             echo -e "${GREEN}DINKY_HOME environment variable configuration completed. Please confirm whether the following configuration is correct：${RESET}"

script/bin/init_flink_dependences.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -x
 
 CURRENT_FLINK_FULL_VERSION=$1
 FLINK_VERSION_SCAN=$2
@@ -16,26 +15,30 @@ if [ -z "$CURRENT_FLINK_FULL_VERSION" ] || [ -z "$FLINK_VERSION_SCAN" ] || [ -z
   exit 1
 fi
 
-if [ -f "$DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" ]; then
-  echo -e "${YELLOW}$DINKY_TMP_DIR ALREADY EXISTS flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz file，To ensure completeness, delete first ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz Download the file again${RESET}"
-  rm -rf ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz
-  if [ -d "$DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}" ]; then
-    echo -e "${YELLOW}The flink directory already exists, delete it $DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}"
-    rm -rf $DINKY_TMP_DIR/flink-${CURRENT_FLINK_FULL_VERSION}
+FLINK_STORE_DIR=${DINKY_TMP_DIR}/flink-download
+
+mkdir -p ${FLINK_STORE_DIR}
+
+if [ -f "${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" ]; then
+  echo -e "${YELLOW}${FLINK_STORE_DIR} ALREADY EXISTS flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz file，To ensure completeness, delete first ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz Download the file again${RESET}"
+  rm -rf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz
+  if [ -d "${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}" ]; then
+    echo -e "${YELLOW}The flink directory already exists, delete it ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}"
+    rm -rf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}
   fi
 fi
 
 try_tsinghua_mirror() {
     local tsinghua_url="https://mirrors.tuna.tsinghua.edu.cn/apache/flink/flink-${CURRENT_FLINK_FULL_VERSION}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz"
     local apache_url="https://archive.apache.org/dist/flink/flink-${CURRENT_FLINK_FULL_VERSION}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz"
 
-    echo -e "${GREEN}Start downloading the Flink-${FLINK_VERSION_SCAN} installation package... Store it in the ${DINKY_TMP_DIR} directory${RESET}"
-    if download_file "$tsinghua_url" "$DINKY_TMP_DIR"; then
+    echo -e "${GREEN}Start downloading the Flink-${FLINK_VERSION_SCAN} installation package... Store it in the ${FLINK_STORE_DIR} directory${RESET}"
+    if download_file "$tsinghua_url" "${FLINK_STORE_DIR}"; then
         echo -e "${BLUE}The address of the currently downloaded Flink installation package is：${tsinghua_url}${RESET}"
         return 0
     else
         echo -e "${YELLOW}File not found in Tsinghua University mirror, try downloading from Apache official source...${RESET}"
-        if download_file "$apache_url" "$DINKY_TMP_DIR"; then
+        if download_file "$apache_url" "${FLINK_STORE_DIR}"; then
             echo -e "${BLUE}The address of the currently downloaded Flink installation package is：${apache_url}${RESET}"
             return 0
         else
@@ -53,7 +56,7 @@ fi
 echo -e "${GREEN}Flink installation package download completed。${RESET}"
 echo -e "\n${GREEN}===============================================================${RESET}\n"
 echo -e "${GREEN}Start decompressing the Flink installation package...${RESET}"
-tar -zxvf ${DINKY_TMP_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz -C ${DINKY_TMP_DIR}/
+tar -zxvf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz -C ${FLINK_STORE_DIR}/
 if [ $? -eq 0 ]; then
     echo -e "${GREEN}Flink installation package decompression completed。${RESET}"
 else
@@ -63,8 +66,8 @@ fi
 
 echo -e "\n${GREEN}===============================================================${RESET}\n"
 
-flink_dir_tmp=$(ls -n ${DINKY_TMP_DIR} | grep '^d' | grep flink | awk '{print $9}')
-full_flink_dir_tmp="${DINKY_TMP_DIR}/${flink_dir_tmp}"
+flink_dir_tmp=$(ls -n ${FLINK_STORE_DIR} | grep '^d' | grep flink | awk '{print $9}')
+full_flink_dir_tmp="${FLINK_STORE_DIR}/${flink_dir_tmp}"
 echo -e "${BLUE}Unzipped directory name：${full_flink_dir_tmp}${RESET}"
 
 
@@ -102,4 +105,6 @@ ls -l ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/
 
 echo -e "${YELLOW}Please check the above dependent files。${RESET}"
 
+rm -rf ${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}*
+
 echo -e "${GREEN}The basic dependency processing is completed, please perform subsequent operations according to the actual situation.${RESET}"

script/bin/init_hadoop_dependences.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-set -x
 
 EXTENDS_HOME=$1
 

script/bin/init_jdbc_driver.sh

@@ -1,13 +1,12 @@
 #!/bin/bash
-set -x
 
 DINKY_LIB_DIR=$1
 
 echo -e "${GREEN}Start downloading the mysql driver package...${RESET}"
 if [ -f "${DINKY_LIB_DIR}/mysql-connector-j-8.4.0.jar" ]; then
     echo -e "${YELLOW}mysql The driver package already exists, no need to download it again. Skip this step。${RESET}"
 else
-    download_file https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.4.0/mysql-connector-j-8.4.0.jar "${DINKY_LIB}"
+    download_file https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.4.0/mysql-connector-j-8.4.0.jar "${DINKY_LIB_DIR}"
     echo -e "${GREEN}Download is complete, please verify. The downloaded file storage address is： ${DINKY_LIB_DIR}/mysql-connector-j-8.4.0.jar${RESET}"
     if [ -f "${DINKY_LIB_DIR}/mysql-connector-j-8.4.0.jar" ]; then
         echo -e "${GREEN}mysql driver package downloaded successfully。${RESET}"