From 6e9ab0d49f5c67032f1823a4049f54ecb7be7983 Mon Sep 17 00:00:00 2001 From: Zzm0809 Date: Tue, 17 Dec 2024 03:33:42 +0000 Subject: [PATCH 1/4] Spotless Apply --- .../SqlTask/PushDolphin/index.tsx | 2 +- .../CenterTabContent/SqlTask/index.tsx | 29 ++++++++++++------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin/index.tsx b/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin/index.tsx index 70c959532c..4b3a29acbf 100644 --- a/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin/index.tsx +++ b/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin/index.tsx @@ -46,7 +46,7 @@ import { InitPushDolphinParams } from '@/types/Studio/init.d'; import { Button, Form, Tag } from 'antd'; import { DefaultOptionType } from 'antd/es/select'; import React, { useEffect } from 'react'; -import {TaskState} from "@/pages/DataStudio/type"; +import { TaskState } from '@/pages/DataStudio/type'; type PushDolphinProps = { onCancel: () => void; diff --git a/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/index.tsx b/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/index.tsx index ab6e5d2953..50b3200eec 100644 --- a/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/index.tsx +++ b/dinky-web/src/pages/DataStudio/CenterTabContent/SqlTask/index.tsx @@ -81,7 +81,12 @@ import { import { l } from '@/utils/intl'; import { editor } from 'monaco-editor'; import { DataStudioActionType } from '@/pages/DataStudio/data.d'; -import {getDataByParams, handleOption, handlePutDataJson, queryDataByParams} from '@/services/BusinessCrud'; +import { + getDataByParams, + handleOption, + handlePutDataJson, + queryDataByParams +} from '@/services/BusinessCrud'; import { API_CONSTANTS } from '@/services/endpoints'; import { Jobs, LineageDetailInfo } from '@/types/DevOps/data'; import { lockTask, matchLanguage } from '@/pages/DataStudio/function'; @@ -98,8 +103,12 @@ import { ResourceInfo } from '@/types/RegCenter/data'; import { buildResourceTreeDataAtTreeForm } from '@/pages/RegCenter/Resource/components/FileTree/function'; import { ProFormDependency } from '@ant-design/pro-form'; import { SavePoint } from '@/pages/DataStudio/CenterTabContent/SqlTask/SavePoint'; -import {DolphinTaskDefinition, DolphinTaskGroupInfo, DolphinTaskMinInfo} from "@/types/Studio/data"; -import PushDolphin from "@/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin"; +import { + DolphinTaskDefinition, + DolphinTaskGroupInfo, + DolphinTaskMinInfo +} from '@/types/Studio/data'; +import PushDolphin from '@/pages/DataStudio/CenterTabContent/SqlTask/PushDolphin'; export type FlinkSqlProps = { showDesc: boolean; @@ -663,15 +672,15 @@ export const SqlTask = memo((props: FlinkSqlProps & any) => { ); let dolphinTaskGroup: DolphinTaskGroupInfo[] | undefined = undefined; - if (dolphinTaskDefinition?.projectCode){ - dolphinTaskGroup = await queryDataByParams< - DolphinTaskGroupInfo[] - >(API_CONSTANTS.SCHEDULER_QUERY_TASK_GROUP, { - projectCode: dolphinTaskDefinition?.projectCode || undefined - }); + if (dolphinTaskDefinition?.projectCode) { + dolphinTaskGroup = await queryDataByParams( + API_CONSTANTS.SCHEDULER_QUERY_TASK_GROUP, + { + projectCode: dolphinTaskDefinition?.projectCode || undefined + } + ); } - const formValuesInfo = dolphinTaskDefinition ? JSON.parse(JSON.stringify(dolphinTaskDefinition)) : {}; From 3f208f752195f1d9f83f4cdb51286de081e905b3 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Thu, 19 Dec 2024 14:42:02 +0800 Subject: [PATCH 2/4] Optimize the execution logic of the script. --- script/bin/auto.sh | 17 +++++++++------ script/bin/init_flink_dependences.sh | 31 +++++++++++++++++++--------- script/bin/init_jdbc_driver.sh | 6 ++++-- script/bin/init_tools_main.sh | 30 +++++++++++++-------------- 4 files changed, 50 insertions(+), 34 deletions(-) diff --git a/script/bin/auto.sh b/script/bin/auto.sh index 0397a85c3d..0db680e9ac 100644 --- a/script/bin/auto.sh +++ b/script/bin/auto.sh @@ -140,7 +140,6 @@ function wait_start_process() { for code in "${success_status_codes[@]}"; do if [ "$health_status" == "$code" ]; then echo -ne "\r[==================================================] 100%\n" - echo -e "${GREEN}Application started completed.${RESET}" return 0 fi done @@ -149,11 +148,11 @@ function wait_start_process() { local filled_length=$((progress * bar_length / 100)) local empty_length=$((bar_length - filled_length)) local bar=$(printf '>%.0s' $(seq 1 $filled_length))$(printf ' %.0s' $(seq 1 $empty_length)) - echo -ne "\r[${bar}] ${progress}% (time consuming: ${formatted_time})" + local processing_inline="\r[${bar}] ${progress}% (time consuming: ${formatted_time})" + echo -ne "${processing_inline}" sleep $delay done - echo -ne "\r[==================================================] 100% (time consuming: ${formatted_time})\n" - echo -e "${RED}Application start failed. Please check the log for details.${RESET}" + echo -ne "${processing_inline}\n" return 1 } @@ -218,7 +217,10 @@ start() { nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none -cp "${CLASS_PATH}" org.dinky.Dinky ${JAR_PARAMS_OPT} > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 & PID=$! echo "${PID}" >"${PID_PATH}"/${PID_FILE} - wait_start_process + if ! wait_start_process; then + echo -e "${RED}Application start failed. Please check the log for details. you can execute tail -fn1000 ${DINKY_LOG_PATH}/dinky-start.log to watch the log ${RESET}" + exit 1 + fi echo -e "${GREEN}........................................Start Dinky Successfully........................................${RESET}" echo -e "${GREEN}current log path : ${DINKY_LOG_PATH}/dinky-start.log , you can execute tail -fn1000 ${DINKY_LOG_PATH}/dinky-start.log to watch the log${RESET}" else @@ -243,7 +245,10 @@ startWithJmx() { if [ -z "$pid" ]; then nohup java ${PARAMS_OPT} ${JVM_OPTS} ${OOM_OPT} ${GC_OPT} -Xverify:none "${JMX}" -cp "${CLASS_PATH}" org.dinky.Dinky ${JAR_PARAMS_OPT} > ${DINKY_LOG_PATH}/dinky-start.log 2>&1 & PID=$! - wait_start_process + if ! wait_start_process; then + echo -e "${RED}Application start failed. Please check the log for details. you can execute tail -fn1000 ${DINKY_LOG_PATH}/dinky-start.log to watch the log ${RESET}" + exit 1 + fi echo -e "$GREEN........................................Start Dinky with Jmx Successfully........................................$RESET" updatePid diff --git a/script/bin/init_flink_dependences.sh b/script/bin/init_flink_dependences.sh index b17be3aff0..a19510afa7 100644 --- a/script/bin/init_flink_dependences.sh +++ b/script/bin/init_flink_dependences.sh @@ -28,27 +28,38 @@ if [ -f "${FLINK_STORE_DIR}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.t fi fi -try_tsinghua_mirror() { +try_mirrors_download_file() { local tsinghua_url="https://mirrors.tuna.tsinghua.edu.cn/apache/flink/flink-${CURRENT_FLINK_FULL_VERSION}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" + local aliyun_url="https://mirrors.aliyun.com/apache/flink/flink-${CURRENT_FLINK_FULL_VERSION}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" local apache_url="https://archive.apache.org/dist/flink/flink-${CURRENT_FLINK_FULL_VERSION}/flink-${CURRENT_FLINK_FULL_VERSION}-bin-scala_2.12.tgz" - echo -e "${GREEN}Start downloading the Flink-${FLINK_VERSION_SCAN} installation package... Store it in the ${FLINK_STORE_DIR} directory${RESET}" - if download_file "$tsinghua_url" "${FLINK_STORE_DIR}"; then - echo -e "${BLUE}The address of the currently downloaded Flink installation package is：${tsinghua_url}${RESET}" + + exec_tsinghua_result=$(download_file "$tsinghua_url" "${FLINK_STORE_DIR}") + if [ ! "$exec_tsinghua_result" ]; then + echo -e "${BLUE}The tsinghua address of the currently downloaded Flink installation package is：${tsinghua_url}${RESET}" return 0 else - echo -e "${YELLOW}File not found in Tsinghua University mirror, try downloading from Apache official source...${RESET}" - if download_file "$apache_url" "${FLINK_STORE_DIR}"; then - echo -e "${BLUE}The address of the currently downloaded Flink installation package is：${apache_url}${RESET}" + echo -e "${YELLOW}Failed to download from Tsinghua mirror, try downloading from Aliyun mirror...${RESET}" + exec_aliyun_result=$(download_file "$aliyun_url" "${FLINK_STORE_DIR}") + if [ "$exec_aliyun_result" ]; then + echo -e "${BLUE}The aliyun address of the currently downloaded Flink installation package is：${aliyun_url}${RESET}" return 0 else - echo -e "${RED}Downloading from Apache official source also failed, please check the network or download manually。${RESET}" - return 1 + echo -e "${YELLOW}Failed to download from Aliyun mirror too, try downloading from Apache official source...${RESET}" + + exec_apache_result=$(download_file "$apache_url" "${FLINK_STORE_DIR}") + if [ "$exec_apache_result" ]; then + echo -e "${RED}Downloading from Apache official source also failed, please check the network or download manually。${RESET}" + return 1 + else + echo -e "${BLUE}The apache address of the currently downloaded Flink installation package is：${apache_url}${RESET}" + return 0 + fi fi fi } -if ! try_tsinghua_mirror; then +if ! try_mirrors_download_file; then exit 0 fi diff --git a/script/bin/init_jdbc_driver.sh b/script/bin/init_jdbc_driver.sh index ca9fd9c952..6521a7e1c7 100644 --- a/script/bin/init_jdbc_driver.sh +++ b/script/bin/init_jdbc_driver.sh @@ -3,8 +3,10 @@ DINKY_LIB_DIR=$1 echo -e "${GREEN}Start downloading the mysql driver package...${RESET}" -if [ -f "${DINKY_LIB_DIR}/mysql-connector-j-8.4.0.jar" ]; then - echo -e "${YELLOW}mysql The driver package already exists, no need to download it again. Skip this step。${RESET}" +# 执行查看文件是否存在 +exec_result=$(ll "${DINKY_LIB_DIR}"/mysql-connector-j-8.4.0.jar) +if [ "$exec_result" ]; then + echo -e "${YELLOW}The mysql driver package already exists, no need to download it again. Skip this step。${RESET}" else download_file https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.4.0/mysql-connector-j-8.4.0.jar "${DINKY_LIB_DIR}" echo -e "${GREEN}Download is complete, please verify. The downloaded file storage address is： ${DINKY_LIB_DIR}/mysql-connector-j-8.4.0.jar${RESET}" diff --git a/script/bin/init_tools_main.sh b/script/bin/init_tools_main.sh index d8757652f6..6ddbced579 100644 --- a/script/bin/init_tools_main.sh +++ b/script/bin/init_tools_main.sh @@ -51,24 +51,17 @@ function get_home_path() { [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" done DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - RETURN_HOME_PATH=$(dirname "$DIR") -} - - - -if [ -z "${DINKY_HOME}" ]; then - echo -e "${RED}DINKY_HOME environment variable is not set. Attempting to determine the correct path...${RESET}" - get_home_path - APP_HOME="${RETURN_HOME_PATH}" -else - get_home_path - if [ "${DINKY_HOME}" != "${RETURN_HOME_PATH}" ]; then - echo -e "${YELLOW}DINKY_HOME is not equal to the current path, use new path to init: ${RETURN_HOME_PATH}${RESET}" - APP_HOME="${RETURN_HOME_PATH}" + local possible_path=$(dirname "$DIR") + if [ -d "$possible_path" ]; then # Verify that the path obtained is a directory, increasing robustness + RETURN_HOME_PATH="$possible_path" else - echo -e "${GREEN}DINKY_HOME is already set to: ${DINKY_HOME}${RESET}" + echo -e "${RED}Calculated path $possible_path is not a valid directory. Please check the script location or deployment setup.${RESET}" + exit 1 fi -fi +} + +get_home_path +APP_HOME="${RETURN_HOME_PATH}" echo -e "${GREEN}Dinky root path: ${APP_HOME} ${RESET}" @@ -131,7 +124,12 @@ function download_file() { target_file_dir=$2 echo -e "${GREEN}Start downloading $source_url to $target_file_dir...${RESET}" wget -P "${target_file_dir}" "${source_url}" + if [ $? -ne 0 ]; then + echo -e "${RED}Failed to download $source_url to $target_file_dir. Please check the network connection and try again.${RESET}" + return 1 + fi echo -e "${GREEN}Download completed. The downloaded file storage address is: $target_file_dir ${RESET}" + return 0 } export -f download_file From d5b597cf20907e4278815f19ad59b67b5d2bb5e9 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Thu, 19 Dec 2024 14:44:09 +0800 Subject: [PATCH 3/4] Optimize the execution logic of the script. --- script/bin/init_jdbc_driver.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/bin/init_jdbc_driver.sh b/script/bin/init_jdbc_driver.sh index 6521a7e1c7..d088e8b667 100644 --- a/script/bin/init_jdbc_driver.sh +++ b/script/bin/init_jdbc_driver.sh @@ -3,7 +3,7 @@ DINKY_LIB_DIR=$1 echo -e "${GREEN}Start downloading the mysql driver package...${RESET}" -# 执行查看文件是否存在 +# Run the command to check whether the file exists exec_result=$(ll "${DINKY_LIB_DIR}"/mysql-connector-j-8.4.0.jar) if [ "$exec_result" ]; then echo -e "${YELLOW}The mysql driver package already exists, no need to download it again. Skip this step。${RESET}" From 7cbcea669c31ec33e57c7d5cc11e302da8ee7716 Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Thu, 19 Dec 2024 16:15:39 +0800 Subject: [PATCH 4/4] Optimize the execution logic of the script. --- script/bin/init_flink_dependences.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/bin/init_flink_dependences.sh b/script/bin/init_flink_dependences.sh index a19510afa7..0f0e9b80d0 100644 --- a/script/bin/init_flink_dependences.sh +++ b/script/bin/init_flink_dependences.sh @@ -111,6 +111,10 @@ echo -e "${GREEN}Process flink-state-processor-api ...${RESET}" cp -r ${full_flink_dir_tmp}/opt/flink-state-processor-api*.jar ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ echo -e "${GREEN}Processing completed。${RESET}" +echo -e "${GREEN}Process flink-s3-fs-presto ...${RESET}" +cp -r ${full_flink_dir_tmp}/opt/flink-s3-fs-presto*.jar ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ +echo -e "${GREEN}Processing completed。${RESET}" + echo -e "${GREEN} ================= List files in the ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/ directory ==============${RESET}" ls -l ${EXTENDS_HOME}/flink${FLINK_VERSION_SCAN}/