From 596fa5395a937c785138d0638da941817f26ba32 Mon Sep 17 00:00:00 2001 From: gaoyan Date: Tue, 24 Sep 2024 09:36:08 +0800 Subject: [PATCH 1/6] Fix deploy doc (#3829) Co-authored-by: gaoyan1998 Co-authored-by: GH Action - Upstream Sync --- docs/docs/deploy_guide/normal_deploy.mdx | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docs/docs/deploy_guide/normal_deploy.mdx b/docs/docs/deploy_guide/normal_deploy.mdx index 7387538937..89b5300088 100644 --- a/docs/docs/deploy_guide/normal_deploy.mdx +++ b/docs/docs/deploy_guide/normal_deploy.mdx @@ -11,8 +11,6 @@ title: 常规部署 Dinky 采用 mysql 作为后端的存储库，mysql 支持 5.7+。这里假设你已经安装了 mysql 。首先需要创建 Dinky 的后端数据库，这里以配置文件中默认库创建。 -在 Dinky 根目录 sql 文件夹下分别放置了 dinky-mysql.sql 、 upgrade/${version}_schema/mysql/ddl 和 dml。如果第一次部署，可以直接将 sql/dinky-mysql.sql 文件在 dinky 数据库下执行。（如果之前已经部署，那 upgrade 目录下存放了各版本的升级 sql ，根据版本号按需执行即可） - @@ -55,14 +53,7 @@ flush privileges; #### 第一次部署 - -```sql -#首先登录 mysql -mysql -h fdw1 -udinky -pdinky -mysql> use dinky; -mysql> source /opt/dinky/sql/dinky-mysql.sql -``` - +Dinky使用Flyway进行数据库版本管理，在第一次部署时，无需手动建表， Flyway 会自动完成。用户只需配置好数据库连接信息，其他配置文件默认即可。 ## Dinky 部署 From 0f52cb25e8ab964c5be3f630d8643c213881c1a5 Mon Sep 17 00:00:00 2001 From: gaoyan Date: Tue, 24 Sep 2024 09:37:49 +0800 Subject: [PATCH 2/6] Fix null pointer exception occurs in alert (#3828) Co-authored-by: gaoyan1998 Co-authored-by: GH Action - Upstream Sync --- .../src/main/java/org/dinky/job/handler/JobAlertHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dinky-admin/src/main/java/org/dinky/job/handler/JobAlertHandler.java b/dinky-admin/src/main/java/org/dinky/job/handler/JobAlertHandler.java index 61a263e485..b285061e00 100644 --- a/dinky-admin/src/main/java/org/dinky/job/handler/JobAlertHandler.java +++ b/dinky-admin/src/main/java/org/dinky/job/handler/JobAlertHandler.java @@ -249,7 +249,8 @@ private void executeAlertAction(Facts facts, AlertRuleDTO alertRuleDTO) throws E if (!Asserts.isNull(task.getAlertGroup())) { // 获取任务的责任人和维护人对应的用户信息|Get the responsible person and maintainer of the task - User ownerInfo = userCache.get(task.getFirstLevelOwner()); + Integer owner = task.getFirstLevelOwner(); + User ownerInfo = owner == null ? null : userCache.get(owner); List maintainerInfo = Lists.newArrayList(); if (CollectionUtils.isNotEmpty(task.getSecondLevelOwners())) { for (Integer secondLevelOwner : task.getSecondLevelOwners()) { From 81db6e499a6afc326bf8e02c8d986cbd01bc6bbb Mon Sep 17 00:00:00 2001 From: ZackYoung Date: Tue, 24 Sep 2024 16:56:17 +0800 Subject: [PATCH 3/6] [Optimize]Optimize some code (#3831) Co-authored-by: zackyoungh --- .../main/java/org/dinky/aop/LogAspect.java | 3 + .../context/FlinkUdfPathContextHolder.java | 11 ++- .../org/dinky/data/result/SelectResult.java | 3 +- .../org/dinky/gateway/yarn/YarnGateway.java | 22 +++-- .../src/components/Flink/FlinkDag/index.tsx | 7 +- .../components/Flink/OptionsSelect/index.tsx | 2 +- dinky-web/src/components/Icons/HomeIcon.tsx | 99 +++++++++++++++++++ .../BottomContainer/Result/index.tsx | 10 +- .../DataStudio/BottomContainer/index.tsx | 3 +- .../pages/DataStudio/LeftContainer/index.tsx | 11 ++- .../RightContainer/JobConfig/index.tsx | 6 +- .../pages/DataStudio/RightContainer/index.tsx | 11 ++- dinky-web/src/pages/DataStudio/data.d.tsx | 6 ++ dinky-web/src/pages/DataStudio/index.tsx | 24 ++--- .../components/AlertInstanceList/index.tsx | 9 +- .../components/ConfigurationList/index.tsx | 15 +-- .../ConfigurationForm/FlinkK8s/contants.tsx | 3 +- .../components/ConfigurationModal/index.tsx | 10 +- .../components/InstanceList/index.tsx | 44 ++++----- .../src/pages/RegCenter/Document/constans.ts | 4 + 20 files changed, 221 insertions(+), 82 deletions(-) diff --git a/dinky-admin/src/main/java/org/dinky/aop/LogAspect.java b/dinky-admin/src/main/java/org/dinky/aop/LogAspect.java index 656d082dc0..732e0bf0ce 100644 --- a/dinky-admin/src/main/java/org/dinky/aop/LogAspect.java +++ b/dinky-admin/src/main/java/org/dinky/aop/LogAspect.java @@ -103,6 +103,9 @@ protected void handleCommonLogic(final JoinPoint joinPoint, final Exception e, O // *========数据库日志=========*// OperateLog operLog = new OperateLog(); Result result = JsonUtils.toBean(jsonResult, new TypeReference>() {}); + if (result == null) { + result = Result.failed(); + } operLog.setStatus(result.isSuccess() ? BusinessStatus.SUCCESS.ordinal() : BusinessStatus.FAIL.ordinal()); // 请求的地址 diff --git a/dinky-common/src/main/java/org/dinky/context/FlinkUdfPathContextHolder.java b/dinky-common/src/main/java/org/dinky/context/FlinkUdfPathContextHolder.java index 0b00a6b9ac..febbfdbcb1 100644 --- a/dinky-common/src/main/java/org/dinky/context/FlinkUdfPathContextHolder.java +++ b/dinky-common/src/main/java/org/dinky/context/FlinkUdfPathContextHolder.java @@ -20,17 +20,21 @@ package org.dinky.context; import java.io.File; +import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * @since 0.7.0 */ public class FlinkUdfPathContextHolder { + private static final List PYTHON_FILE_SUFFIX = + Arrays.asList(".zip", ".py", ".pyc", ".pyo", ".pyd", ".pyw", ".pyz", ".pyzw"); private final Set UDF_PATH_CONTEXT = new HashSet<>(); private final Set OTHER_PLUGINS_PATH_CONTEXT = new HashSet<>(); - private final Set PYTHON_UDF_FILE = new HashSet<>(); private final Set FILES = new HashSet<>(); public void addUdfPath(File file) { @@ -54,7 +58,10 @@ public Set getUdfFile() { } public Set getPyUdfFile() { - return PYTHON_UDF_FILE; + return getAllFileSet().stream() + .filter(file -> PYTHON_FILE_SUFFIX.stream() + .anyMatch(suffix -> file.getName().endsWith(suffix))) + .collect(Collectors.toSet()); } public Set getOtherPluginsFiles() { diff --git a/dinky-core/src/main/java/org/dinky/data/result/SelectResult.java b/dinky-core/src/main/java/org/dinky/data/result/SelectResult.java index 1705d7d3b3..b7ea0f1aaf 100644 --- a/dinky-core/src/main/java/org/dinky/data/result/SelectResult.java +++ b/dinky-core/src/main/java/org/dinky/data/result/SelectResult.java @@ -31,6 +31,7 @@ import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.collection.ListUtil; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -42,6 +43,7 @@ @Slf4j @Setter @Getter +@NoArgsConstructor public class SelectResult extends AbstractResult implements IResult { private String jobID; @@ -65,7 +67,6 @@ public SelectResult( this.columns = columns; this.jobID = jobID; this.success = success; - // this.endTime = LocalDateTime.now(); this.isDestroyed = false; } diff --git a/dinky-gateway/src/main/java/org/dinky/gateway/yarn/YarnGateway.java b/dinky-gateway/src/main/java/org/dinky/gateway/yarn/YarnGateway.java index 5277eba2d2..13d69af6ab 100644 --- a/dinky-gateway/src/main/java/org/dinky/gateway/yarn/YarnGateway.java +++ b/dinky-gateway/src/main/java/org/dinky/gateway/yarn/YarnGateway.java @@ -112,6 +112,7 @@ public YarnGateway(GatewayConfig config) { super(config); } + @Override public void init() { initConfig(); initYarnClient(); @@ -192,7 +193,7 @@ private void initYarnClient() { hadoopUserName = "hdfs"; } - // 设置 yarn 提交的用户名 + // Set the username for the yarn submission String yarnUser = configuration.get(CustomerConfigureOptions.YARN_APPLICATION_USER); if (StrUtil.isNotBlank(yarnUser)) { UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(yarnUser)); @@ -211,6 +212,7 @@ private Path getYanConfigFilePath(String path) { return new Path(URI.create(config.getClusterConfig().getHadoopConfigPath() + "/" + path)); } + @Override public SavePointResult savepointCluster(String savePoint) { if (Asserts.isNull(yarnClient)) { init(); @@ -221,6 +223,7 @@ public SavePointResult savepointCluster(String savePoint) { return runClusterSavePointResult(savePoint, applicationId, clusterDescriptor); } + @Override public SavePointResult savepointJob(String savePoint) { if (Asserts.isNull(yarnClient)) { init(); @@ -253,13 +256,14 @@ private void autoCancelCluster(ClusterClient clusterClient) { Thread.sleep(3000); clusterClient.shutDownCluster(); } catch (InterruptedException e) { - e.printStackTrace(); + logger.error(e.getMessage()); } finally { clusterClient.close(); } }); } + @Override public TestResult test() { try { initConfig(); @@ -365,13 +369,12 @@ protected YarnClusterDescriptor createYarnClusterDescriptorWithJar(FlinkUdfPathC } protected YarnClusterDescriptor createInitYarnClusterDescriptor() { - YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor( + return new YarnClusterDescriptor( configuration, yarnConfiguration, yarnClient, YarnClientYarnClusterInformationRetriever.create(yarnClient), true); - return yarnClusterDescriptor; } protected String getWebUrl(ClusterClient clusterClient, YarnResult result) @@ -464,7 +467,7 @@ public String getLatestJobManageHost(String appId, String oldJobManagerHost) { HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration); if (HighAvailabilityMode.ZOOKEEPER == highAvailabilityMode) { - configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, appId); + configuration.set(HighAvailabilityOptions.HA_CLUSTER_ID, appId); String zkQuorum = configuration.getValue(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM); if (zkQuorum == null || StringUtils.isBlank(zkQuorum)) { @@ -495,13 +498,13 @@ public String getLatestJobManageHost(String appId, String oldJobManagerHost) { } } } catch (Exception e) { - e.printStackTrace(); + logger.error("", e); } finally { if (Asserts.isNotNull(zooKeeper)) { try { zooKeeper.close(); } catch (InterruptedException e) { - e.printStackTrace(); + logger.error("", e); } } } @@ -515,12 +518,11 @@ public String getLatestJobManageHost(String appId, String oldJobManagerHost) { * Creates a ZooKeeper path of the form "/a/b/.../z". */ private static String generateZookeeperPath(String... paths) { - final String result = Arrays.stream(paths) + + return Arrays.stream(paths) .map(YarnGateway::trimSlashes) .filter(s -> !s.isEmpty()) .collect(Collectors.joining("/", "/", "")); - - return result; } private static String trimSlashes(String input) { diff --git a/dinky-web/src/components/Flink/FlinkDag/index.tsx b/dinky-web/src/components/Flink/FlinkDag/index.tsx index efa2a685bd..ad421c8479 100644 --- a/dinky-web/src/components/Flink/FlinkDag/index.tsx +++ b/dinky-web/src/components/Flink/FlinkDag/index.tsx @@ -168,9 +168,10 @@ function getMaxWidthAndDepth(edges: CusEdge[]): { maxWidth: number; maxDepth: nu graph[sourceCell].push(targetCell); }); - const maxSource = Object.keys(sourceCount).reduce((a, b) => - sourceCount[a] > sourceCount[b] ? a : b - ); + const maxSource = + Object.keys(sourceCount).length < 1 + ? '1' + : Object.keys(sourceCount).reduce((a, b) => (sourceCount[a] > sourceCount[b] ? a : b)); const maxWidth = sourceCount[maxSource]; const visited: Record = {}; diff --git a/dinky-web/src/components/Flink/OptionsSelect/index.tsx b/dinky-web/src/components/Flink/OptionsSelect/index.tsx index c209c1247c..e2b52e84ec 100644 --- a/dinky-web/src/components/Flink/OptionsSelect/index.tsx +++ b/dinky-web/src/components/Flink/OptionsSelect/index.tsx @@ -41,7 +41,7 @@ const FlinkOptionsSelect = (props: FlinkOptionsProps) => { return ( renderTemplateDropDown(item) }} + fieldProps={{ dropdownRender: (item) => renderTemplateDropDown(item), virtual: false }} /> ); }; diff --git a/dinky-web/src/components/Icons/HomeIcon.tsx b/dinky-web/src/components/Icons/HomeIcon.tsx index 16250e802f..a7350c0230 100644 --- a/dinky-web/src/components/Icons/HomeIcon.tsx +++ b/dinky-web/src/components/Icons/HomeIcon.tsx @@ -501,6 +501,105 @@ export const ClusterConfigIcon = (props: any) => { ); }; +export const K8sIcon = (props: any) => { + const imgStyle = props.style || {}; + return ( + <> + ( + + + + + )} + /> + + ); +}; +export const HadoopIcon = (props: any) => { + const imgStyle = props.style || {}; + return ( + <> + ( + + + + + + + + + + + + + )} + /> + + ); +}; export const BatchJobIcon = (props: any) => { const imgStyle = props.style || {}; diff --git a/dinky-web/src/pages/DataStudio/BottomContainer/Result/index.tsx b/dinky-web/src/pages/DataStudio/BottomContainer/Result/index.tsx index 6b0a9e8dcc..338356afe8 100644 --- a/dinky-web/src/pages/DataStudio/BottomContainer/Result/index.tsx +++ b/dinky-web/src/pages/DataStudio/BottomContainer/Result/index.tsx @@ -37,8 +37,9 @@ import { Button, Empty, Input, InputRef, Space, Table, Tabs, Tooltip } from 'ant import { ColumnsType, ColumnType } from 'antd/es/table'; import { FilterConfirmProps } from 'antd/es/table/interface'; import { DataIndex } from 'rc-table/es/interface'; -import { useEffect, useRef, useState } from 'react'; +import { useRef, useState } from 'react'; import { connect } from 'umi'; +import { useAsyncEffect } from 'ahooks'; type Data = { [c: string]: any; @@ -131,7 +132,6 @@ const Result = (props: any) => { return; } - const params = currentTabs.params; const consoleData = currentTabs.console; if (consoleData.result && !isRefresh) { setData(consoleData.result); @@ -161,7 +161,7 @@ const Result = (props: any) => { } ); const data = tableData.data; - if (data.success) { + if (tableData.success && data?.success) { consoleData.result = data; setData(data); } else { @@ -175,10 +175,10 @@ const Result = (props: any) => { setLoading(false); }; - useEffect(() => { + useAsyncEffect(async () => { setData({}); setDataList([]); - loadData(initIsRefresh); + await loadData(initIsRefresh); }, [currentTabs?.console?.refreshResult, currentTabs?.console?.refreshResults, currentTabs?.id]); const getColumns = (columns: string[] = []) => { diff --git a/dinky-web/src/pages/DataStudio/BottomContainer/index.tsx b/dinky-web/src/pages/DataStudio/BottomContainer/index.tsx index 1b48ed71cd..be78823731 100644 --- a/dinky-web/src/pages/DataStudio/BottomContainer/index.tsx +++ b/dinky-web/src/pages/DataStudio/BottomContainer/index.tsx @@ -35,9 +35,10 @@ import { l } from '@/utils/intl'; import { connect } from '@@/exports'; import { ConfigProvider, Space, Tabs } from 'antd'; import React, { useEffect, useState } from 'react'; +import { LayoutSize } from '@/pages/DataStudio/data.d'; export type BottomContainerProps = { - size: number; + size: LayoutSize; height: number | string; }; diff --git a/dinky-web/src/pages/DataStudio/LeftContainer/index.tsx b/dinky-web/src/pages/DataStudio/LeftContainer/index.tsx index eb9e051c4b..219674b7d3 100644 --- a/dinky-web/src/pages/DataStudio/LeftContainer/index.tsx +++ b/dinky-web/src/pages/DataStudio/LeftContainer/index.tsx @@ -27,9 +27,10 @@ import { LeftSide } from '@/pages/DataStudio/route'; import { connect } from '@@/exports'; import { Tabs } from 'antd'; import React, { useContext } from 'react'; +import { LayoutSize } from '@/pages/DataStudio/data.d'; export type LeftContainerProps = { - size: number; + size: LayoutSize; leftContainer: StateType['leftContainer']; rightContainer: StateType['rightContainer']; }; @@ -44,7 +45,9 @@ const LeftContainer: React.FC = (props: any) => { } = props; const btn = useContext(BtnContext); const themeValue = useThemeValue(); - const MAX_WIDTH = size.width - 2 * VIEW.leftToolWidth - rightContainer.width - 700; + + const rightContainerWidth = rightContainer.selectKey === '' ? 0 : rightContainer.width; + const maxWidth = size.width - 2 * VIEW.leftToolWidth - rightContainerWidth - 50; /** * Sidebar size changes * @param width @@ -70,7 +73,7 @@ const LeftContainer: React.FC = (props: any) => { * Sidebar maximization */ const handleMaxsize = () => { - handleReSizeChange(MAX_WIDTH); + handleReSizeChange(maxWidth); }; /** @@ -87,7 +90,7 @@ const LeftContainer: React.FC = (props: any) => { visible: leftContainer.selectKey !== '', defaultSize: { width: leftContainer.width, height: leftContainer.height }, minWidth: 225, - maxWidth: MAX_WIDTH, + maxWidth: maxWidth, enable: { right: true }, btnGroup: btn[leftContainer.selectKey] ? btn[leftContainer.selectKey].map((item: CircleDataStudioButtonProps) => ( diff --git a/dinky-web/src/pages/DataStudio/RightContainer/JobConfig/index.tsx b/dinky-web/src/pages/DataStudio/RightContainer/JobConfig/index.tsx index 6c82386ae0..504a727028 100644 --- a/dinky-web/src/pages/DataStudio/RightContainer/JobConfig/index.tsx +++ b/dinky-web/src/pages/DataStudio/RightContainer/JobConfig/index.tsx @@ -402,7 +402,7 @@ const JobConfig = (props: any) => { { /> @@ -454,7 +454,7 @@ const JobConfig = (props: any) => { handleNameChange(e.target.value, index)} placeholder={l('pages.datastudio.label.udf.name')} - style={{ width: calculatorWidth(rightContainer.width) - 80 }} + style={{ width: calculatorWidth(rightContainer.width) - 90 }} /> diff --git a/dinky-web/src/pages/DataStudio/RightContainer/index.tsx b/dinky-web/src/pages/DataStudio/RightContainer/index.tsx index 6a50524780..5d8aa24863 100644 --- a/dinky-web/src/pages/DataStudio/RightContainer/index.tsx +++ b/dinky-web/src/pages/DataStudio/RightContainer/index.tsx @@ -26,10 +26,11 @@ import { RightSide } from '@/pages/DataStudio/route'; import { l } from '@/utils/intl'; import { connect } from '@@/exports'; import { Tabs } from 'antd'; -import React from 'react'; +import React, { useEffect } from 'react'; +import { LayoutSize } from '@/pages/DataStudio/data.d'; export type RightContainerProps = { - size: number; + size: LayoutSize; bottomHeight: number; }; const RightContainer: React.FC = (prop: any) => { @@ -43,7 +44,11 @@ const RightContainer: React.FC = (prop: any) => { updateSelectRightKey, tabs } = prop; - const maxWidth = size.width - 2 * VIEW.leftToolWidth - leftContainer.width - 600; + + const leftContainerWidth = leftContainer.selectKey === '' ? 0 : leftContainer.width; + const maxWidth = size.width - 2 * VIEW.leftToolWidth - leftContainerWidth - 50; + + console.log(leftContainer); return ( = (props: any) => { queryTaskData, queryTaskSortTypeData, updateToolContentHeight, - updateBottomHeight, querySessionData, queryEnv, updateCenterContentHeight, @@ -81,16 +80,17 @@ const DataStudio: React.FC = (props: any) => { const persist = app._store.persist; const { fullscreen } = useEditor(); - const getClientSize = () => ({ - width: document.documentElement.clientWidth, - height: document.documentElement.clientHeight, - contentHeight: - document.documentElement.clientHeight - - VIEW.headerNavHeight - - VIEW.headerHeight - - VIEW.footerHeight - - VIEW.otherHeight - }); + const getClientSize = () => + ({ + width: document.documentElement.clientWidth, + height: document.documentElement.clientHeight, + contentHeight: + document.documentElement.clientHeight - + VIEW.headerNavHeight - + VIEW.headerHeight - + VIEW.footerHeight - + VIEW.otherHeight + }) as LayoutSize; const [size, setSize] = useState(getClientSize()); diff --git a/dinky-web/src/pages/RegCenter/Alert/AlertInstance/components/AlertInstanceList/index.tsx b/dinky-web/src/pages/RegCenter/Alert/AlertInstance/components/AlertInstanceList/index.tsx index baae203d16..723615cdef 100644 --- a/dinky-web/src/pages/RegCenter/Alert/AlertInstance/components/AlertInstanceList/index.tsx +++ b/dinky-web/src/pages/RegCenter/Alert/AlertInstance/components/AlertInstanceList/index.tsx @@ -40,8 +40,9 @@ import { ProList } from '@ant-design/pro-components'; import { ActionType } from '@ant-design/pro-table'; import { Descriptions, Input, Modal, Space, Tag, Tooltip } from 'antd'; import DescriptionsItem from 'antd/es/descriptions/Item'; -import React, { useEffect, useRef, useState } from 'react'; +import React, { useRef, useState } from 'react'; import AlertTypeChoose from '../AlertTypeChoose'; +import { useAsyncEffect } from 'ahooks'; const AlertInstanceList: React.FC = () => { /** @@ -65,11 +66,11 @@ const AlertInstanceList: React.FC = () => { ); }; - useEffect(() => { - queryAlertInstanceList(); + useAsyncEffect(async () => { + await queryAlertInstanceList(); }, []); - const executeAndCallbackRefresh = async (callback: () => void) => { + const executeAndCallbackRefresh = async (callback: () => Promise) => { setAlertInstanceState((prevState) => ({ ...prevState, loading: true })); await callback(); setAlertInstanceState((prevState) => ({ ...prevState, loading: false })); diff --git a/dinky-web/src/pages/RegCenter/Cluster/Configuration/components/ConfigurationList/index.tsx b/dinky-web/src/pages/RegCenter/Cluster/Configuration/components/ConfigurationList/index.tsx index e722060099..4aa5c88d1f 100644 --- a/dinky-web/src/pages/RegCenter/Cluster/Configuration/components/ConfigurationList/index.tsx +++ b/dinky-web/src/pages/RegCenter/Cluster/Configuration/components/ConfigurationList/index.tsx @@ -22,7 +22,7 @@ import { EditBtn } from '@/components/CallBackButton/EditBtn'; import { EnableSwitchBtn } from '@/components/CallBackButton/EnableSwitchBtn'; import { NormalDeleteBtn } from '@/components/CallBackButton/NormalDeleteBtn'; import { RunningBtn } from '@/components/CallBackButton/RunningBtn'; -import { ClusterConfigIcon } from '@/components/Icons/HomeIcon'; +import { HadoopIcon, K8sIcon } from '@/components/Icons/HomeIcon'; import { DataAction } from '@/components/StyledComponents'; import { Authorized, HasAuthority } from '@/hooks/useAccess'; import ConfigurationModal from '@/pages/RegCenter/Cluster/Configuration/components/ConfigurationModal'; @@ -45,7 +45,8 @@ import { l } from '@/utils/intl'; import { CheckCircleOutlined, ExclamationCircleOutlined, HeartTwoTone } from '@ant-design/icons'; import { ActionType, ProList } from '@ant-design/pro-components'; import { Button, Descriptions, Input, Modal, Space, Tag, Tooltip } from 'antd'; -import { useEffect, useRef, useState } from 'react'; +import { useRef, useState } from 'react'; +import { useAsyncEffect } from 'ahooks'; export default () => { /** @@ -62,8 +63,8 @@ export default () => { ); }; - useEffect(() => { - queryClusterConfigList(); + useAsyncEffect(async () => { + await queryClusterConfigList(); }, []); /** @@ -71,7 +72,7 @@ export default () => { * @param {() => void} callback * @returns {Promise} */ - const executeAndCallbackRefresh = async (callback: () => void) => { + const executeAndCallbackRefresh = async (callback: () => Promise): Promise => { setClusterConfigState((prevState) => ({ ...prevState, loading: true })); await callback(); await queryClusterConfigList(); @@ -253,13 +254,15 @@ export default () => { */ const renderData = (list: Cluster.Config[]) => list.map((item: Cluster.Config) => { + const Icon = item.type === 'yarn-application' ? HadoopIcon : K8sIcon; return { subTitle: renderDataSubTitle(item), actions: {renderDataActionButton(item)}, avatar: ( - = (props) open={visible} modalProps={{ onCancel: handleCancel, - bodyStyle: { - maxHeight: '70vh', - overflowY: 'auto', - overflowX: 'hidden' + styles: { + body: { + maxHeight: '70vh', + overflowY: 'auto', + overflowX: 'hidden' + } } }} title={value.id ? l('rc.cc.modify') : l('rc.cc.create')} diff --git a/dinky-web/src/pages/RegCenter/Cluster/Instance/components/InstanceList/index.tsx b/dinky-web/src/pages/RegCenter/Cluster/Instance/components/InstanceList/index.tsx index 2893059031..397a82fb91 100644 --- a/dinky-web/src/pages/RegCenter/Cluster/Instance/components/InstanceList/index.tsx +++ b/dinky-web/src/pages/RegCenter/Cluster/Instance/components/InstanceList/index.tsx @@ -55,6 +55,7 @@ import { Col, Descriptions, Divider, + Flex, Input, List, Row, @@ -68,7 +69,7 @@ import { useState } from 'react'; import EllipsisMiddle from '@/components/Typography/EllipsisMiddle'; import { isContainsChinese } from '@/utils/function'; -const { Text, Paragraph, Link } = Typography; +const { Paragraph, Link } = Typography; export default () => { /** @@ -92,7 +93,7 @@ export default () => { * @param {() => void} callback * @returns {Promise} */ - const executeAndCallback = async (callback: () => void) => { + const executeAndCallback = async (callback: () => Promise): Promise => { setClusterInstanceStatus((prevState) => ({ ...prevState, loading: true })); await callback(); setClusterInstanceStatus((prevState) => ({ ...prevState, loading: false })); @@ -215,7 +216,7 @@ export default () => { return ( <> - +
{l('rc.ci.jma')}: {renderWebUiRedirect(item)} @@ -240,31 +241,30 @@ export default () => { />
- - - handleChangeEnable(item)} - disabled={!HasAuthority(PermissionConstants.REGISTRATION_CLUSTER_INSTANCE_EDIT)} - /> - - {CLUSTER_TYPE_OPTIONS().find((record) => item.type === record.value)?.label} - - : } - color={item.status === 1 ? 'success' : 'warning'} - > - {item.status === 1 - ? l('global.table.status.normal') - : l('global.table.status.abnormal')} - - {renderActionButton(item)} + + handleChangeEnable(item)} + disabled={!HasAuthority(PermissionConstants.REGISTRATION_CLUSTER_INSTANCE_EDIT)} + /> + + {CLUSTER_TYPE_OPTIONS().find((record) => item.type === record.value)?.label} + + : } + color={item.status === 1 ? 'success' : 'warning'} + > + {item.status === 1 + ? l('global.table.status.normal') + : l('global.table.status.abnormal')} + + ); }; diff --git a/dinky-web/src/pages/RegCenter/Document/constans.ts b/dinky-web/src/pages/RegCenter/Document/constans.ts index cc1ca00d3a..1820e7708c 100644 --- a/dinky-web/src/pages/RegCenter/Document/constans.ts +++ b/dinky-web/src/pages/RegCenter/Document/constans.ts @@ -87,6 +87,10 @@ export const VERSIONS = [ text: 'Flink-1.19', value: '1.19' }, + { + text: 'Flink-1.20', + value: '1.20' + }, { text: 'All Versions', value: 'All Versions' From 1f23c38b8cfdddd36b495e4fadcce731ea11587b Mon Sep 17 00:00:00 2001 From: Zzm0809 <934230207@qq.com> Date: Tue, 24 Sep 2024 17:14:28 +0800 Subject: [PATCH 4/6] [Optimize][Web]Optimize task list layout (#3830) Signed-off-by: Zzm0809 <934230207@qq.com> --- .../pages/DataStudio/LeftContainer/Project/JobTree/index.tsx | 2 +- .../src/pages/DataStudio/MiddleContainer/StudioEditor/index.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dinky-web/src/pages/DataStudio/LeftContainer/Project/JobTree/index.tsx b/dinky-web/src/pages/DataStudio/LeftContainer/Project/JobTree/index.tsx index 385f685d97..e341c5616e 100644 --- a/dinky-web/src/pages/DataStudio/LeftContainer/Project/JobTree/index.tsx +++ b/dinky-web/src/pages/DataStudio/LeftContainer/Project/JobTree/index.tsx @@ -299,7 +299,7 @@ const JobTree: React.FC = (props) => { {data.length ? ( onNodeClick(info)} onRightClick={onRightClick} diff --git a/dinky-web/src/pages/DataStudio/MiddleContainer/StudioEditor/index.tsx b/dinky-web/src/pages/DataStudio/MiddleContainer/StudioEditor/index.tsx index 86ca2accf8..a4565b8867 100644 --- a/dinky-web/src/pages/DataStudio/MiddleContainer/StudioEditor/index.tsx +++ b/dinky-web/src/pages/DataStudio/MiddleContainer/StudioEditor/index.tsx @@ -182,7 +182,7 @@ const StudioEditor: React.FC = (props) => { style={{ position: 'absolute', top: 15, - right: '10%', + right: '15%', boxShadow: '0 0 10px #ccc' }} > From d9366a9474eb6de3f47f5ebd34cd6768bcd7339b Mon Sep 17 00:00:00 2001 From: ZackYoung Date: Tue, 24 Sep 2024 18:21:58 +0800 Subject: [PATCH 5/6] [Feature][Admin]Add global token (#3832) Co-authored-by: zackyoungh --- .../java/org/dinky/controller/DownloadController.java | 9 ++++++--- .../dinky/resource/impl/HttpFsDataOutputStream.java | 4 +++- .../src/main/java/org/dinky/data/enums/Status.java | 3 +++ .../java/org/dinky/data/model/SystemConfiguration.java | 5 ++++- .../src/main/resources/i18n/messages_en_US.properties | 2 ++ .../src/main/resources/i18n/messages_zh_CN.properties | 2 ++ dinky-flink/dinky-flink-1.20/pom.xml | 10 +++++----- 7 files changed, 25 insertions(+), 10 deletions(-) diff --git a/dinky-admin/src/main/java/org/dinky/controller/DownloadController.java b/dinky-admin/src/main/java/org/dinky/controller/DownloadController.java index 0cbd5eb07d..7f5e05c9bc 100644 --- a/dinky-admin/src/main/java/org/dinky/controller/DownloadController.java +++ b/dinky-admin/src/main/java/org/dinky/controller/DownloadController.java @@ -43,6 +43,7 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @@ -66,7 +67,6 @@ @Api(tags = "UDF & App Jar Controller") @RequestMapping("/download") public class DownloadController { - // todo: Controller has injection risk @GetMapping("downloadDepJar/{taskId}") @ApiOperation("Download UDF Jar") public void downloadJavaUDF(@PathVariable Integer taskId, HttpServletResponse resp) { @@ -131,12 +131,15 @@ public void downloadFromRs(String path, HttpServletResponse resp) { ServletUtil.write(resp, inputStream); } - // todo: There is a risk of injection in this interface @PostMapping("uploadFromRsByLocal") @ApiOperation("Upload From Resource By Local") @SaIgnore - public Result uploadFromRs(String path, @RequestParam("file") MultipartFile file) { + public Result uploadFromRs( + String path, @RequestParam("file") MultipartFile file, @RequestHeader("token") String token) { SystemConfiguration systemConfiguration = SystemConfiguration.getInstances(); + if (!systemConfiguration.getDinkyToken().getValue().equals(token)) { + return Result.failed("token is not correct"); + } if (!systemConfiguration.getResourcesEnable().getValue() || !systemConfiguration.getResourcesModel().getValue().equals(ResourcesModelEnum.LOCAL)) { return Result.failed("resources model is not local or resources is not enable"); diff --git a/dinky-app/dinky-app-base/src/main/java/org/dinky/resource/impl/HttpFsDataOutputStream.java b/dinky-app/dinky-app-base/src/main/java/org/dinky/resource/impl/HttpFsDataOutputStream.java index 821925d5be..653f99f19e 100644 --- a/dinky-app/dinky-app-base/src/main/java/org/dinky/resource/impl/HttpFsDataOutputStream.java +++ b/dinky-app/dinky-app-base/src/main/java/org/dinky/resource/impl/HttpFsDataOutputStream.java @@ -63,8 +63,10 @@ public void flush() throws IOException { } private void sendFile() { + SystemConfiguration systemConfiguration = SystemConfiguration.getInstances(); try (HttpResponse httpResponse = HttpUtil.createPost( - SystemConfiguration.getInstances().getDinkyAddr().getValue() + "/download/uploadFromRsByLocal") + systemConfiguration.getDinkyAddr().getValue() + "/download/uploadFromRsByLocal") + .header("token", systemConfiguration.getDinkyToken().getValue()) .form("file", file) .form("path", path.toString()) .execute()) { diff --git a/dinky-common/src/main/java/org/dinky/data/enums/Status.java b/dinky-common/src/main/java/org/dinky/data/enums/Status.java index e9d2cee6f5..bab1c383b6 100644 --- a/dinky-common/src/main/java/org/dinky/data/enums/Status.java +++ b/dinky-common/src/main/java/org/dinky/data/enums/Status.java @@ -350,6 +350,9 @@ public enum Status { SYS_ENV_SETTINGS_DINKYADDR(116, "sys.env.settings.dinkyAddr"), SYS_ENV_SETTINGS_DINKYADDR_NOTE(117, "sys.env.settings.dinkyAddr.note"), + SYS_ENV_SETTINGS_DINKYTOKEN(116, "sys.env.settings.dinkyToken"), + SYS_ENV_SETTINGS_DINKYTOKEN_NOTE(117, "sys.env.settings.dinkyToken.note"), + SYS_ENV_SETTINGS_JOB_RESEND_DIFF_SECOND(118, "sys.env.settings.jobResendDiffSecond"), SYS_ENV_SETTINGS_JOB_RESEND_DIFF_SECOND_NOTE(119, "sys.env.settings.jobResendDiffSecond.note"), diff --git a/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java b/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java index 0c4290a70f..3244d2cf85 100644 --- a/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java +++ b/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java @@ -119,11 +119,14 @@ public static Configuration.OptionBuilder key(Status status) { .stringType() .defaultValue("python3") .note(Status.SYS_ENV_SETTINGS_PYTHONHOME_NOTE); - private final Configuration dinkyAddr = key(Status.SYS_ENV_SETTINGS_DINKYADDR) .stringType() .defaultValue(System.getProperty("dinkyAddr")) .note(Status.SYS_ENV_SETTINGS_DINKYADDR_NOTE); + private final Configuration dinkyToken = key(Status.SYS_ENV_SETTINGS_DINKYTOKEN_NOTE) + .stringType() + .defaultValue("efda1551-7958-4e0f-80a8-dfd107df3e38") + .note(Status.SYS_ENV_SETTINGS_DINKYTOKEN); private final Configuration jobReSendDiffSecond = key(Status.SYS_ENV_SETTINGS_JOB_RESEND_DIFF_SECOND) .intType() diff --git a/dinky-common/src/main/resources/i18n/messages_en_US.properties b/dinky-common/src/main/resources/i18n/messages_en_US.properties index 7ca751290e..ed40a4ec95 100644 --- a/dinky-common/src/main/resources/i18n/messages_en_US.properties +++ b/dinky-common/src/main/resources/i18n/messages_en_US.properties @@ -200,6 +200,8 @@ sys.env.settings.pythonHome=Python Env variables sys.env.settings.pythonHome.note=Python environment variables, used to submit Python tasks and build Python Udf sys.env.settings.dinkyAddr=Dinky Address sys.env.settings.dinkyAddr.note=This address is an accessible Dinky address, such as http://127.0.0.1:8888 +sys.env.settings.dinkyToken=Dinky Token +sys.env.settings.dinkyToken.note=This token is used to access some interfaces within Dinky, such as uploading resources sys.env.settings.jobResendDiffSecond=Alert anti-resend interval sys.env.settings.jobResendDiffSecond.note=During this interval, when the Alert information sent reaches the configured value of [Maximum number of alarm resend prevention], after reaching the threshold, the Alert information will no longer be sent; unit: seconds sys.env.settings.diffMinuteMaxSendCount=Maximum number of alarms to prevent resending diff --git a/dinky-common/src/main/resources/i18n/messages_zh_CN.properties b/dinky-common/src/main/resources/i18n/messages_zh_CN.properties index 321f0ff425..50f82d25d3 100644 --- a/dinky-common/src/main/resources/i18n/messages_zh_CN.properties +++ b/dinky-common/src/main/resources/i18n/messages_zh_CN.properties @@ -200,6 +200,8 @@ sys.env.settings.pythonHome=Python 环境变量 sys.env.settings.pythonHome.note=Python 环境变量,用于提交 Python 任务以及构建 Python Udf sys.env.settings.dinkyAddr=Dinky 地址 sys.env.settings.dinkyAddr.note=该地址为可访问的 Dinky 地址，如 http://127.0.0.1:8888 +sys.env.settings.dinkyToken=Dinky Token +sys.env.settings.dinkyToken.note = 此 token 用于访问 Dinky 内的一些接口，如上传资源等 sys.env.settings.jobResendDiffSecond=告警防重发间隔 sys.env.settings.jobResendDiffSecond.note=在此间隔内,发送告警信息达到 [告警防重发最大条数] 配置的值时,达到阈值后,不再发送告警信息; 单位：秒 sys.env.settings.diffMinuteMaxSendCount=告警防重发最大条数 diff --git a/dinky-flink/dinky-flink-1.20/pom.xml b/dinky-flink/dinky-flink-1.20/pom.xml index 1db51f7273..24a263c5b1 100644 --- a/dinky-flink/dinky-flink-1.20/pom.xml +++ b/dinky-flink/dinky-flink-1.20/pom.xml @@ -84,6 +84,11 @@ flink-connector-kafka 3.2.0-1.19 + + org.apache.doris + flink-doris-connector-1.20 + 24.0.0 + org.apache.flink flink-shaded-guava @@ -138,11 +143,6 @@ commons-cli ${commons.version} - - org.apache.doris - flink-doris-connector-1.19 - 1.6.2 - org.apache.flink flink-runtime-web From cf80e450ff3d62dfac509f4a9d6bb5a04040fd00 Mon Sep 17 00:00:00 2001 From: zhuxt2015 <594754793@qq.com> Date: Wed, 25 Sep 2024 10:51:46 +0800 Subject: [PATCH 6/6] [Improvement][data studio]oracle timestamp column type order is changed to precede time column (#3821) --- .../main/java/org/dinky/metadata/convert/OracleTypeConvert.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dinky-metadata/dinky-metadata-oracle/src/main/java/org/dinky/metadata/convert/OracleTypeConvert.java b/dinky-metadata/dinky-metadata-oracle/src/main/java/org/dinky/metadata/convert/OracleTypeConvert.java index 5f1bb59cd4..78e9115e6b 100644 --- a/dinky-metadata/dinky-metadata-oracle/src/main/java/org/dinky/metadata/convert/OracleTypeConvert.java +++ b/dinky-metadata/dinky-metadata-oracle/src/main/java/org/dinky/metadata/convert/OracleTypeConvert.java @@ -36,8 +36,8 @@ public OracleTypeConvert() { this.convertMap.clear(); register("char", ColumnType.STRING); register("date", ColumnType.LOCAL_DATETIME); - register("time", ColumnType.TIME); register("timestamp", ColumnType.TIMESTAMP); + register("time", ColumnType.TIME); register("number", OracleTypeConvert::convertNumber); register("float", ColumnType.JAVA_LANG_FLOAT); register("clob", ColumnType.STRING);