diff --git a/dinky-admin/pom.xml b/dinky-admin/pom.xml index 3f377b0cb66..59823819164 100644 --- a/dinky-admin/pom.xml +++ b/dinky-admin/pom.xml @@ -51,6 +51,12 @@ cn.dev33 sa-token-jwt + + + cn.hutool + hutool-jwt + + com.alibaba @@ -170,6 +176,20 @@ org.dinky dinky-core + + + cn.hutool + hutool-crypto + + + cn.hutool + hutool-http + + + cn.hutool + hutool-json + + org.dinky diff --git a/dinky-admin/src/test/java/org/dinky/utils/SqlUtilTest.java b/dinky-admin/src/test/java/org/dinky/utils/SqlUtilTest.java index afa7bf3583a..e1f6b336dac 100644 --- a/dinky-admin/src/test/java/org/dinky/utils/SqlUtilTest.java +++ b/dinky-admin/src/test/java/org/dinky/utils/SqlUtilTest.java @@ -19,6 +19,10 @@ package org.dinky.utils; +import static org.dinky.data.enums.Status.SYS_FLINK_SETTINGS_SQLSEPARATOR; + +import org.dinky.data.model.SystemConfiguration; + import org.assertj.core.api.Assertions; import org.junit.Ignore; import org.junit.Test; @@ -45,4 +49,15 @@ public void testRemoveNote() { Assertions.assertThat(removedNoteSql).isNotNull(); Assertions.assertThat(removedNoteSql).isNotEqualTo(testSql); } + + @Test + public void getStatements() { + String sql = "set 'state.savepoints.dir' = 'hdfs://namenode:9000/tmp/checkpoint'; --ddd\n" + + "set 'state.checkpoints.dir' = 'hdfs://namenode:9000/tmp/checkpoint'; --dd \n" + + "create table abc ;\n"; + SystemConfiguration.getInstances() + .setConfiguration(SYS_FLINK_SETTINGS_SQLSEPARATOR.getKey(), ";\\s*(?:\\n|--.*)"); + String[] statements = SqlUtil.getStatements(sql); + Assertions.assertThat(statements.length).isEqualTo(3); + } } diff --git a/dinky-app/dinky-app-base/pom.xml b/dinky-app/dinky-app-base/pom.xml index e10b85176cc..973e1eb192f 100644 --- a/dinky-app/dinky-app-base/pom.xml +++ b/dinky-app/dinky-app-base/pom.xml @@ -93,8 +93,143 @@ cn.hutool hutool-all + + + org.dinky + dinky-metadata-base + + + * + * + + + + + org.dinky + dinky-metadata-clickhouse + + + * + * + + + + + org.dinky + dinky-metadata-doris + + + * + * + + + + + org.dinky + dinky-metadata-hive + + + * + * + + + + + org.dinky + dinky-metadata-mysql + + + * + * + + + + + org.dinky + dinky-metadata-oracle + + + * + * + + + + + org.dinky + dinky-metadata-phoenix + + + * + * + + + + + org.dinky + dinky-metadata-postgresql + + + * + * + + + + + org.dinky + dinky-metadata-presto + + + * + * + + + + + org.dinky + dinky-metadata-sqlserver + + + * + * + + + + + org.dinky + dinky-metadata-starrocks + + + * + * + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + false + + + + + shade + + package + + + + + + + + + + flink-single-version diff --git a/dinky-app/dinky-app-base/src/main/java/org/dinky/app/util/FlinkAppUtil.java b/dinky-app/dinky-app-base/src/main/java/org/dinky/app/util/FlinkAppUtil.java index abcb3162bf0..9ac3c9ffe7c 100644 --- a/dinky-app/dinky-app-base/src/main/java/org/dinky/app/util/FlinkAppUtil.java +++ b/dinky-app/dinky-app-base/src/main/java/org/dinky/app/util/FlinkAppUtil.java @@ -79,7 +79,6 @@ public static void monitorFlinkTask(Executor executor, int taskId) { public static void monitorFlinkTask(JobClient jobClient, int taskId) { boolean isRun = true; String jobId = jobClient.getJobID().toHexString(); - try { while (isRun) { String jobStatus = jobClient.getJobStatus().get().toString(); diff --git a/dinky-cdc/dinky-cdc-core/src/main/java/org/dinky/cdc/utils/FlinkStatementUtil.java b/dinky-cdc/dinky-cdc-core/src/main/java/org/dinky/cdc/utils/FlinkStatementUtil.java index 8154acdd98b..05a2c2cd308 100644 --- a/dinky-cdc/dinky-cdc-core/src/main/java/org/dinky/cdc/utils/FlinkStatementUtil.java +++ b/dinky-cdc/dinky-cdc-core/src/main/java/org/dinky/cdc/utils/FlinkStatementUtil.java @@ -35,7 +35,7 @@ private FlinkStatementUtil() {} public static String getCDCInsertSql(Table table, String targetName, String sourceName) { StringBuilder sb = new StringBuilder("INSERT INTO "); - sb.append(targetName); + sb.append("`").append(targetName).append("`"); sb.append(" SELECT\n"); for (int i = 0; i < table.getColumns().size(); i++) { sb.append(" "); diff --git a/dinky-client/dinky-client-base/src/main/java/org/dinky/sql/FlinkQuery.java b/dinky-client/dinky-client-base/src/main/java/org/dinky/sql/FlinkQuery.java index f0e457ed1f1..ce7d6e74217 100644 --- a/dinky-client/dinky-client-base/src/main/java/org/dinky/sql/FlinkQuery.java +++ b/dinky-client/dinky-client-base/src/main/java/org/dinky/sql/FlinkQuery.java @@ -19,6 +19,8 @@ package org.dinky.sql; +import org.dinky.data.model.SystemConfiguration; + /** * FlinkQuery * @@ -27,7 +29,7 @@ public class FlinkQuery { public static String separator() { - return ";\n"; + return SystemConfiguration.getInstances().getSqlSeparator(); } public static String defaultCatalog() { diff --git a/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java b/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java index dbdcaf5fb07..3148363180c 100644 --- a/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java +++ b/dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java @@ -68,7 +68,7 @@ public static Configuration.OptionBuilder key(Status status) { .note(Status.SYS_FLINK_SETTINGS_USERESTAPI_NOTE); private final Configuration sqlSeparator = key(Status.SYS_FLINK_SETTINGS_SQLSEPARATOR) .stringType() - .defaultValue(";\\n") + .defaultValue(";\\s*(?:\\n|--.*)") .note(Status.SYS_FLINK_SETTINGS_SQLSEPARATOR_NOTE); private final Configuration jobIdWait = key(Status.SYS_FLINK_SETTINGS_JOBIDWAIT) .intType() diff --git a/dinky-common/src/main/java/org/dinky/utils/SqlUtil.java b/dinky-common/src/main/java/org/dinky/utils/SqlUtil.java index 5e5703dac19..46f6c7e578f 100644 --- a/dinky-common/src/main/java/org/dinky/utils/SqlUtil.java +++ b/dinky-common/src/main/java/org/dinky/utils/SqlUtil.java @@ -45,7 +45,7 @@ public static String[] getStatements(String sql, String sqlSeparator) { return new String[0]; } - String[] splits = sql.replace(";\r\n", ";\n").split(sqlSeparator); + String[] splits = sql.replace("\r\n", "\n").split(sqlSeparator); String lastStatement = splits[splits.length - 1].trim(); if (lastStatement.endsWith(SEMICOLON)) { splits[splits.length - 1] = lastStatement.substring(0, lastStatement.length() - 1); diff --git a/dinky-core/src/main/java/org/dinky/constant/FlinkSQLConstant.java b/dinky-core/src/main/java/org/dinky/constant/FlinkSQLConstant.java index 0db0b7f5c2a..dc79bd26544 100644 --- a/dinky-core/src/main/java/org/dinky/constant/FlinkSQLConstant.java +++ b/dinky-core/src/main/java/org/dinky/constant/FlinkSQLConstant.java @@ -19,21 +19,25 @@ package org.dinky.constant; +import org.dinky.sql.FlinkQuery; + /** * FlinkSQLConstant * * @since 2021/5/25 15:51 */ -public interface FlinkSQLConstant { +public class FlinkSQLConstant { + private FlinkSQLConstant() {} + /** 分隔符 */ - String SEPARATOR = ";\n"; + public static final String SEPARATOR = FlinkQuery.separator(); /** DDL 类型 */ - String DDL = "DDL"; + public static final String DDL = "DDL"; /** DML 类型 */ - String DML = "DML"; + public static final String DML = "DML"; /** DATASTREAM 类型 */ - String DATASTREAM = "DATASTREAM"; + public static final String DATASTREAM = "DATASTREAM"; /** The define identifier of FlinkSQL Variable */ - String VARIABLES = ":="; + public static final String VARIABLES = ":="; } diff --git a/dinky-core/src/main/java/org/dinky/trans/ddl/CreateCDCSourceOperation.java b/dinky-core/src/main/java/org/dinky/trans/ddl/CreateCDCSourceOperation.java index a04352b7b6f..8829d926b4c 100644 --- a/dinky-core/src/main/java/org/dinky/trans/ddl/CreateCDCSourceOperation.java +++ b/dinky-core/src/main/java/org/dinky/trans/ddl/CreateCDCSourceOperation.java @@ -123,9 +123,9 @@ public TableResult execute(Executor executor) { // 分库分表所有表结构都是一样的，取出列表中第一个表名即可 String schemaTableName = table.getSchemaTableNameList().get(0); // 真实的表名 + String realSchemaName = schemaTableName.split("\\.")[0]; String tableName = schemaTableName.split("\\.")[1]; - table.setColumns(driver.listColumnsSortByPK(schemaName, tableName)); - table.setColumns(driver.listColumnsSortByPK(schemaName, table.getName())); + table.setColumns(driver.listColumnsSortByPK(realSchemaName, tableName)); schemaList.add(schema); if (null != sinkDriver) { diff --git a/dinky-gateway/src/main/java/org/dinky/gateway/yarn/YarnGateway.java b/dinky-gateway/src/main/java/org/dinky/gateway/yarn/YarnGateway.java index d8eacc34dff..8989fef7f76 100644 --- a/dinky-gateway/src/main/java/org/dinky/gateway/yarn/YarnGateway.java +++ b/dinky-gateway/src/main/java/org/dinky/gateway/yarn/YarnGateway.java @@ -35,7 +35,6 @@ import org.dinky.gateway.result.TestResult; import org.dinky.gateway.result.YarnResult; import org.dinky.utils.FlinkJsonUtil; -import org.dinky.utils.ThreadUtil; import org.apache.flink.client.deployment.ClusterRetrieveException; import org.apache.flink.client.program.ClusterClient; @@ -340,32 +339,33 @@ protected String getWebUrl(ClusterClient clusterClient, YarnResul && counts-- > 0) { Thread.sleep(1000); } - // 睡眠2秒，防止application快速识别抛出的错误 - ThreadUtil.sleep(2000); - ApplicationReport applicationReport = yarnClient.getApplicationReport(clusterClient.getClusterId()); - if (applicationReport.getYarnApplicationState() != YarnApplicationState.RUNNING) { - String logUrl = yarnClient - .getContainers(applicationReport.getCurrentApplicationAttemptId()) - .get(0) - .getLogUrl(); - String log = ReUtil.getGroup1(HTML_TAG_REGEX, HttpUtil.get(logUrl + "/jobmanager.log?start=-10000")); - logger.error("\n\nHistory log url is: {}\n\n ", logUrl); - throw new RuntimeException( - "Yarn application state is not running, please check yarn cluster status. Log content:\n" + log); - } - webUrl = applicationReport.getOriginalTrackingUrl(); + webUrl = clusterClient.getWebInterfaceURL(); final List jobDetailsList = new ArrayList<>(); while (jobDetailsList.isEmpty() && counts-- > 0) { + ApplicationReport applicationReport = yarnClient.getApplicationReport(clusterClient.getClusterId()); + if (applicationReport.getYarnApplicationState() != YarnApplicationState.RUNNING) { + String log = getYarnContainerLog(applicationReport); + throw new RuntimeException( + "Yarn application state is not running, please check yarn cluster status. Log content:\n" + + log); + } + // 睡眠1秒，防止flink因为依赖或其他问题导致任务秒挂 Thread.sleep(1000); - String url = yarnClient .getApplicationReport(clusterClient.getClusterId()) .getTrackingUrl() + JobsOverviewHeaders.URL.substring(1); String json = HttpUtil.get(url); - MultipleJobsDetails jobsDetails = FlinkJsonUtil.toBean(json, JobsOverviewHeaders.getInstance()); - jobDetailsList.addAll(jobsDetails.getJobs()); + try { + MultipleJobsDetails jobsDetails = FlinkJsonUtil.toBean(json, JobsOverviewHeaders.getInstance()); + jobDetailsList.addAll(jobsDetails.getJobs()); + } catch (Exception e) { + Thread.sleep(1000); + String log = getYarnContainerLog(applicationReport); + logger.error("Yarn application state is not running, please check yarn cluster status. Log content:\n" + + log); + } if (!jobDetailsList.isEmpty()) { break; } @@ -380,4 +380,15 @@ protected String getWebUrl(ClusterClient clusterClient, YarnResul } return webUrl; } + + protected String getYarnContainerLog(ApplicationReport applicationReport) throws YarnException, IOException { + String logUrl = yarnClient + .getContainers(applicationReport.getCurrentApplicationAttemptId()) + .get(0) + .getLogUrl(); + String content = HttpUtil.get(logUrl + "/jobmanager.log?start=-10000"); + String log = ReUtil.getGroup1(HTML_TAG_REGEX, content); + logger.info("\n\nHistory log url is: {}\n\n ", logUrl); + return log; + } } diff --git a/dinky-web/config/routes.ts b/dinky-web/config/routes.ts index 852bcea1a6b..48ebdd3821e 100644 --- a/dinky-web/config/routes.ts +++ b/dinky-web/config/routes.ts @@ -41,8 +41,14 @@ export default [ ] }, { - path: '/', - redirect: '/datastudio' + path: '/', + redirect: '/redirect', + }, + { + path: '/redirect', + component: './Other/Redirect', + layout: false, + hideInMenu: true, }, // { // path: '/home', diff --git a/dinky-web/src/app.tsx b/dinky-web/src/app.tsx index e038aab6d75..9e9926fd5cd 100644 --- a/dinky-web/src/app.tsx +++ b/dinky-web/src/app.tsx @@ -37,6 +37,7 @@ import { FullScreenProvider } from './hooks/useEditor'; import { errorConfig } from './requestErrorConfig'; import { getDataByParamsReturnResult } from './services/BusinessCrud'; import { API } from './services/data'; +import Redirect from './pages/Other/Redirect'; // const isDev = process.env.NODE_ENV === "development"; const loginPath = API_CONSTANTS.LOGIN_PATH; @@ -208,7 +209,7 @@ export const dva = { * 动态修改默认跳转路由 */ const patch = (oldRoutes: any, routes: SysMenu[]) => { - oldRoutes[1].routes = oldRoutes[1].routes.map( + oldRoutes[1].routes = oldRoutes[1]?.routes?.map( (route: { routes: { path: any; element: JSX.Element }[]; path: string }) => { if (route.routes?.length) { const redirect = routes?.filter((r) => r.path.startsWith(route.path)); @@ -230,7 +231,9 @@ const patch = (oldRoutes: any, routes: SysMenu[]) => { */ export function patchClientRoutes({ routes }: { routes: SysMenu[] }) { // 根据 extraRoutes 对 routes 做一些修改 - patch(routes, extraRoutes); + if(extraRoutes.length){ + patch(routes, extraRoutes); + } } /*** diff --git a/dinky-web/src/components/Flink/FlinkDag/index.tsx b/dinky-web/src/components/Flink/FlinkDag/index.tsx index 32b74c7b77e..01ce0836d91 100644 --- a/dinky-web/src/components/Flink/FlinkDag/index.tsx +++ b/dinky-web/src/components/Flink/FlinkDag/index.tsx @@ -87,7 +87,7 @@ const RenderCheckpoint = (id: string, checkPoints: any) => { label: y, children: ( { return { title: z, diff --git a/dinky-web/src/global.less b/dinky-web/src/global.less index 7269a05ae1c..927094bd199 100644 --- a/dinky-web/src/global.less +++ b/dinky-web/src/global.less @@ -408,7 +408,7 @@ h5 { .ant-tabs-nav { .ant-tabs-nav-wrap { .ant-tabs-tab { - padding: 4px 8px; + padding: 8px 8px; } .ant-tabs-tab-active { diff --git a/dinky-web/src/pages/AuthCenter/Role/components/AssignMenu/index.tsx b/dinky-web/src/pages/AuthCenter/Role/components/AssignMenu/index.tsx index dd35e2291a9..56c70c51e3a 100644 --- a/dinky-web/src/pages/AuthCenter/Role/components/AssignMenu/index.tsx +++ b/dinky-web/src/pages/AuthCenter/Role/components/AssignMenu/index.tsx @@ -116,6 +116,35 @@ const AssignMenu: React.FC = (props) => { setRoleAssignMenu((prevState) => ({ ...prevState, selectValue: selectKeys })); }; + const treeData = buildMenuTree( + sortTreeData(roleAssignMenu.menuTreeData.menus), + roleAssignMenu.searchValue + ); + + const treeToArray = (list, newArr = []) => { + list.forEach((item) => { + const { children } = item; + if (children) { + if (children.length) { + newArr.push(item); + return treeToArray(children, newArr); + } + } + newArr.push(item); + }); + return newArr; + }; + + const filterHalfKeys: any = useCallback( + (keys) => { + const treeArray = treeToArray(treeData); + return keys.filter((key) => + treeArray.some((tree) => tree.value == key && !tree.isLeaf) + ); + }, + [treeData] + ); + return ( = (props) => { onCheck(keys)} + onCheck={(keys, e) => { + onCheck(keys?.concat(e.halfCheckedKeys)); + }} multiple={true} className={'treeList'} - treeData={buildMenuTree( - sortTreeData(roleAssignMenu.menuTreeData.menus), - roleAssignMenu.searchValue - )} + treeData={treeData} /> diff --git a/dinky-web/src/pages/DataStudio/LeftContainer/Project/JobTree/index.tsx b/dinky-web/src/pages/DataStudio/LeftContainer/Project/JobTree/index.tsx index f5e0cacd43c..d59107d8efb 100644 --- a/dinky-web/src/pages/DataStudio/LeftContainer/Project/JobTree/index.tsx +++ b/dinky-web/src/pages/DataStudio/LeftContainer/Project/JobTree/index.tsx @@ -159,7 +159,7 @@ const JobTree: React.FC = (props) => { selectedKeys={selectKey} onExpand={onExpand} treeData={data} - autoExpandParent={autoExpandParent} + // autoExpandParent={autoExpandParent} /> ) : ( { activeKey === item.key ? fullscreen ? document.body.clientHeight - : props.centerContentHeight - 40 + : props.centerContentHeight - 48 : 0 } /> diff --git a/dinky-web/src/pages/DataStudio/model.ts b/dinky-web/src/pages/DataStudio/model.ts index 3bfb84737ef..25ec3ed70f4 100644 --- a/dinky-web/src/pages/DataStudio/model.ts +++ b/dinky-web/src/pages/DataStudio/model.ts @@ -52,8 +52,8 @@ export const VIEW = { marginTop: 84, topHeight: 35.6, bottomHeight: 100, - rightMargin: 32, - leftMargin: 36, + rightMargin: 42, + leftMargin: 42, midMargin: 44, otherHeight: 0, paddingInline: 50 diff --git a/dinky-web/src/pages/Other/Redirect/index.tsx b/dinky-web/src/pages/Other/Redirect/index.tsx new file mode 100644 index 00000000000..6c0605a0c76 --- /dev/null +++ b/dinky-web/src/pages/Other/Redirect/index.tsx @@ -0,0 +1,40 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { SysMenu } from '@/types/AuthCenter/data'; +import { Navigate, useModel } from 'umi'; + +const Redirect = () => { + const { initialState, _ } = useModel('@@initialState'); + + console.log(initialState); + + const filterMenus = (menus: SysMenu[]) => { + return menus?.filter((menu) => menu.type !== 'F'); + }; + let extraRoutes = filterMenus(initialState?.currentUser?.menuList); + + if (initialState?.currentUser?.user?.superAdminFlag) { + return ; + } + + return ; +}; + +export default Redirect; diff --git a/pom.xml b/pom.xml index eb86626455f..20565858074 100644 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,7 @@ 1.6.2 2.3.0 2.9.0 - 5.13.1.202206130422-r + 5.13.2.202306221912-r 5.9.1 4.1.0 5.12.4