diff --git a/README.md b/README.md index 7634191d..a174ee03 100644 --- a/README.md +++ b/README.md @@ -35,16 +35,16 @@ The Dimension node, Transform node, Sink node and [Visualis](https://github.com/        Supports multi-version management, full life cycle management, monitoring alarm, checkpoint and savepoint management capabilities of streaming jobs. -![prod center](docs/images/stream_product_center.png) +![prod center](docs/images/stream_product_center_en.png)        Running information page: -![Running information](docs/images/stream_job_detail.png) +![Running information](docs/images/stream_job_detail_en.png)        Configurations page： -![Configurations](docs/images/stream_job_config_1.png) -![Configurations](docs/images/stream_job_config_2.png) +![Configurations](docs/images/stream_job_config_en_1.png) +![Configurations](docs/images/stream_job_config_en_2.png)        For more features, please refer to: [User Manual](docs/en_US/userManual/StreamisUserManual.md). diff --git a/bin/upgrade.sh b/bin/upgrade.sh new file mode 100644 index 00000000..3a71a8ee --- /dev/null +++ b/bin/upgrade.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# +# Copyright 2022 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Use to upgrade from 0.1.0 to 0.2.0 + +if [ -f "~/.bashrc" ];then + echo "Warning! user bashrc file does not exist." +else + source ~/.bashrc +fi + +shellDir=`dirname $0` +workDir=`cd ${shellDir}/..;pwd` + +interact_echo(){ + while [ 1 ]; do + read -p "$1 (Y/N)" yn + if [[ "${yn}x" == "Yx" ]] || [[ "${yn}x" == "yx" ]]; then + return 0 + elif [[ "${yn}x" == "Nx" ]] || [[ "${yn}x" == "nx" ]]; then + return 1 + else + echo "Unknown choose: [$yn], please choose again." + fi + done +} + +interact_echo "Are you sure the current version of Streamis is 0.1.0 and need to upgrade to 0.2.0 ?" +if [[ $? == 0 ]]; then + source ${workDir}/conf/db.sh + echo "<------ Will connect to [${MYSQL_HOST}:${MYSQL_PORT}] to upgrade the tables in database... ------>" + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 << EOF 1>/dev/null + /*Modify the table column*/ + ALTER TABLE \`linkis_stream_job\` MODIFY COLUMN \`project_name\` varchar(100) DEFAULT NULL; + ALTER TABLE \`linkis_stream_job\` MODIFY COLUMN \`name\` varchar(200) DEFAULT NULL; + ALTER TABLE \`linkis_stream_project\` MODIFY COLUMN \`name\` varchar(100) DEFAULT NULL; + ALTER TABLE \`linkis_stream_task\` MODIFY COLUMN \`job_id\` varchar(200) DEFAULT NULL; + ALTER TABLE \`linkis_stream_task\` MODIFY COLUMN \`linkis_job_id\` varchar(200) DEFAULT NULL; + + ALTER TABLE \`linkis_stream_project\` ADD create_time datetime DEFAULT NULL; + ALTER TABLE \`linkis_stream_project\` ADD last_update_by varchar(50) DEFAULT NULL; + ALTER TABLE \`linkis_stream_project\` ADD last_update_time datetime DEFAULT NULL; + ALTER TABLE \`linkis_stream_project\` ADD is_deleted tinyint unsigned DEFAULT 0; + + /*Add indexes into the tables*/ + ALTER TABLE \`linkis_stream_job\` ADD UNIQUE KEY(\`project_name\`, \`name\`); + ALTER TABLE \`linkis_stream_job_version\` ADD UNIQUE KEY(\`job_id\`, \`version\`); + + /*Add new tables*/ + DROP TABLE IF EXISTS \`linkis_stream_project_privilege\`; + CREATE TABLE \`linkis_stream_project_privilege\` ( + \`id\` bigint(20) NOT NULL AUTO_INCREMENT, + \`project_id\` bigint(20) NOT NULL, + \`user_name\` varchar(100) NOT NULL, + \`privilege\` tinyint(1) DEFAULT '0' NOT NULL COMMENT '1:发布权限 ，2:编辑权限 ，3:查看权限', + PRIMARY KEY (\`id\`) USING BTREE + ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='项目权限表'; + + DROP TABLE IF EXISTS \`linkis_stream_job_config_def\`; + CREATE TABLE \`linkis_stream_job_config_def\` ( + \`id\` bigint(20) NOT NULL AUTO_INCREMENT, + \`key\` varchar(100) COLLATE utf8_bin NOT NULL, + \`name\` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT 'Equals option', + \`type\` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT 'NONE' COMMENT 'def type, NONE: 0, INPUT: 1, SELECT: 2', + \`sort\` int(10) DEFAULT '0' COMMENT 'In order to sort the configurations that have the same level', + \`description\` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT 'Description of configuration', + \`validate_type\` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT 'Method the validate the configuration', + \`validate_rule\` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT 'Value of validation rule', + \`style\` varchar(200) COLLATE utf8_bin DEFAULT '' COMMENT 'Display style', + \`visiable\` tinyint(1) NOT NULL DEFAULT '1' COMMENT '0: hidden, 1: display', + \`level\` tinyint(1) NOT NULL DEFAULT '1' COMMENT '0: root, 1: leaf', + \`unit\` varchar(25) COLLATE utf8_bin DEFAULT NULL COMMENT 'Unit symbol', + \`default_value\` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT 'Default value', + \`ref_values\` varchar(200) COLLATE utf8_bin DEFAULT '', + \`parent_ref\` bigint(20) DEFAULT NULL COMMENT 'Parent key of configuration def', + \`required\` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'If the value of configuration is necessary', + \`is_temp\` tinyint(1) DEFAULT '0' COMMENT 'Temp configuration', + PRIMARY KEY (\`id\`), + UNIQUE KEY \`config_def_key\` (\`key\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + DROP TABLE IF EXISTS \`linkis_stream_job_config\`; + CREATE TABLE \`linkis_stream_job_config\` ( + \`job_id\` bigint(20) NOT NULL, + \`job_name\` varchar(200) COLLATE utf8_bin NOT NULL COMMENT 'Just store the job name', + \`key\` varchar(100) COLLATE utf8_bin NOT NULL, + \`value\` varchar(500) COLLATE utf8_bin NOT NULL, + \`ref_def_id\` bigint(20) DEFAULT NULL COMMENT 'Refer to id in config_def table', + PRIMARY KEY (\`job_id\`,\`key\`), + KEY \`config_def_id\` (\`ref_def_id\`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + /*Execute dml*/ + source ${workDir}/db/streamis_dml.sql + + /*Data migration*/ + INSERT INTO \`linkis_stream_job_config\`(\`key\`, \`value\`, \`job_id\`, \`job_name\`, \`ref_def_id\`) SELECT ov.config_key, ov.config_value, ov.job_id, ov.job_name, d.id as refer_id from linkis_stream_configuration_config_value ov left join linkis_stream_job_config_def d on ov.config_key = d.key WHERE ov.config_value IS NOT NULL AND ov.job_name IS NOT NULL GROUP BY ov.job_id,ov.config_key; + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.memory" WHERE \`key\` = "flink.taskmanager.memory"; + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.cpus" WHERE \`key\` = "flink.taskmanager.cpu.cores"; + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.cpus" WHERE \`key\` = "wds.linkis.flink.taskManager.cpus"; + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.numberOfTaskSlots" WHERE \`key\` = "flink.taskmanager.numberOfTaskSlots"; + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.app.parallelism" WHERE \`key\` = "wds.linkis.engineconn.flink.app.parallelism"; + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.jobmanager.memory" WHERE \`key\` = "flink.jobmanager.memory"; + UPDATE linkis_stream_job_config c SET \`ref_def_id\` = (SELECT d.id FROM linkis_stream_job_config_def d WHERE d.\`key\` = c.\`key\`) WHERE c.ref_def_id IS NULL; + SELECT @flink_extra_param_id:=id FROM linkis_stream_job_config_def WHERE \`key\` = "wds.linkis.flink.custom"; + UPDATE linkis_stream_job_config SET ref_def_id = @flink_extra_param_id WHERE ref_def_id IS NULL; + + /*Drop tables*/ + /*DROP TABLE \`linkis_stream_configuration_config_key\`*/ + /*DROP TABLE \`linkis_stream_configuration_config_value\`*/ + + /*update tables data*/ + delimiter %% + + create procedure update_project() + BEGIN + -- 声明变量 + DECLARE projectname varchar(50); + DECLARE done INT default 0; + + -- 创建游标，并设置游标所指的数据 + DECLARE cur CURSOR for + SELECT distinct j.project_name from linkis_stream_job j; + -- 游标执行完，即遍历结束。设置done的值为1 + DECLARE CONTINUE HANDLER for not FOUND set done = 1; + -- 开启游标 + open cur; + -- 执行循环 + posLoop: + LOOP + -- 从游标中取出projectname + FETCH cur INTO projectname ; + -- 如果done的值为1，即遍历结束，结束循环 + IF done = 1 THEN + LEAVE posLoop; + -- 注意，if语句需要添加END IF结束IF + END IF; + insert into linkis_stream_project(\`name\`,\`create_by\`,\`create_time\`) values (projectname,\'system\',now()); + -- 关闭循环 + END LOOP posLoop; + -- 关闭游标 + CLOSE cur; + -- 关闭分隔标记 + END %% + + create procedure update_project_privilege() + BEGIN + -- 声明变量 + DECLARE projectid bigint(20); + DECLARE create_by varchar(50); + DECLARE done INT default 0; + + -- 创建游标，并设置游标所指的数据 + DECLARE cur CURSOR for + SELECT distinct p.id,j.create_by from linkis_stream_project p,linkis_stream_job j where p.name =j.project_name ; + -- 游标执行完，即遍历结束。设置done的值为1 + DECLARE CONTINUE HANDLER for not FOUND set done = 1; + -- 开启游标 + open cur; + -- 执行循环 + posLoop: + LOOP + -- 从游标中取出id + FETCH cur INTO projectid ,create_by; + -- 如果done的值为1，即遍历结束，结束循环 + IF done = 1 THEN + LEAVE posLoop; + -- 注意，if语句需要添加END IF结束IF + END IF; + + insert into linkis_stream_project_privilege (project_id ,user_name ,privilege) values (projectid,create_by,2); + -- 关闭循环 + END LOOP posLoop; + -- 关闭游标 + CLOSE cur; + -- 关闭分隔标记 + END %% + delimiter ; + + call update_project; + call update_project_privilege; + + drop PROCEDURE update_project; + drop PROCEDURE update_project_privilege; + +EOF + echo "<------ End to upgrade ------>" +fi + + + diff --git a/docs/en_US/0.2.0/development/StreamisUpgradeDocumentation.md b/docs/en_US/0.2.0/development/StreamisUpgradeDocumentation.md new file mode 100644 index 00000000..db816d7a --- /dev/null +++ b/docs/en_US/0.2.0/development/StreamisUpgradeDocumentation.md @@ -0,0 +1,44 @@ +Streamis upgrade document. This article mainly introduces the upgrade steps of adapting DSS1.1.0 and linkis1.1.1 based on the original installation of Streamis service. The biggest difference between Streamis 0.2.0 and Streamis 0.1.0 is that it accesses DSS appconn and optimizes the start and stop of jobs. + +# 1. Work before upgrading streamis +Before upgrading Streamis, please install linkis1.1.1 and DSS1.1.0 or above, and ensure that the linkis Flink engine and DSS can be used normally. For the installation of DSS and linkis, please refer to [dss & linkis one click installation and deployment document](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2/DSS%E5%8D%95%E6%9C%BA%E9%83%A8%E7%BD%B2%E6%96%87%E6%A1%A3.md). + +# 2. Streamis upgrade steps + +## Install streamisappconn +1) Delete the old version of StreamisAppconn package + +Enter the following directory, find the appconn folder of streamis and delete it, if any: +```shell script +{DSS_Install_HOME}/dss/dss-appconns +``` + +2) StreamisAppconn installation deployment + +To install the DSS StreamisAppConn plug-in. Please refer to: [StreamisAppConn plug-in installation document](development/StreamisAppConnInstallationDocument.md) + +## Installing the Streamis backend +Update Lib in the obtained installation package to the path 'streamis-server/lib' under the streamis installation directory, and the file contents under 'streamis-server/conf' can be updated as needed. + +Enter the installation directory and execute the update script to complete the update of database table structure and data: +```shell script +cd {Streamis_Install_HOME} +sh bin/upgrade.sh +``` + +Then complete the update and restart of the Streamis server through the following command: +```shell script +cd {Streamis_Install_HOME}/streamis-server +sh bin/stop-streamis-server.sh +sh bin/start-streamis-server.sh +``` + +##Installing the Streamis front end +First delete the front-end directory folder of the old version, and then replace it with the new front-end installation package. +``` +mkdir ${STREAMIS_FRONT_PATH} +cd ${STREAMIS_FRONT_PATH} +#1.Delete front-end directory folder +#2.Place the front-end package +unzip streamis-${streamis-version}.zip +``` \ No newline at end of file diff --git a/docs/en_US/userManual/StreamisUserManual.md b/docs/en_US/userManual/StreamisUserManual.md index 4ebe42a8..974a2765 100644 --- a/docs/en_US/userManual/StreamisUserManual.md +++ b/docs/en_US/userManual/StreamisUserManual.md @@ -10,7 +10,7 @@ The entry path is **Home-DSS component application-Enter Streamis** -![Streamis entrance](../../images/create_stream_product_center.png) +![Streamis entrance](../../images/create_stream_product_center_en.png)
Picture 2.1 Streamis entrance]
## 3. Core indicators @@ -19,7 +19,7 @@ The entry path is **Home-DSS component application-Enter Streamis**          The core indicator shows the status summary of the Flink tasks uploaded to the project for execution. There are temporarily 7 states, showing the state name and the number of tasks in that state. The specific content is as shown in the figure below. -![Core indicators](../../images/home_page.png) +![Core indicators](../../images/home_page_en.png)
Picture 3.1 Core indicators
# 4. Job management @@ -94,7 +94,7 @@ The entry path is **Home-DSS component application-Enter Streamis**          Click **"job name"** of a task in the task list to provide the function of managing the task, or click **"three dots"** to the left of the name to call up the specific function configuration entry, as shown below: -![Configuration job](../../images/job_list.png) +![Configuration job](../../images/job_list_en.png)
Picture4.3 Configuration job

@@ -109,9 +109,13 @@ The entry path is **Home-DSS component application-Enter Streamis**
+Click batch operation, and multiple job tasks can be restarted. Restart and snapshot will generate a snapshot and then restart. Restart directly will not generate a snapshot. + +![jobbulk_operate](../../images/jobbulk_operate_en.png) + ### 4.3.1. Job summary: -![Operating condition](../../images/stream_job_detail.png) +![Operating condition](../../images/stream_job_detail_en.png)
Picture 4.4 Job summary

@@ -122,7 +126,7 @@ The entry path is **Home-DSS component application-Enter Streamis** ### 4.3.2. Job history: -![Execution history](../../images/stream_job_history.png) +![Execution history](../../images/stream_job_history_en.png)
Picture 4.5 Job history

@@ -134,8 +138,8 @@ The entry path is **Home-DSS component application-Enter Streamis** ### 4.3.3. Job config: -![Configuration](../../images/stream_job_config_1.png) -![Configuration](../../images/stream_job_config_2.png) +![Configuration](../../images/stream_job_config_en_1.png) +![Configuration](../../images/stream_job_config_en_2.png)
Picture 4.6 Job config

@@ -159,7 +163,7 @@ The entry path is **Home-DSS component application-Enter Streamis** **Flink Jar Job details** -![Job details](../../images/stream_job_flinkjar_jobcontent.png) +![Job details](../../images/stream_job_flinkjar_jobcontent_en.png)
Picture 4.7 Flink Jar Job details

@@ -171,7 +175,7 @@ The entry path is **Home-DSS component application-Enter Streamis** **Flink SQL job details** -![Job details](../../images/stream_job_flinksql_jobcontent.png) +![Job details](../../images/stream_job_flinksql_jobcontent_en.png)
Picture 4.8 Flink SQL job details

@@ -192,10 +196,10 @@ The entry path is **Home-DSS component application-Enter Streamis**
-![Engineering Documents Home Page](../../images/project_source_file_list.png) +![Engineering Documents Home Page](../../images/project_source_file_list_en.png)
Picture 5.1 Engineering Documents Home Page

-![Upload project file](../../images/project_source_file_import.png) +![Upload project file](../../images/project_source_file_import_en.png)
Picture 5.2 Upload project file
diff --git a/docs/images/create_stream_product_center_en.png b/docs/images/create_stream_product_center_en.png new file mode 100644 index 00000000..952d11af Binary files /dev/null and b/docs/images/create_stream_product_center_en.png differ diff --git a/docs/images/home_page_en.png b/docs/images/home_page_en.png new file mode 100644 index 00000000..28b0399a Binary files /dev/null and b/docs/images/home_page_en.png differ diff --git a/docs/images/job_list_en.png b/docs/images/job_list_en.png new file mode 100644 index 00000000..487bf502 Binary files /dev/null and b/docs/images/job_list_en.png differ diff --git a/docs/images/jobbulk_operate_en.png b/docs/images/jobbulk_operate_en.png new file mode 100644 index 00000000..6c064cbf Binary files /dev/null and b/docs/images/jobbulk_operate_en.png differ diff --git a/docs/images/project_source_file_import_en.png b/docs/images/project_source_file_import_en.png new file mode 100644 index 00000000..01b3419c Binary files /dev/null and b/docs/images/project_source_file_import_en.png differ diff --git a/docs/images/project_source_file_list_en.png b/docs/images/project_source_file_list_en.png new file mode 100644 index 00000000..84590e46 Binary files /dev/null and b/docs/images/project_source_file_list_en.png differ diff --git a/docs/images/stream_job_config_en_1.png b/docs/images/stream_job_config_en_1.png new file mode 100644 index 00000000..40d403db Binary files /dev/null and b/docs/images/stream_job_config_en_1.png differ diff --git a/docs/images/stream_job_config_en_2.png b/docs/images/stream_job_config_en_2.png new file mode 100644 index 00000000..466f73ed Binary files /dev/null and b/docs/images/stream_job_config_en_2.png differ diff --git a/docs/images/stream_job_detail_en.png b/docs/images/stream_job_detail_en.png new file mode 100644 index 00000000..d7a7e3ba Binary files /dev/null and b/docs/images/stream_job_detail_en.png differ diff --git a/docs/images/stream_job_flinkjar_jobcontent_en.png b/docs/images/stream_job_flinkjar_jobcontent_en.png new file mode 100644 index 00000000..6e02623b Binary files /dev/null and b/docs/images/stream_job_flinkjar_jobcontent_en.png differ diff --git a/docs/images/stream_job_flinksql_jobcontent_en.png b/docs/images/stream_job_flinksql_jobcontent_en.png new file mode 100644 index 00000000..16e5faee Binary files /dev/null and b/docs/images/stream_job_flinksql_jobcontent_en.png differ diff --git a/docs/images/stream_job_history_en.png b/docs/images/stream_job_history_en.png new file mode 100644 index 00000000..d6e075ad Binary files /dev/null and b/docs/images/stream_job_history_en.png differ diff --git a/docs/images/stream_product_center_en.png b/docs/images/stream_product_center_en.png new file mode 100644 index 00000000..477fea4f Binary files /dev/null and b/docs/images/stream_product_center_en.png differ diff --git "a/docs/zh_CN/0.2.0/development/Streamis\345\215\207\347\272\247\346\226\207\346\241\243.md" "b/docs/zh_CN/0.2.0/development/Streamis\345\215\207\347\272\247\346\226\207\346\241\243.md" new file mode 100644 index 00000000..cf5a89b8 --- /dev/null +++ "b/docs/zh_CN/0.2.0/development/Streamis\345\215\207\347\272\247\346\226\207\346\241\243.md" @@ -0,0 +1,45 @@ +Streamis 升级文档，本文主要介绍在原有安装Streamis服务的基础上适配DSS1.1.0和Linkis1.1.1的升级步骤，Streamis0.2.0相对与Streamis0.1.0版本最大的区别在于接入了DSS AppConn，对job的启停做了优化。 + +# 1.升级Streamis前的工作 +您在升级Streamis之前，请先安装 Linkis1.1.1 和 DSS1.1.0 及以上版本，并且保证 Linkis Flink 引擎 和 DSS 可以正常使用，DSS 和 Linkis 安装，可参照 [DSS & Linkis 一键安装部署文档](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2/DSS%E5%8D%95%E6%9C%BA%E9%83%A8%E7%BD%B2%E6%96%87%E6%A1%A3.md)。 + +# 2.Streamis升级步骤 + +## 安装StreamisAppConn + +1）删除旧版本StreamisAppConn包 + +进入下列目录，找到streamis的appconn文件夹并删除，如果存在的话： +```shell script +{DSS_Install_HOME}/dss/dss-appconns +``` + +2）StreamisAppConn安装部署 + +安装 DSS StreamisAppConn 插件，请参考: [StreamisAppConn 插件安装文档](development/StreamisAppConn安装文档.md) + +## 安装Streamis后端 +将获取到的安装包中lib更新到Streamis安装目录下的路径 `streamis-server/lib` 中,`streamis-server/conf`下的文件内容可根据需要进行更新。 + +进入安装目录下，执行更新脚本，完成对数据库表结构和数据的更新： +```shell script +cd {Streamis_Install_HOME} +sh bin/upgrade.sh +``` + +再通过以下命令完成 Streamis Server 的更新重启： +```shell script +cd {Streamis_Install_HOME}/streamis-server +sh bin/stop-streamis-server.sh +sh bin/start-streamis-server.sh +``` + +## 安装Streamis前端 +先删除旧版本前端目录文件夹，再替换为新的前端安装包 +``` +mkdir ${STREAMIS_FRONT_PATH} +cd ${STREAMIS_FRONT_PATH} +#1.删除前端目录文件夹 +#2.放置前端包 +unzip streamis-${streamis-version}.zip +``` \ No newline at end of file diff --git a/images/en_US/readme/architecture.png b/images/en_US/readme/architecture.png index 15e081e2..36a4a1f6 100644 Binary files a/images/en_US/readme/architecture.png and b/images/en_US/readme/architecture.png differ diff --git a/images/zh_CN/readme/architecture.png b/images/zh_CN/readme/architecture.png index ee9c9569..872891e4 100644 Binary files a/images/zh_CN/readme/architecture.png and b/images/zh_CN/readme/architecture.png differ