Merge pull request #41 from WeDataSphere/dev-0.2.0-docs-appconn

upgrade document

README.md

@@ -35,16 +35,16 @@ The Dimension node, Transform node, Sink node and [Visualis](https://github.com/
 
        Supports multi-version management, full life cycle management, monitoring alarm, checkpoint and savepoint management capabilities of streaming jobs.
 
-![prod center](docs/images/stream_product_center.png)
+![prod center](docs/images/stream_product_center_en.png)
 
        Running information page:
 
-![Running information](docs/images/stream_job_detail.png)
+![Running information](docs/images/stream_job_detail_en.png)
 
        Configurations page：
 
-![Configurations](docs/images/stream_job_config_1.png)
-![Configurations](docs/images/stream_job_config_2.png)
+![Configurations](docs/images/stream_job_config_en_1.png)
+![Configurations](docs/images/stream_job_config_en_2.png)
 
        For more features, please refer to: [User Manual](docs/en_US/userManual/StreamisUserManual.md).
 

bin/upgrade.sh

@@ -0,0 +1,205 @@
+#!/usr/bin/env bash
+#
+# Copyright 2022 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Use to upgrade from 0.1.0 to 0.2.0
+
+if [ -f "~/.bashrc" ];then
+  echo "Warning! user bashrc file does not exist."
+else
+  source ~/.bashrc
+fi
+
+shellDir=`dirname $0`
+workDir=`cd ${shellDir}/..;pwd`
+
+interact_echo(){
+  while [ 1 ]; do
+    read -p "$1 (Y/N)" yn
+    if [[ "${yn}x" == "Yx" ]] || [[ "${yn}x" == "yx" ]]; then
+      return 0
+    elif [[ "${yn}x" == "Nx" ]] || [[ "${yn}x" == "nx" ]]; then
+      return 1
+    else
+      echo "Unknown choose: [$yn], please choose again."
+    fi
+  done
+}
+
+interact_echo "Are you sure the current version of Streamis is 0.1.0 and need to upgrade to 0.2.0 ?"
+if [[ $? == 0 ]]; then
+  source ${workDir}/conf/db.sh
+  echo "<------ Will connect to [${MYSQL_HOST}:${MYSQL_PORT}] to upgrade the tables in database... ------>"
+  mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 << EOF 1>/dev/null
+    /*Modify the table column*/
+    ALTER TABLE \`linkis_stream_job\` MODIFY COLUMN \`project_name\` varchar(100) DEFAULT NULL;
+    ALTER TABLE \`linkis_stream_job\` MODIFY COLUMN \`name\` varchar(200) DEFAULT NULL;
+    ALTER TABLE \`linkis_stream_project\` MODIFY COLUMN \`name\` varchar(100) DEFAULT NULL;
+    ALTER TABLE \`linkis_stream_task\` MODIFY COLUMN \`job_id\` varchar(200) DEFAULT NULL;
+    ALTER TABLE \`linkis_stream_task\` MODIFY COLUMN \`linkis_job_id\` varchar(200) DEFAULT NULL;
+
+    ALTER TABLE \`linkis_stream_project\` ADD create_time datetime DEFAULT NULL;
+    ALTER TABLE \`linkis_stream_project\` ADD last_update_by varchar(50) DEFAULT NULL;
+    ALTER TABLE \`linkis_stream_project\` ADD last_update_time datetime DEFAULT NULL;
+    ALTER TABLE \`linkis_stream_project\` ADD is_deleted tinyint unsigned DEFAULT 0;
+
+    /*Add indexes into the tables*/
+    ALTER TABLE \`linkis_stream_job\` ADD UNIQUE KEY(\`project_name\`, \`name\`);
+    ALTER TABLE \`linkis_stream_job_version\` ADD UNIQUE KEY(\`job_id\`, \`version\`);
+
+    /*Add new tables*/
+    DROP TABLE IF EXISTS \`linkis_stream_project_privilege\`;
+    CREATE TABLE \`linkis_stream_project_privilege\` (
+      \`id\` bigint(20) NOT NULL AUTO_INCREMENT,
+      \`project_id\` bigint(20) NOT NULL,
+      \`user_name\` varchar(100) NOT NULL,
+      \`privilege\` tinyint(1) DEFAULT '0' NOT NULL COMMENT '1:发布权限 ，2:编辑权限 ，3:查看权限',
+      PRIMARY KEY (\`id\`) USING BTREE
+    ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='项目权限表';
+
+    DROP TABLE IF EXISTS \`linkis_stream_job_config_def\`;
+    CREATE TABLE \`linkis_stream_job_config_def\` (
+      \`id\` bigint(20) NOT NULL AUTO_INCREMENT,
+      \`key\` varchar(100) COLLATE utf8_bin NOT NULL,
+      \`name\` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT 'Equals option',
+      \`type\` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT 'NONE' COMMENT 'def type, NONE: 0, INPUT: 1, SELECT: 2',
+      \`sort\` int(10) DEFAULT '0' COMMENT 'In order to sort the configurations that have the same level',
+      \`description\` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT 'Description of configuration',
+      \`validate_type\` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT 'Method the validate the configuration',
+      \`validate_rule\` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT 'Value of validation rule',
+      \`style\` varchar(200) COLLATE utf8_bin DEFAULT '' COMMENT 'Display style',
+      \`visiable\` tinyint(1) NOT NULL DEFAULT '1' COMMENT '0: hidden, 1: display',
+      \`level\` tinyint(1) NOT NULL DEFAULT '1' COMMENT '0: root, 1: leaf',
+      \`unit\` varchar(25) COLLATE utf8_bin DEFAULT NULL COMMENT 'Unit symbol',
+      \`default_value\` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT 'Default value',
+      \`ref_values\` varchar(200) COLLATE utf8_bin DEFAULT '',
+      \`parent_ref\` bigint(20) DEFAULT NULL COMMENT 'Parent key of configuration def',
+      \`required\` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'If the value of configuration is necessary',
+      \`is_temp\` tinyint(1) DEFAULT '0' COMMENT 'Temp configuration',
+      PRIMARY KEY (\`id\`),
+      UNIQUE KEY \`config_def_key\` (\`key\`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+    DROP TABLE IF EXISTS \`linkis_stream_job_config\`;
+    CREATE TABLE \`linkis_stream_job_config\` (
+      \`job_id\` bigint(20) NOT NULL,
+      \`job_name\` varchar(200) COLLATE utf8_bin NOT NULL COMMENT 'Just store the job name',
+      \`key\` varchar(100) COLLATE utf8_bin NOT NULL,
+      \`value\` varchar(500) COLLATE utf8_bin NOT NULL,
+      \`ref_def_id\` bigint(20) DEFAULT NULL COMMENT 'Refer to id in config_def table',
+      PRIMARY KEY (\`job_id\`,\`key\`),
+      KEY \`config_def_id\` (\`ref_def_id\`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+    /*Execute dml*/
+    source ${workDir}/db/streamis_dml.sql
+
+    /*Data migration*/
+    INSERT INTO \`linkis_stream_job_config\`(\`key\`, \`value\`, \`job_id\`, \`job_name\`, \`ref_def_id\`) SELECT ov.config_key, ov.config_value, ov.job_id, ov.job_name, d.id as refer_id from linkis_stream_configuration_config_value ov left join linkis_stream_job_config_def d on ov.config_key = d.key WHERE ov.config_value IS NOT NULL AND ov.job_name IS NOT NULL GROUP BY ov.job_id,ov.config_key;
+    UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.memory" WHERE \`key\` = "flink.taskmanager.memory";
+    UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.cpus" WHERE \`key\` = "flink.taskmanager.cpu.cores";
+    UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.cpus" WHERE \`key\` = "wds.linkis.flink.taskManager.cpus";
+    UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.numberOfTaskSlots" WHERE \`key\` = "flink.taskmanager.numberOfTaskSlots";
+    UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.app.parallelism" WHERE \`key\` = "wds.linkis.engineconn.flink.app.parallelism";
+    UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.jobmanager.memory" WHERE \`key\` = "flink.jobmanager.memory";
+    UPDATE linkis_stream_job_config c SET \`ref_def_id\` = (SELECT d.id FROM linkis_stream_job_config_def d WHERE d.\`key\` = c.\`key\`) WHERE c.ref_def_id IS NULL;
+    SELECT @flink_extra_param_id:=id FROM linkis_stream_job_config_def WHERE \`key\` = "wds.linkis.flink.custom";
+    UPDATE linkis_stream_job_config SET ref_def_id = @flink_extra_param_id WHERE ref_def_id IS NULL;
+
+    /*Drop tables*/
+    /*DROP TABLE \`linkis_stream_configuration_config_key\`*/
+    /*DROP TABLE \`linkis_stream_configuration_config_value\`*/
+
+    /*update tables data*/
+    delimiter %%
+
+    create procedure update_project()
+    BEGIN
+        -- 声明变量
+        DECLARE projectname varchar(50);
+        DECLARE done INT default 0;
+
+    -- 创建游标，并设置游标所指的数据
+        DECLARE cur CURSOR for
+            SELECT distinct j.project_name from linkis_stream_job j;
+    -- 游标执行完，即遍历结束。设置done的值为1
+        DECLARE CONTINUE HANDLER for not FOUND set done = 1;
+    -- 开启游标
+        open cur;
+    -- 执行循环
+        posLoop:
+        LOOP
+                -- 从游标中取出projectname
+            FETCH cur INTO projectname ;
+            -- 如果done的值为1，即遍历结束，结束循环
+            IF done = 1 THEN
+                LEAVE posLoop;
+    -- 注意，if语句需要添加END IF结束IF
+            END IF;
+            insert into linkis_stream_project(\`name\`,\`create_by\`,\`create_time\`) values (projectname,\'system\',now());
+    -- 关闭循环
+        END LOOP posLoop;
+    -- 关闭游标
+        CLOSE cur;
+    -- 关闭分隔标记
+    END %%
+
+    create procedure update_project_privilege()
+    BEGIN
+        -- 声明变量
+        DECLARE projectid bigint(20);
+        DECLARE create_by varchar(50);
+        DECLARE done INT default 0;
+
+    -- 创建游标，并设置游标所指的数据
+        DECLARE cur CURSOR for
+            SELECT distinct p.id,j.create_by  from linkis_stream_project p,linkis_stream_job j where p.name =j.project_name ;
+    -- 游标执行完，即遍历结束。设置done的值为1
+        DECLARE CONTINUE HANDLER for not FOUND set done = 1;
+    -- 开启游标
+        open cur;
+    -- 执行循环
+        posLoop:
+        LOOP
+                -- 从游标中取出id
+            FETCH cur INTO projectid ,create_by;
+            -- 如果done的值为1，即遍历结束，结束循环
+            IF done = 1 THEN
+                LEAVE posLoop;
+    -- 注意，if语句需要添加END IF结束IF
+            END IF;
+
+            insert into linkis_stream_project_privilege (project_id ,user_name ,privilege) values (projectid,create_by,2);
+    -- 关闭循环
+        END LOOP posLoop;
+    -- 关闭游标
+        CLOSE cur;
+    -- 关闭分隔标记
+    END %%
+    delimiter ;
+
+    call update_project;
+    call update_project_privilege;
+
+    drop PROCEDURE update_project;
+    drop PROCEDURE update_project_privilege;
+
+EOF
+  echo "<------ End to upgrade ------>"
+fi
+
+
+

docs/en_US/0.2.0/development/StreamisUpgradeDocumentation.md

@@ -0,0 +1,44 @@
+Streamis upgrade document. This article mainly introduces the upgrade steps of adapting DSS1.1.0 and linkis1.1.1 based on the original installation of Streamis service. The biggest difference between Streamis 0.2.0 and Streamis 0.1.0 is that it accesses DSS appconn and optimizes the start and stop of jobs.
+
+# 1. Work before upgrading streamis
+Before upgrading Streamis, please install linkis1.1.1 and DSS1.1.0 or above, and ensure that the linkis Flink engine and DSS can be used normally. For the installation of DSS and linkis, please refer to [dss & linkis one click installation and deployment document](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2/DSS%E5%8D%95%E6%9C%BA%E9%83%A8%E7%BD%B2%E6%96%87%E6%A1%A3.md).
+
+# 2. Streamis upgrade steps
+
+## Install streamisappconn
+1) Delete the old version of StreamisAppconn package
+
+Enter the following directory, find the appconn folder of streamis and delete it, if any:
+```shell script
+{DSS_Install_HOME}/dss/dss-appconns
+```
+
+2) StreamisAppconn installation deployment
+
+To install the DSS StreamisAppConn plug-in. Please refer to: [StreamisAppConn plug-in installation document](development/StreamisAppConnInstallationDocument.md)
+
+## Installing the Streamis backend
+Update Lib in the obtained installation package to the path 'streamis-server/lib' under the streamis installation directory, and the file contents under 'streamis-server/conf' can be updated as needed.
+
+Enter the installation directory and execute the update script to complete the update of database table structure and data:
+```shell script
+cd {Streamis_Install_HOME}
+sh bin/upgrade.sh
+```
+
+Then complete the update and restart of the Streamis server through the following command:
+```shell script
+cd {Streamis_Install_HOME}/streamis-server
+sh bin/stop-streamis-server.sh 
+sh bin/start-streamis-server.sh 
+```
+
+##Installing the Streamis front end
+First delete the front-end directory folder of the old version, and then replace it with the new front-end installation package.
+```
+mkdir ${STREAMIS_FRONT_PATH}
+cd ${STREAMIS_FRONT_PATH}
+#1.Delete front-end directory folder
+#2.Place the front-end package
+unzip streamis-${streamis-version}.zip
+```

docs/en_US/userManual/StreamisUserManual.md

@@ -10,7 +10,7 @@
 
 The entry path is **Home-DSS component application-Enter Streamis**
 
-![Streamis entrance](../../images/create_stream_product_center.png)
+![Streamis entrance](../../images/create_stream_product_center_en.png)
 <center>Picture 2.1 Streamis entrance]</center>
 
 ## 3. Core indicators
@@ -19,7 +19,7 @@ The entry path is **Home-DSS component application-Enter Streamis**
 
 &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;The core indicator shows the status summary of the Flink tasks uploaded to the project for execution. There are temporarily 7 states, showing the state name and the number of tasks in that state. The specific content is as shown in the figure below.
 
-![Core indicators](../../images/home_page.png)
+![Core indicators](../../images/home_page_en.png)
 <center>Picture 3.1 Core indicators</center>
 
 # 4. Job management
@@ -94,7 +94,7 @@ The entry path is **Home-DSS component application-Enter Streamis**
 
 &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Click **"job name"** of a task in the task list to provide the function of managing the task, or click **"three dots"** to the left of the name to call up the specific function configuration entry, as shown below:
 
-![Configuration job](../../images/job_list.png)
+![Configuration job](../../images/job_list_en.png)
 <center>Picture4.3 Configuration job</center> 
 
 <br/>
@@ -109,9 +109,13 @@ The entry path is **Home-DSS component application-Enter Streamis**
 
 <br/>
 
+Click batch operation, and multiple job tasks can be restarted. Restart and snapshot  will generate a snapshot and then restart. Restart directly will not generate a snapshot.
+
+![jobbulk_operate](../../images/jobbulk_operate_en.png)
+
 ### 4.3.1. Job summary:
 
-![Operating condition](../../images/stream_job_detail.png)
+![Operating condition](../../images/stream_job_detail_en.png)
 <center> Picture 4.4 Job summary</center> 
 
 <br/>
@@ -122,7 +126,7 @@ The entry path is **Home-DSS component application-Enter Streamis**
 
 ### 4.3.2. Job history:
 
-![Execution history](../../images/stream_job_history.png)
+![Execution history](../../images/stream_job_history_en.png)
 <center>Picture 4.5 Job history</center> 
 
 <br/>
@@ -134,8 +138,8 @@ The entry path is **Home-DSS component application-Enter Streamis**
 
 ### 4.3.3. Job config:
 
-![Configuration](../../images/stream_job_config_1.png)
-![Configuration](../../images/stream_job_config_2.png)
+![Configuration](../../images/stream_job_config_en_1.png)
+![Configuration](../../images/stream_job_config_en_2.png)
 <center>Picture 4.6 Job config</center> 
 
 <br/>
@@ -159,7 +163,7 @@ The entry path is **Home-DSS component application-Enter Streamis**
 
 **Flink Jar Job details**
 
-![Job details](../../images/stream_job_flinkjar_jobcontent.png)
+![Job details](../../images/stream_job_flinkjar_jobcontent_en.png)
 <center>Picture 4.7 Flink Jar Job details</center> 
 
 <br/>
@@ -171,7 +175,7 @@ The entry path is **Home-DSS component application-Enter Streamis**
 
 **Flink SQL job details**
 
-![Job details](../../images/stream_job_flinksql_jobcontent.png)
+![Job details](../../images/stream_job_flinksql_jobcontent_en.png)
 <center>Picture 4.8 Flink SQL job details</center> 
 
 <br/>
@@ -192,10 +196,10 @@ The entry path is **Home-DSS component application-Enter Streamis**
 
 <br/>
 
-![Engineering Documents Home Page](../../images/project_source_file_list.png)
+![Engineering Documents Home Page](../../images/project_source_file_list_en.png)
 <center>Picture 5.1 Engineering Documents Home Page</center>
 
 <br/>
 
-![Upload project file](../../images/project_source_file_import.png)
+![Upload project file](../../images/project_source_file_import_en.png)
 <center>Picture 5.2 Upload project file</center>