EasyML 1.2.0 version, which include tensorflow support and other new …

…improvements in release document.

QuickStart.md

@@ -188,6 +188,12 @@ Because the hadoop cluster network communication depend on ssh, we need to confi
 
 ### Init Mysql database
  * Run `sh init_mysql.sh` to prepare the databases for *Oozie* and *EasyML Studio* 
+### Add Tensorflow Support
+Our EML installation package in version 1.2.0 or above could support Tensorflow mode.Follow the steps bellow If you want to have a try.
+
+ * Enter the `hadoop-master` container via  docker exec -it hadoop-master /bin/bash 
+ * Run `sh /root/config-tensorflow.sh` to config the environment that tensorflow need.
+ * If you want to config the environment yourself, you can do use the  tensorflow folder scripts in EasyML Project.
 ### Start HDFS，spark and Yarn service
  * Enter the *hadoop-master* container via `docker exec -it hadoop-master /bin/bash` (a vital important command to enter every container)
  * Run `sh /root/start-hadoop.sh` to start hadoop and spark service

mysql/studio/account.sql

@@ -1,22 +1,20 @@
 /*
 Navicat MySQL Data Transfer
 
-Source Server         : BDA(烟台)
 Source Server Version : 50717
-Source Host           : 10.20.13.7:3306
 Source Database       : studio
 
 Target Server Type    : MYSQL
 Target Server Version : 50717
 File Encoding         : 65001
 
-Date: 2017-10-18 16:18:47
+Date: 2018-03-29 10:36:38
 */
 
 SET FOREIGN_KEY_CHECKS=0;
 
 -- ----------------------------
--- Table structure for account
+-- Table structure for `account`
 -- ----------------------------
 DROP TABLE IF EXISTS `account`;
 CREATE TABLE `account` (

mysql/studio/category.sql

@@ -1,22 +1,20 @@
 /*
 Navicat MySQL Data Transfer
 
-Source Server         : BDA(烟台)
 Source Server Version : 50717
-Source Host           : 10.20.13.7:3306
 Source Database       : studio
 
 Target Server Type    : MYSQL
 Target Server Version : 50717
 File Encoding         : 65001
 
-Date: 2017-10-18 16:27:48
+Date: 2018-03-29 10:36:46
 */
 
 SET FOREIGN_KEY_CHECKS=0;
 
 -- ----------------------------
--- Table structure for category
+-- Table structure for `category`
 -- ----------------------------
 DROP TABLE IF EXISTS `category`;
 CREATE TABLE `category` (

mysql/studio/dataset.sql

@@ -1,22 +1,20 @@
 /*
 Navicat MySQL Data Transfer
 
-Source Server         : BDA(烟台)
 Source Server Version : 50717
-Source Host           : 10.20.13.7:3306
 Source Database       : studio
 
 Target Server Type    : MYSQL
 Target Server Version : 50717
 File Encoding         : 65001
 
-Date: 2017-10-18 16:27:57
+Date: 2018-03-29 10:36:02
 */
 
 SET FOREIGN_KEY_CHECKS=0;
 
 -- ----------------------------
--- Table structure for dataset
+-- Table structure for `dataset`
 -- ----------------------------
 DROP TABLE IF EXISTS `dataset`;
 CREATE TABLE `dataset` (
@@ -37,4 +35,5 @@ CREATE TABLE `dataset` (
 -- ----------------------------
 -- Records of dataset
 -- ----------------------------
-INSERT INTO `dataset` VALUES ('Dataset', 'DataSet', 'System Data', '[email protected]', '/EML/Data/Dataset', '0', null, '0.1', '2017-04-29 16:34:01', '', 'General');
+INSERT INTO `dataset` VALUES ('A2EEE4A9-EA84-47D2-9EAA-CC8CE701FD49', 'Dataset', '0A0F402F-670F-4696-9D9C-42F0E0D665A00', '[email protected]', '/EML/Data/A2EEE4A9-EA84-47D2-9EAA-CC8CE701FD49', '0', null, '0.1', '2018-03-28 18:40:08', '', '');
+INSERT INTO `dataset` VALUES ('EA15288C-40D5-4E44-884E-0B0C508651BF', 'mnist', '0A0F402F-670F-4696-9D9C-42F0E0D665A00', '[email protected]', '/EML/Data/EA15288C-40D5-4E44-884E-0B0C508651BF', '0', null, '0.1', '2018-03-28 18:39:27', '', '');

mysql/studio/jobcron.sql

@@ -1,22 +1,20 @@
 /*
 Navicat MySQL Data Transfer
 
-Source Server         : BDA(烟台)
 Source Server Version : 50717
-Source Host           : 10.20.13.7:3306
 Source Database       : studio
 
 Target Server Type    : MYSQL
 Target Server Version : 50717
 File Encoding         : 65001
 
-Date: 2017-10-18 16:28:07
+Date: 2018-03-29 10:36:53
 */
 
 SET FOREIGN_KEY_CHECKS=0;
 
 -- ----------------------------
--- Table structure for jobcron
+-- Table structure for `jobcron`
 -- ----------------------------
 DROP TABLE IF EXISTS `jobcron`;
 CREATE TABLE `jobcron` (

mysql/studio/moduleversion.sql

@@ -1,22 +1,20 @@
 /*
 Navicat MySQL Data Transfer
 
-Source Server         : BDA(烟台)
 Source Server Version : 50717
-Source Host           : 10.20.13.7:3306
 Source Database       : studio
 
 Target Server Type    : MYSQL
 Target Server Version : 50717
 File Encoding         : 65001
 
-Date: 2017-10-18 16:28:16
+Date: 2018-03-29 10:37:05
 */
 
 SET FOREIGN_KEY_CHECKS=0;
 
 -- ----------------------------
--- Table structure for moduleversion
+-- Table structure for `moduleversion`
 -- ----------------------------
 DROP TABLE IF EXISTS `moduleversion`;
 CREATE TABLE `moduleversion` (

mysql/studio/supernode.sql

@@ -1,22 +1,20 @@
 /*
 Navicat MySQL Data Transfer
 
-Source Server         : BDA(烟台)
 Source Server Version : 50717
-Source Host           : 10.20.13.7:3306
 Source Database       : studio
 
 Target Server Type    : MYSQL
 Target Server Version : 50717
 File Encoding         : 65001
 
-Date: 2017-10-18 16:28:57
+Date: 2018-03-29 10:37:25
 */
 
 SET FOREIGN_KEY_CHECKS=0;
 
 -- ----------------------------
--- Table structure for supernode
+-- Table structure for `supernode`
 -- ----------------------------
 DROP TABLE IF EXISTS `supernode`;
 CREATE TABLE `supernode` (

src/main/java/eml/studio/client/controller/DBController.java

@@ -9,6 +9,7 @@
 import java.util.logging.Logger;
 
 import eml.studio.client.mvp.AppController;
+import eml.studio.client.mvp.presenter.MonitorPresenter.View;
 import eml.studio.client.mvp.presenter.Presenter;
 import eml.studio.client.rpc.CategoryService;
 import eml.studio.client.rpc.CategoryServiceAsync;
@@ -71,7 +72,7 @@ public class DBController {
 	 * @return  If submit is success
 	 * @throws CommandParseException
 	 */
-	public boolean submitUploadProgram2DB(final Presenter presenter, final UploadFileModule fileUploader,
+	public boolean submitUploadProgram2DB(final Presenter presenter,  final View view, final UploadFileModule fileUploader,
 			Program program, final DescribeGrid grid)
 					throws CommandParseException {
 		String name = grid.getText("Name");
@@ -107,6 +108,11 @@ public void onFailure(Throwable caught) {
 			@Override
 			public void onSuccess(Program result) {
 				fileUploader.setUpLoadProgram(result);
+				ProgramLeaf node = new ProgramLeaf(result);
+				ProgramTreeLoader.addContextMenu( view.getProgramTree(),
+						node);
+				ProgramTreeLoader.addProgramLeaf( view.getProgramTree(),
+						node,AppController.email);
 				logger.info("Program insert to DB:"+result.getCategory());
 			}
 		});
@@ -121,7 +127,7 @@ public void onSuccess(Program result) {
 	 * @return  If submit is success
 	 * @throws CommandParseException
 	 */
-	public boolean submitUploadDataset2DB(final Presenter presenter,final UploadFileModule fileUploader
+	public boolean submitUploadDataset2DB(final Presenter presenter,final View view, final UploadFileModule fileUploader
 			,Dataset dataset, DescribeGrid grid)
 					throws CommandParseException {
 		String name = grid.getText("Name");
@@ -145,6 +151,12 @@ public void onFailure(Throwable caught) {
 			@Override
 			public void onSuccess(Dataset result) {
 				fileUploader.setUpLoadDataset(result);
+				DatasetLeaf node = new DatasetLeaf(result);
+				DatasetTreeLoader.addContextMenu( view.getDatasetTree()
+						,node);
+				DatasetTreeLoader.addDatasetLeaf( view.getDatasetTree(),
+						node,AppController.email);
+				logger.info("Dataset insert to DB:"+result.getCategory());
 			}
 		});
 		return true;
@@ -446,12 +458,14 @@ public static String[] getProgramPanelValue(final Program program, boolean isUpd
 		String TypeString = program.getType();
 		logger.info(TypeString.toLowerCase());
 		if ("单机".equals(TypeString.toLowerCase()) || "standalone".equals(TypeString.toLowerCase()))
-			values[i] = Constants.studioUIMsg.standalone() + "/" + Constants.studioUIMsg.distributed()+"/ETL";
+			values[i] = Constants.studioUIMsg.standalone() + "/" + Constants.studioUIMsg.distributed()+"/ETL"+"/Tensorflow";
 		if ("spark".equals(TypeString) || "分布式".equals(TypeString) ||TypeString.equals("distributed")||TypeString.toLowerCase().equals("distributed")) {
-			values[i] = Constants.studioUIMsg.distributed() + "/" + Constants.studioUIMsg.standalone() + "/ETL";
+			values[i] = Constants.studioUIMsg.distributed() + "/" + Constants.studioUIMsg.standalone() + "/ETL"+"/Tensorflow";
 		}
 		if ("etl".equals(TypeString))
-			values[i] = "ETL/"+Constants.studioUIMsg.distributed() + "/" + Constants.studioUIMsg.standalone();
+			values[i] = "ETL/"+Constants.studioUIMsg.distributed() + "/" + Constants.studioUIMsg.standalone()+"/Tensorflow";
+		if("tensorflow".equals(TypeString))
+			values[i] = "Tensorflow/"+Constants.studioUIMsg.distributed() + "/" + Constants.studioUIMsg.standalone()+"/ETL";
 
 		i ++;
 
@@ -482,6 +496,16 @@ public static String[] getProgramPanelValue(final Program program, boolean isUpd
 		values[i++] = program.getDescription();
 		values[i++] = program.getCommandline();
 
+		String tensorflowMode = program.getTensorflowMode();
+		if("单机".equals(tensorflowMode) || "standalone".equals(tensorflowMode))
+			values[i] = Constants.studioUIMsg.standalone()+"/"+Constants.studioUIMsg.modelDistributed()+"/"+Constants.studioUIMsg.dataDistributed();
+		else if("模型分布".equals(tensorflowMode) || "model distributed".equals(tensorflowMode))
+			values[i] = Constants.studioUIMsg.modelDistributed()+"/"+Constants.studioUIMsg.standalone()+"/"+Constants.studioUIMsg.dataDistributed();
+		else if("数据分布".equals(tensorflowMode) || "data distributed".equals(tensorflowMode))
+			values[i] = Constants.studioUIMsg.dataDistributed()+"/"+Constants.studioUIMsg.standalone()+"/"+Constants.studioUIMsg.modelDistributed();
+		else 
+			values[i]= Constants.studioUIMsg.standalone()+"/"+Constants.studioUIMsg.modelDistributed()+"/"+Constants.studioUIMsg.dataDistributed();
+
 		return values;
 	}
 
@@ -507,10 +531,10 @@ public static String[] getDatasetPanelValue(final Dataset dataset, boolean isUpd
 
 		values[1] = null;
 		String TypeString = dataset.getContenttype();
-		if ("General".equals(TypeString)) values[2] = "General/TSV/CSV";
-		if ("TSV".equals(TypeString)) values[2] = "TSV/General/TSV";
-		if ("CSV".equals(TypeString)) values[2] = "CSV/General/TSV";
-		else values[2] = "General/TSV/CSV";
+		if (TypeString == null || TypeString.equals("") || "General".equals(TypeString)) values[2] = "General/CSV/TSV/JSON";
+		else if ("TSV".equals(TypeString)) values[2] = "TSV/General/CSV/JSON";
+		else if ("CSV".equals(TypeString)) values[2] = "CSV/General/TSV/JSON";
+		else values[2] = "JSON/General/TSV/CSV";
 		values[0] = dataset.getName();
 		values[5] = AppController.email;
 		values[6] = dataset.getDescription();

src/main/java/eml/studio/client/i18n/StudioUIMessages.java

@@ -8,7 +8,7 @@
 import com.google.gwt.i18n.client.Messages;
 
 public interface StudioUIMessages extends Messages {
-	
+
 	String yes();
 	String no();
 	String submit();
@@ -30,6 +30,8 @@ public interface StudioUIMessages extends Messages {
 	String systemData();
 	String preprocess();
 	String standalone();
+	String modelDistributed();
+	String dataDistributed();
 	String distributed();
 	String evaluation();
 	String transformation();
@@ -87,6 +89,7 @@ public interface StudioUIMessages extends Messages {
 	String moduleType();
 	String programable();
 	String moduleCMDFormat();
+	String tensorflowMode();
 
 	String dataName();
 	String dataCategory();

src/main/java/eml/studio/client/ui/menu/JobDeleteMenu.java

@@ -5,9 +5,11 @@
  */
 package eml.studio.client.ui.menu;
 
+import eml.studio.client.mvp.AppController;
 import eml.studio.client.rpc.JobService;
 import eml.studio.client.rpc.JobServiceAsync;
 import eml.studio.client.ui.tree.JobLeaf;
+import eml.studio.client.util.Constants;
 
 import com.google.gwt.core.client.GWT;
 import com.google.gwt.user.client.Command;
@@ -26,21 +28,27 @@ public static MenuItem create(final JobLeaf node) {
 			@Override
 			public void execute() {
 				String id = node.getModule().getJobId();
-				boolean y = Window.confirm("Are you sure you want to delete?");
-				if (y) {
-					JobServiceAsync srv = GWT.create(JobService.class);
-					srv.deleteJob(id, new AsyncCallback<Void>() {
-
-						@Override
-						public void onFailure(Throwable caught) {
-							Window.alert(caught.getMessage());
-						}
-
-						@Override
-						public void onSuccess(Void result) {
-							node.delete();
-						}
-					});
+				boolean isExampleDir = node.getParentItem().getText().equals(Constants.studioUIMsg.examples());
+				if(!isExampleDir && node.getModule().getAccount().equals(AppController.email) && node.getModule().getIsExample())
+					Window.alert("The job is example job, please don't delete it in my task!");
+				else
+				{
+					boolean y = Window.confirm("Are you sure you want to delete?");
+					if (y) {
+						JobServiceAsync srv = GWT.create(JobService.class);
+						srv.deleteJob(id, new AsyncCallback<Void>() {
+
+							@Override
+							public void onFailure(Throwable caught) {
+								Window.alert(caught.getMessage());
+							}
+
+							@Override
+							public void onSuccess(Void result) {
+								node.delete();
+							}
+						});
+					}
 				}
 				this.component.getContextMenu().hide();
 			}

src/main/java/eml/studio/client/ui/panel/EditProgramPanel.java

@@ -39,21 +39,21 @@ public EditProgramPanel(String emailAccount,final ProgramTree tree,final Program
 		super();
 		labarr = new String[][]{
 				{"Name","Category","Type","Programable","Isdeterministic","Version","CreateDate",
-					"Owner","Description","CommandLine"},
+					"Owner","Description","CommandLine","TensorflowMode"},
 					{Constants.studioUIMsg.moduleName(), Constants.studioUIMsg.moduleCategory(), 
 						Constants.studioUIMsg.moduleType(), Constants.studioUIMsg.programable(),
 						Constants.studioUIMsg.moduleDeterminacy(),Constants.studioUIMsg.moduleVersion(),
 						Constants.studioUIMsg.moduleCreateTime(), Constants.studioUIMsg.moduleOwner(),
-						Constants.studioUIMsg.dataDescription(),Constants.studioUIMsg.moduleCMDFormat()},
+						Constants.studioUIMsg.dataDescription(),Constants.studioUIMsg.moduleCMDFormat(),Constants.studioUIMsg.tensorflowMode()},
 						{"textbox","tree","listbox","listbox","listbox","textbox","textbox",
-							"textbox","textarea","textarea"},
-							{"true","true","true","true","true","true","false","false","true","true"},
+							"textbox","textarea","textarea","listbox"},
+							{"true","true","true","true","true","true","false","false","true","true","true"},
 							{"", Constants.studioUIMsg.chooseCategory(),
-								Constants.studioUIMsg.standalone() + "/" + Constants.studioUIMsg.distributed()+"/ETL",
+								Constants.studioUIMsg.standalone() + "/" + Constants.studioUIMsg.distributed()+"/ETL"+"/Tensorflow",
 								Constants.studioUIMsg.no()+"/" + Constants.studioUIMsg.yes(),
 								Constants.studioUIMsg.no()+"/" + Constants.studioUIMsg.yes(),
-								"0.1", TimeUtils.timeNow(), AppController.email, "", ""},
-								{"left","left","left","left","left","left","left","left","left","right"}
+								"0.1", TimeUtils.timeNow(), AppController.email, "", "",Constants.studioUIMsg.standalone()+"/"+Constants.studioUIMsg.modelDistributed()+"/"+Constants.studioUIMsg.dataDistributed()},
+								{"left","left","left","left","left","left","left","left","left","right","right"}
 		};
 		owner = emailAccount;
 		this.tree = tree;