add e2e test for mysql plugin (#273)

* add e2e test for mysql plugin

* fix comments and `mvn` conflicting issue.(will merge commits after the approval)

mysql-plugin/pom.xml

@@ -26,7 +26,7 @@
   <name>Mysql plugin</name>
   <artifactId>mysql-plugin</artifactId>
   <modelVersion>4.0.0</modelVersion>
-
+  
   <dependencies>
     <dependency>
       <groupId>io.cdap.cdap</groupId>
@@ -89,11 +89,101 @@
     </dependency>
   </dependencies>
   <build>
+    <testSourceDirectory>${testSourceLocation}</testSourceDirectory>
     <plugins>
       <plugin>
         <groupId>io.cdap</groupId>
         <artifactId>cdap-maven-plugin</artifactId>
       </plugin>
     </plugins>
   </build>
+  <profiles>
+    <profile>
+      <id>e2e-tests</id>
+      <properties>
+        <testSourceLocation>src/e2e-test/java</testSourceLocation>
+      </properties>
+      <build>
+        <testResources>
+          <testResource>
+            <directory>src/e2e-test/resources</directory>
+          </testResource>
+        </testResources>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <skipTests>true</skipTests>
+            </configuration>
+          </plugin>
+
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <configuration>
+              <includes>
+                <include>TestRunner.java</include>
+              </includes>
+            </configuration>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>integration-test</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+
+          <plugin>
+            <groupId>net.masterthought</groupId>
+            <artifactId>maven-cucumber-reporting</artifactId>
+            <version>5.5.0</version>
+
+            <executions>
+              <execution>
+                <id>execution</id>
+                <phase>verify</phase>
+                <goals>
+                  <goal>generate</goal>
+                </goals>
+                <configuration>
+                  <projectName>Cucumber Reports</projectName> <!-- Replace with project name -->
+                  <outputDirectory>target/cucumber-reports/advanced-reports</outputDirectory>
+                  <buildNumber>1</buildNumber>
+                  <skip>false</skip>
+                  <inputDirectory>${project.build.directory}/cucumber-reports</inputDirectory>
+                  <jsonFiles> <!-- supports wildcard or name pattern -->
+                    <param>**/*.json</param>
+                  </jsonFiles> <!-- optional, defaults to outputDirectory if not specified -->
+                  <classificationDirectory>${project.build.directory}/cucumber-reports</classificationDirectory>
+                  <checkBuildResult>true</checkBuildResult>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+
+      <dependencies>
+        <dependency>
+          <groupId>io.cdap.tests.e2e</groupId>
+          <artifactId>cdap-e2e-framework</artifactId>
+          <version>0.0.1-SNAPSHOT</version>
+          <scope>test</scope>
+        </dependency>
+        <dependency>
+          <groupId>ch.qos.logback</groupId>
+          <artifactId>logback-core</artifactId>
+          <version>1.2.8</version>
+          <scope>runtime</scope>
+        </dependency>
+        <dependency>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+          <version>27.0.1-jre</version>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
 </project>

mysql-plugin/src/e2e-test/features/mysql/MySql.feature

@@ -0,0 +1,63 @@
+#
+# Copyright © 2022 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+
+@Mysql
+Feature: Mysql - Verify Mysql source data transfer
+  @MYSQL_SOURCE_TEST @MYSQL_SINK_TEST
+  Scenario: To verify data is getting transferred from Mysql to Mysql successfully
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "MySQL" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "MySQL" from the plugins list as: "Sink"
+    Then Connect plugins: "MySQL" and "MySQL2" to establish connection
+    Then Navigate to the properties page of plugin: "MySQL"
+    Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driver"
+    Then Replace input plugin property: "host" with value: "host"
+    Then Replace input plugin property: "port" with value: "port"
+    Then Replace input plugin property: "user" with value: "username"
+    Then Replace input plugin property: "password" with value: "password"
+    Then Enter input plugin property: "referenceName" with value: "sourceRef"
+    Then Replace input plugin property: "database" with value: "database"
+    Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
+    Then Click on the Get Schema button
+    Then Verify the Output Schema matches the Expected Schema: "outputSchema"
+    Then Validate "MySQL" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "MySQL2"
+    Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driver"
+    Then Replace input plugin property: "host" with value: "host"
+    Then Replace input plugin property: "port" with value: "port"
+    Then Replace input plugin property: "database" with value: "database"
+    Then Replace input plugin property: "tableName" with value: "targetTable"
+    Then Replace input plugin property: "user" with value: "username"
+    Then Replace input plugin property: "password" with value: "password"
+    Then Enter input plugin property: "referenceName" with value: "targetRef"
+    Then Validate "MySQL2" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Verify the preview of pipeline is "success"
+    Then Click on preview data for MySQL sink
+    Then Verify preview output schema matches the outputSchema captured in properties
+    Then Close the preview data
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Get count of no of records transferred to target MySQL Table
+    Then Validate records transferred to target table is equal to number of records from source table

mysql-plugin/src/e2e-test/java/io/cdap/plugin/MysqlClient.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright © 2021 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.plugin;
+
+import io.cdap.e2e.utils.PluginPropertyUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+/**
+ *  MySQL client.
+ */
+public class MysqlClient {
+  private static final String host =  PluginPropertyUtils.pluginProp("host");
+  private static final int port = Integer.parseInt(PluginPropertyUtils.pluginProp("port"));
+  private static final String database = PluginPropertyUtils.pluginProp("database");
+
+  private static Connection getMysqlConnection() throws SQLException, ClassNotFoundException {
+    Class.forName("com.mysql.cj.jdbc.Driver");
+    String username = PluginPropertyUtils.pluginProp("username");
+    String password = PluginPropertyUtils.pluginProp("password");
+    return DriverManager.getConnection("jdbc:mysql://" + host + ":" + port + "/" + database, username, password);
+  }
+
+  public static int countRecord(String table) throws SQLException, ClassNotFoundException {
+    String countQuery = "SELECT COUNT(*) as total FROM " + table;
+    try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement();
+         ResultSet rs = statement.executeQuery(countQuery)) {
+      int num = 0;
+      while (rs.next()) {
+        num = (rs.getInt(1));
+      }
+      return num;
+    }
+  }
+
+  public static void createSourceTable(String sourceTable) throws SQLException, ClassNotFoundException {
+    try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement()) {
+      String createSourceTableQuery = "CREATE TABLE IF NOT EXISTS " + sourceTable +
+        "(id int, lastName varchar(255), PRIMARY KEY (id))";
+      statement.executeUpdate(createSourceTableQuery);
+
+      // Truncate table to clean the data of last failure run.
+      String truncateSourceTableQuery = "TRUNCATE TABLE " + sourceTable;
+      statement.executeUpdate(truncateSourceTableQuery);
+
+      // Insert dummy data.
+      statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" +
+                                "VALUES (1, 'Simpson')");
+      statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" +
+                                "VALUES (2, 'McBeal')");
+      statement.executeUpdate("INSERT INTO " + sourceTable + " (id, lastName)" +
+                                "VALUES (3, 'Flinstone')");
+    }
+  }
+
+  public static void createTargetTable(String targetTable) throws SQLException, ClassNotFoundException {
+    try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement()) {
+      String createTargetTableQuery = "CREATE TABLE IF NOT EXISTS " + targetTable +
+        "(id int, lastName varchar(255), PRIMARY KEY (id))";
+      statement.executeUpdate(createTargetTableQuery);
+      // Truncate table to clean the data of last failure run.
+      String truncateTargetTableQuery = "TRUNCATE TABLE " + targetTable;
+      statement.executeUpdate(truncateTargetTableQuery);
+    }
+  }
+
+  public static void dropTables(String[] tables) throws SQLException, ClassNotFoundException {
+    try (Connection connect = getMysqlConnection(); Statement statement = connect.createStatement()) {
+      for (String table : tables) {
+        String dropTableQuery = "Drop Table " + table;
+        statement.executeUpdate(dropTableQuery);
+      }
+    }
+  }
+}

mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright © 2021 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.plugin.common.stepsdesign;
+
+import io.cdap.e2e.utils.PluginPropertyUtils;
+import io.cdap.plugin.MysqlClient;
+import io.cucumber.java.After;
+import io.cucumber.java.Before;
+
+import java.sql.SQLException;
+
+/**
+ * MYSQL test hooks.
+ */
+public class TestSetupHooks {
+
+  @Before(order = 1)
+  public static void overrideUserAndPasswordIfProvided() {
+    String username = System.getenv("username");
+    if (username != null && !username.isEmpty()) {
+        PluginPropertyUtils.addPluginProp("username", username);
+    }
+    String password = System.getenv("password");
+    if (password != null && !password.isEmpty()) {
+      PluginPropertyUtils.addPluginProp("password", password);
+    }
+  }
+
+  @Before(order = 1, value = "@MYSQL_SOURCE_TEST")
+  public static void setSelectQuery() {                
+    String sourceTable =  PluginPropertyUtils.pluginProp("sourceTable");
+    PluginPropertyUtils.addPluginProp("selectQuery",
+                                      PluginPropertyUtils.pluginProp("selectQuery").
+                                        replace("${table}", sourceTable));
+  }
+
+  @Before(order = 2, value = "@MYSQL_SOURCE_TEST")
+  public static void createTables() throws SQLException, ClassNotFoundException {
+    MysqlClient.createSourceTable(PluginPropertyUtils.pluginProp("sourceTable"));
+    MysqlClient.createTargetTable(PluginPropertyUtils.pluginProp("targetTable"));
+  }
+
+  @After(order = 2, value = "@MYSQL_SINK_TEST")
+  public static void dropTables() throws SQLException, ClassNotFoundException {
+    MysqlClient.dropTables(new String[]{PluginPropertyUtils.pluginProp("sourceTable"),
+      PluginPropertyUtils.pluginProp("targetTable")});
+  }
+
+}

mysql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/package-info.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright © 2022 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * Package contains the locators for the Joiner plugin.
+ */
+package io.cdap.plugin.common.stepsdesign;

mysql-plugin/src/e2e-test/java/io/cdap/plugin/mysql/runners/TestRunner.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright © 2022 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package io.cdap.plugin.mysql.runners;
+
+import io.cucumber.junit.Cucumber;
+import io.cucumber.junit.CucumberOptions;
+import org.junit.runner.RunWith;
+
+/**
+ * Test Runner to execute Mysql plugin test cases.
+ */
+@RunWith(Cucumber.class)
+@CucumberOptions(
+  features = {"src/e2e-test/features"},
+  glue = {"stepsdesign", "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.mysql.stepsdesign"},
+  tags = {"@Mysql"},
+  plugin = {"pretty", "html:target/cucumber-html-report/mysql",
+    "json:target/cucumber-reports/cucumber-mysql.json",
+    "junit:target/cucumber-reports/cucumber-mysql.xml"}
+)
+public class TestRunner {
+}