Bigtable E2E coverage as per ITN.

pom.xml

@@ -854,6 +854,12 @@
       <version>0.2.0</version>
     </dependency>
     <!-- End: dependency used by the Dataplex connector -->
+    <!-- https://mvnrepository.com/artifact/com.google.cloud/google-cloud-bigtable -->
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-bigtable</artifactId>
+      <version>1.17.1</version>
+    </dependency>
   </dependencies>
 
   <build>

src/e2e-test/bigtable/BigTableToBigTable.feature

@@ -0,0 +1,130 @@
+@BigTable
+Feature: BigTable source - Verification of BigTable to BigTable successful data transfer without using connections
+
+  @BIGTABLE_SOURCE_TEST @BIGTABLE_SINK_TEST
+  Scenario: To verify data is getting transferred from BigTable source table to BigTable sink table
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "Bigtable" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "Bigtable" from the plugins list as: "Sink"
+    Then Connect plugins: "Bigtable" and "Bigtable2" to establish connection
+    Then Navigate to the properties page of plugin: "Bigtable"
+    Then Enter input plugin property: "referenceName" with value: "CBTSourceReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableInstance"
+    Then Enter input plugin property: "table" with value: "bigtableSourceTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsourceMappings"
+    Then Select Macro action of output schema property: "outputSchemaMacroInput" and set the value to "cbtSourceOutputSchema"
+    Then Validate "Bigtable" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Bigtable2"
+    Then Enter input plugin property: "referenceName" with value: "CBTSinkReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableTargetInstance"
+    Then Enter input plugin property: "table" with value: "bigtableTargetTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsinkMappings"
+    Then Validate "Bigtable" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Enter runtime argument value "cbtSourceOutputSchema" for key "cbtSourceOutputSchema"
+    Then Run the preview of pipeline with runtime arguments
+    Then Wait till pipeline preview is in running state
+    Then Open and capture pipeline preview logs
+    Then Verify the preview run status of pipeline in the logs is "succeeded"
+    Then Close the pipeline logs
+    Then Close the preview
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Enter runtime argument value "cbtSourceOutputSchema" for key "cbtSourceOutputSchema"
+    Then Run the Pipeline in Runtime with runtime arguments
+    Then Wait till pipeline is in running state
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate OUT record count is equal to IN record count
+
+  @BIGTABLE_SOURCE_TEST @EXISTING_BIGTABLE_SINK
+  Scenario: To verify data is getting transferred from BigTable source table to  existing BigTable sink
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "Bigtable" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "Bigtable" from the plugins list as: "Sink"
+    Then Connect plugins: "Bigtable" and "Bigtable2" to establish connection
+    Then Navigate to the properties page of plugin: "Bigtable"
+    Then Enter input plugin property: "referenceName" with value: "CBTSourceReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableInstance"
+    Then Enter input plugin property: "table" with value: "bigtableSourceTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsourceMappings"
+    Then Select Macro action of output schema property: "outputSchemaMacroInput" and set the value to "cbtSourceOutputSchema"
+    Then Validate "Bigtable" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Bigtable2"
+    Then Enter input plugin property: "referenceName" with value: "CBTSinkReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableTargetInstance"
+    Then Enter input plugin property: "table" with value: "bigtableTargetExistingTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsinkMappings"
+    Then Validate "Bigtable" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Enter runtime argument value "cbtSourceOutputSchema" for key "cbtSourceOutputSchema"
+    Then Run the preview of pipeline with runtime arguments
+    Then Wait till pipeline preview is in running state
+    Then Open and capture pipeline preview logs
+    Then Verify the preview run status of pipeline in the logs is "succeeded"
+    Then Close the pipeline logs
+    Then Close the preview
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Enter runtime argument value "cbtSourceOutputSchema" for key "cbtSourceOutputSchema"
+    Then Run the Pipeline in Runtime with runtime arguments
+    Then Wait till pipeline is in running state
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate OUT record count is equal to IN record count
+
+  @BIGTABLE_SOURCE_TEST @BIGTABLE_SINK_TEST
+  Scenario: To verify data is getting transferred from not existing BigTable source table to BigTable sink table
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "Bigtable" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "Bigtable" from the plugins list as: "Sink"
+    Then Connect plugins: "Bigtable" and "Bigtable2" to establish connection
+    Then Navigate to the properties page of plugin: "Bigtable"
+    Then Enter input plugin property: "referenceName" with value: "CBTSourceReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableInstance"
+    Then Enter input plugin property: "table" with value: "bigtableSourceTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsourceMappings"
+    Then Select Macro action of output schema property: "outputSchemaMacroInput" and set the value to "cbtSourceOutputSchema"
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Bigtable2"
+    Then Enter input plugin property: "referenceName" with value: "CBTSinkReferenceName"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Enter input plugin property: "instance" with value: "bigtableTargetInstance"
+    Then Enter input plugin property: "table" with value: "bigtableTargetTable"
+    Then Replace input plugin property: "keyAlias" with value: "id"
+    Then Enter key value pairs for plugin property: "columnMappings" with values from json: "cbtsinkMappings"
+    Then Validate "Bigtable" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Enter runtime argument value "cbtSourceOutputSchema" for key "cbtSourceOutputSchema"
+    Then Run the preview of pipeline with runtime arguments
+    Then Wait till pipeline preview is in running state
+    Then Open and capture pipeline preview logs
+    Then Verify the preview run status of pipeline in the logs is "succeeded"
+    Then Close the pipeline logs
+    Then Close the preview
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Enter runtime argument value "cbtSourceOutputSchema" for key "cbtSourceOutputSchema"
+    Then Run the Pipeline in Runtime with runtime arguments
+    Then Wait till pipeline is in running state
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate OUT record count is equal to IN record count

src/e2e-test/java/io/cdap/plugin/bigtable/runners/TestRunner.java

@@ -0,0 +1,22 @@
+package io.cdap.plugin.bigtable.runners;
+
+
+import io.cucumber.junit.Cucumber;
+import io.cucumber.junit.CucumberOptions;
+import org.junit.runner.RunWith;
+
+/**
+ * Test Runner to execute Bigtable testcases.
+ */
+@RunWith(Cucumber.class)
+@CucumberOptions(
+        features = {"src/e2e-test/features"},
+        glue = {"io.cdap.plugin.bigtable.stepsdesign", "io.cdap.plugin.common.stepsdesign", "stepsdesign"},
+        tags = {"@BigTable"},
+        monochrome = true,
+        plugin = {"pretty", "html:target/cucumber-html-report/bigtable",
+                "json:target/cucumber-reports/cucumber-bigtable.json",
+                "junit:target/cucumber-reports/cucumber-bigtable.xml"}
+)
+public class TestRunner {
+}

src/e2e-test/java/io/cdap/plugin/bigtable/runners/package-info.java

@@ -0,0 +1 @@
+package io.cdap.plugin.bigtable.runners;

src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java

@@ -15,6 +15,7 @@
  */
 package io.cdap.plugin.common.stepsdesign;
 
+
 import com.google.cloud.bigquery.BigQueryException;
 import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.StorageException;
@@ -23,6 +24,7 @@
 import io.cdap.e2e.utils.BigQueryClient;
 import io.cdap.e2e.utils.PluginPropertyUtils;
 import io.cdap.e2e.utils.StorageClient;
+import io.cdap.plugin.utils.BigTableClient;
 import io.cdap.plugin.utils.PubSubClient;
 import io.cdap.plugin.utils.SpannerClient;
 import io.cucumber.java.After;
@@ -48,6 +50,12 @@
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
+import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient;
+import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings;
+import org.apache.hadoop.hbase.client.Connection;
+
+
+
 
 /**
  * GCP test hooks.
@@ -69,6 +77,22 @@ public class TestSetupHooks {
   public static String spannerTargetTable = StringUtils.EMPTY;
   public static boolean firstSpannerTestFlag = true;
   public static String datasetName = PluginPropertyUtils.pluginProp("dataset");
+  public static boolean firstBigTableTestFlag = true;
+  public static String bigtableInstance = StringUtils.EMPTY;
+  public static String bigtableCluster = StringUtils.EMPTY;
+
+  public static String bigtableSourceTable = StringUtils.EMPTY;
+  public static String bigtableTargetInstance = StringUtils.EMPTY;
+  public static String bigtableTargetCluster = StringUtils.EMPTY;
+
+  public static String bigtableTargetTable = StringUtils.EMPTY;
+  public static String bigtableExistingTargetTable = StringUtils.EMPTY;
+
+  public static BigtableInstanceAdminSettings instanceAdminSettings;
+  public static  BigtableInstanceAdminClient adminClient;
+  public static Connection bigTableConnection ;
+  public static Connection bigTableExistingTargetTableConnection ;
+
 
   @Before(order = 1)
   public static void overrideServiceAccountFilePathIfProvided() {
@@ -1296,4 +1320,112 @@ private static String createGCSBucketLifeCycle() throws IOException, URISyntaxEx
   public static void createBucketWithLifeCycle() throws IOException, URISyntaxException {
     gcsTargetBucketName = createGCSBucketLifeCycle();
     BeforeActions.scenario.write("GCS target bucket name - " + gcsTargetBucketName); }
-}
+
+
+  @Before(order = 1, value = "@BIGTABLE_SOURCE_TEST")
+  public static void testGoogleBigtableSetup() throws IOException, URISyntaxException {
+     String projectId = PluginPropertyUtils.pluginProp("projectId");
+    instanceAdminSettings =
+            BigtableInstanceAdminSettings.newBuilder().setProjectId(projectId).build();
+    adminClient = BigtableInstanceAdminClient.create(instanceAdminSettings);
+
+    if (firstBigTableTestFlag) {
+      Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+        try {
+          BigTableClient.deleteInstance(adminClient, bigtableInstance);
+          BeforeActions.scenario.write("Bigtable instance--" + bigtableInstance + " deleted successfully");
+          bigtableInstance = StringUtils.EMPTY;
+        } catch (Exception e) {
+          if (e.getMessage().contains("NOT FOUND")) {
+            BeforeActions.scenario.write("BigTable instance--" + bigtableInstance + " does not exist.");
+          }
+        }
+      }));
+      firstBigTableTestFlag = false;
+
+      bigtableInstance = "e2e-inst-" + (int) (Math.random() * Integer.MAX_VALUE);
+      bigtableCluster = "e2e-clstr-" + (int) (Math.random() * Integer.MAX_VALUE);
+      try {
+        BigTableClient.createBigTableInstance(adminClient, bigtableInstance, bigtableCluster);
+        BeforeActions.scenario.write("BigTable instance--" + bigtableInstance + " created successfully");
+        bigTableConnection = BigTableClient.connect(projectId, bigtableInstance, null);
+        //Source table creation
+        bigtableSourceTable = "e2e-src-table-" + (int) (Math.random() * Integer.MAX_VALUE);
+        BigTableClient.createTables(bigTableConnection, bigtableSourceTable, null);
+        BeforeActions.scenario.write("sourceTable--" + bigtableSourceTable + " created successfully");
+        BigTableClient.populateData(bigTableConnection, bigtableSourceTable);
+        BeforeActions.scenario.write("sourceTable--" + bigtableSourceTable + " populated successfully");
+        PluginPropertyUtils.addPluginProp("bigtableInstance", bigtableInstance);
+        PluginPropertyUtils.addPluginProp("bigtableCluster", bigtableCluster);
+        PluginPropertyUtils.addPluginProp("bigtableSourceTable", bigtableSourceTable);
+      } catch (Exception e) {
+        e.printStackTrace();
+      }
+
+    }
+
+  }
+
+  @Before(order = 2, value = "@BIGTABLE_SINK_TEST")
+  public static void setTempTargetBigTableInstanceAndTableName() {
+    bigtableTargetInstance=bigtableInstance;
+    bigtableTargetCluster=bigtableCluster;
+    bigtableTargetTable="e2e_target_table_"
+            + UUID.randomUUID().toString().substring(0, 10).replaceAll("-", "_");
+    PluginPropertyUtils.addPluginProp("bigtableTargetInstance", bigtableTargetInstance);
+    PluginPropertyUtils.addPluginProp("bigtableTargetCluster", bigtableTargetCluster);
+    PluginPropertyUtils.addPluginProp("bigtableTargetTable", bigtableTargetTable);
+    BeforeActions.scenario.write("BigTable Target instance name - " + bigtableTargetInstance);
+    BeforeActions.scenario.write("BigTable Target cluster name - " + bigtableTargetCluster);
+    BeforeActions.scenario.write("BigTable Target table name - " + bigtableTargetTable);
+  }
+
+  @After(order = 2, value = "@BIGTABLE_SINK_TEST")
+  public static void emptyTargetBigTableInstanceAndTableName() {
+    PluginPropertyUtils.removePluginProp("bigtableTargetInstance");
+    PluginPropertyUtils.removePluginProp("bigtableTargetCluster");
+    PluginPropertyUtils.removePluginProp("bigtableTargetTable");
+    bigtableTargetInstance=StringUtils.EMPTY;
+    bigtableTargetCluster=StringUtils.EMPTY;
+    bigtableTargetTable=StringUtils.EMPTY;
+  }
+
+  @Before(order = 2, value = "@EXISTING_BIGTABLE_SINK")
+  public static void makeExistingBigTableInstanceAndTableName() {
+    try {
+      String projectId = PluginPropertyUtils.pluginProp("projectId");
+      instanceAdminSettings =
+              BigtableInstanceAdminSettings.newBuilder().setProjectId(projectId).build();
+      adminClient = BigtableInstanceAdminClient.create(instanceAdminSettings);
+      bigtableTargetInstance = bigtableInstance;
+      bigtableTargetCluster = bigtableCluster;
+      bigtableExistingTargetTable = "e2e_target_table_"
+              + UUID.randomUUID().toString().substring(0, 10).replaceAll("-", "_");
+
+      BigTableClient.createBigTableInstance(adminClient, bigtableTargetInstance, bigtableTargetCluster);
+      bigTableExistingTargetTableConnection =
+              BigTableClient.connect(projectId, bigtableTargetInstance, null);
+      BigTableClient.createTables(bigTableExistingTargetTableConnection, null, bigtableExistingTargetTable);
+      PluginPropertyUtils.addPluginProp("bigtableTargetInstance", bigtableTargetInstance);
+      PluginPropertyUtils.addPluginProp("bigtableTargetCluster", bigtableTargetCluster);
+      PluginPropertyUtils.addPluginProp("bigtableTargetExistingTable", bigtableExistingTargetTable);
+
+      } catch (Exception e) {
+        e.printStackTrace();
+      }
+    }
+
+  @After(order = 2, value = "@EXISTING_BIGTABLE_SINK")
+  public static void emptyExistingBigTableInstanceAndTableName() {
+    try {
+      PluginPropertyUtils.removePluginProp("bigtableTargetInstance");
+      PluginPropertyUtils.removePluginProp("bigtableTargetCluster");
+      PluginPropertyUtils.removePluginProp("bigtableTargetExistingTable");
+      bigtableTargetInstance=StringUtils.EMPTY;
+      bigtableTargetCluster=StringUtils.EMPTY;
+      bigtableExistingTargetTable=StringUtils.EMPTY;
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+}