Creating new annotation and new workflow for spanner staging tests (#…

…2053) * Creating a new annotation for spanner staging tests * Creating a new github action: * Adding the annotation to textImportPipeline
GoogleCloudPlatform · Dec 11, 2024 · d0b0bb0 · d0b0bb0
1 parent 6a50180
commit d0b0bb0
Show file tree

Hide file tree

Showing 10 changed files with 134 additions and 7 deletions.
diff --git a/.github/workflows/spanner-staging-tests.yml b/.github/workflows/spanner-staging-tests.yml
@@ -0,0 +1,51 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Spanner Staging integration tests
+
+on:
+  workflow_dispatch:
+
+permissions: read-all
+
+jobs:
+  spanner_java_integration_tests_templates:
+    name: Spanner Dataflow Templates Integration Tests
+    timeout-minutes: 180
+    # Run on any runner that matches all the specified runs-on values.
+    runs-on: [ self-hosted, it ]
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
+    - name: Setup Environment
+      id: setup-env
+      uses: ./.github/actions/setup-env
+    - name: Run Integration Tests
+      run: |
+        ./cicd/run-it-tests \
+        --modules-to-build="ALL" \
+        --it-region="us-central1" \
+        --it-project="cloud-teleport-testing" \
+        --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
+        --it-private-connectivity="datastream-private-connect-us-central1" \
+        --it-spanner-host="https://staging-wrenchworks.sandbox.googleapis.com/"
+    - name: Upload Integration Tests Report
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+      if: always() # always run even if the previous step fails
+      with:
+        name: surefire-test-results
+        path: '**/surefire-reports/TEST-*.xml'
+        retention-days: 1
+    - name: Cleanup Java Environment
+      uses: ./.github/actions/cleanup-java-env
diff --git a/cicd/cmd/run-it-smoke-tests/main.go b/cicd/cmd/run-it-smoke-tests/main.go
@@ -66,6 +66,7 @@ func main() {
 		flags.ArtifactBucket(),
 		flags.StageBucket(),
 		flags.PrivateConnectivity(),
+		flags.SpannerHost(),
 		flags.FailureMode(),
 		flags.RetryFailures(),
 		flags.StaticOracleHost(),

diff --git a/cicd/cmd/run-it-tests/main.go b/cicd/cmd/run-it-tests/main.go
@@ -67,6 +67,7 @@ func main() {
 		flags.StageBucket(),
 		flags.HostIp(),
 		flags.PrivateConnectivity(),
+		flags.SpannerHost(),
 		flags.FailureMode(),
 		flags.RetryFailures(),
 		flags.StaticOracleHost(),

diff --git a/cicd/internal/flags/it-flags.go b/cicd/internal/flags/it-flags.go
@@ -101,10 +101,10 @@ func PrivateConnectivity() string {
 }
 
 func SpannerHost() string {
-	if dSpannerHost == "" {
-		return "-DspannerHost=" + "https://staging-wrenchworks.sandbox.googleapis.com/"
+	if dSpannerHost != "" {
+		return "-DspannerHost=" + dSpannerHost
 	}
-	return "-DspannerHost=" + dSpannerHost
+	return ""
 }
 
 func FailureMode() string {

diff --git a/metadata/src/main/java/com/google/cloud/teleport/metadata/SpannerStagingTest.java b/metadata/src/main/java/com/google/cloud/teleport/metadata/SpannerStagingTest.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.cloud.teleport.metadata;
+
+/** Annotation that marks the test as a Spanner staging test. */
+public @interface SpannerStagingTest {}
diff --git a/pom.xml b/pom.xml
@@ -106,6 +106,7 @@
     <load.tests>com.google.cloud.teleport.metadata.TemplateLoadTest</load.tests>
     <direct-runner.tests>com.google.cloud.teleport.metadata.DirectRunnerTest</direct-runner.tests>
     <excluded.spanner.tests></excluded.spanner.tests>
+    <spanner.staging.tests>com.google.cloud.teleport.metadata.SpannerStagingTest</spanner.staging.tests>
 
     <licenseHeaderFile>JAVA_LICENSE_HEADER</licenseHeaderFile>
   </properties>
@@ -458,6 +459,56 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>spannerStagingIntegrationTests</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <!-- Skip coverage checks, unit tests are skipped -->
+        <jacoco.skip>true</jacoco.skip>
+        <!-- Some modules may yield no integration tests -->
+        <failIfNoTests>false</failIfNoTests>
+        <!-- Parallelism settings. Default is 2, set to consider methods -->
+        <itParallelismType>classesAndMethods</itParallelismType>
+        <itParallelism>2</itParallelism>
+      </properties>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>${surefire.version}</version>
+            <configuration combine.self="override">
+              <systemProperties>
+                <property>
+                  <name>beamPythonVersion</name>
+                  <value>${beam-python.version}</value>
+                </property>
+                <property>
+                  <name>beamJavaVersion</name>
+                  <value>${beam.version}</value>
+                </property>
+                <property>
+                  <name>beamMavenRepo</name>
+                  <value>${beam-maven-repo}</value>
+                </property>
+              </systemProperties>
+              <includes>
+                <include>**/*.java</include>
+              </includes>
+              <groups>
+                ${spanner.staging.tests}
+              </groups>
+              <reuseForks>true</reuseForks>
+              <parallel>${itParallelismType}</parallel>
+              <threadCount>${itParallelism}</threadCount>
+              <trimStackTrace>false</trimStackTrace>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>templatesLoadTests</id>
       <activation>

diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/ExportPipelineIT.java b/v1/src/test/java/com/google/cloud/teleport/spanner/ExportPipelineIT.java
@@ -23,6 +23,7 @@
 
 import com.google.cloud.spanner.Dialect;
 import com.google.cloud.spanner.Mutation;
+import com.google.cloud.teleport.metadata.SpannerStagingTest;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -47,7 +48,7 @@
 import org.junit.runners.Parameterized;
 
 /** Integration test for {@link ExportPipeline Spanner to GCS Avro} template. */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SpannerStagingTest.class})
 @TemplateIntegrationTest(ExportPipeline.class)
 @RunWith(Parameterized.class)
 public class ExportPipelineIT extends SpannerTemplateITBase {

diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/ImportPipelineIT.java b/v1/src/test/java/com/google/cloud/teleport/spanner/ImportPipelineIT.java
@@ -22,6 +22,7 @@
 
 import com.google.cloud.spanner.Dialect;
 import com.google.cloud.spanner.Struct;
+import com.google.cloud.teleport.metadata.SpannerStagingTest;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -44,7 +45,7 @@
 import org.junit.runners.Parameterized;
 
 /** Integration test for {@link ImportPipeline} classic template. */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SpannerStagingTest.class})
 @TemplateIntegrationTest(ImportPipeline.class)
 @RunWith(Parameterized.class)
 public class ImportPipelineIT extends SpannerTemplateITBase {

diff --git a/v1/src/test/java/com/google/cloud/teleport/templates/SpannerToTextIT.java b/v1/src/test/java/com/google/cloud/teleport/templates/SpannerToTextIT.java
@@ -22,6 +22,7 @@
 
 import com.google.cloud.spanner.Dialect;
 import com.google.cloud.spanner.Mutation;
+import com.google.cloud.teleport.metadata.SpannerStagingTest;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -43,7 +44,7 @@
 import org.junit.runners.Parameterized;
 
 /** Integration test for {@link SpannerToText Spanner to GCS Text} template. */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SpannerStagingTest.class})
 @TemplateIntegrationTest(SpannerToText.class)
 @RunWith(Parameterized.class)
 public class SpannerToTextIT extends SpannerTemplateITBase {

diff --git a/v1/src/test/java/com/google/cloud/teleport/templates/TextImportPipelineIT.java b/v1/src/test/java/com/google/cloud/teleport/templates/TextImportPipelineIT.java
@@ -23,6 +23,7 @@
 
 import com.google.cloud.spanner.Dialect;
 import com.google.cloud.spanner.Struct;
+import com.google.cloud.teleport.metadata.SpannerStagingTest;
 import com.google.cloud.teleport.metadata.TemplateIntegrationTest;
 import com.google.cloud.teleport.spanner.TextImportPipeline;
 import com.google.common.collect.ImmutableList;
@@ -47,7 +48,7 @@
 import org.junit.runners.JUnit4;
 
 /** Integration test for {@link TextImportPipeline}. */
-@Category(TemplateIntegrationTest.class)
+@Category({TemplateIntegrationTest.class, SpannerStagingTest.class})
 @TemplateIntegrationTest(TextImportPipeline.class)
 @RunWith(JUnit4.class)
 public final class TextImportPipelineIT extends TemplateTestBase {