diff --git a/.github/workflows/spanner-staging-tests.yml b/.github/workflows/spanner-staging-tests.yml new file mode 100644 index 0000000000..51c22a991e --- /dev/null +++ b/.github/workflows/spanner-staging-tests.yml @@ -0,0 +1,51 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Spanner Staging integration tests + +on: + workflow_dispatch: + +permissions: read-all + +jobs: + spanner_java_integration_tests_templates: + name: Spanner Dataflow Templates Integration Tests + timeout-minutes: 180 + # Run on any runner that matches all the specified runs-on values. + runs-on: [ self-hosted, it ] + steps: + - name: Checkout Code + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 + - name: Setup Environment + id: setup-env + uses: ./.github/actions/setup-env + - name: Run Integration Tests + run: | + ./cicd/run-it-tests \ + --modules-to-build="ALL" \ + --it-region="us-central1" \ + --it-project="cloud-teleport-testing" \ + --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \ + --it-private-connectivity="datastream-private-connect-us-central1" \ + --it-spanner-host="https://staging-wrenchworks.sandbox.googleapis.com/" + - name: Upload Integration Tests Report + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + if: always() # always run even if the previous step fails + with: + name: surefire-test-results + path: '**/surefire-reports/TEST-*.xml' + retention-days: 1 + - name: Cleanup Java Environment + uses: ./.github/actions/cleanup-java-env \ No newline at end of file diff --git a/cicd/cmd/run-it-smoke-tests/main.go b/cicd/cmd/run-it-smoke-tests/main.go index e5fa462235..5a2fc17fc1 100644 --- a/cicd/cmd/run-it-smoke-tests/main.go +++ b/cicd/cmd/run-it-smoke-tests/main.go @@ -66,6 +66,7 @@ func main() { flags.ArtifactBucket(), flags.StageBucket(), flags.PrivateConnectivity(), + flags.SpannerHost(), flags.FailureMode(), flags.RetryFailures(), flags.StaticOracleHost(), diff --git a/cicd/cmd/run-it-tests/main.go b/cicd/cmd/run-it-tests/main.go index 24b8145d63..8830edfb9b 100644 --- a/cicd/cmd/run-it-tests/main.go +++ b/cicd/cmd/run-it-tests/main.go @@ -67,6 +67,7 @@ func main() { flags.StageBucket(), flags.HostIp(), flags.PrivateConnectivity(), + flags.SpannerHost(), flags.FailureMode(), flags.RetryFailures(), flags.StaticOracleHost(), diff --git a/cicd/internal/flags/it-flags.go b/cicd/internal/flags/it-flags.go index 1aa55e9a08..6937353983 100644 --- a/cicd/internal/flags/it-flags.go +++ b/cicd/internal/flags/it-flags.go @@ -101,10 +101,10 @@ func PrivateConnectivity() string { } func SpannerHost() string { - if dSpannerHost == "" { - return "-DspannerHost=" + "https://staging-wrenchworks.sandbox.googleapis.com/" + if dSpannerHost != "" { + return "-DspannerHost=" + dSpannerHost } - return "-DspannerHost=" + dSpannerHost + return "" } func FailureMode() string { diff --git a/metadata/src/main/java/com/google/cloud/teleport/metadata/SpannerStagingTest.java b/metadata/src/main/java/com/google/cloud/teleport/metadata/SpannerStagingTest.java new file mode 100644 index 0000000000..48947f267d --- /dev/null +++ b/metadata/src/main/java/com/google/cloud/teleport/metadata/SpannerStagingTest.java @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.metadata; + +/** Annotation that marks the test as a Spanner staging test. */ +public @interface SpannerStagingTest {} diff --git a/pom.xml b/pom.xml index 297ac1ae3d..04b1d3d6d1 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,7 @@ com.google.cloud.teleport.metadata.TemplateLoadTest com.google.cloud.teleport.metadata.DirectRunnerTest + com.google.cloud.teleport.metadata.SpannerStagingTest JAVA_LICENSE_HEADER @@ -458,6 +459,56 @@ + + spannerStagingIntegrationTests + + false + + + + true + + false + + classesAndMethods + 2 + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + + beamPythonVersion + ${beam-python.version} + + + beamJavaVersion + ${beam.version} + + + beamMavenRepo + ${beam-maven-repo} + + + + **/*.java + + + ${spanner.staging.tests} + + true + ${itParallelismType} + ${itParallelism} + false + + + + + templatesLoadTests diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/ExportPipelineIT.java b/v1/src/test/java/com/google/cloud/teleport/spanner/ExportPipelineIT.java index bdeb29a5fd..3ae9bbdb83 100644 --- a/v1/src/test/java/com/google/cloud/teleport/spanner/ExportPipelineIT.java +++ b/v1/src/test/java/com/google/cloud/teleport/spanner/ExportPipelineIT.java @@ -23,6 +23,7 @@ import com.google.cloud.spanner.Dialect; import com.google.cloud.spanner.Mutation; +import com.google.cloud.teleport.metadata.SpannerStagingTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; import java.io.IOException; import java.util.ArrayList; @@ -47,7 +48,7 @@ import org.junit.runners.Parameterized; /** Integration test for {@link ExportPipeline Spanner to GCS Avro} template. */ -@Category(TemplateIntegrationTest.class) +@Category({TemplateIntegrationTest.class, SpannerStagingTest.class}) @TemplateIntegrationTest(ExportPipeline.class) @RunWith(Parameterized.class) public class ExportPipelineIT extends SpannerTemplateITBase { diff --git a/v1/src/test/java/com/google/cloud/teleport/spanner/ImportPipelineIT.java b/v1/src/test/java/com/google/cloud/teleport/spanner/ImportPipelineIT.java index 87d45a2dcb..8a0a75778d 100644 --- a/v1/src/test/java/com/google/cloud/teleport/spanner/ImportPipelineIT.java +++ b/v1/src/test/java/com/google/cloud/teleport/spanner/ImportPipelineIT.java @@ -22,6 +22,7 @@ import com.google.cloud.spanner.Dialect; import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SpannerStagingTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -44,7 +45,7 @@ import org.junit.runners.Parameterized; /** Integration test for {@link ImportPipeline} classic template. */ -@Category(TemplateIntegrationTest.class) +@Category({TemplateIntegrationTest.class, SpannerStagingTest.class}) @TemplateIntegrationTest(ImportPipeline.class) @RunWith(Parameterized.class) public class ImportPipelineIT extends SpannerTemplateITBase { diff --git a/v1/src/test/java/com/google/cloud/teleport/templates/SpannerToTextIT.java b/v1/src/test/java/com/google/cloud/teleport/templates/SpannerToTextIT.java index 5240495a16..b75c057793 100644 --- a/v1/src/test/java/com/google/cloud/teleport/templates/SpannerToTextIT.java +++ b/v1/src/test/java/com/google/cloud/teleport/templates/SpannerToTextIT.java @@ -22,6 +22,7 @@ import com.google.cloud.spanner.Dialect; import com.google.cloud.spanner.Mutation; +import com.google.cloud.teleport.metadata.SpannerStagingTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; import java.io.IOException; import java.util.ArrayList; @@ -43,7 +44,7 @@ import org.junit.runners.Parameterized; /** Integration test for {@link SpannerToText Spanner to GCS Text} template. */ -@Category(TemplateIntegrationTest.class) +@Category({TemplateIntegrationTest.class, SpannerStagingTest.class}) @TemplateIntegrationTest(SpannerToText.class) @RunWith(Parameterized.class) public class SpannerToTextIT extends SpannerTemplateITBase { diff --git a/v1/src/test/java/com/google/cloud/teleport/templates/TextImportPipelineIT.java b/v1/src/test/java/com/google/cloud/teleport/templates/TextImportPipelineIT.java index 3f43b14ec0..e2893bcf83 100644 --- a/v1/src/test/java/com/google/cloud/teleport/templates/TextImportPipelineIT.java +++ b/v1/src/test/java/com/google/cloud/teleport/templates/TextImportPipelineIT.java @@ -23,6 +23,7 @@ import com.google.cloud.spanner.Dialect; import com.google.cloud.spanner.Struct; +import com.google.cloud.teleport.metadata.SpannerStagingTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; import com.google.cloud.teleport.spanner.TextImportPipeline; import com.google.common.collect.ImmutableList; @@ -47,7 +48,7 @@ import org.junit.runners.JUnit4; /** Integration test for {@link TextImportPipeline}. */ -@Category(TemplateIntegrationTest.class) +@Category({TemplateIntegrationTest.class, SpannerStagingTest.class}) @TemplateIntegrationTest(TextImportPipeline.class) @RunWith(JUnit4.class) public final class TextImportPipelineIT extends TemplateTestBase {