Merge branch 'data-integrations:develop' into develop

Vipinofficial11 · Mar 23, 2023 · abe93d7 · abe93d7
2 parents f8a56c0 + 40e7a9c
commit abe93d7
Show file tree

Hide file tree

Showing 35 changed files with 1,950 additions and 86 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,66 @@
+# Copyright © 2022 Cask Data, Inc.
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#  use this file except in compliance with the License. You may obtain a copy of
+#  the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations under
+#  the License.
+
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# Note: Any changes to this workflow would be used only after merging into develop.
+name: Build with unit tests
+
+on:
+  workflow_run:
+    workflows:
+      - Trigger build
+    types:
+      - completed
+
+jobs:
+  build:
+    runs-on: k8s-runner-build
+
+    if: ${{ github.event.workflow_run.conclusion != 'skipped' }}
+
+    steps:
+      # Pinned 1.0.0 version
+      - uses: haya14busa/action-workflow_run-status@967ed83efa565c257675ed70cfe5231f062ddd94
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.workflow_run.head_sha }}
+
+      - name: Cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-${{ github.workflow }}
+
+      - name: Build with Maven
+        run: mvn clean test -fae -T 2 -B -V -DcloudBuild -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
+      - name: Archive build artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: Build debug files
+          path: |
+            **/target/rat.txt
+            **/target/surefire-reports/*
+
+      - name: Surefire Report
+        # Pinned 3.5.2 version
+        uses: mikepenz/action-junit-report@16a9560bd02f11e7e3bf6b3e2ef6bba6c9d07c32
+        if: always()
+        with:
+          report_paths: '**/target/surefire-reports/TEST-*.xml'
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          detailed_summary: true
+          commit: ${{ github.event.workflow_run.head_sha }}
+          check_name: Test Report
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -0,0 +1,104 @@
+#  Copyright © 2022 Cask Data, Inc.
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#  use this file except in compliance with the License. You may obtain a copy of
+#  the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations under
+#  the License.
+
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+# Note: Any changes to this workflow would be used only after merging into develop
+name: Build e2e tests
+
+on:
+  workflow_run:
+    workflows:
+      - Trigger build
+    types:
+      - completed
+
+jobs:
+  build:
+    runs-on: k8s-runner-e2e
+
+    if: ${{ github.event.workflow_run.conclusion != 'skipped' }}
+
+    steps:
+      - uses: haya14busa/action-workflow_run-status@967ed83efa565c257675ed70cfe5231f062ddd94
+      - uses: actions/checkout@v3
+        with:
+          path: plugin
+          ref: ${{ github.event.workflow_run.head_sha }}
+
+      - name: Checkout e2e test repo
+        uses: actions/checkout@v3
+        with:
+          repository: cdapio/cdap-e2e-tests
+          path: e2e
+
+      - name: Cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ github.workflow }}-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-${{ github.workflow }}
+
+      - name: Get Secrets from GCP Secret Manager
+        id: 'secrets'
+        uses: 'google-github-actions/get-secretmanager-secrets@v0'
+        with:
+          secrets: |-
+            SALESFORCE_MARKETING_CLIENT_ID:cdapio-github-builds/SALESFORCE_MARKETING_CLIENT_ID
+            SALESFORCE_MARKETING_CLIENT_SECRET:cdapio-github-builds/SALESFORCE_MARKETING_CLIENT_SECRET
+            SALESFORCE_MARKETING_BASE_URI:cdapio-github-builds/SALESFORCE_MARKETING_BASE_URI
+            SALESFORCE_MARKETING_SOAP_API_ENDPOINT:cdapio-github-builds/SALESFORCE_MARKETING_SOAP_API_ENDPOINT
+            SALESFORCE_MARKETING_REST_API_ENDPOINT:cdapio-github-builds/SALESFORCE_MARKETING_REST_API_ENDPOINT
+            PROJECT_ID:cdapio-github-builds/PROJECT_ID
+            SALESFORCE_MARKETING_DATASET:cdapio-github-builds/SALESFORCE_MARKETING_DATASET
+
+      - name: Run tests
+        run: python3 e2e/src/main/scripts/run_e2e_test.py
+        env:
+            SALESFORCE_MARKETING_CLIENT_ID: ${{ steps.secrets.outputs.SALESFORCE_MARKETING_CLIENT_ID }}
+            SALESFORCE_MARKETING_CLIENT_SECRET: ${{ steps.secrets.outputs.SALESFORCE_MARKETING_CLIENT_SECRET }}
+            SALESFORCE_MARKETING_BASE_URI: ${{ steps.secrets.outputs.SALESFORCE_MARKETING_BASE_URI }}
+            SALESFORCE_MARKETING_SOAP_API_ENDPOINT: ${{ steps.secrets.outputs.SALESFORCE_MARKETING_SOAP_API_ENDPOINT }}
+            SALESFORCE_MARKETING_REST_API_ENDPOINT: ${{ steps.secrets.outputs.SALESFORCE_MARKETING_REST_API_ENDPOINT }}
+            PROJECT_ID: ${{ steps.secrets.outputs.PROJECT_ID }}
+            SALESFORCE_MARKETING_DATASET: ${{ steps.secrets.outputs.SALESFORCE_MARKETING_DATASET }}
+
+      - name: Upload report
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: Cucumber report
+          path: ./plugin/target/cucumber-reports
+
+      - name: Upload debug files
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: Debug files
+          path: ./**/target/e2e-debug
+
+      - name: Upload reports to GCS
+        uses: google-github-actions/upload-cloud-storage@v0
+        if: always()
+        with:
+          path: ./plugin/target/cucumber-reports
+          destination: e2e-tests-cucumber-reports/${{ github.event.repository.name }}/${{ github.ref }}
+
+      - name: github-status-action
+        uses: Sibz/github-status-action@67af1f4042a5a790681aad83c44008ca6cfab83d
+        if: always()
+        with:
+          authToken: ${{ secrets.GITHUB_TOKEN }}
+          state: success
+          context: Cucumber report
+          sha: ${{github.event.pull_request.head.sha || github.sha}}
+
diff --git a/.github/workflows/trigger.yml b/.github/workflows/trigger.yml
@@ -0,0 +1,46 @@
+#  Copyright © 2022 Cask Data, Inc.
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#  use this file except in compliance with the License. You may obtain a copy of
+#  the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations under
+#  the License.
+
+# This workflow will trigger build.yml only when needed.
+# This way we don't flood main workflow run list
+# Note that build.yml from develop will be used even for PR builds
+# Also it will have access to the proper GITHUB_SECRET
+
+name: Trigger build
+
+on:
+  push:
+    branches: [ develop, release/** ]
+  pull_request:
+    branches: [ develop, release/** ]
+    types: [ opened, synchronize, reopened, labeled ]
+  workflow_dispatch:
+
+jobs:
+  trigger:
+    runs-on: k8s-runner-e2e
+
+    # We allow builds:
+    # 1) When triggered manually
+    # 2) When it's a merge into a branch
+    # 3) For PRs that are labeled as build and
+    #  - It's a code change
+    #  - A build label was just added
+    # A bit complex, but prevents builds when other labels are manipulated.
+    if: >
+      github.event_name == 'workflow_dispatch'
+      || github.event_name == 'push'
+      || (contains(github.event.pull_request.labels.*.name, 'build')
+         && (github.event.action != 'labeled' || github.event.label.name == 'build')
+         )
+    steps:
+      - name: Trigger build
+        run: echo Maven build will be triggered now
diff --git a/SECURITY.md b/SECURITY.md
@@ -0,0 +1,8 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+To report a security issue, please use [https://g.co/vulnz](https://g.co/vulnz).
+We use g.co/vulnz for our intake, and do coordination and disclosure here on
+GitHub (including using GitHub Security Advisory). The Google Security Team will
+respond within 5 working days of your report on g.co/vulnz.