Teradata workflow files

Created workflow files for teradata connector test, documentation builds
Teradata · May 6, 2024 · 19e7e14 · 19e7e14
1 parent d3864c2
commit 19e7e14
Show file tree

Hide file tree

Showing 7 changed files with 306 additions and 3 deletions.
diff --git a/.github/workflows/airbyte-ci-teradata-ci.yaml b/.github/workflows/airbyte-ci-teradata-ci.yaml
@@ -0,0 +1,73 @@
+name: airbyte-ci-teradata-ci
+on:
+  push:
+    branches:
+      - td_develop
+  workflow_dispatch:
+permissions:
+  contents: write
+  pages: write
+  id-token: write
+jobs:
+  teradata-tests:
+    name: "Run Airbyte Teradata Destination Connector Unit and Integration Tests"
+    runs-on: "ubuntu-22.04"
+    steps:
+      - name: "Checkout Source Code"
+        uses: actions/checkout@v4
+        with:
+          ref: td_develop
+          persist-credentials: false
+      - name: "Install PIP"
+        run: |
+          sudo apt-get update
+          sudo apt install python3-pip
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: "Install Java 17"
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+      - name: "Install Node 16"
+        uses: actions/setup-node@v4
+        with:
+          node-version: '16'
+      - name: Install jq
+        run: sudo apt install -y jq
+      - name: "Install Docker and Docker Compose"
+        run: |
+          sudo apt-get update
+          sudo apt-get install ca-certificates curl gnupg lsb-release
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+          sudo chmod a+r /etc/apt/keyrings/docker.gpg
+          echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin
+          sudo chmod 666 /var/run/docker.sock
+      - name: Download Test artifacts from Teradata S3
+        run: |
+          mkdir airbyte-integrations/connectors/destination-teradata/secrets
+          aws s3 cp s3://${{ vars.NON_SSL_CSAE_ENV_AWS_PATH }} airbyte-integrations/connectors/destination-teradata/secrets
+          aws s3 cp s3://${{ vars.SSL_CSAE_ENV_AWS_PATH }} airbyte-integrations/connectors/destination-teradata/secrets
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+      - name: Install airbyte-ci binary
+        id: install-airbyte-ci
+        shell: bash
+        run: |
+          curl -sSL  https://connectors.airbyte.com/files/airbyte-ci/releases/ubuntu/3.0.2/airbyte-ci --output airbyte-ci-bin
+          sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
+          sudo chmod +x /usr/local/bin/airbyte-ci
+      - name: Test connectors
+        run: |
+          airbyte-ci connectors --name=destination-teradata test
+          mkdir integrationtestreport
+          ls -al ./airbyte-integrations/connectors/destination-teradata/build/reports/tests/integrationTestJava
+          cp -r  ./airbyte-integrations/connectors/destination-teradata/build/reports/tests/integrationTestJava integrationtestreport
+      
diff --git a/.github/workflows/airbyte-ci-teradata-ci_branch.yaml b/.github/workflows/airbyte-ci-teradata-ci_branch.yaml
@@ -0,0 +1,73 @@
+name: airbyte-ci-teradata-ci-branch
+on:
+  push:
+    branches-ignore:
+      - td_develop
+  workflow_dispatch:
+permissions:
+  contents: write
+  pages: write
+  id-token: write
+jobs:
+  teradata-tests:
+    name: "Run Airbyte Teradata Destination Connector Unit and Integration Tests"
+    runs-on: "ubuntu-22.04"
+    steps:
+      - name: "Checkout Source Code"
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ vars.BRANCH }}
+          persist-credentials: false
+      - name: "Install PIP"
+        run: |
+          sudo apt-get update
+          sudo apt install python3-pip
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: "Install Java 17"
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+      - name: "Install Node 16"
+        uses: actions/setup-node@v4
+        with:
+          node-version: '16'
+      - name: Install jq
+        run: sudo apt install -y jq
+      - name: "Install Docker and Docker Compose"
+        run: |
+          sudo apt-get update
+          sudo apt-get install ca-certificates curl gnupg lsb-release
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+          sudo chmod a+r /etc/apt/keyrings/docker.gpg
+          echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin
+          sudo chmod 666 /var/run/docker.sock
+      - name: Download Test artifacts from Teradata S3
+        run: |
+          mkdir airbyte-integrations/connectors/destination-teradata/secrets
+          aws s3 cp s3://${{ vars.NON_SSL_CSAE_ENV_AWS_PATH }} airbyte-integrations/connectors/destination-teradata/secrets
+          aws s3 cp s3://${{ vars.SSL_CSAE_ENV_AWS_PATH }} airbyte-integrations/connectors/destination-teradata/secrets
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+      - name: Install airbyte-ci binary
+        id: install-airbyte-ci
+        shell: bash
+        run: |
+          curl -sSL  https://connectors.airbyte.com/files/airbyte-ci/releases/ubuntu/2.11.0/airbyte-ci --output airbyte-ci-bin
+          sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci
+          sudo chmod +x /usr/local/bin/airbyte-ci
+      - name: Test connectors
+        run: |
+          airbyte-ci connectors --name=destination-teradata test
+          mkdir integrationtestreport
+          ls -al ./airbyte-integrations/connectors/destination-teradata/build/reports/tests/integrationTestJava
+          cp -r  ./airbyte-integrations/connectors/destination-teradata/build/reports/tests/integrationTestJava integrationtestreport
+      
diff --git a/.github/workflows/airbyte-ci-teradata-master.yaml b/.github/workflows/airbyte-ci-teradata-master.yaml
@@ -0,0 +1,60 @@
+name: airbyte-ci-teradata-master
+on:
+  workflow_dispatch:
+permissions:
+  contents: write
+  pages: write
+  id-token: write
+jobs:
+  teradata-tests:
+    name: "Run Airbyte Teradata Destination Connector Unit and Integration Tests"
+    runs-on: "ubuntu-22.04"
+    steps:
+      - name: "Checkout Source Code"
+        uses: actions/checkout@v4
+        with:
+          ref: master
+          persist-credentials: false
+      - name: "Install PIP"
+        run: |
+          sudo apt-get update
+          sudo apt install python3-pip
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: "Install Java 17"
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+      - name: "Install Node 16"
+        uses: actions/setup-node@v4
+        with:
+          node-version: '16'
+      - name: Install jq
+        run: sudo apt install -y jq
+      - name: "Install Docker and Docker Compose"
+        run: |
+          sudo apt-get update
+          sudo apt-get install ca-certificates curl gnupg lsb-release
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+          sudo chmod a+r /etc/apt/keyrings/docker.gpg
+          echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin
+          sudo chmod 666 /var/run/docker.sock
+      - name: "Download Test artifacts from Teradata S3"
+        run: |
+          mkdir airbyte-integrations/connectors/destination-teradata/secrets
+          aws s3 cp s3://${{ vars.NON_SSL_CSAE_ENV_AWS_PATH }} airbyte-integrations/connectors/destination-teradata/secrets
+          aws s3 cp s3://${{ vars.SSL_CSAE_ENV_AWS_PATH }} airbyte-integrations/connectors/destination-teradata/secrets
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+      - name: "Run Teradata Destination Connector Integration Tests"
+        run: |
+          REMOTE_CONNECTOR_CATALOG_URL=https://github.com/SatishChGit/AirbyteEnv/blob/main/oss_registry.json
+          ./gradlew :airbyte-integrations:connectors:destination-teradata:integrationTest
diff --git a/.github/workflows/airbyte-ci-teradata.yml b/.github/workflows/airbyte-ci-teradata.yml
@@ -0,0 +1,78 @@
+name: airbyte-ci-teradata
+on:
+  push:
+    branches:
+      - td_develop
+  workflow_dispatch:
+  schedule:
+  - cron: "0 6 * * *"
+permissions:
+  contents: write
+  pages: write
+  id-token: write
+jobs:
+  teradata-tests:
+    name: "Run Airbyte Teradata Destination Connector Unit and Integration Tests"
+    runs-on: "ubuntu-22.04"
+    steps:
+      - name: "Checkout Source Code"
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ vars.BRANCH }}
+          persist-credentials: false
+      - name: "Install PIP"
+        run: |
+          sudo apt-get update
+          sudo apt install python3-pip
+      - name: "Install Python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: "Install Java 17"
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+      - name: "Install Node 16"
+        uses: actions/setup-node@v4
+        with:
+          node-version: '16'
+      - name: Install jq
+        run: sudo apt install -y jq
+      - name: "Install Docker and Docker Compose"
+        run: |
+          sudo apt-get update
+          sudo apt-get install ca-certificates curl gnupg lsb-release
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+          sudo chmod a+r /etc/apt/keyrings/docker.gpg
+          echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin
+          sudo chmod 666 /var/run/docker.sock
+      - name: Download Test artifacts from Teradata S3
+        run: |
+          mkdir airbyte-integrations/connectors/destination-teradata/secrets
+          aws s3 cp s3://${{ vars.NON_SSL_CSAE_ENV_AWS_PATH }} airbyte-integrations/connectors/destination-teradata/secrets
+          aws s3 cp s3://${{ vars.SSL_CSAE_ENV_AWS_PATH }} airbyte-integrations/connectors/destination-teradata/secrets
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+      - name: "Run Teradata Destination Connector Integration Tests"
+        run:  |
+          REMOTE_CONNECTOR_CATALOG_URL=https://github.com/SatishChGit/AirbyteEnv/blob/main/oss_registry.json
+          ./gradlew :airbyte-integrations:connectors:destination-teradata:integrationTest
+      - name: Build GITHUB Page for Teradata Airbyte Connector Integration Tests Report
+        if: always()
+        uses: actions/jekyll-build-pages@v1
+        with:
+         source: airbyte-integrations/connectors/destination-teradata/build/reports/tests/integrationTestJava
+         destination: ./_site
+      - name: Upload Teradata Airbyte Connector Integration Tests Report to GITHUB page
+        if: always()
+        uses: actions/upload-pages-artifact@v2
+      - name: Publish Teradata Airbyte Connector Integration Tests Report to GITHUB page
+        if: always()
+        id: deployment
+        uses: actions/deploy-pages@v2
diff --git a/.github/workflows/sync_fork.yml b/.github/workflows/sync_fork.yml
@@ -0,0 +1,17 @@
+name: Sync Fork
+
+on:
+  schedule:
+    - cron: '30 20,2 * * *' # At every 20.30 (8.30 PM) and 2.30 AM UTC timezone - 12.00 AM IST and 8 AM
+  workflow_dispatch: # on button click
+
+jobs:
+  sync:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: tgymnich/[email protected]
+        with:
+          base: master
+          head: master
diff --git a/...ata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java b/...ata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java
@@ -107,7 +107,7 @@ public void createTableIfNotExists(final JdbcDatabase database, final String sch
   @Override
   public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) {
     return String.format(
-        "CREATE SET TABLE %s.%s, FALLBACK ( %s VARCHAR(256), %s JSON, %s TIMESTAMP(6)) " +
+        "CREATE TABLE %s.%s, FALLBACK ( %s VARCHAR(256), %s JSON, %s TIMESTAMP(6)) " +
             " UNIQUE PRIMARY INDEX (%s) ",
         schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA,
         JavaBaseConstants.COLUMN_NAME_EMITTED_AT, JavaBaseConstants.COLUMN_NAME_AB_ID);

diff --git a/.../java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java b/.../java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java
@@ -86,6 +86,7 @@ void initEnvironment() throws Exception {
           configJson.get("env_region").asText(),
           configJson.get("env_password").asText());
       response = teradataHttpClient.createEnvironment(request, token).get();
+      LOGGER.info("Environemnt " + configJson.get("env_name").asText() + " is created successfully ");
     } else if (response.state() == EnvironmentResponse.State.STOPPED) {
       var request = new EnvironmentRequest(name, new OperationRequest("start"));
       teradataHttpClient.startEnvironment(request, token);
@@ -100,8 +101,9 @@ void initEnvironment() throws Exception {
   void cleanupEnvironment() throws ExecutionException, InterruptedException {
     try {
       TeradataHttpClient teradataHttpClient = new TeradataHttpClient(configJson.get("env_url").asText());
-      var request = new EnvironmentRequest(configJson.get("env_name").asText(), new OperationRequest("stop"));
-      teradataHttpClient.stopEnvironment(request, configJson.get("env_token").asText());
+      var request = new DeleteEnvironmentRequest(configJson.get("env_name").asText());
+      teradataHttpClient.deleteEnvironment(request, configJson.get("env_token").asText());
+      LOGGER.info("Environemnt " + configJson.get("env_name").asText() + " is deleted successfully ");
     } catch (BaseException be) {
       LOGGER.error("Environemnt " + configJson.get("env_name").asText() + " is not available. " + be.getMessage());
     }