Merge pull request #2 from TykTechnologies/add-api-linting

added API linting workflow

.github/scripts/validate_fields.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Ensure the ENV environment variable is set
+if [ -z "$CURR_ENV" ]; then
+    echo "Error: ENV environment variable has been set to $CURR_ENV."
+    exit 1
+else
+    echo "Info: ENV environment variable has been set to $CURR_ENV."
+fi
+
+# Ensure the DIRECTORY environment variable is set
+if [ -z "$DIRECTORY" ]; then
+    echo "Error: DIRECTORY environment variable is not set."
+    exit 1
+else
+    echo "Info: DIRECTORY environment variable has been set to $DIRECTORY."
+fi
+
+for file in $(find . -name "*.json" ! -name ".tyk.json"); do
+    # echo "Validating API/Policy definition $file"
+    # # Check if target_url is valid and not empty
+    # if jq -e '.proxy.target_url | length > 0' "$file" > /dev/null; then
+    #   echo "$file contains a valid proxy.target_url."
+    #   target_url=$(jq -r '.proxy.target_url' "$file")
+    #   echo "proxy.target_url: $target_url"
+    # else
+    #   echo "$file does NOT contain a valid proxy.target_url."
+    #   exit 1
+    # fi
+
+    # Check if either JWT auth or or MutuatTLS is enabled
+    echo "Validating if the correct authentication mechanisms are enabled"
+    if jq -e '.api_definition.enable_jwt == true or .api_definition.auth_configs.use_mutual_tls_auth == true' "$file" > /dev/null; then
+        echo "$file has either JWT auth or MutualTLS auth set to true."
+    else
+        echo "$file does NOT have either JWT auth or MutualTLS auth set to true."
+        exit 1
+    fi
+
+done

.github/workflows/tyk-dev.yml

@@ -0,0 +1,16 @@
+# Tyk developer pipeline -- perform validation and linting of API definitions and policies if needed for dev env APIs.
+# This dev workflow will triggered if any PRs have been made specifically to the dev directory.
+name: Tyk Development Workflow
+
+# Execute workflow on dev pull requests
+on:
+    pull_request:
+        paths:
+            - 'dev/**'
+
+jobs:
+    # Run linter and validation workflow
+    tyk-lint:
+        uses: ./.github/workflows/tyk-lint.yml
+        with:
+            environment: 'dev'

.github/workflows/tyk-env-promotion.yml

@@ -0,0 +1,44 @@
+name: Tyk assets environment promotion
+
+# Perform the env promotion only on push to main branch
+on:
+    push:
+      branches: [ main ]
+
+jobs:
+    env-promotion:
+        runs-on: ubuntu-latest
+
+        steps:
+        # Check out the current repo and fetch only the current commits (JTBD)
+        - name: 'Checkout Repository'
+          uses: actions/checkout@v4
+          with:
+            fetch-depth: 1
+
+        # - name: Create .tyk.json
+        #   run: |
+        #     cd ./stg
+        #     echo '{' > .tyk.json
+        #     echo '  "type": "apidef",' >> .tyk.json
+        #     echo '  "files": [' >> .tyk.json
+        #     find . -type f -name '*.json' -path './apis/*' -exec echo '    {"file": "{}"},' \; | sed '$ s/,$//' >> .tyk.json
+        #     echo '  ],' >> .tyk.json
+        #     echo '  "policies": [' >> .tyk.json
+        #     find . -type f -name '*.json' -path './policies/*' -exec echo '    {"file": "{}"},' \; | sed '$ s/,$//' >> .tyk.json
+        #     echo '  ],' >> .tyk.json
+        #     echo '  "assets": [' >> .tyk.json
+        #     find . -type f -name '*.json' -path './assets/*' -exec echo '    {"file": "{}"},' \; | sed '$ s/,$//' >> .tyk.json
+        #     echo '  ]' >> .tyk.json
+        #     echo '}' >> .tyk.json
+        #     cat .tyk.json
+
+        # - name: Sync with Tyk
+        #   env:
+        #     TYK_SYNC_REPO: ${{ vars.TYK_SYNC_REPO }}
+        #     TYK_SYNC_VERSION: ${{ vars.TYK_SYNC_VERSION }}
+        #     TYK_DASHBOARD_URL: ${{ secrets.TYK_DASHBOARD_URL }}
+        #     TYK_DASHBOARD_SECRET: ${{ secrets.TYK_DASHBOARD_SECRET }}
+        #   run: |
+        #     docker run ${TYK_SYNC_REPO}:${TYK_SYNC_VERSION} version
+        #     docker run -v ${{ github.workspace }}:/app/data ${TYK_SYNC_REPO}:${TYK_SYNC_VERSION} sync --path /app/data --dashboard ${TYK_DASHBOARD_URL} --secret ${TYK_DASHBOARD_SECRET}

.github/workflows/tyk-lint.yml

@@ -0,0 +1,75 @@
+# Basic Tyk API and Policies schema validation and linter triggered on development assets. The idea of this workflow is to validate specific field requirements 
+# or enforce governance to make sure specific custom plugins are used or formats 
+name: Tyk Schema Validation / Linter
+
+# Perform the Tyk schema validation only on PR requests
+on:
+  workflow_dispatch:
+    inputs:
+      environment: 
+        type: choice
+        options:
+          - dev
+          - stg
+          - prod
+  workflow_call:
+    inputs:
+      environment:
+        type: string
+
+jobs:
+  schema-linter-and-validation:
+    runs-on: ubuntu-latest
+
+    steps:
+    # Check out the current repo and fetch only the current commits (JTBD)
+    - name: 'Checkout Repository'
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+
+    # List the content that exists within the repo to validate the files
+    - name: 'List Repository Contents'
+      run: |
+        ls -la
+        pwd
+
+    # Install JQ library used to introspect the API and Policy definitions
+    - name: 'Install JQ Library'
+      uses: dcarbone/install-jq-action@v2
+    - name: 'Check JQ Library'
+      run: |
+        which jq
+        jq --version
+    
+    # Determine the environment to lint run linter against
+    - name: 'Determine Environment'
+      id: determine_environment
+      run: |
+          if [[ "${{ github.event_name }}" == "workflow_call" ]]; then
+            # Workflow called with an input
+            echo "environment=${{ inputs.environment }}" >> $GITHUB_ENV
+          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            # Workflow called with an input
+            echo "environment=${{ inputs.environment }}" >> $GITHUB_ENV
+          else
+            # Pull request called with an input
+            echo "environment=${{ inputs.environment }}" >> $GITHUB_ENV
+          fi
+    
+    # Tyk API Linting / Validation using Github Action Library
+    - name: 'Linter / validation using Spectral / Stoplight'
+      uses: stoplightio/spectral-action@latest
+      with:
+        file_glob: ${{ env.environment }}/apis/api-*.json
+        spectral_ruleset: ${{ env.environment }}/tykapi-ruleset.yaml
+      continue-on-error: false
+
+    # Tyk API Linting / Validation using JQ Library
+    - name: 'Linter / validation using JQ'
+      env: 
+        DIRECTORY: "./infrastructure/${{ env.environment }}/apis/"
+        CURR_ENV: ${{ env.environment }}
+      run: |
+        chmod +x ./.github/scripts/validate_fields.sh
+        ./.github/scripts/validate_fields.sh

.github/workflows/tyk-production.yml

@@ -0,0 +1,56 @@
+# Please refer to staging for an example on automating production level assets.
+name: Generate Production Tyk Assets
+
+# Perform the env promotion only on push to main branch
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  actions: read
+  checks: write
+
+jobs:
+    set-up-prod-tyk-assets:
+        runs-on: ubuntu-latest
+
+        steps:
+        # Check out the current repo and fetch only the current commits (JTBD)
+        - name: 'Checkout Repository'
+          uses: actions/checkout@v4
+          with:
+            fetch-depth: 1
+
+        - name: 'Perform workflow to generate production assets'
+          run: |
+            echo "Hello, World!"
+        
+        # # Check for modified files
+        # - name: Check for modified files
+        #   id: git-check
+        #   run: |
+        #     if [ -n "$(git status --porcelain)" ]; then
+        #       echo "modified=true" >> $GITHUB_ENV
+        #     else
+        #       echo "modified=false" >> $GITHUB_ENV
+        #     fi
+
+        # # Push changes to remote repository
+        # - name: Commit changes
+        #   if: env.modified == 'true'
+        #   env:
+        #     ORG_NAME: ${{ secrets.ORG_NAME }}
+        #     ORG_EMAIL: ${{ secrets.ORG_EMAIL }}
+        #   run: |
+        #     git config --global user.name "$ORG_NAME"
+        #     git config --global user.email "$ORG_EMAIL"
+        #     git add .
+        #     git commit -am "CI: Update staging assets"
+        #     git push
+
+    # # Promote to staging env
+    # tyk-env-promotion:
+    #   uses: ./.github/workflows/tyk-env-promotion.yml
+    #   with:
+    #       environment: 'stg'
+

.github/workflows/tyk-staging.yml

@@ -1,14 +1,19 @@
 # Basic env promotion with GH secret store substituion triggered on successful PR reviews and approval with a push to main branch.
 # This workflow will substitution env specific values for both staging and prod and deploy them as required.
-name: API and Policy Promotion
+name: Generate Staging Tyk Assets
 
 # Perform the env promotion only on push to main branch
 on:
-    push:
-      branches: [ TargetURL-Replacement ]
+  workflow_dispatch:
+  push:
+    paths:
+      - 'dev/**'
+    branches: 
+      - main
 
 jobs:
-  staging-env-promotion:
+  # Set up staging assets
+  set-up-stg-tyk-assets:
     runs-on: ubuntu-latest
 
     steps:
@@ -56,7 +61,7 @@ jobs:
           new_file="stg-$(basename "$file")"
           # Replace proxy.target_url with the GitHub secret value and write to the new file
           echo ${{secrets.STG_US_PROXY_TARGET_URL}} | sed 's/./& /g'
-          jq --arg url "$STG_US_PROXY_TARGET_URL" '.proxy.target_url = $url' "$file" > "../stg/$new_file"
+          jq --arg url "$STG_US_PROXY_TARGET_URL" '.proxy.target_url = $url' "$file" > "../stg/apis/$new_file"
           echo "Updated target_url in $new_file"
         done
         # List the content that exists within the repo to validate the files
@@ -69,6 +74,7 @@ jobs:
         # cd ./stg
         # ls -la
         # pwd
+        # Tyk Classic API Definitions
         for file in $(find . -name "stg-api*.json"); do
           echo "Adding config_data to Tyk Classic API Definition: $file"
           #echo "Config_Data Before:"
@@ -79,6 +85,7 @@ jobs:
           mv tmp_api.json $file
           echo "Updated config_data in $file"
         done
+        # OAS Definitions
         # for file in $(find . -name "stg*oas*.json"); do
         #   echo "Processing Tyk OAS Definitions"
         #   # Create a new file path in the stg directory with a stg- prefix
@@ -98,6 +105,7 @@ jobs:
         STG_US_MTLS_ROOT_CERT: ${{ secrets.STG_US_MTLS_ROOT_CERT }}
       run: |
         cd ./stg
+        # Tyk Classic API Definitions
         for file in $(find . -name "stg-api*.json"); do
           mtls_api=$(jq '.api_definition.use_mutual_tls_auth' $file)
           if [ "$mtls_api" = "true" ]; then
@@ -107,7 +115,7 @@ jobs:
             echo "Updated root cert(s) in $file"
           fi
         done
-        #OAS
+        # OAS Definitions
         # for file in $(find . -name "*oas*.json"); do
         #   echo "Processing Tyk OAS Definitions"
         #   # Create a new file path in the stg directory with a stg- prefix
@@ -126,6 +134,12 @@ jobs:
         #   fi
         # done
 
+    # Run linter and validation workflow
+    - name: Staging Linter
+      uses: ./.github/workflows/tyk-lint.yml
+      with:
+          environment: 'stg'
+
     # List repo content post sub
     # - name: 'List Repository Contents Post-Substitution'
     #   run: |
@@ -155,4 +169,10 @@ jobs:
         git config --global user.email "$ORG_EMAIL"
         git add .
         git commit -am "CI: Update staging assets"
-        git push
+        git push
+
+  # Promote to staging env
+  tyk-env-promotion:
+    uses: ./.github/workflows/tyk-env-promotion.yml
+    with:
+        environment: 'stg'