build_test.yml

name: Build and Test

on:
    push:
        branches:
            - main
        tags:
            - v*
    pull_request:
        branches:
            - main
            - prep-**
    workflow_dispatch:
        inputs:
          logLevel:

            default: warning
            description: "Log level"
            required: true
          tags:
            description: "Test scenario tags"
jobs:
  lint:
    name: Check linting
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.8'
      - name: Display Python version
        run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
      - name: Upgrade setuptools, pip and wheel
        run: python -m pip install -U setuptools pip wheel
      - name: Install tox
        run: python -m pip install tox
      - name: Set PY
        run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
      - uses: actions/cache@v1
        with:
          path: ~/.cache/pre-commit
          key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Run fix_lint
        run: python -m tox -e fix_lint

  test:
   name: Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
   needs: lint
   runs-on: ${{ matrix.os.image_name }}
   strategy:
    fail-fast: false
    matrix:
      os:
       - image_name: ubuntu-latest
         download_name: manylinux_x86_64
       - image_name: macos-latest
         download_name: macosx_x86_64
       - image_name: windows-2019
         download_name: win_amd64
      python-version: ["3.8"]
      cloud-provider: [aws, azure, gcp]
   steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Display Python version
      run: python -c "import sys; print(sys.version)"
    - name: Setup parameters file
      shell: bash
      env:
        PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
      run: |
        gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
        .github/workflows/parameters/parameters_${{ matrix.cloud-provider }}.py.gpg > tests/parameters.py
    - name: Upgrade setuptools, pip and wheel
      run: python -m pip install -U setuptools pip wheel
    - name: Install tox
      run: python -m pip install tox
    - name: List installed packages
      run: python -m pip freeze
    - name: Run tests
      run: python -m tox -e "py${PYTHON_VERSION/\./}" --skip-missing-interpreters false
      env:
        PYTHON_VERSION: ${{ matrix.python-version }}
        PYTEST_ADDOPTS: -vvv --color=yes --tb=short
        TOX_PARALLEL_NO_SPINNER: 1
    - name: Combine coverages
      run: python -m tox -e coverage --skip-missing-interpreters false
      shell: bash
    - uses: actions/upload-artifact@v2
      with:
        name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
        path: |
          .tox/.coverage
          .tox/coverage.xml

  test_connector_regression:
   name: Connector Regression Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
   needs: lint
   runs-on: ${{ matrix.os.image_name }}
   strategy:
    fail-fast: false
    matrix:
      os:
       - image_name: ubuntu-latest
         download_name: manylinux_x86_64
      python-version: ["3.8"]
      cloud-provider: [aws]
   steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Display Python version
      run: python -c "import sys; print(sys.version)"
    - name: Setup parameters file
      shell: bash
      env:
        PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
      run: |
        gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
        .github/workflows/parameters/parameters_${{ matrix.cloud-provider }}.py.gpg > tests/connector_regression/test/parameters.py
    - name: Upgrade setuptools, pip and wheel
      run: python -m pip install -U setuptools pip wheel
    - name: Install tox
      run: python -m pip install tox
    - name: List installed packages
      run: python -m pip freeze
    - name: Run tests
      run: python -m tox -e connector_regression --skip-missing-interpreters false
      env:
        PYTEST_ADDOPTS: -vvv --color=yes --tb=short
        TOX_PARALLEL_NO_SPINNER: 1

  combine-coverage:
    if: ${{ success() || failure() }}
    name: Combine coverage
    needs: [test, test_connector_regression]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: actions/download-artifact@v2
        with:
          path: artifacts
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.8'
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Upgrade setuptools and pip
        run: python -m pip install -U setuptools pip wheel
      - name: Install tox
        run: python -m pip install tox
      - name: Collect all coverages to one dir
        run: |
          python -c '
          from pathlib import Path
          import shutil
          src_dir = Path("artifacts")
          dst_dir = Path(".") / ".tox"
          dst_dir.mkdir()
          for src_file in src_dir.glob("*/.coverage"):
              dst_file = dst_dir / f".coverage.{src_file.parent.name[9:]}"
              print(f"{src_file} copy to {dst_file}")
              shutil.copy(str(src_file), str(dst_file))'
      - name: Combine coverages
        run: python -m tox -e coverage
      - name: Publish html coverage
        uses: actions/upload-artifact@v2
        with:
          name: overall_cov_html
          path: .tox/htmlcov
      - name: Publish xml coverage
        uses: actions/upload-artifact@v2
        with:
          name: overall_cov_xml
          path: .tox/coverage.xml
      - uses: codecov/codecov-action@v1
        with:
          file: .tox/coverage.xml