ci.yml

name: Build, test and Release
on:
  push:
    branches:
      - master
      - '*.*'
  pull_request:
  release:
    types:
      - published

jobs:
  build:
    strategy:
      matrix:
        python-version: [3.7, 3.8, 3.9, "3.10"]
        include:
          - python-version: 3.7
            tox-py: py37
          - python-version: 3.8
            tox-py: py38
          - python-version: 3.9
            tox-py: py39
          - python-version: "3.10"
            tox-py: py310
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m venv venv
        . venv/bin/activate
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        pip install tox

    - name: Lint with flake8
      run: |
        pip install flake8
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=W503
    - name: Run tox
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        . venv/bin/activate
        tox -e ${{ matrix.tox-py }}

    - uses: codecov/codecov-action@v1
      name: Report code coverage  

  debian-build:
    # Build debian packages
    strategy:
      matrix:
        suite: [focal]
        include:
          - suite: focal
            os-version: ubuntu-20.04
      fail-fast: false
    runs-on: ${{ matrix.os-version }}
    steps:
    - uses: actions/checkout@v3
    - name: Build Debian Package
      run: |
        version=$(cat VERSION)
        version_exists=$(cat debian/changelog | grep "($VERSION)" || echo "")
        if [ -z "$version_exists" ]
        then
            echo "Updating debian changelog..."
            cd packaging/docker-build
            docker-compose build release && docker-compose run release
            cd ../..
        fi
        cd packaging/docker-build
        docker-compose build "cassandra-medusa-builder-${{ matrix.suite }}" \
            && docker-compose run "cassandra-medusa-builder-${{ matrix.suite }}"
        
        cd ../../packages
        if [ -f "cassandra-medusa_${version}-0~${{ matrix.suite }}0_amd64.deb" ]; then
          echo "${suite} debian package was successfully built"
        else
          echo "Error: no packages found for ${{ matrix.suite }}"
          exit 1
        fi
    - name: Install Debian Package and run Medusa
      run: |
        version=$(cat VERSION)
        cd packages
        sudo dpkg -i cassandra-medusa_${version}-0~${{ matrix.suite }}0_amd64.deb
        medusa
        exit $?
      
  integration-tests:
    needs: build
    continue-on-error: ${{ matrix.experimental }}
    strategy:
        fail-fast: false
        matrix:
          python-version: ["3.10"]
          it-backend: [local, s3, gcs, minio, azure, azure-hierarchical]
          # IBM not included by default due to lite plan quota being easily exceeded
          #it-backend: [local, s3, gcs, minio, ibm, azure]
          cassandra-version: [2.2.19, 3.11.11, 4.0.0, 'github:apache/trunk']
          include:
            - cassandra-version: 2.2.19
              experimental: false
            - cassandra-version: 3.11.11
              experimental: false
            - cassandra-version: 4.0.0
              experimental: false
            - cassandra-version: 'github:apache/trunk'
              experimental: true
          exclude:
            # excludes unnecessary combinations
            - it-backend: s3
              cassandra-version: 2.2.19
            - it-backend: s3
              cassandra-version: 'github:apache/trunk'
            - it-backend: gcs
              cassandra-version: 2.2.19
            - it-backend: gcs
              cassandra-version: 'github:apache/trunk'
            - it-backend: gcs
              python-version: "3.10"
            - it-backend: minio
              cassandra-version: 2.2.19
            - it-backend: minio
              cassandra-version: 'github:apache/trunk'
            - it-backend: minio
              python-version: "3.10"
            - it-backend: azure
              cassandra-version: 2.2.19
            - it-backend: azure
              cassandra-version: 'github:apache/trunk'
            - it-backend: azure
              python-version: "3.10"
            - it-backend: azure-hierarchical
              cassandra-version: 2.2.19
            - it-backend: azure-hierarchical
              cassandra-version: 3.11.11
            - it-backend: azure-hierarchical
              cassandra-version: 'github:apache/trunk'
            - it-backend: azure-hierarchical
              python-version: "3.10"

    runs-on: ubuntu-20.04
    services:
      minio:
        image: lazybit/minio
        ports:
          - 9000:9000
        env:
          MINIO_ACCESS_KEY: minio_key
          MINIO_SECRET_KEY: minio_secret
        volumes:
          - ${{ github.workspace }}/../data:/data
        options: --name=minio --health-cmd "curl http://localhost:9000/minio/health/live"
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Setup Java Action
      uses: actions/setup-java@v1
      with:
        java-version: '8.0.252'
        architecture: x64
    - name: Install dependencies
      run: |
        python -m venv venv
        . venv/bin/activate
        sudo apt-get remove azure-cli
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        pip install -r requirements-test.txt
        pip install ccm
        case '${{ matrix.it-backend }}' in
          'azure'|'azure-hierarchical')
            pip install -r requirements-azure.txt
            ;;
          'ibm'|'minio'|'s3')
            pip install -r requirements-s3.txt
            ;;
        esac

    - name: Check if integration tests can run
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.BUCKET_KEY }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.BUCKET_SECRET }}
      run: |
        if [[ ( -n "${AWS_ACCESS_KEY_ID}" && "${{ matrix.it-backend }}" == "s3" ) \
          || ( "${{ matrix.it-backend }}" == "local" ) \
          || ( "${{ matrix.it-backend }}" == "minio" ) \
          || ( -n '${{ secrets.MEDUSA_GCS_CREDENTIALS }}' && "${{ matrix.it-backend }}" == "gcs" ) \
          || ( -n '${{ secrets.MEDUSA_IBM_CREDENTIALS }}' && "${{ matrix.it-backend }}" == "ibm" ) \
          || ( -n '${{ secrets.MEDUSA_AZURE_CREDENTIALS }}' && "${{ matrix.it-backend }}" == "azure" ) \
          || ( -n '${{ secrets.MEDUSA_AZURE_HIERARCHICAL_CREDENTIALS }}' && "${{ matrix.it-backend }}" == "azure-hierarchical" ) ]];
        then
          echo "IT_CAN_RUN=yes" >> $GITHUB_ENV
        else
          echo "IT_CAN_RUN=no" >> $GITHUB_ENV
        fi 

    - name: Run integration tests
      if: ${{ env.IT_CAN_RUN == 'yes' }}
      env:
        AWS_ACCESS_KEY_ID: ${{ secrets.BUCKET_KEY }}
        AWS_SECRET_ACCESS_KEY: ${{ secrets.BUCKET_SECRET }}
      run: |
        set -e
        . venv/bin/activate
        if [[ "${{ matrix.it-backend }}" == "ibm" ]];
        then
          # Prevent awscli from using AWS secrets in case we're running against IBM cloud
          unset AWS_ACCESS_KEY_ID
          unset AWS_SECRET_ACCESS_KEY
        fi
        # Write GCS service account credentials to a file
        mkdir ~/.aws
        # This fake cluster needs to be created first so that the integration tests pass in GH actions. Don't ask me why...
        ccm create test_cluster -v binary:3.11.4 -n 1 --vnodes
        ccm node1 updateconf 'storage_port: 7011'
        ccm node1 updateconf 'concurrent_reads: 4'
        ccm node1 updateconf 'concurrent_writes: 4'
        ccm node1 updateconf 'concurrent_counter_writes: 4'
        ccm node1 updateconf 'num_tokens: 4'
        sed -i 's/#MAX_HEAP_SIZE="4G"/MAX_HEAP_SIZE="256m"/' ~/.ccm/test_cluster/node1/conf/cassandra-env.sh
        sed -i 's/#HEAP_NEWSIZE="800M"/HEAP_NEWSIZE="200M"/' ~/.ccm/test_cluster/node1/conf/cassandra-env.sh
        ccm start -v
        ccm showlastlog|tail -100
        ccm stop
        if [ "${{ matrix.it-backend }}" == "s3" ]
        then
          # AWS S3 Storage tests
          ./run_integration_tests.sh -v --s3 --no-local --cassandra-version=${{ matrix.cassandra-version }}
        elif [ "${{ matrix.it-backend }}" == "gcs" ]
        then
          # Google Cloud Storage tests
          echo '${{ secrets.MEDUSA_GCS_CREDENTIALS }}' > ~/medusa_credentials.json
          ./run_integration_tests.sh -v --gcs --no-local --cassandra-version=${{ matrix.cassandra-version }}
        elif [ "${{ matrix.it-backend }}" == "ibm" ]
        then
          # IBM Cloud Object Storage tests
          printf "%s" '${{ secrets.MEDUSA_IBM_CREDENTIALS }}' > ~/.aws/ibm_credentials
          ./run_integration_tests.sh -v --ibm --no-local --cassandra-version=${{ matrix.cassandra-version }}
        elif [ "${{ matrix.it-backend }}" == "minio" ]
        then
          # MinIO Object Storage tests
          unset AWS_ACCESS_KEY_ID
          unset AWS_SECRET_ACCESS_KEY
          wget https://dl.min.io/client/mc/release/linux-amd64/mc
          chmod +x mc
          ./mc alias set minio http://127.0.0.1:9000 minio_key minio_secret
          ./mc mb minio/medusa-dev
          cp ./tests/resources/minio/minio_credentials ~/.aws/minio_credentials
          ./run_integration_tests.sh -v --minio --no-local --cassandra-version=${{ matrix.cassandra-version }} 
        elif [ "${{ matrix.it-backend }}" == "azure" ]
        then
          # Azure Blob Storage tests
          printf "%s" '${{ secrets.MEDUSA_AZURE_CREDENTIALS }}' > ~/medusa_azure_credentials.json
          ./run_integration_tests.sh -v --azure --no-local --cassandra-version=${{ matrix.cassandra-version }} 
        elif [ "${{ matrix.it-backend }}" == "azure-hierarchical" ]
        then
          # Azure Blob Storage with hierarchical namespace tests
          printf "%s" '${{ secrets.MEDUSA_AZURE_HIERARCHICAL_CREDENTIALS }}' > ~/medusa_azure_credentials.json
          ./run_integration_tests.sh -v --azure --no-local --cassandra-version=${{ matrix.cassandra-version }} 
        else
        # Local storage tests
          ./run_integration_tests.sh -v --cassandra-version=${{ matrix.cassandra-version }}
        fi

        # Move and convert the coverage analysis file to XML
        mv tests/integration/.coverage .
        coverage xml

    - uses: codecov/codecov-action@v1
      name: Report code coverage

    - name: Setup tmate session
      if: "failure()"
      uses: mxschmitt/action-tmate@v3
      timeout-minutes: 30

  publish-docker-master:
    needs: [debian-build, build, integration-tests]
    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Set up QEMU
      uses: docker/setup-qemu-action@v2
      with:
        platforms: 'arm64,arm'
    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v2
    - name: Login to DockerHub
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.K8SSANDRA_DOCKER_HUB_USERNAME }}
        password: ${{ secrets.K8SSANDRA_DOCKER_HUB_PASSWORD }}
    - name: Set git parsed values
      id: vars
      shell: bash
      run: |
        echo "sha_short=$(git rev-parse --short=8 ${{ github.sha }})" >> $GITHUB_OUTPUT
    - name: Build Medusa
      run: |
        echo "Publishing release $(git rev-parse --short HEAD) in Docker Hub"
        python setup.py build
    - name: Build image and push
      id: docker_build
      uses: docker/build-push-action@v4
      with:
        file: k8s/Dockerfile
        context: .
        push: true
        tags: k8ssandra/medusa:${{ steps.vars.outputs.sha_short}},k8ssandra/medusa:master
        platforms: linux/amd64,linux/arm64
        cache-from: type=local,src=/tmp/.buildx-cache
        cache-to: type=local,dest=/tmp/.buildx-cache
    
  publish-debian:
    # We can only release if the build above succeeded first
    needs: [debian-build, build, integration-tests]
    if: github.event_name == 'release' && github.event.action == 'published'
    strategy:
      matrix:
        suite: [focal]
        include:
          - suite: focal
            os-version: ubuntu-20.04
      fail-fast: false
    runs-on: ${{ matrix.os-version }}
    steps:
    - uses: actions/checkout@v3
    - name: Install dependencies
      run: |
        sudo curl -L "https://github.com/docker/compose/releases/download/1.25.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
        sudo chmod +x /usr/local/bin/docker-compose
        sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
    - name: Build Debian
      run: |
        version=$(cat VERSION)
        echo "VERSION=$version" >> $GITHUB_ENV
        cd packaging/docker-build
        docker-compose build "cassandra-medusa-builder-${{ matrix.suite }}" \
            && docker-compose run "cassandra-medusa-builder-${{ matrix.suite }}"
        
    - name: Push Debian to Cloudsmith
      id: push-deb
      uses: cloudsmith-io/action@master
      with:
        api-key: ${{ secrets.CLOUDSMITH_API_KEY }}
        command: 'push'
        format: 'deb'
        owner: 'thelastpickle'
        repo: 'medusa'
        distro: 'ubuntu'
        release: ${{ matrix.suite }}
        republish: 'true'
        file: "packages/cassandra-medusa_${{ env.VERSION }}-0~${{ matrix.suite }}0_amd64.deb"

  publish-docker:
    needs: [publish-debian]
    if: github.event_name == 'release' && github.event.action == 'published'
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Set up QEMU
      uses: docker/setup-qemu-action@v2
      with:
        platforms: 'arm64,arm'
    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v1
    - name: Login to DockerHub
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.K8SSANDRA_DOCKER_HUB_USERNAME }}
        password: ${{ secrets.K8SSANDRA_DOCKER_HUB_PASSWORD }}
    - name: Set git parsed values
      id: vars
      shell: bash
      run: |
        echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT
    - name: Build Medusa
      run: |
        echo "Publishing release ${{ steps.vars.outputs.version}} in Docker Hub"
        python setup.py build
    - name: Build image and push
      id: docker_build
      uses: docker/build-push-action@v4
      with:
        file: k8s/Dockerfile
        context: .
        push: false
        tags: k8ssandra/medusa:${{ steps.vars.outputs.version}}
        platforms: linux/amd64,linux/arm64
        cache-from: type=local,src=/tmp/.buildx-cache
        cache-to: type=local,dest=/tmp/.buildx-cache

  publish-pypi:
    # We can only release if the build above succeeded first
    needs: [publish-docker]
    if: github.event_name == 'release' && github.event.action == 'published'
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python 3.7
      uses: actions/setup-python@v1
      with:
        python-version: 3.7
    - name: Install dependencies
      run: |
        python -m venv venv
        . venv/bin/activate
        python -m pip install --upgrade pip
    - name: Build and publish to pypi
      env:
        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
      run: |
        . venv/bin/activate
        pip install setuptools wheel twine
        python setup.py sdist bdist_wheel
        twine upload dist/*