unit-and-integration-tests-unified.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: "Unit & Integration tests CI"
on:
  push:
    paths-ignore:
      - '**/*.md'
      - 'dev/**'
      - 'distribution/bin/**'
      - 'docs/**'
      - 'examples/**/jupyter-notebooks/**'
      - 'web-console/**'
      - 'website/**'
    branches:
      - master
      - '[0-9]+.[0-9]+.[0-9]+' # release branches
      - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches
  pull_request:
    paths-ignore:
      - '**/*.md'
      - 'dev/**'
      - 'distribution/bin/**'
      - 'docs/**'
      - 'examples/**/jupyter-notebooks/**'
      - 'web-console/**'
      - 'website/**'
    branches:
      - master
      - '[0-9]+.[0-9]+.[0-9]+' # release branches
      - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches

concurrency:
  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.run_id }}' # group workflows only on pull_requests and not on branch commits
  cancel-in-progress: true

env:
  MYSQL_DRIVER_CLASSNAME: com.mysql.jdbc.Driver # Used to set druid config in docker image for revised ITs
  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5

jobs:
  set-env-var:
    name: Set env var
    runs-on: ubuntu-latest
    outputs:
      DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ steps.image_name.outputs.image_name }}
      BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ steps.it_enabled.outputs.enabled }}
      DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }}
      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
    steps:
      - name: Set image name env var
        id: image_name
        run: |
          echo "::set-output name=image_name::org.apache.druid.integration-tests/test:${{ env.DRUID_PREVIOUS_VERSION }}"
      - name: Set env for enabling backward compatibility it
        id: it_enabled
        run: |
          if [ -n "${{ env.DRUID_PREVIOUS_VERSION }}" ]; then
            echo "::set-output name=enabled::true"
          else
            echo "::set-output name=enabled::false"
          fi

  build:
    needs: set-env-var
    name: "build (jdk${{ matrix.jdk }})"
    strategy:
      fail-fast: false
      matrix:
        # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429
        jdk: [ '11', '17', '21.0.4' ]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout branch
        uses: actions/checkout@v4

      # skip the "cache: maven" step from setup-java. We explicitly use a
      # different cache key since we cannot reuse it across commits.
      - uses: actions/setup-java@v4
        with:
          distribution: 'zulu'
          java-version: ${{ matrix.jdk }}

      # the build step produces SNAPSHOT artifacts into the local maven repository,
      # we include github.sha in the cache key to make it specific to that build/jdk
      - name: Cache Maven m2 repository
        id: maven
        uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ github.sha }}
          restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }}

      - name: Cache targets
        id: target
        uses: actions/cache@v4
        with:
          path: |
            ./**/target
          key: maven-${{ runner.os }}-${{ matrix.jdk }}-targets-${{ github.sha }}

      - name: Cache image
        id: docker_container
        uses: actions/cache@v4
        with:
          key: druid-container-jdk${{ matrix.jdk }}.tar.gz-${{ github.sha }}
          path: |
            ./druid-container-jdk${{ matrix.jdk }}.tar.gz
            ./integration-tests-ex/image/target/env.sh

      - name: Cache previous version image
        id: docker_container_previous_version
        uses: actions/cache@v4
        with:
          key: druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }}
          path: |
            ./druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz

      - name: Maven build
        id: maven_build
        run: |
          ./it.sh ci

      - name: Container build
        env:
          BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
          DRUID_PREVIOUS_VERSION: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}
          DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
          DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
        run: |
          ./it.sh image
          source ./integration-tests-ex/image/target/env.sh
          docker tag $DRUID_IT_IMAGE_NAME $DRUID_IT_IMAGE_NAME-jdk${{ matrix.jdk }}

      - name: Save docker container to archive
        run: |
          source ./integration-tests-ex/image/target/env.sh
          echo $DRUID_IT_IMAGE_NAME
          docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ matrix.jdk }}.tar.gz

      - name: Save previous version docker image
        if: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }}
        run: |
          docker tag ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}
          echo ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
          docker save "${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz    

  unit-tests-phase2:
    strategy:
      fail-fast: false
      matrix:
        # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429
        jdk: [ '11', '21.0.4' ]
    name: "unit tests (jdk${{ matrix.jdk }})"
    uses: ./.github/workflows/unit-tests.yml
    needs: unit-tests
    if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }}
    with:
      jdk: ${{ matrix.jdk }}

  unit-tests:
    name: "unit tests (jdk17)"
    uses: ./.github/workflows/unit-tests.yml
    needs: build
    with:
      jdk: 17

  standard-its:
    needs: unit-tests
    if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }}
    uses: ./.github/workflows/standard-its.yml

  revised-its:
    needs: [unit-tests, set-env-var]
    if: ${{ always() && (needs.unit-tests.result == 'success' || needs.unit-tests.outputs.continue_tests) }}
    uses: ./.github/workflows/revised-its.yml
    with:
      BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
      DRUID_PREVIOUS_VERSION: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}
      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
      DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}