ci.yml

name: ci

on:
  push:
    branches:
      - master
  pull_request:

env:
  # An envar that signals to tests we are executing in the CI environment
  CONTINUOUS_INTEGRATION: true
  # maven.wagon.rto is in millis, defaults to 30m
  MAVEN_OPTS: "-Xmx512M -XX:+ExitOnOutOfMemoryError -Dmaven.wagon.rto=60000"
  MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError -Dmaven.wagon.rto=60000"
  MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all"
  MAVEN_TEST: "-B -Dair.check.skip-all --fail-at-end"
  RETRY: .github/bin/retry

# Cancel previous PR builds.
concurrency:
  # Cancel all workflow runs except latest within a concurrency group. This is achieved by defining a concurrency group for the PR.
  # Non-PR builds have singleton concurrency groups.
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.number || github.sha }}
  cancel-in-progress: true

jobs:
  maven-checks:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        java-version:
          - 11
          - 16
    timeout-minutes: 45
    steps:
      - name: Free Disk Space
        run: |
          df -h
          sudo apt-get clean
          df -h
      - uses: actions/checkout@v2
      - uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: ${{ matrix.java-version }}
      - name: Maven Checks
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY ./mvnw install -B -V -T C1 -DskipTests -P ci -pl '!:trino-server-rpm'
      - name: Test Server RPM
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY bash -c './mvnw verify -B -P ci -pl :trino-server-rpm || find core/trino-server-rpm/ -exec ls -ald {} +'
      - name: Clean Maven Output
        run: ./mvnw clean -pl '!:trino-server,!:trino-cli'
      - name: Test Docker Image
        run: core/docker/build-local.sh

  error-prone-checks:
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: 11
      - name: Maven Install
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
      - name: Error Prone Checks
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          # Run Error Prone on one module with a retry to ensure all runtime dependencies are fetched
          $RETRY ./mvnw ${MAVEN_TEST} -T C1 clean test-compile -P errorprone-compiler -pl ':trino-spi'
          # The main Error Prone run
          ./mvnw ${MAVEN_TEST} -T C1 clean test-compile -P errorprone-compiler \
            -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'

  web-ui-checks:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v2
      - name: Web UI Checks
        run: core/trino-main/bin/check_webui.sh

  test-jdbc-compatibility:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0 # checkout tags so version in Manifest is set properly
      - uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: 11
      - name: Maven Install
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -pl '!:trino-test-jdbc-compatibility-old-driver,!:trino-docs,!:trino-server,!:trino-server-rpm'
      - name: Test old JDBC vs current server
        run: testing/trino-test-jdbc-compatibility-old-driver/bin/run_tests.sh
      - name: Test current JDBC vs old server
        if: always()
        run: ./mvnw test ${MAVEN_TEST} -pl :trino-test-jdbc-compatibility-old-server

  hive-tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        config:
          - config-empty
          - config-hdp3
          # TODO: config-cdh5
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: 11
      - name: Install Hive Module
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl :trino-hive-hadoop2
      - name: Run Hive Tests
        run: |
          source testing/trino-product-tests/conf/product-tests-${{ matrix.config }}.sh &&
            plugin/trino-hive-hadoop2/bin/run_hive_tests.sh
      - name: Run Hive S3 Tests
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESSKEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETKEY }}
          S3_BUCKET: "presto-ci-test"
          S3_BUCKET_ENDPOINT: "s3.us-east-2.amazonaws.com"
        run: |
          if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then
            source testing/trino-product-tests/conf/product-tests-${{ matrix.config }}.sh &&
              plugin/trino-hive-hadoop2/bin/run_hive_s3_tests.sh
          fi
      - name: Run Hive Glue Tests
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESSKEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETKEY }}
          AWS_REGION: us-east-2
        run: |
          if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then
            ./mvnw test ${MAVEN_TEST} -pl :trino-hive -P test-hive-glue
          fi
      - name: Run Hive Azure ABFS Access Key Tests
        if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
        env:
          ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
          ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
          ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
        run: |
          if [ "${ABFS_CONTAINER}" != "" ]; then
            source testing/trino-product-tests/conf/product-tests-${{ matrix.config }}.sh &&
              plugin/trino-hive-hadoop2/bin/run_hive_abfs_access_key_tests.sh
          fi
      - name: Run Hive Azure ABFS OAuth Tests
        if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
        env:
          ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
          ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
          ABFS_OAUTH_ENDPOINT: ${{ secrets.AZURE_ABFS_OAUTH_ENDPOINT }}
          ABFS_OAUTH_CLIENTID: ${{ secrets.AZURE_ABFS_OAUTH_CLIENTID }}
          ABFS_OAUTH_SECRET: ${{ secrets.AZURE_ABFS_OAUTH_SECRET }}
        run: |
          if [ -n "$ABFS_CONTAINER" ]; then
            source testing/trino-product-tests/conf/product-tests-${{ matrix.config }}.sh &&
              plugin/trino-hive-hadoop2/bin/run_hive_abfs_oauth_tests.sh
          fi
      - name: Run Hive Azure WASB Tests
        if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
        env:
          WASB_CONTAINER: ${{ secrets.AZURE_WASB_CONTAINER }}
          WASB_ACCOUNT: ${{ secrets.AZURE_WASB_ACCOUNT }}
          WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESSKEY }}
        run: |
          if [ "${WASB_CONTAINER}" != "" ]; then
            source testing/trino-product-tests/conf/product-tests-${{ matrix.config }}.sh &&
              plugin/trino-hive-hadoop2/bin/run_hive_wasb_tests.sh
          fi
      - name: Run Hive Azure ADL Tests
        if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
        env:
          ADL_NAME: ${{ secrets.AZURE_ADL_NAME }}
          ADL_CLIENT_ID: ${{ secrets.AZURE_ADL_CLIENTID }}
          ADL_CREDENTIAL: ${{ secrets.AZURE_ADL_CREDENTIAL }}
          ADL_REFRESH_URL: ${{ secrets.AZURE_ADL_REFRESHURL }}
        run: |
          if [ "${ADL_NAME}" != "" ]; then
            source testing/trino-product-tests/conf/product-tests-${{ matrix.config }}.sh &&
              plugin/trino-hive-hadoop2/bin/run_hive_adl_tests.sh
          fi
      - name: Run Hive Alluxio Tests
        run: |
          source testing/trino-product-tests/conf/product-tests-${{ matrix.config }}.sh &&
            plugin/trino-hive-hadoop2/bin/run_hive_alluxio_tests.sh

  test-other-modules:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: 11
      - name: Maven Install
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
      - name: Maven Tests
        run: |
          ./mvnw test ${MAVEN_TEST} -pl '
            !:trino-main,
            !:trino-tests,
            !:trino-raptor-legacy,
            !:trino-accumulo,
            !:trino-cassandra,
            !:trino-hive,!:trino-orc,!:trino-parquet,
            !:trino-mongodb,!:trino-kafka,!:trino-elasticsearch,
            !:trino-redis,
            !:trino-sqlserver,!:trino-postgresql,!:trino-mysql,!:trino-memsql,
            !:trino-oracle,
            !:trino-kudu,
            !:trino-iceberg,!:trino-druid,
            !:trino-phoenix,!:trino-phoenix5,
            !:trino-docs,!:trino-server,!:trino-server-rpm,
            !:trino-test-jdbc-compatibility-old-server,
            !:trino-bigquery'

  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        modules:
          - ":trino-main"
          - ":trino-tests"
          - ":trino-tests -P ci-only"
          - ":trino-raptor-legacy"
          - ":trino-accumulo"
          - ":trino-cassandra"
          - ":trino-hive,:trino-orc"
          - ":trino-hive,:trino-parquet -P test-parquet"
          - ":trino-mongodb,:trino-kafka,:trino-elasticsearch,:trino-elasticsearch -P test-stats"
          - ":trino-redis"
          - ":trino-sqlserver,:trino-postgresql,:trino-mysql"
          - ":trino-oracle"
          - ":trino-kudu"
          - ":trino-iceberg,:trino-druid"
          - ":trino-phoenix,:trino-phoenix5"
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: 11
      - name: Maven Install
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl $(echo '${{ matrix.modules }}' | cut -d' ' -f1)
      - name: Maven Tests
        run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }}

  test-memsql:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: 11
      - name: Cleanup node
        # This is required as a virtual environment update 20210219.1 left too little space for MemSQL to work
        run: .github/bin/cleanup-node.sh
      - name: Maven Install
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl :trino-memsql
      - name: Memsql Tests
        env:
          MEMSQL_LICENSE: ${{ secrets.MEMSQL_LICENSE }}
        run: |
          if [ "${MEMSQL_LICENSE}" != "" ]; then
            ./mvnw test ${MAVEN_TEST} -pl :trino-memsql -Dmemsql.license=${MEMSQL_LICENSE}
          fi

  test-bigquery:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: 11
      - name: Maven Install
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl :trino-bigquery
      - name: Basic BigQuery Tests
        run: ./mvnw test ${MAVEN_TEST} -pl :trino-bigquery
      - name: Cloud BigQuery Tests
        env:
          BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CREDENTIALS_KEY }}
        run: |
          if [ "${BIGQUERY_CREDENTIALS_KEY}" != "" ]; then
            ./mvnw test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests -Dbigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}"
          fi
      - name: Cloud BigQuery Case Insensitive Mapping Tests
        env:
          BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CREDENTIALS_KEY }}
        run: |
          if [ "${BIGQUERY_CREDENTIALS_KEY}" != "" ]; then
            ./mvnw test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests-case-insensitive-mapping -Dbigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}"
          fi

  pt:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        config:
          - default
          - hdp3
          - cdh5
        suite:
          - suite-1
          - suite-2
          - suite-3
          # suite-4 does not exist
          - suite-5
        exclude:
          - config: cdh5
            ignore exclusion if: >-
              ${{ github.event_name != 'pull_request'
               || contains(github.event.pull_request.labels.*.name, 'tests:all')
               || contains(github.event.pull_request.labels.*.name, 'tests:hive')
               }}

          - config: default
            ignore exclusion if: >-
              ${{ github.event_name != 'pull_request'
               || contains(github.event.pull_request.labels.*.name, 'tests:all')
               || contains(github.event.pull_request.labels.*.name, 'tests:hive')
               }}

        ignore exclusion if:
          # Do not use this property outside of the matrix configuration.
          #
          # This is added to all matrix entries so they may be conditionally
          # excluded by adding them to the excludes list with a GHA expression
          # for this property.
          # - If the expression evaluates to true, it will never match the a
          #   actual value of the property, and will therefore not be excluded.
          # - If the expression evaluates to false, it will match the actual
          #   value of the property, and the exclusion will apply normally.
          - false
        include:
          # this suite ignores is not meant to be run with different configs
          - config: default
            suite: suite-6-non-generic
          # this suite ignores is not meant to be run with different configs
          - config: default
            suite: suite-7-non-generic
          # this suite ignores is not meant to be run with different configs
          - config: default
            suite: suite-8-non-generic
          # this suite ignores is not meant to be run with different configs
          - config: default
            suite: suite-tpcds
          # this suite ignores is not meant to be run with different configs
          - config: default
            suite: suite-oauth2
    # PT Launcher's timeout defaults to 2h, account for preparation steps (compilation) and add some margin
    timeout-minutes: 140
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-java@v2
        with:
          distribution: 'zulu'
          java-version: 11
      - name: Maven Install
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server-rpm'
      - name: Free Disk Space
        run: |
          docker image prune -af
          sudo apt-get clean
          rm -rf ~/.m2/repository
      - name: Product Tests
        run: |
          bin/ptl suite run \
            --suite ${{ matrix.suite }} --config config-${{ matrix.config }} --no-bind --logs-dir logs/ --timeout 2h