test.yml

name: 🧪 Tests
run-name: 🧪 Tests (${{ github.event_name }})

on:
  workflow_dispatch:
  workflow_call:
  pull_request:
    branches: [main]
  push:
    branches: [main]

concurrency:
  group: ${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  branch_tag: ${{ format('dev-{0}', github.head_ref || github.ref_name) }}

jobs:
  check_changes:
    name: Check changed files
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: read
    outputs:
      pr_changes: ${{ steps.pr_changes.outputs.changes }}
      push_changes: ${{ steps.push_changes.outputs.changes }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v2
        if: github.event_name == 'pull_request'
        id: pr_changes
        with:
          filters: |
            dcpy:
              - dcpy/**
              - pyproject.toml
              - admin/run_environment/**
            template:
              - .github/workflows/test.yml
              - .github/workflows/build.yml
              - .github/workflows/template_test.yml
              - products/template/**
              - dcpy/**
              - admin/run_environment/**
            qa:
              - apps/qa/**
              - dcpy/**
              - admin/run_environment/**
            cbbr:
              - products/cbbr/**
              - admin/run_environment/**
            checkbook:
              - products/checkbook/**
              - admin/run_environment/**
            zap:
              - products/zap-opendata/**
              - admin/run_environment/**
            docker:
              - admin/run_environment/**
            docker_base:
              - admin/run_environment/docker/base/**
      - uses: fvankrieken/paths-filter@master
        if: github.event_name == 'pull_request'
        id: push_changes
        with:
          filters: |
            docker:
              - admin/run_environment/**
            docker_base:
              - admin/run_environment/docker/base/**
          treat-pr-as-push: true

  determine_image_logic:
    # See https://github.com/NYCPlanning/data-engineering/wiki/Docker-Image-Management for convoluted logic for docker images
    # For our "base" image and 3 "children" images, determines whether 
    #   1. what appropriate tag is ("latest" or "dev-{ branch name }")
    #   2. if "dev", whether to build or use existing based on if there are changes in latest push
    # Exception is the compile_python_requirements branch, which runs weekly and always rebuilds everything
    name: Determine which docker images to build/use
    needs: check_changes
    runs-on: ubuntu-22.04
    outputs:
      base_tag: ${{ steps.tag.outputs.base }}
      child_tag: ${{ steps.tag.outputs.child }}
      base_action: ${{ steps.action.outputs.base }}
      child_action: ${{ steps.action.outputs.child }}
    steps:
    - name: set image tags
      id: tag
      run: |
        base="${{ 
          (github.head_ref == 'compile_python_reqs' || contains(needs.check_changes.outputs.pr_changes, 'docker_base'))
          && env.branch_tag || 'latest'
        }}"
        child="${{ 
          (github.head_ref == 'compile_python_reqs' || contains(needs.check_changes.outputs.pr_changes, 'docker'))
          && env.branch_tag || 'latest'
        }}"
        echo "base tag: \"$base\""
        echo "child tag: \"$child\""
        echo "base=$base" >> $GITHUB_OUTPUT
        echo "child=$child" >> $GITHUB_OUTPUT
    - name: set actions
      id: action
      run: |
        base="${{ 
          (
            github.head_ref == 'compile_python_reqs'
            || (
              contains(needs.check_changes.outputs.pr_changes, 'docker_base')
              && contains(needs.check_changes.outputs.push_changes, 'docker_base')
            )
          )
          && 'Build' || 'Use existing'
        }}"
        child="${{ 
          (
            github.head_ref == 'compile_python_reqs'
            || (
              contains(needs.check_changes.outputs.pr_changes, 'docker')
              && contains(needs.check_changes.outputs.push_changes, 'docker')
            )
          )
          && 'Build' || 'Use existing'
        }}"
        echo "base action: $base"
        echo "child action: $child"
        echo "base=$base" >> $GITHUB_OUTPUT
        echo "child=$child" >> $GITHUB_OUTPUT

  docker_base_image:
    name: >
      ${{ 
        format(
          '{0} base:{1} docker image',
          needs.determine_image_logic.outputs.base_action, 
          needs.determine_image_logic.outputs.base_tag
        )
      }}
    needs: determine_image_logic
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
        if: needs.determine_image_logic.outputs.base_action == 'Build'
      - name: Load Secrets
        if: needs.determine_image_logic.outputs.base_action == 'Build'
        uses: 1password/load-secrets-action@v1
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
          DOCKER_USERNAME: "op://Data Engineering/Dockerhub/username"
          DOCKER_PASSWORD: "op://Data Engineering/Dockerhub/password"
      - name: publish images
        if: needs.determine_image_logic.outputs.base_action == 'Build'
        run: ./admin/ops/docker_build_and_publish.sh base "${{ needs.determine_image_logic.outputs.base_tag }}"

  docker_child_images:
    name: >
      ${{ 
        format(
          '{0} {1}:{2} docker image',
          needs.determine_image_logic.outputs.child_action, 
          matrix.image,
          needs.determine_image_logic.outputs.child_tag
        )
      }}
    needs: 
      - determine_image_logic
      - docker_base_image
    runs-on: ubuntu-22.04
    strategy: 
      matrix: 
        image:
          - build-base
          - build-geosupport
          - dev
    steps:
      - uses: actions/checkout@v4
        if: needs.determine_image_logic.outputs.child_action == 'Build'
      - name: Load Secrets
        if: needs.determine_image_logic.outputs.child_action == 'Build'
        uses: 1password/load-secrets-action@v1
        with:
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
          DOCKER_USERNAME: "op://Data Engineering/Dockerhub/username"
          DOCKER_PASSWORD: "op://Data Engineering/Dockerhub/password"
      - name: publish images
        if: needs.determine_image_logic.outputs.child_action == 'Build'
        run: ./admin/ops/docker_build_and_publish.sh "${{ matrix.image }}" "${{ needs.determine_image_logic.outputs.base_tag }}" "${{ needs.determine_image_logic.outputs.child_tag }}"

  test:
    uses: ./.github/workflows/test_helper.yml
    needs: 
      - check_changes
      - determine_image_logic
      - docker_child_images
    with:
      image_tag: ${{ needs.determine_image_logic.outputs.child_tag }}
      path_filter: ${{ needs.check_changes.outputs.pr_changes }}
    secrets: inherit

  build:
    needs: docker_child_images
    # only run build as part of tests in the case of the PRs generated by python_compile_requirements.yml
    if: ${{ github.head_ref == 'compile_python_reqs' }}
    uses: ./.github/workflows/build.yml
    secrets: inherit
    with:
      dataset_name: all
      build_note: check newly compiled python requirements
      recipe_file: recipe
      dev_image: true