ci-testing-deploy.yml

name: CI
# NOTE: this CI workflow is running a number of unit/integration/system tests jobs
# and also builds a set of "test" docker images (required for integration/system tests)
# if all tests are passing, and the CI runs on the master branch, then it will also deploy
# the "test" docker images to dockerhub registry
#
# in PRs: only the jobs that needs to run will be run by detecting file changes
#
# NOTE2: to keep the github branch protection simpler, the required jobs are the ones named:
# unit-tests, integration-tests, and system-tests

on:
  push:
    branches:
      - "*"
    tags-ignore:
      - "*"
  pull_request:
    branches:
      - "*"
  # https://github.blog/changelog/2023-02-08-pull-request-merge-queue-public-beta/
  merge_group:
    branches:
      - "master"

  workflow_dispatch:
    inputs:
      tests:
        description: tests to run
        required: true
        default: "all"
        type: choice
        options:
          - unit-tests
          - integration-tests
          - system-tests
          - all

env:
  # NOTE: 'COLUMNS' is a shell env var that represents the width (number of columns)
  #       of the terminal or command-line interface in characters.
  COLUMNS: 120

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  changes:
    name: detect file changes since last push
    runs-on: ubuntu-latest
    # Set job outputs to values from filter step
    outputs:
      aws-library: ${{ steps.filter.outputs.aws-library }}
      dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }}
      models-library: ${{ steps.filter.outputs.models-library }}
      common-library: ${{ steps.filter.outputs.common-library }}
      notifications-library: ${{ steps.filter.outputs.notifications-library }}
      postgres-database: ${{ steps.filter.outputs.postgres-database }}
      service-integration: ${{ steps.filter.outputs.service-integration }}
      service-library: ${{ steps.filter.outputs.service-library }}
      settings-library: ${{ steps.filter.outputs.settings-library }}
      simcore-sdk: ${{ steps.filter.outputs.simcore-sdk }}
      agent: ${{ steps.filter.outputs.agent }}
      api: ${{ steps.filter.outputs.api }}
      api-server: ${{ steps.filter.outputs.api-server }}
      autoscaling: ${{ steps.filter.outputs.autoscaling }}
      catalog: ${{ steps.filter.outputs.catalog }}
      clusters-keeper: ${{ steps.filter.outputs.clusters-keeper }}
      dask-sidecar: ${{ steps.filter.outputs.dask-sidecar }}
      datcore-adapter: ${{ steps.filter.outputs.datcore-adapter }}
      director: ${{ steps.filter.outputs.director }}
      director-v2: ${{ steps.filter.outputs.director-v2 }}
      dynamic-sidecar: ${{ steps.filter.outputs.dynamic-sidecar }}
      efs-guardian: ${{ steps.filter.outputs.efs-guardian }}
      invitations: ${{ steps.filter.outputs.invitations }}
      migration: ${{ steps.filter.outputs.migration }}
      payments: ${{ steps.filter.outputs.payments }}
      dynamic-scheduler: ${{ steps.filter.outputs.dynamic-scheduler }}
      resource-usage-tracker: ${{ steps.filter.outputs.resource-usage-tracker }}
      static-webserver: ${{ steps.filter.outputs.static-webserver }}
      storage: ${{ steps.filter.outputs.storage }}
      webserver: ${{ steps.filter.outputs.webserver }}
      anything: ${{ steps.filter.outputs.anything }}
      anything-py: ${{ steps.filter.outputs.anything-py }}
      anything-js: ${{ steps.filter.outputs.anything-js }}
    steps:
      - uses: actions/checkout@v4
      # For pull requests it's not necessary to checkout the code
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            aws-library:
              - 'packages/aws-library/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            dask-task-models-library:
              - 'packages/dask-task-models-library/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            models-library:
              - 'packages/models-library/**'
              - 'packages/postgres-database/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            common-library:
              - 'packages/common-library/**'
            notifications-library:
              - 'packages/notifications-library/**'
              - 'packages/postgres-database/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            postgres-database:
              - 'packages/postgres-database/**'
              - 'packages/pytest-simcore/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            service-integration:
              - 'packages/models-library/**'
              - 'packages/pytest-simcore/**'
              - 'packages/service-integration/**'
              - 'services/docker-compose*'
            service-library:
              - 'packages/pytest-simcore/**'
              - 'packages/service-library/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            settings-library:
              - 'packages/pytest-simcore/**'
              - 'packages/settings-library/**'
              - 'services/docker-compose*'
            simcore-sdk:
              - 'packages/**'
              - 'services/docker-compose*'
            agent:
              - 'packages/**'
              - 'services/agent/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            api:
              - 'api/**'
            api-server:
              - 'packages/**'
              - 'services/api-server/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            autoscaling:
              - 'packages/**'
              - 'services/autoscaling/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            catalog:
              - 'packages/**'
              - 'services/catalog/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            clusters-keeper:
              - 'packages/**'
              - 'services/clusters-keeper/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            dask-sidecar:
              - 'packages/**'
              - 'services/dask-sidecar/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            datcore-adapter:
              - 'packages/**'
              - 'services/datcore-adapter/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            director:
              - 'packages/**'
              - 'services/director/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            director-v2:
              - 'packages/**'
              - 'services/director-v2/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            dynamic-sidecar:
              - 'packages/**'
              - 'services/dynamic-sidecar/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            efs-guardian:
              - 'packages/**'
              - 'services/efs-guardian/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            invitations:
              - 'packages/**'
              - 'services/invitations/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            migration:
              - 'packages/**'
              - 'services/migration/**'
              - 'services/docker-compose*'
            payments:
                - 'packages/**'
                - 'services/payments/**'
                - 'services/docker-compose*'
                - 'scripts/mypy/*'
                - 'mypy.ini'
            dynamic-scheduler:
                - 'packages/**'
                - 'services/dynamic-scheduler/**'
                - 'services/docker-compose*'
                - 'scripts/mypy/*'
                - 'mypy.ini'
            resource-usage-tracker:
              - 'packages/**'
              - 'services/resource-usage-tracker/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            static-webserver:
              - 'services/static-webserver/**'
              - 'services/docker-compose*'
            storage:
              - 'packages/**'
              - 'services/storage/**'
              - 'services/docker-compose*'
              - 'scripts/mypy/*'
              - 'mypy.ini'
            webserver:
              - 'packages/**'
              - 'services/web/**'
              - 'services/docker-compose*'
            anything:
              - 'packages/**'
              - 'services/**'
              - 'tests/**'
            anything-py:
              - '**/*.py'
            anything-js:
              - '**/*.js'
  build-test-images:
    # this step comes first, so that it is executed as first job in push calls
    # in PR calls this step is anyway skipped
    needs: changes
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    name: "[build] docker images"
    steps:
      - name: Remove unused software
        run: |
          echo "Available storage before:"
          sudo df -h
          echo
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          echo "Available storage after:"
          sudo df -h
          echo
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      - name: show system environs
        run: ./ci/helpers/show_system_versions.bash
      - name: build images
        run: |
          export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
          mkdir --parents /${{ runner.temp }}/build
          make build local-dest=/${{ runner.temp }}/build
      - name: upload build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
          path: /${{ runner.temp }}/build

  unit-test-webserver-01:
    needs: changes
    if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 01"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/web/server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/webserver.bash typecheck
      - name: test isolated
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/webserver.bash test_isolated
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/webserver.bash test_with_db 01
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-webserver-02:
    needs: changes
    if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 02"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/web/server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: test
        run: ./ci/github/unit-testing/webserver.bash test_with_db 02
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-webserver-03:
    needs: changes
    if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[unit] webserver 03"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/web/server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install webserver
        run: ./ci/github/unit-testing/webserver.bash install
      - name: test
        run: ./ci/github/unit-testing/webserver.bash test_with_db 03
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-webserver-04:
      needs: changes
      if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
      timeout-minutes: 25 # if this timeout gets too small, then split the tests
      name: "[unit] webserver 04"
      runs-on: ${{ matrix.os }}
      strategy:
        matrix:
          python: ["3.11"]
          os: [ubuntu-22.04]
        fail-fast: false
      steps:
        - uses: actions/checkout@v4
        - name: setup docker buildx
          id: buildx
          uses: docker/setup-buildx-action@v3
          with:
            driver: docker-container
        - name: setup python environment
          uses: actions/setup-python@v5
          with:
            python-version: ${{ matrix.python }}
        - name: install uv
          uses: astral-sh/setup-uv@v4
          with:
            version: "0.4.x"
            enable-cache: false
            cache-dependency-glob: "**/web/server/requirements/ci.txt"
        - name: show system version
          run: ./ci/helpers/show_system_versions.bash
        - name: install webserver
          run: ./ci/github/unit-testing/webserver.bash install
        - name: test
          run: ./ci/github/unit-testing/webserver.bash test_with_db 04
        - uses: codecov/codecov-action@v5
          env:
            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
          with:
            flags: unittests #optional
        - name: Upload test results to Codecov
          if: ${{ !cancelled() }}
          uses: codecov/test-results-action@v1
          with:
            token: ${{ secrets.CODECOV_TOKEN }}

  unit-test-storage:
    needs: changes
    if: ${{ needs.changes.outputs.storage == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] storage"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/storage/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/storage.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/storage.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/storage.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-agent:
    needs: changes
    if: ${{ needs.changes.outputs.agent == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] agent"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/agent/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/agent.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/agent.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/agent.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-api:
    needs: changes
    if: ${{ needs.changes.outputs.api == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] api"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/api/tests/requirements.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install api
        run: ./ci/github/unit-testing/api.bash install
      - name: test
        run: ./ci/github/unit-testing/api.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-api-server:
    needs: changes
    if: ${{ needs.changes.outputs.api-server == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] api-server"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/api-server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/api-server.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/api-server.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/api-server.bash test
      - name: OAS backwards compatibility check
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/api-server.bash openapi-diff
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-autoscaling:
    needs: changes
    if: ${{ needs.changes.outputs.autoscaling == 'true' || github.event_name == 'push' }}
    timeout-minutes: 22 # temporary: mypy takes a huge amount of time to run here, maybe we should cache it
    name: "[unit] autoscaling"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/autoscaling/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/autoscaling.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/autoscaling.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/autoscaling.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-catalog:
    needs: changes
    if: ${{ needs.changes.outputs.catalog == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] catalog"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/catalog/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/catalog.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/catalog.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/catalog.bash test
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/catalog/test_failures
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-clusters-keeper:
    needs: changes
    if: ${{ needs.changes.outputs.clusters-keeper == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] clusters-keeper"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/clusters-keeper/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv
          source .venv/bin/activate && \
          pushd services/clusters-keeper && \
          make install-ci
      - name: typecheck
        run: |
          source .venv/bin/activate && \
          uv pip install mypy && \
          pushd services/clusters-keeper && \
          make mypy
      - name: test
        if: ${{ !cancelled() }}
        run: |
          source .venv/bin/activate && \
          pushd services/clusters-keeper && \
          make test-ci-unit
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-datcore-adapter:
    needs: changes
    if: ${{ needs.changes.outputs.datcore-adapter == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] datcore-adapter"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/datcore-adapter/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/datcore-adapter.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/datcore-adapter.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/datcore-adapter.bash test
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/datcore-adapter/test_failures
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-director:
    needs: changes
    if: ${{ needs.changes.outputs.director == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] director"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/director/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/director.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/director.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/director.bash test
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director/test_failures
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-director-v2:
    needs: changes
    if: ${{ needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] director-v2"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/director-v2/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/director-v2.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/director-v2.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/director-v2.bash test
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-aws-library:
    needs: changes
    if: ${{ needs.changes.outputs.aws-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] aws-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/aws-library/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/aws-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/aws-library.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/aws-library.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-dask-task-models-library:
    needs: changes
    if: ${{ needs.changes.outputs.dask-task-models-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dask-task-models-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/dask-task-models-library/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dask-task-models-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dask-task-models-library.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/dask-task-models-library.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-dask-sidecar:
    needs: changes
    if: ${{ needs.changes.outputs.dask-sidecar == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dask-sidecar"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/dask-sidecar/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dask-sidecar.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dask-sidecar.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/dask-sidecar.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-payments:
    needs: changes
    if: ${{ needs.changes.outputs.payments == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] payments"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/payments/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/payments.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/payments.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/payments.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-dynamic-scheduler:
    needs: changes
    if: ${{ needs.changes.outputs.dynamic-scheduler == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dynamic-scheduler"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/dynamic-scheduler/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dynamic-scheduler.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dynamic-scheduler.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/dynamic-scheduler.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-resource-usage-tracker:
    needs: changes
    if: ${{ needs.changes.outputs.resource-usage-tracker == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] resource-usage-tracker"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/resource-usage-tracker/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv
          source .venv/bin/activate && \
          pushd services/resource-usage-tracker && \
          make install-ci
      - name: typecheck
        run: |
          source .venv/bin/activate && \
          pushd services/resource-usage-tracker && \
          make mypy
      - name: test
        if: ${{ !cancelled() }}
        run: |
          source .venv/bin/activate && \
          pushd services/resource-usage-tracker && \
          make test-ci-unit
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-dynamic-sidecar:
    needs: changes
    if: ${{ needs.changes.outputs.dynamic-sidecar == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] dynamic-sidecar"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/dynamic-sidecar/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/dynamic-sidecar.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/dynamic-sidecar.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/dynamic-sidecar.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-efs-guardian:
    needs: changes
    if: ${{ needs.changes.outputs.efs-guardian == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] efs-guardian"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/efs-guardian/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: |
          make devenv
          source .venv/bin/activate && \
          pushd services/efs-guardian && \
          make install-ci
      - name: typecheck
        run: |
          source .venv/bin/activate && \
          uv pip install mypy && \
          pushd services/efs-guardian && \
          make mypy
      - name: test
        if: ${{ !cancelled() }}
        run: |
          source .venv/bin/activate && \
          pushd services/efs-guardian && \
          make test-ci-unit
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-python-linting:
    needs: changes
    if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] python-linting"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11", "3.12"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/ci/helpers/requirements.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/python-linting.bash install
      - name: test
        run: ./ci/github/unit-testing/python-linting.bash test

  unit-test-postgres-database:
    needs: changes
    if: ${{ needs.changes.outputs.postgres-database == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] postgres-database"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/postgres-database/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/postgres-database.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/postgres-database.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/postgres-database.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-invitations:
    needs: changes
    if: ${{ needs.changes.outputs.invitations == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] invitations"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/notifications-library/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/invitations.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/invitations.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/invitations.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-service-integration:
    needs: changes
    if: ${{ needs.changes.outputs.service-integration == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] service-integration"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/service-integration/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/service-integration.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/service-integration.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/service-integration.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-service-library:
    needs: changes
    if: ${{ needs.changes.outputs.service-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] service-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/service-library/requirements/ci*.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/service-library.bash install_all
      - name: typecheck
        run: ./ci/github/unit-testing/service-library.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/service-library.bash test_all
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-settings-library:
    needs: changes
    if: ${{ needs.changes.outputs.settings-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] settings-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/settings-library/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/settings-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/settings-library.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/settings-library.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-models-library:
    needs: changes
    if: ${{ needs.changes.outputs.models-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] models-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/models-library/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/models-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/models-library.bash typecheck
      - name: test
        run: ./ci/github/unit-testing/models-library.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-common-library:
    needs: changes
    if: ${{ needs.changes.outputs.common-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] common-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: yezz123/setup-uv@v4
      - uses: actions/cache@v4
        id: cache-uv
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/common-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/common-library.bash typecheck
      - name: test
        run: ./ci/github/unit-testing/common-library.bash test
      - uses: codecov/codecov-action@v5
        with:
          flags: unittests #optional

  unit-test-notifications-library:
    needs: changes
    if: ${{ needs.changes.outputs.notifications-library == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit]notifications-library"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
        docker_buildx: [v0.10.4]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: ${{ matrix.docker_buildx }}
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/notifications-library/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/notifications-library.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/notifications-library.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/notifications-library.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-test-simcore-sdk:
    needs: changes
    if: ${{ needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }}
    timeout-minutes: 18 # if this timeout gets too small, then split the tests
    name: "[unit] simcore-sdk"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/simcore-sdk/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/unit-testing/simcore-sdk.bash install
      - name: typecheck
        run: ./ci/github/unit-testing/simcore-sdk.bash typecheck
      - name: test
        if: ${{ !cancelled() }}
        run: ./ci/github/unit-testing/simcore-sdk.bash test
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: unittests #optional


  unit-tests:
    # NOTE: this is a github required status check!
    if: ${{ always() }}
    needs:
      [
        unit-test-agent,
        unit-test-api-server,
        unit-test-api,
        unit-test-autoscaling,
        unit-test-catalog,
        unit-test-clusters-keeper,
        unit-test-dask-sidecar,
        unit-test-aws-library,
        unit-test-dask-task-models-library,
        unit-test-datcore-adapter,
        unit-test-director-v2,
        unit-test-director,
        unit-test-dynamic-sidecar,
        unit-test-efs-guardian,
        unit-test-models-library,
        unit-test-common-library,
        unit-test-notifications-library,
        unit-test-payments,
        unit-test-dynamic-scheduler,
        unit-test-postgres-database,
        unit-test-python-linting,
        unit-test-resource-usage-tracker,
        unit-test-service-integration,
        unit-test-service-library,
        unit-test-settings-library,
        unit-test-simcore-sdk,
        unit-test-storage,
        unit-test-webserver-01,
        unit-test-webserver-02,
        unit-test-webserver-03,
        unit-test-webserver-04,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: a previous unit-test job failed
        if: ${{ contains(join(needs.*.result, ','), 'failure') }}
        run: |
          echo "::error title=ERROR::one of the unit-tests failed!"
          echo "${{ join(needs.*.result, ',') }}"
          exit 1
      - name: all the previous unit-tests were run successfully or skipped
        if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
        run: echo "::notice All good!"

  integration-test-webserver-01:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] webserver 01"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/web/server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/webserver.bash install
      - name: test
        run: ./ci/github/integration-testing/webserver.bash test 01
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/web/server/test_failures
      - name: cleanup
        if: ${{ !cancelled() }}
        run: ./ci/github/integration-testing/webserver.bash clean_up
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: integrationtests #optional

  integration-test-webserver-02:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] webserver 02"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/web/server/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/webserver.bash install
      - name: test
        run: ./ci/github/integration-testing/webserver.bash test 02
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/web/server/test_failures
      - name: cleanup
        if: ${{ !cancelled() }}
        run: ./ci/github/integration-testing/webserver.bash clean_up
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: integrationtests #optional

  integration-test-director-v2-01:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push'}}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] director-v2 01"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/director-v2/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/director-v2.bash install
      - name: test
        run: ./ci/github/integration-testing/director-v2.bash test 01
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - name: cleanup
        if: ${{ !cancelled() }}
        run: ./ci/github/integration-testing/director-v2.bash clean_up
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: integrationtests #optional

  integration-test-director-v2-02:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] director-v2 02"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    env:
      # NOTE: DIRECTOR_DEFAULT_MAX_* used for integration-tests that include `director` service
      DIRECTOR_DEFAULT_MAX_MEMORY: 268435456
      DIRECTOR_DEFAULT_MAX_NANO_CPUS: 10000000
      DIRECTOR_TRACING: null
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup rclone docker volume plugin
        run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/director-v2/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/director-v2.bash install
      - name: test
        run: ./ci/github/integration-testing/director-v2.bash test 02
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/director-v2/test_failures
      - name: cleanup
        if: ${{ !cancelled() }}
        run: ./ci/github/integration-testing/director-v2.bash clean_up
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: integrationtests #optional

  integration-test-dynamic-sidecar:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.dynamic-sidecar == 'true' || github.event_name == 'push'}}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] dynamic-sidecar"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/dynamic-sidecar/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/dynamic-sidecar.bash install
      - name: test
        run: ./ci/github/integration-testing/dynamic-sidecar.bash test 01
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./services/dynamic-sidecar/test_failures
      - name: cleanup
        if: ${{ !cancelled() }}
        run: ./ci/github/integration-testing/dynamic-sidecar.bash clean_up
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: integrationtests #optional

  integration-test-simcore-sdk:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[int] simcore-sdk"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: install rclone
        run: sudo ./ci/github/helpers/install_rclone.bash
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/simcore-sdk/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/integration-testing/simcore-sdk.bash install
      - name: test
        run: ./ci/github/integration-testing/simcore-sdk.bash test
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./packages/simcore-sdk/test_failures
      - name: cleanup
        if: ${{ !cancelled() }}
        run: ./ci/github/integration-testing/simcore-sdk.bash clean_up
      - uses: codecov/codecov-action@v5
        if: ${{ !cancelled() }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          flags: integrationtests

  integration-tests:
    # NOTE: this is a github required status check!
    if: ${{ always() }}
    needs:
      [
        integration-test-director-v2-01,
        integration-test-director-v2-02,
        integration-test-dynamic-sidecar,
        integration-test-simcore-sdk,
        integration-test-webserver-01,
        integration-test-webserver-02,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: a previous integration-test job failed
        if: ${{ contains(join(needs.*.result, ','), 'failure') }}
        run: |
          echo "::error title=ERROR::one of the integration-tests failed!"
          exit 1
      - name: all the previous integration-tests were run successfully or skipped
        if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
        run: echo "::notice All good!"

  system-test-public-api:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 25 # if this timeout gets too small, then split the tests
    name: "[sys] public api"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/public-api/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: ./ci/github/system-testing/public-api.bash install
      - name: test
        run: ./ci/github/system-testing/public-api.bash test
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./test_failures
      - name: cleanup
        if: ${{ !cancelled() }}
        run: ./ci/github/system-testing/public-api.bash clean_up

  system-test-swarm-deploy:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] deploy simcore"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/swarm-deploy/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/system-testing/swarm-deploy.bash install
      - name: test
        run: ./ci/github/system-testing/swarm-deploy.bash test
      - name: dump services setting schemas
        run: export DOCKER_REGISTRY=local; export DOCKER_IMAGE_TAG=production; make settings-schema.json
      - name: upload services settings schemas
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_services_settings_schemas
          path: ./services/**/settings-schema.json
      - name: upload failed tests logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./test_failures
      - name: cleanup
        if: ${{ !cancelled() }}
        run: ./ci/github/system-testing/swarm-deploy.bash clean_up

  system-test-e2e:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] e2e"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        node: [14]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - uses: actions/setup-node@v4.1.0
        with:
          node-version: ${{ matrix.node }}
          cache: "npm"
          cache-dependency-path: "tests/e2e/package-lock.json"
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/e2e/requirements/requirements.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: setup
        run: ./ci/github/system-testing/e2e.bash install
      - name: test
        run: ./ci/github/system-testing/e2e.bash test
      - name: dump docker logs
        id: docker_logs_dump
        if: ${{ !cancelled() }}
        run: ./ci/github/system-testing/e2e.bash dump_docker_logs
      - name: upload docker logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./tests/e2e/test_failures
      - name: upload screenshots
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_screenshots
          path: tests/e2e/screenshots
      - name: upload e2e logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_logs
          path: tests/e2e/logs
      - name: cleanup
        if: ${{ !cancelled() }}
        run: ./ci/github/system-testing/e2e.bash clean_up

  system-test-e2e-playwright:
    needs: [changes, build-test-images]
    if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] e2e-playwright"
    runs-on: ${{ matrix.os }}
    # NOTE: this is an interesting way, but generate a load of issues like not having docker installed, etc etc.
    # container:
    #   image: mcr.microsoft.com/playwright/python:v1.39.0-jammy
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/e2e-playwright/requirements/ci.txt"
      - name: expose github runtime for buildx
        uses: crazy-max/ghaction-github-runtime@v3
      - name: download docker images
        uses: actions/download-artifact@v4
        with:
          name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
          path: /${{ runner.temp }}/build
      - name: load docker images
        run: make load-images local-src=/${{ runner.temp }}/build
      - name: prepare devenv
        run: make devenv
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: setup
        run: |
          ./ci/github/system-testing/e2e-playwright.bash install
      - name: test
        run: |
          ./ci/github/system-testing/e2e-playwright.bash test
      - name: dump docker logs
        id: docker_logs_dump
        if: ${{ !cancelled() }}
        run: ./ci/github/system-testing/e2e-playwright.bash dump_docker_logs
      - name: upload docker logs
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_docker_logs
          path: ./tests/e2e-playwright/test_failures
      - name: upload tracing if failed
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}_tracing
          path: tests/e2e-playwright/test-results

  system-test-environment-setup:
    timeout-minutes: 30 # if this timeout gets too small, then split the tests
    name: "[sys] environment setup"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
          cache-dependency-glob: "**/environment-setup/requirements/ci.txt"
      - name: show system version
        run: ./ci/helpers/show_system_versions.bash
      - name: install
        run: ./ci/github/system-testing/environment-setup.bash install
      - name: test
        run: ./ci/github/system-testing/environment-setup.bash test
      - name: cleanup
        if: ${{ !cancelled() }}
        run: ./ci/github/system-testing/environment-setup.bash clean_up

  system-tests:
    # NOTE: this is a github required status check!
    if: ${{ always() }}
    needs:
      [
        system-test-e2e,
        system-test-e2e-playwright,
        system-test-environment-setup,
        system-test-public-api,
        system-test-swarm-deploy,
        system-api-specs,
        system-backwards-compatibility
      ]
    runs-on: ubuntu-latest
    steps:
      - name: a previous system-test job failed
        if: ${{ contains(join(needs.*.result, ','), 'failure') }}
        run: |
          echo "::error title=ERROR::one of the system-tests failed!"
          exit 1
      - name: all the previous system-tests were run successfully or skipped
        if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
        run: echo "::notice All good!"

  deploy:
    name: deploy to dockerhub
    if: github.event_name == 'push'
    needs: [unit-tests, integration-tests, system-tests]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python: ["3.11"]
        os: [ubuntu-22.04]
      fail-fast: false
    env:
      # secrets can be set in settings/secrets on github
      DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
    steps:
      - uses: actions/checkout@v4
      - name: setup docker buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          driver: docker-container
      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
      - name: download docker images with retry
        uses: Wandalen/wretry.action@master
        with:
          action: actions/download-artifact@v4
          with: |
            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
            path: /${{ runner.temp }}/build
          attempt_limit: 5
          attempt_delay: 1000
      - name: load docker images
        run: |
          make load-images local-src=/${{ runner.temp }}/build
      - name: set owner variable
        run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
      - if: github.ref == 'refs/heads/master'
        name: deploy master image
        env:
          TAG_PREFIX: master-github
        run: ./ci/deploy/dockerhub-deploy.bash -n
      - if: contains(github.ref, 'refs/heads/hotfix_v')
        name: deploy release hotfix image
        env:
          TAG_PREFIX: hotfix-github
        run: ./ci/deploy/dockerhub-deploy.bash -n
      - if: contains(github.ref, 'refs/heads/hotfix_staging_')
        name: deploy staging hotfix image
        env:
          TAG_PREFIX: hotfix-staging-github
        run: ./ci/deploy/dockerhub-deploy.bash -n

  system-api-specs:
    needs: [changes]
    if: ${{ needs.changes.outputs.anything-py == 'true' && github.event_name == 'push' && github.event.pull_request != null }}
    timeout-minutes: 10
    name: "[sys] check api-specs are up to date"
    runs-on: ubuntu-latest
    steps:
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
      - name: checkout source branch
        uses: actions/checkout@v4
      - name: Regenerate specs and check
        run: |
          uv venv .venv && source .venv/bin/activate
          make openapi-specs
          ./ci/github/helpers/openapi-specs-diff.bash diff \
            https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/refs/heads/${{ github.event.pull_request.head.ref }} \
            .

  system-backwards-compatibility:
    needs: [changes, system-api-specs]
    if: ${{ needs.changes.outputs.anything-py == 'true' && github.event_name == 'push' && github.event.pull_request != null }}
    timeout-minutes: 10
    name: "[sys] api-server backwards compatibility"
    runs-on: ubuntu-latest
    steps:
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
      - name: checkout
        uses: actions/checkout@v4
      - name: check api-server backwards compatibility
        run: |
          ./scripts/openapi-diff.bash breaking --fail-on ERR\
            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.event.pull_request.base.ref }}/services/api-server/openapi.json \
            /specs/services/api-server/openapi.json

  api-spec-backwards-compatibility:
    needs: [changes, system-api-specs]
    if: ${{ needs.changes.outputs.anything-py == 'true' && github.event_name == 'push' && github.event.pull_request != null }}
    continue-on-error: true
    timeout-minutes: 10
    name: "api-specs-backwards-compatibility"
    runs-on: ubuntu-latest
    steps:
      - name: setup python environment
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "0.4.x"
          enable-cache: false
      - name: checkout
        uses: actions/checkout@v4
      - name: Check openapi-specs backwards compatibility
        run: |
          ./ci/github/helpers/openapi-specs-diff.bash breaking \
            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.event.pull_request.base.ref }} \
            .