Skip to content

♻️ reroute get project inactivity via dynamic-scheduler #36384

♻️ reroute get project inactivity via dynamic-scheduler

♻️ reroute get project inactivity via dynamic-scheduler #36384

name: CI
# NOTE: this CI workflow is running a number of unit/integration/system tests jobs
# and also builds a set of "test" docker images (required for integration/system tests)
# if all tests are passing, and the CI runs on the master branch, then it will also deploy
# the "test" docker images to dockerhub registry
#
# in PRs: only the jobs that needs to run will be run by detecting file changes
#
# NOTE2: to keep the github branch protection simpler, the required jobs are the ones named:
# unit-tests, integration-tests, and system-tests
on:
push:
branches:
- "*"
tags-ignore:
- "*"
pull_request:
branches:
- "*"
# https://github.blog/changelog/2023-02-08-pull-request-merge-queue-public-beta/
merge_group:
branches:
- "master"
workflow_dispatch:
inputs:
tests:
description: tests to run
required: true
default: "all"
type: choice
options:
- unit-tests
- integration-tests
- system-tests
- all
env:
# NOTE: 'COLUMNS' is a shell env var that represents the width (number of columns)
# of the terminal or command-line interface in characters.
COLUMNS: 120
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
changes:
name: detect file changes since last push
runs-on: ubuntu-latest
# Set job outputs to values from filter step
outputs:
aws-library: ${{ steps.filter.outputs.aws-library }}
dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }}
models-library: ${{ steps.filter.outputs.models-library }}
common-library: ${{ steps.filter.outputs.common-library }}
notifications-library: ${{ steps.filter.outputs.notifications-library }}
postgres-database: ${{ steps.filter.outputs.postgres-database }}
service-integration: ${{ steps.filter.outputs.service-integration }}
service-library: ${{ steps.filter.outputs.service-library }}
settings-library: ${{ steps.filter.outputs.settings-library }}
simcore-sdk: ${{ steps.filter.outputs.simcore-sdk }}
agent: ${{ steps.filter.outputs.agent }}
api: ${{ steps.filter.outputs.api }}
api-server: ${{ steps.filter.outputs.api-server }}
autoscaling: ${{ steps.filter.outputs.autoscaling }}
catalog: ${{ steps.filter.outputs.catalog }}
clusters-keeper: ${{ steps.filter.outputs.clusters-keeper }}
dask-sidecar: ${{ steps.filter.outputs.dask-sidecar }}
datcore-adapter: ${{ steps.filter.outputs.datcore-adapter }}
director: ${{ steps.filter.outputs.director }}
director-v2: ${{ steps.filter.outputs.director-v2 }}
dynamic-sidecar: ${{ steps.filter.outputs.dynamic-sidecar }}
efs-guardian: ${{ steps.filter.outputs.efs-guardian }}
invitations: ${{ steps.filter.outputs.invitations }}
migration: ${{ steps.filter.outputs.migration }}
payments: ${{ steps.filter.outputs.payments }}
dynamic-scheduler: ${{ steps.filter.outputs.dynamic-scheduler }}
resource-usage-tracker: ${{ steps.filter.outputs.resource-usage-tracker }}
static-webserver: ${{ steps.filter.outputs.static-webserver }}
storage: ${{ steps.filter.outputs.storage }}
webserver: ${{ steps.filter.outputs.webserver }}
anything: ${{ steps.filter.outputs.anything }}
anything-py: ${{ steps.filter.outputs.anything-py }}
anything-js: ${{ steps.filter.outputs.anything-js }}
steps:
- uses: actions/checkout@v4
# For pull requests it's not necessary to checkout the code
- uses: dorny/paths-filter@v3
id: filter
with:
filters: |
aws-library:
- 'packages/aws-library/**'
- 'packages/pytest-simcore/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
dask-task-models-library:
- 'packages/dask-task-models-library/**'
- 'packages/pytest-simcore/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
models-library:
- 'packages/models-library/**'
- 'packages/postgres-database/**'
- 'packages/pytest-simcore/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
common-library:
- 'packages/common-library/**'
notifications-library:
- 'packages/notifications-library/**'
- 'packages/postgres-database/**'
- 'packages/pytest-simcore/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
postgres-database:
- 'packages/postgres-database/**'
- 'packages/pytest-simcore/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
service-integration:
- 'packages/models-library/**'
- 'packages/pytest-simcore/**'
- 'packages/service-integration/**'
- 'services/docker-compose*'
service-library:
- 'packages/pytest-simcore/**'
- 'packages/service-library/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
settings-library:
- 'packages/pytest-simcore/**'
- 'packages/settings-library/**'
- 'services/docker-compose*'
simcore-sdk:
- 'packages/**'
- 'services/docker-compose*'
agent:
- 'packages/**'
- 'services/agent/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
api:
- 'api/**'
api-server:
- 'packages/**'
- 'services/api-server/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
autoscaling:
- 'packages/**'
- 'services/autoscaling/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
catalog:
- 'packages/**'
- 'services/catalog/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
clusters-keeper:
- 'packages/**'
- 'services/clusters-keeper/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
dask-sidecar:
- 'packages/**'
- 'services/dask-sidecar/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
datcore-adapter:
- 'packages/**'
- 'services/datcore-adapter/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
director:
- 'packages/**'
- 'services/director/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
director-v2:
- 'packages/**'
- 'services/director-v2/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
dynamic-sidecar:
- 'packages/**'
- 'services/dynamic-sidecar/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
efs-guardian:
- 'packages/**'
- 'services/efs-guardian/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
invitations:
- 'packages/**'
- 'services/invitations/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
migration:
- 'packages/**'
- 'services/migration/**'
- 'services/docker-compose*'
payments:
- 'packages/**'
- 'services/payments/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
dynamic-scheduler:
- 'packages/**'
- 'services/dynamic-scheduler/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
resource-usage-tracker:
- 'packages/**'
- 'services/resource-usage-tracker/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
static-webserver:
- 'services/static-webserver/**'
- 'services/docker-compose*'
storage:
- 'packages/**'
- 'services/storage/**'
- 'services/docker-compose*'
- 'scripts/mypy/*'
- 'mypy.ini'
webserver:
- 'packages/**'
- 'services/web/**'
- 'services/docker-compose*'
anything:
- 'packages/**'
- 'services/**'
- 'tests/**'
anything-py:
- '**/*.py'
anything-js:
- '**/*.js'
build-test-images:
# this step comes first, so that it is executed as first job in push calls
# in PR calls this step is anyway skipped
needs: changes
if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
timeout-minutes: 30
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
name: "[build] docker images (excluding frontend)"
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: expose github runtime for buildx
uses: crazy-max/ghaction-github-runtime@v3
- name: show system environs
run: ./ci/helpers/show_system_versions.bash
- name: build images
run: |
export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
mkdir --parents /${{ runner.temp }}/build
make build local-dest=/${{ runner.temp }}/build exclude=static-webserver
- name: upload build artifacts
uses: actions/upload-artifact@v4
with:
name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
path: /${{ runner.temp }}/build
build-test-images-frontend:
# this step comes first, so that it is executed as first job in push calls
# in PR calls this step is anyway skipped
needs: changes
if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
timeout-minutes: 30
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
name: "[build] docker images (frontend-only)"
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: expose github runtime for buildx
uses: crazy-max/ghaction-github-runtime@v3
- name: show system environs
run: ./ci/helpers/show_system_versions.bash
- name: build images
run: |
export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
mkdir --parents /${{ runner.temp }}/build
make build local-dest=/${{ runner.temp }}/build target=static-webserver
- name: upload build artifacts
uses: actions/upload-artifact@v4
with:
name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-frontend
path: /${{ runner.temp }}/build
unit-test-webserver-01:
needs: changes
if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
timeout-minutes: 25 # if this timeout gets too small, then split the tests
name: "[unit] webserver 01"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/web/server/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install webserver
run: ./ci/github/unit-testing/webserver.bash install
- name: typecheck
run: ./ci/github/unit-testing/webserver.bash typecheck
- name: test isolated
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/webserver.bash test_isolated
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/webserver.bash test_with_db 01
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-webserver-02:
needs: changes
if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
timeout-minutes: 25 # if this timeout gets too small, then split the tests
name: "[unit] webserver 02"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/web/server/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install webserver
run: ./ci/github/unit-testing/webserver.bash install
- name: test
run: ./ci/github/unit-testing/webserver.bash test_with_db 02
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-webserver-03:
needs: changes
if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
timeout-minutes: 25 # if this timeout gets too small, then split the tests
name: "[unit] webserver 03"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/web/server/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install webserver
run: ./ci/github/unit-testing/webserver.bash install
- name: test
run: ./ci/github/unit-testing/webserver.bash test_with_db 03
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-webserver-04:
needs: changes
if: ${{ needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
timeout-minutes: 25 # if this timeout gets too small, then split the tests
name: "[unit] webserver 04"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/web/server/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install webserver
run: ./ci/github/unit-testing/webserver.bash install
- name: test
run: ./ci/github/unit-testing/webserver.bash test_with_db 04
- uses: codecov/codecov-action@v5
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/test-results-action@v1
with:
token: ${{ secrets.CODECOV_TOKEN }}
unit-test-storage:
needs: changes
if: ${{ needs.changes.outputs.storage == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] storage"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/storage/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/storage.bash install
- name: typecheck
run: ./ci/github/unit-testing/storage.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/storage.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-agent:
needs: changes
if: ${{ needs.changes.outputs.agent == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] agent"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install rclone
run: sudo ./ci/github/helpers/install_rclone.bash
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/agent/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/agent.bash install
- name: typecheck
run: ./ci/github/unit-testing/agent.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/agent.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-api:
needs: changes
if: ${{ needs.changes.outputs.api == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] api"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/api/tests/requirements.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install api
run: ./ci/github/unit-testing/api.bash install
- name: test
run: ./ci/github/unit-testing/api.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-api-server:
needs: changes
if: ${{ needs.changes.outputs.api-server == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] api-server"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/api-server/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/api-server.bash install
- name: typecheck
run: ./ci/github/unit-testing/api-server.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/api-server.bash test
- name: OAS backwards compatibility check
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/api-server.bash openapi-diff
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-autoscaling:
needs: changes
if: ${{ needs.changes.outputs.autoscaling == 'true' || github.event_name == 'push' }}
timeout-minutes: 22 # temporary: mypy takes a huge amount of time to run here, maybe we should cache it
name: "[unit] autoscaling"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/autoscaling/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/autoscaling.bash install
- name: typecheck
run: ./ci/github/unit-testing/autoscaling.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/autoscaling.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-catalog:
needs: changes
if: ${{ needs.changes.outputs.catalog == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] catalog"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/catalog/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/catalog.bash install
- name: typecheck
run: ./ci/github/unit-testing/catalog.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/catalog.bash test
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./services/catalog/test_failures
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-clusters-keeper:
needs: changes
if: ${{ needs.changes.outputs.clusters-keeper == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] clusters-keeper"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/clusters-keeper/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: |
make devenv
source .venv/bin/activate && \
pushd services/clusters-keeper && \
make install-ci
- name: typecheck
run: |
source .venv/bin/activate && \
uv pip install mypy && \
pushd services/clusters-keeper && \
make mypy
- name: test
if: ${{ !cancelled() }}
run: |
source .venv/bin/activate && \
pushd services/clusters-keeper && \
make test-ci-unit
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-datcore-adapter:
needs: changes
if: ${{ needs.changes.outputs.datcore-adapter == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] datcore-adapter"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/datcore-adapter/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/datcore-adapter.bash install
- name: typecheck
run: ./ci/github/unit-testing/datcore-adapter.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/datcore-adapter.bash test
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./services/datcore-adapter/test_failures
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-director:
needs: changes
if: ${{ needs.changes.outputs.director == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] director"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/director/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/director.bash install
- name: typecheck
run: ./ci/github/unit-testing/director.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/director.bash test
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./services/director/test_failures
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-director-v2:
needs: changes
if: ${{ needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] director-v2"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/director-v2/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/director-v2.bash install
- name: typecheck
run: ./ci/github/unit-testing/director-v2.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/director-v2.bash test
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./services/director-v2/test_failures
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-aws-library:
needs: changes
if: ${{ needs.changes.outputs.aws-library == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] aws-library"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/aws-library/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/aws-library.bash install
- name: typecheck
run: ./ci/github/unit-testing/aws-library.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/aws-library.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-dask-task-models-library:
needs: changes
if: ${{ needs.changes.outputs.dask-task-models-library == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] dask-task-models-library"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/dask-task-models-library/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/dask-task-models-library.bash install
- name: typecheck
run: ./ci/github/unit-testing/dask-task-models-library.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/dask-task-models-library.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-dask-sidecar:
needs: changes
if: ${{ needs.changes.outputs.dask-sidecar == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] dask-sidecar"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/dask-sidecar/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/dask-sidecar.bash install
- name: typecheck
run: ./ci/github/unit-testing/dask-sidecar.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/dask-sidecar.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-payments:
needs: changes
if: ${{ needs.changes.outputs.payments == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] payments"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/payments/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/payments.bash install
- name: typecheck
run: ./ci/github/unit-testing/payments.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/payments.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-dynamic-scheduler:
needs: changes
if: ${{ needs.changes.outputs.dynamic-scheduler == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] dynamic-scheduler"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/dynamic-scheduler/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/dynamic-scheduler.bash install
- name: typecheck
run: ./ci/github/unit-testing/dynamic-scheduler.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/dynamic-scheduler.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-resource-usage-tracker:
needs: changes
if: ${{ needs.changes.outputs.resource-usage-tracker == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] resource-usage-tracker"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/resource-usage-tracker/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: |
make devenv
source .venv/bin/activate && \
pushd services/resource-usage-tracker && \
make install-ci
- name: typecheck
run: |
source .venv/bin/activate && \
pushd services/resource-usage-tracker && \
make mypy
- name: test
if: ${{ !cancelled() }}
run: |
source .venv/bin/activate && \
pushd services/resource-usage-tracker && \
make test-ci-unit
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-dynamic-sidecar:
needs: changes
if: ${{ needs.changes.outputs.dynamic-sidecar == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] dynamic-sidecar"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/dynamic-sidecar/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/dynamic-sidecar.bash install
- name: typecheck
run: ./ci/github/unit-testing/dynamic-sidecar.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/dynamic-sidecar.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-efs-guardian:
needs: changes
if: ${{ needs.changes.outputs.efs-guardian == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] efs-guardian"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/efs-guardian/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: |
make devenv
source .venv/bin/activate && \
pushd services/efs-guardian && \
make install-ci
- name: typecheck
run: |
source .venv/bin/activate && \
uv pip install mypy && \
pushd services/efs-guardian && \
make mypy
- name: test
if: ${{ !cancelled() }}
run: |
source .venv/bin/activate && \
pushd services/efs-guardian && \
make test-ci-unit
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-python-linting:
needs: changes
if: ${{ needs.changes.outputs.anything-py == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] python-linting"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11", "3.12"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/ci/helpers/requirements.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/python-linting.bash install
- name: test
run: ./ci/github/unit-testing/python-linting.bash test
unit-test-postgres-database:
needs: changes
if: ${{ needs.changes.outputs.postgres-database == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] postgres-database"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/postgres-database/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/postgres-database.bash install
- name: typecheck
run: ./ci/github/unit-testing/postgres-database.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/postgres-database.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-invitations:
needs: changes
if: ${{ needs.changes.outputs.invitations == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] invitations"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/notifications-library/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/invitations.bash install
- name: typecheck
run: ./ci/github/unit-testing/invitations.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/invitations.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-service-integration:
needs: changes
if: ${{ needs.changes.outputs.service-integration == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] service-integration"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/service-integration/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/service-integration.bash install
- name: typecheck
run: ./ci/github/unit-testing/service-integration.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/service-integration.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-service-library:
needs: changes
if: ${{ needs.changes.outputs.service-library == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] service-library"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/service-library/requirements/ci*.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/service-library.bash install_all
- name: typecheck
run: ./ci/github/unit-testing/service-library.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/service-library.bash test_all
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-settings-library:
needs: changes
if: ${{ needs.changes.outputs.settings-library == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] settings-library"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/settings-library/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/settings-library.bash install
- name: typecheck
run: ./ci/github/unit-testing/settings-library.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/settings-library.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-models-library:
needs: changes
if: ${{ needs.changes.outputs.models-library == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] models-library"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/models-library/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/models-library.bash install
- name: typecheck
run: ./ci/github/unit-testing/models-library.bash typecheck
- name: test
run: ./ci/github/unit-testing/models-library.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-common-library:
needs: changes
if: ${{ needs.changes.outputs.common-library == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] common-library"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: yezz123/setup-uv@v4
- uses: actions/cache@v4
id: cache-uv
with:
path: ~/.cache/uv
key: ${{ runner.os }}-${{ github.job }}-python-${{ matrix.python }}-uv
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/common-library.bash install
- name: typecheck
run: ./ci/github/unit-testing/common-library.bash typecheck
- name: test
run: ./ci/github/unit-testing/common-library.bash test
- uses: codecov/codecov-action@v5
with:
flags: unittests #optional
unit-test-notifications-library:
needs: changes
if: ${{ needs.changes.outputs.notifications-library == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit]notifications-library"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
docker_buildx: [v0.10.4]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
version: ${{ matrix.docker_buildx }}
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/notifications-library/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/notifications-library.bash install
- name: typecheck
run: ./ci/github/unit-testing/notifications-library.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/notifications-library.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-test-simcore-sdk:
needs: changes
if: ${{ needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }}
timeout-minutes: 18 # if this timeout gets too small, then split the tests
name: "[unit] simcore-sdk"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: install rclone
run: sudo ./ci/github/helpers/install_rclone.bash
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/simcore-sdk/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/unit-testing/simcore-sdk.bash install
- name: typecheck
run: ./ci/github/unit-testing/simcore-sdk.bash typecheck
- name: test
if: ${{ !cancelled() }}
run: ./ci/github/unit-testing/simcore-sdk.bash test
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: unittests #optional
unit-tests:
# NOTE: this is a github required status check!
if: ${{ always() }}
needs:
[
unit-test-agent,
unit-test-api-server,
unit-test-api,
unit-test-autoscaling,
unit-test-catalog,
unit-test-clusters-keeper,
unit-test-dask-sidecar,
unit-test-aws-library,
unit-test-dask-task-models-library,
unit-test-datcore-adapter,
unit-test-director-v2,
unit-test-director,
unit-test-dynamic-sidecar,
unit-test-efs-guardian,
unit-test-models-library,
unit-test-common-library,
unit-test-notifications-library,
unit-test-payments,
unit-test-dynamic-scheduler,
unit-test-postgres-database,
unit-test-python-linting,
unit-test-resource-usage-tracker,
unit-test-service-integration,
unit-test-service-library,
unit-test-settings-library,
unit-test-simcore-sdk,
unit-test-storage,
unit-test-webserver-01,
unit-test-webserver-02,
unit-test-webserver-03,
unit-test-webserver-04,
]
runs-on: ubuntu-latest
steps:
- name: a previous unit-test job failed
if: ${{ contains(join(needs.*.result, ','), 'failure') }}
run: |
echo "::error title=ERROR::one of the unit-tests failed!"
echo "${{ join(needs.*.result, ',') }}"
exit 1
- name: all the previous unit-tests were run successfully or skipped
if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
run: echo "::notice All good!"
integration-test-webserver-01:
needs: [changes, build-test-images]
if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[int] webserver 01"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
# FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
- name: download docker images with retry
uses: Wandalen/wretry.action@master
with:
action: actions/download-artifact@v4
with: |
name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
path: /${{ runner.temp }}/build
attempt_limit: 5
attempt_delay: 1000
- name: load docker images
run: make load-images local-src=/${{ runner.temp }}/build
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/web/server/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/integration-testing/webserver.bash install
- name: test
run: ./ci/github/integration-testing/webserver.bash test 01
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./services/web/server/test_failures
- name: cleanup
if: ${{ !cancelled() }}
run: ./ci/github/integration-testing/webserver.bash clean_up
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: integrationtests #optional
integration-test-webserver-02:
needs: [changes, build-test-images]
if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.webserver == 'true' || github.event_name == 'push' }}
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[int] webserver 02"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: expose github runtime for buildx
uses: crazy-max/ghaction-github-runtime@v3
# FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
- name: download docker images with retry
uses: Wandalen/wretry.action@master
with:
action: actions/download-artifact@v4
with: |
name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
path: /${{ runner.temp }}/build
attempt_limit: 5
attempt_delay: 1000
- name: load docker images
run: make load-images local-src=/${{ runner.temp }}/build
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/web/server/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/integration-testing/webserver.bash install
- name: test
run: ./ci/github/integration-testing/webserver.bash test 02
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./services/web/server/test_failures
- name: cleanup
if: ${{ !cancelled() }}
run: ./ci/github/integration-testing/webserver.bash clean_up
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: integrationtests #optional
integration-test-director-v2-01:
needs: [changes, build-test-images]
if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push'}}
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[int] director-v2 01"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: expose github runtime for buildx
uses: crazy-max/ghaction-github-runtime@v3
# FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
- name: download docker images with retry
uses: Wandalen/wretry.action@master
with:
action: actions/download-artifact@v4
with: |
name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
path: /${{ runner.temp }}/build
attempt_limit: 5
attempt_delay: 1000
- name: load docker images
run: make load-images local-src=/${{ runner.temp }}/build
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/director-v2/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/integration-testing/director-v2.bash install
- name: test
run: ./ci/github/integration-testing/director-v2.bash test 01
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./services/director-v2/test_failures
- name: cleanup
if: ${{ !cancelled() }}
run: ./ci/github/integration-testing/director-v2.bash clean_up
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: integrationtests #optional
integration-test-director-v2-02:
needs: [changes, build-test-images]
if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.director-v2 == 'true' || github.event_name == 'push' }}
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[int] director-v2 02"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
env:
# NOTE: DIRECTOR_DEFAULT_MAX_* used for integration-tests that include `director` service
DIRECTOR_DEFAULT_MAX_MEMORY: 268435456
DIRECTOR_DEFAULT_MAX_NANO_CPUS: 10000000
DIRECTOR_TRACING: null
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup rclone docker volume plugin
run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: expose github runtime for buildx
uses: crazy-max/ghaction-github-runtime@v3
# FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
- name: download docker images with retry
uses: Wandalen/wretry.action@master
with:
action: actions/download-artifact@v4
with: |
name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
path: /${{ runner.temp }}/build
attempt_limit: 5
attempt_delay: 1000
- name: load docker images
run: make load-images local-src=/${{ runner.temp }}/build
- name: install rclone
run: sudo ./ci/github/helpers/install_rclone.bash
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/director-v2/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/integration-testing/director-v2.bash install
- name: test
run: ./ci/github/integration-testing/director-v2.bash test 02
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./services/director-v2/test_failures
- name: cleanup
if: ${{ !cancelled() }}
run: ./ci/github/integration-testing/director-v2.bash clean_up
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: integrationtests #optional
integration-test-dynamic-sidecar:
needs: [changes, build-test-images]
if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.dynamic-sidecar == 'true' || github.event_name == 'push'}}
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[int] dynamic-sidecar"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: expose github runtime for buildx
uses: crazy-max/ghaction-github-runtime@v3
# FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
- name: download docker images with retry
uses: Wandalen/wretry.action@master
with:
action: actions/download-artifact@v4
with: |
name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
path: /${{ runner.temp }}/build
attempt_limit: 5
attempt_delay: 1000
- name: load docker images
run: make load-images local-src=/${{ runner.temp }}/build
- name: install rclone
run: sudo ./ci/github/helpers/install_rclone.bash
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/dynamic-sidecar/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/integration-testing/dynamic-sidecar.bash install
- name: test
run: ./ci/github/integration-testing/dynamic-sidecar.bash test 01
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./services/dynamic-sidecar/test_failures
- name: cleanup
if: ${{ !cancelled() }}
run: ./ci/github/integration-testing/dynamic-sidecar.bash clean_up
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: integrationtests #optional
integration-test-simcore-sdk:
needs: [changes, build-test-images]
if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }}
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[int] simcore-sdk"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: install rclone
run: sudo ./ci/github/helpers/install_rclone.bash
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
# FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
- name: download docker images with retry
uses: Wandalen/wretry.action@master
with:
action: actions/download-artifact@v4
with: |
name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend
path: /${{ runner.temp }}/build
attempt_limit: 5
attempt_delay: 1000
- name: load docker images
run: make load-images local-src=/${{ runner.temp }}/build
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/simcore-sdk/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/integration-testing/simcore-sdk.bash install
- name: test
run: ./ci/github/integration-testing/simcore-sdk.bash test
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./packages/simcore-sdk/test_failures
- name: cleanup
if: ${{ !cancelled() }}
run: ./ci/github/integration-testing/simcore-sdk.bash clean_up
- uses: codecov/codecov-action@v5
if: ${{ !cancelled() }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
flags: integrationtests
integration-tests:
# NOTE: this is a github required status check!
if: ${{ always() }}
needs:
[
integration-test-director-v2-01,
integration-test-director-v2-02,
integration-test-dynamic-sidecar,
integration-test-simcore-sdk,
integration-test-webserver-01,
integration-test-webserver-02,
]
runs-on: ubuntu-latest
steps:
- name: a previous integration-test job failed
if: ${{ contains(join(needs.*.result, ','), 'failure') }}
run: |
echo "::error title=ERROR::one of the integration-tests failed!"
exit 1
- name: all the previous integration-tests were run successfully or skipped
if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
run: echo "::notice All good!"
system-test-public-api:
needs: [changes, build-test-images, build-test-images-frontend]
if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
timeout-minutes: 25 # if this timeout gets too small, then split the tests
name: "[sys] public api"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: expose github runtime for buildx
uses: crazy-max/ghaction-github-runtime@v3
# FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
- name: download docker images with retry
uses: Wandalen/wretry.action@master
with:
action: actions/download-artifact@v4
with: |
pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-*
path: /${{ runner.temp }}/build
attempt_limit: 5
attempt_delay: 1000
- name: load docker images
run: make load-images local-src=/${{ runner.temp }}/build
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/public-api/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: ./ci/github/system-testing/public-api.bash install
- name: test
run: ./ci/github/system-testing/public-api.bash test
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./test_failures
- name: cleanup
if: ${{ !cancelled() }}
run: ./ci/github/system-testing/public-api.bash clean_up
system-test-swarm-deploy:
needs: [changes, build-test-images, build-test-images-frontend]
if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[sys] deploy simcore"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: expose github runtime for buildx
uses: crazy-max/ghaction-github-runtime@v3
# FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
- name: download docker images with retry
uses: Wandalen/wretry.action@master
with:
action: actions/download-artifact@v4
with: |
pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-*
path: /${{ runner.temp }}/build
attempt_limit: 5
attempt_delay: 1000
- name: load docker images
run: make load-images local-src=/${{ runner.temp }}/build
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/swarm-deploy/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/system-testing/swarm-deploy.bash install
- name: test
run: ./ci/github/system-testing/swarm-deploy.bash test
- name: dump services setting schemas
run: export DOCKER_REGISTRY=local; export DOCKER_IMAGE_TAG=production; make settings-schema.json
- name: upload services settings schemas
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_services_settings_schemas
path: ./services/**/settings-schema.json
- name: upload failed tests logs
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./test_failures
- name: cleanup
if: ${{ !cancelled() }}
run: ./ci/github/system-testing/swarm-deploy.bash clean_up
system-test-e2e:
needs: [changes, build-test-images, build-test-images-frontend]
if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[sys] e2e"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
node: [14]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- uses: actions/[email protected]
with:
node-version: ${{ matrix.node }}
cache: "npm"
cache-dependency-path: "tests/e2e/package-lock.json"
- name: expose github runtime for buildx
uses: crazy-max/ghaction-github-runtime@v3
# FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
- name: download docker images with retry
uses: Wandalen/wretry.action@master
with:
action: actions/download-artifact@v4
with: |
pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-*
path: /${{ runner.temp }}/build
attempt_limit: 5
attempt_delay: 1000
- name: load docker images
run: make load-images local-src=/${{ runner.temp }}/build
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/e2e/requirements/requirements.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: setup
run: ./ci/github/system-testing/e2e.bash install
- name: test
run: ./ci/github/system-testing/e2e.bash test
- name: dump docker logs
id: docker_logs_dump
if: ${{ !cancelled() }}
run: ./ci/github/system-testing/e2e.bash dump_docker_logs
- name: upload docker logs
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./tests/e2e/test_failures
- name: upload screenshots
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_screenshots
path: tests/e2e/screenshots
- name: upload e2e logs
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_logs
path: tests/e2e/logs
- name: cleanup
if: ${{ !cancelled() }}
run: ./ci/github/system-testing/e2e.bash clean_up
system-test-e2e-playwright:
needs: [changes, build-test-images, build-test-images-frontend]
if: ${{ needs.changes.outputs.anything == 'true' || github.event_name == 'push' }}
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[sys] e2e-playwright"
runs-on: ${{ matrix.os }}
# NOTE: this is an interesting way, but generate a load of issues like not having docker installed, etc etc.
# container:
# image: mcr.microsoft.com/playwright/python:v1.39.0-jammy
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/e2e-playwright/requirements/ci.txt"
- name: expose github runtime for buildx
uses: crazy-max/ghaction-github-runtime@v3
- name: download docker images
uses: actions/download-artifact@v4
with:
pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-*
path: /${{ runner.temp }}/build
- name: load docker images
run: make load-images local-src=/${{ runner.temp }}/build
- name: prepare devenv
run: make devenv
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: setup
run: |
./ci/github/system-testing/e2e-playwright.bash install
- name: test
run: |
./ci/github/system-testing/e2e-playwright.bash test
- name: dump docker logs
id: docker_logs_dump
if: ${{ !cancelled() }}
run: ./ci/github/system-testing/e2e-playwright.bash dump_docker_logs
- name: upload docker logs
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_docker_logs
path: ./tests/e2e-playwright/test_failures
- name: upload tracing if failed
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}_tracing
path: tests/e2e-playwright/test-results
system-test-environment-setup:
timeout-minutes: 30 # if this timeout gets too small, then split the tests
name: "[sys] environment setup"
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
- name: setup python environment
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: install uv
uses: astral-sh/setup-uv@v4
with:
version: "0.4.x"
enable-cache: false
cache-dependency-glob: "**/environment-setup/requirements/ci.txt"
- name: show system version
run: ./ci/helpers/show_system_versions.bash
- name: install
run: ./ci/github/system-testing/environment-setup.bash install
- name: test
run: ./ci/github/system-testing/environment-setup.bash test
- name: cleanup
if: ${{ !cancelled() }}
run: ./ci/github/system-testing/environment-setup.bash clean_up
system-tests:
# NOTE: this is a github required status check!
if: ${{ always() }}
needs:
[
system-test-e2e,
system-test-e2e-playwright,
system-test-environment-setup,
system-test-public-api,
system-test-swarm-deploy
]
runs-on: ubuntu-latest
steps:
- name: a previous system-test job failed
if: ${{ contains(join(needs.*.result, ','), 'failure') }}
run: |
echo "::error title=ERROR::one of the system-tests failed!"
exit 1
- name: all the previous system-tests were run successfully or skipped
if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
run: echo "::notice All good!"
deploy:
name: deploy to dockerhub
if: github.event_name == 'push'
needs: [unit-tests, integration-tests, system-tests]
runs-on: ${{ matrix.os }}
strategy:
matrix:
python: ["3.11"]
os: [ubuntu-22.04]
fail-fast: false
env:
# secrets can be set in settings/secrets on github
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
steps:
- uses: actions/checkout@v4
- name: setup docker buildx
id: buildx
uses: docker/setup-buildx-action@v3
with:
driver: docker-container
# FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
- name: download docker images with retry
uses: Wandalen/wretry.action@master
with:
action: actions/download-artifact@v4
with: |
pattern: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-*
path: /${{ runner.temp }}/build
attempt_limit: 5
attempt_delay: 1000
- name: load docker images
run: |
make load-images local-src=/${{ runner.temp }}/build
- name: set owner variable
run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
- if: github.ref == 'refs/heads/master'
name: deploy master image
env:
TAG_PREFIX: master-github
run: ./ci/deploy/dockerhub-deploy.bash -n
- if: contains(github.ref, 'refs/heads/hotfix_v')
name: deploy release hotfix image
env:
TAG_PREFIX: hotfix-github
run: ./ci/deploy/dockerhub-deploy.bash -n
- if: contains(github.ref, 'refs/heads/hotfix_staging_')
name: deploy staging hotfix image
env:
TAG_PREFIX: hotfix-staging-github
run: ./ci/deploy/dockerhub-deploy.bash -n