Skip to content

Add ComputeNode, LocalV2 + deprecate RemoteV1 and LocalV1 #635

Add ComputeNode, LocalV2 + deprecate RemoteV1 and LocalV1

Add ComputeNode, LocalV2 + deprecate RemoteV1 and LocalV1 #635

Workflow file for this run

name: Python package
on: [push, pull_request]
# https://stackoverflow.com/a/72408109/6388696
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-concurrency-to-cancel-any-in-progress-job-or-run
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
linting:
name: Run linting/pre-commit checks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- run: pip install pre-commit
- run: pre-commit --version
- run: pre-commit install
- run: pre-commit run --all-files
unit-tests:
needs: [linting]
runs-on: ${{ matrix.platform }}
strategy:
max-parallel: 4
matrix:
platform: [ubuntu-latest, windows-latest, macos-latest]
python-version: ['3.8', '3.9', '3.10', '3.11']
env:
PLATFORM: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v4
- name: Install poetry
run: |
python -m pip install --upgrade pip
pip install poetry
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: poetry
- name: Install dependencies
run: poetry install --with=dev
- name: Setup passwordless SSH access to localhost for tests
# Adapted from https://stackoverflow.com/a/60367309/6388696
if: runner.os == 'Linux'
run: |
ssh-keygen -t ed25519 -f ~/.ssh/testkey -N ''
cat > ~/.ssh/config <<EOF
Host localhost
User $USER
HostName 127.0.0.1
IdentityFile ~/.ssh/testkey
EOF
echo -n 'from="127.0.0.1" ' | cat - ~/.ssh/testkey.pub > ~/.ssh/authorized_keys
chmod og-rw ~
ssh -o 'StrictHostKeyChecking no' localhost id
- name: Test with pytest
run: poetry run pytest --cov=milatools --cov-report=xml --cov-append
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./coverage.xml
flags: unittests
env_vars: PLATFORM,PYTHON
name: codecov-umbrella
fail_ci_if_error: false
mock-slurm-integration-tests:
name: integration tests with a mock slurm cluster
needs: [unit-tests]
runs-on: ${{ matrix.platform }}
strategy:
max-parallel: 5
matrix:
# TODO: We should ideally also run this with Windows/Mac clients and a Linux
# server. Unsure how to set that up with GitHub Actions though.
platform: [ubuntu-latest]
python-version: ['3.8', '3.9', '3.10', '3.11']
# For the action to work, you have to supply a mysql
# service as defined below.
services:
mysql:
image: mysql:8.0
env:
MYSQL_ROOT_PASSWORD: root
ports:
- "8888:3306"
options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
steps:
- uses: actions/checkout@v4
# NOTE: Replacing this with our customized version of
# - uses: koesterlab/setup-slurm-action@v1
- uses: ./.github/custom_setup_slurm_action
timeout-minutes: 5
- name: Test if the slurm cluster is setup correctly
run: srun --nodes=1 --ntasks=1 --cpus-per-task=1 --mem=1G --time=00:01:00 hostname
- name: Setup passwordless SSH access to localhost for tests
# Adapted from https://stackoverflow.com/a/60367309/6388696
run: |
ssh-keygen -t ed25519 -f ~/.ssh/testkey -N ''
cat > ~/.ssh/config <<EOF
Host localhost
User $USER
HostName 127.0.0.1
IdentityFile ~/.ssh/testkey
EOF
echo -n 'from="127.0.0.1" ' | cat - ~/.ssh/testkey.pub > ~/.ssh/authorized_keys
chmod og-rw ~
ssh -o 'StrictHostKeyChecking no' localhost id
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install poetry
poetry install --with=dev
- name: Launch integration tests
run: poetry run pytest --slow --cov=milatools --cov-report=xml --cov-append -vvv --log-level=DEBUG
timeout-minutes: 15
env:
SLURM_CLUSTER: localhost
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./coverage.xml
flags: integrationtests
env_vars: PLATFORM,PYTHON
name: codecov-umbrella
fail_ci_if_error: false
real-slurm-integration-tests:
name: integration tests with a real SLURM cluster
needs: [mock-slurm-integration-tests]
runs-on: self-hosted
strategy:
max-parallel: 5
matrix:
# TODO: We should ideally also run this with Windows/Mac clients and a Linux
# server. Unsure how to set that up with GitHub Actions though.
python-version: ['3.11']
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install poetry
poetry install --with=dev
- name: Launch integration tests
id: self_hosted_integration_tests
run: poetry run pytest --slow -n 5 --cov=milatools --cov-report=xml --cov-append -vvv --log-level=DEBUG
# todo: Seems like the tests time out while waiting for jobs to be allocated!
timeout-minutes: 30
env:
SLURM_CLUSTER: mila
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./coverage.xml
flags: integrationtests
env_vars: PLATFORM,PYTHON
name: codecov-umbrella
fail_ci_if_error: false