Add ComputeNode, LocalV2 + deprecate RemoteV1 and LocalV1
#605
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python package | |
| on: [push, pull_request] | |
| jobs: | |
| linting: | |
| name: Run linting/pre-commit checks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - run: pip install pre-commit | |
| - run: pre-commit --version | |
| - run: pre-commit install | |
| - run: pre-commit run --all-files | |
| unit-tests: | |
| needs: [linting] | |
| runs-on: ${{ matrix.platform }} | |
| strategy: | |
| max-parallel: 4 | |
| matrix: | |
| platform: [ubuntu-latest, windows-latest, macos-latest] | |
| python-version: ['3.8', '3.9', '3.10', '3.11'] | |
| env: | |
| PLATFORM: ${{ matrix.platform }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install poetry | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install poetry | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: poetry | |
| - name: Install dependencies | |
| run: poetry install --with=dev | |
| - name: Setup passwordless SSH access to localhost for tests | |
| # Adapted from https://stackoverflow.com/a/60367309/6388696 | |
| if: runner.os == 'Linux' | |
| run: | | |
| ssh-keygen -t ed25519 -f ~/.ssh/testkey -N '' | |
| cat > ~/.ssh/config <<EOF | |
| Host localhost | |
| User $USER | |
| HostName 127.0.0.1 | |
| IdentityFile ~/.ssh/testkey | |
| EOF | |
| echo -n 'from="127.0.0.1" ' | cat - ~/.ssh/testkey.pub > ~/.ssh/authorized_keys | |
| chmod og-rw ~ | |
| ssh -o 'StrictHostKeyChecking no' localhost id | |
| - name: Test with pytest | |
| run: poetry run pytest --cov=milatools --cov-report=xml --cov-append | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./coverage.xml | |
| flags: unittests | |
| env_vars: PLATFORM,PYTHON | |
| name: codecov-umbrella | |
| fail_ci_if_error: false | |
| mock-slurm-integration-tests: | |
| name: integration tests with a mock slurm cluster | |
| needs: [unit-tests] | |
| runs-on: ${{ matrix.platform }} | |
| strategy: | |
| max-parallel: 5 | |
| matrix: | |
| # TODO: We should ideally also run this with Windows/Mac clients and a Linux | |
| # server. Unsure how to set that up with GitHub Actions though. | |
| platform: [ubuntu-latest] | |
| python-version: ['3.8', '3.9', '3.10', '3.11'] | |
| # For the action to work, you have to supply a mysql | |
| # service as defined below. | |
| services: | |
| mysql: | |
| image: mysql:8.0 | |
| env: | |
| MYSQL_ROOT_PASSWORD: root | |
| ports: | |
| - "8888:3306" | |
| options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # NOTE: Replacing this with our customized version of | |
| # - uses: koesterlab/setup-slurm-action@v1 | |
| - uses: ./.github/custom_setup_slurm_action | |
| timeout-minutes: 5 | |
| - name: Test if the slurm cluster is setup correctly | |
| run: srun --nodes=1 --ntasks=1 --cpus-per-task=1 --mem=1G --time=00:01:00 hostname | |
| - name: Setup passwordless SSH access to localhost for tests | |
| # Adapted from https://stackoverflow.com/a/60367309/6388696 | |
| run: | | |
| ssh-keygen -t ed25519 -f ~/.ssh/testkey -N '' | |
| cat > ~/.ssh/config <<EOF | |
| Host localhost | |
| User $USER | |
| HostName 127.0.0.1 | |
| IdentityFile ~/.ssh/testkey | |
| EOF | |
| echo -n 'from="127.0.0.1" ' | cat - ~/.ssh/testkey.pub > ~/.ssh/authorized_keys | |
| chmod og-rw ~ | |
| ssh -o 'StrictHostKeyChecking no' localhost id | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install poetry | |
| poetry install --with=dev | |
| - name: Launch integration tests | |
| run: poetry run pytest --slow --cov=milatools --cov-report=xml --cov-append -vvv --log-level=DEBUG | |
| timeout-minutes: 5 | |
| env: | |
| SLURM_CLUSTER: localhost | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./coverage.xml | |
| flags: integrationtests | |
| env_vars: PLATFORM,PYTHON | |
| name: codecov-umbrella | |
| fail_ci_if_error: false | |
| real-slurm-integration-tests: | |
| name: integration tests with a real SLURM cluster | |
| needs: [mock-slurm-integration-tests] | |
| runs-on: self-hosted | |
| strategy: | |
| max-parallel: 5 | |
| matrix: | |
| # TODO: We should ideally also run this with Windows/Mac clients and a Linux | |
| # server. Unsure how to set that up with GitHub Actions though. | |
| python-version: ['3.11'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install poetry | |
| poetry install --with=dev | |
| - name: Launch integration tests | |
| id: self_hosted_integration_tests | |
| run: poetry run pytest --slow -n 5 --cov=milatools --cov-report=xml --cov-append -vvv --log-level=DEBUG | |
| # todo: Seems like the tests time out while waiting for jobs to be allocated! | |
| # timeout-minutes: 60 | |
| env: | |
| SLURM_CLUSTER: mila | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./coverage.xml | |
| flags: integrationtests | |
| env_vars: PLATFORM,PYTHON | |
| name: codecov-umbrella | |
| fail_ci_if_error: false |