Skip to content

MLOS DevContainer

MLOS DevContainer #822

Workflow file for this run

name: MLOS DevContainer
on:
workflow_dispatch:
inputs:
tags:
description: Manual MLOS DevContainer run aux info tags
NO_CACHE:
type: boolean
description: Disable caching?
default: false
required: false
push:
branches: [ main ]
pull_request:
branches: [ main ]
release:
types: [ published ]
merge_group:
types: [checks_requested]
schedule:
- cron: "1 0 * * *"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: true
jobs:
DevContainer:
runs-on: ubuntu-latest
permissions:
contents: read
# Here we only test a single (latest) version of python.
env:
DOCKER_BUILDKIT: 1
BUILDKIT_INLINE_CACHE: 1
steps:
- uses: actions/checkout@v4
- name: Get commit messages
id: get-commit-messages
if: github.event_name == 'pull_request'
run: |
# Only look at the last 10 commits, to avoid getting too much data.
# If the message wasn't in there (e.g. because the push was a merge and too large), then we just ignore it.
git fetch --deepen=10
# Handle multiline output:
{
echo 'COMMIT_MESSAGES<<COMMIT_MESSAGES_EOF'
git log --format="%B" -10 ${{ github.event.before }}..${{ github.event.after }} | tee /tmp/commit_messages.txt
echo COMMIT_MESSAGES_EOF
} >> $GITHUB_OUTPUT
- name: Validate tag
if: github.ref_type == 'tag'
# Note: May need to update this for release branches in the future too.
run: |
set -x
git fetch --deepen=100
if ! git branch -a --contains ${{ github.ref_name }} | grep origin/main; then
echo "ERROR: tag ${{ github.ref_name }} doesn't appear to be included in the main branch." >&2
exit 1
fi
if ! echo "${{ github.ref_name }}" | egrep -q '^v([0-9]+\.){2,3}[0-9]+$'; then
echo "ERROR: tag ${{ github.ref_name }} doesn't appear to be a valid version tag." >&2
exit 1
fi
- name: Set NO_CACHE variable based on commit messages and for nightly builds
if: github.event_name == 'schedule' || contains(steps.get-commit-messages.outputs.COMMIT_MESSAGES, 'NO_CACHE=true') || github.event.inputs.NO_CACHE == 'true'
run: |
echo "NO_CACHE=true" >> $GITHUB_ENV
- name: Run pytest with debug logging enabled for nightly runs
if: github.event_name == 'schedule'
run: |
echo "PYTEST_EXTRA_OPTIONS=--log-level=DEBUG" >> $GITHUB_ENV
- name: Log some environment variables for debugging
run: |
set -x
printenv
echo "NO_CACHE: ${NO_CACHE:-}"
echo "EVENT_NAME: ${{ github.event_name }}"
echo "BEFORE: ${{ github.event.before }}"
echo "AFTER: ${{ github.event.after }}"
# echo "COMMIT_MESSAGES: `cat /tmp/commit_messages.txt`"
echo "COMMIT_SHA: ${{ github.sha }}"
echo "git log -1 ${{ github.sha }}: $(git log -1 ${{ github.sha }})"
echo "Manual Trigger Input Tags: ${{ github.event.inputs.tags }}"
echo "Manual Trigger Input NO_CACHE: ${{ github.event.inputs.NO_CACHE }}"
# Output the full event json for debugging.
# cat ${{ github.event_path }}
- name: Build the devcontainer image
timeout-minutes: 15
run: |
set -x
make CONDA_INFO_LEVEL=-v devcontainer
- name: Start the devcontainer in the background
timeout-minutes: 3
run: |
set -x
docker run -d --rm --user root \
--volume /var/run/docker.sock:/var/run/docker.sock \
--env DOCKER_BUILDKIT=$DOCKER_BUILDKIT \
--volume $(pwd):/workspaces/MLOS \
--env CONTAINER_WORKSPACE_FOLDER=/workspaces/MLOS \
--env LOCAL_WORKSPACE_FOLDER=$(pwd) \
--env PYTEST_EXTRA_OPTIONS=$PYTEST_EXTRA_OPTIONS \
--workdir /workspaces/MLOS \
--add-host host.docker.internal:host-gateway \
--name mlos-devcontainer mlos-devcontainer sleep 1800
- name: Fixup vscode uid/gid in the running container
timeout-minutes: 3
run: |
set -x
docker exec --user root mlos-devcontainer groupmod --non-unique -g `id -g` vscode
docker exec --user root mlos-devcontainer usermod --non-unique -u `id -u` -g `id -g` vscode
docker exec --user root mlos-devcontainer chown -R vscode:vscode /home/vscode
- name: Print some debug info from inside the container
run: |
docker exec --user vscode --env USER=vscode mlos-devcontainer printenv
- name: Check that github.com is in the ssh known_hosts file
run: |
docker exec --user vscode --env USER=vscode mlos-devcontainer grep ^github.com /home/vscode/.ssh/known_hosts
- name: Update the conda env in the devcontainer
timeout-minutes: 10
run: |
set -x
docker exec --user vscode --env USER=vscode mlos-devcontainer make CONDA_INFO_LEVEL=-v conda-env
- name: Verify expected version of python in conda env
timeout-minutes: 2
run: |
set -x
docker exec --user vscode --env USER=vscode mlos-devcontainer conda run -n mlos python -c \
'from sys import version_info as vers; assert (vers.major, vers.minor) == (3, 12), f"Unexpected python version: {vers}"'
- name: Check for missing licenseheaders
timeout-minutes: 3
run: |
set -x
docker exec --user vscode --env USER=vscode mlos-devcontainer make CONDA_INFO_LEVEL=-v licenseheaders
# licenseheaders changes the contents of the files, so make this check fail if there are any changes detected
git --no-pager diff --exit-code
- name: Run lint checks
timeout-minutes: 5
run: |
set -x
docker exec --user vscode --env USER=vscode --env MAKEFLAGS=-Oline mlos-devcontainer make CONDA_INFO_LEVEL=-v check
- name: Run tests
timeout-minutes: 10
run: |
set -x
# Simulate test collection in vscode.
test_count=$(docker exec --user vscode --env USER=vscode mlos-devcontainer \
conda run -n mlos python -m pytest -svxl -n auto --collect-only --rootdir /workspaces/MLOS -s --cache-clear \
| grep -c '<Function ')
if [ "${test_count:-0}" -lt 725 ]; then echo "Expected at least 725 tests, got '$test_count'" >&2; exit 1; fi
# Now actually run the tests.
docker exec --user vscode --env USER=vscode mlos-devcontainer make CONDA_INFO_LEVEL=-v test
- name: Generate and test binary distribution files
timeout-minutes: 10
run: |
set -x
docker exec --user vscode --env USER=vscode mlos-devcontainer make CONDA_INFO_LEVEL=-v dist dist-test
- name: Test rebuilding the devcontainer in the devcontainer
# FIXME:
# timeout-minutes: 3
timeout-minutes: 10
run: |
set -x
git --no-pager diff --exit-code
docker exec --user vscode --env USER=vscode mlos-devcontainer make CONDA_INFO_LEVEL=-v devcontainer
- name: Generate docs and test check them
run: |
set -x
docker exec --user vscode --env USER=vscode --env MAKEFLAGS=-Oline mlos-devcontainer make CONDA_INFO_LEVEL=-v doc
# Make sure we can publish the coverage report.
rm -f doc/build/html/htmlcov/.gitignore
- uses: actions/upload-artifact@v4
if: github.ref == 'refs/heads/main'
with:
name: docs
path: doc/build/html
- name: Publish package to Test PyPi
if: github.event_name == 'release' && github.ref_type == 'tag'
run: |
if [ -n "${{ secrets.PYPI_TEST_USERNAME }}" ]; then
docker exec --user vscode --env USER=vscode --env MAKEFLAGS=-Oline \
--env TWINE_USERNAME=${{ secrets.PYPI_TEST_USERNAME }} --env TWINE_PASSWORD=${{ secrets.PYPI_TEST_PASSWORD }} \
mlos-devcontainer make CONDA_INFO_LEVEL=-v publish-test-pypi
fi
- name: Publish package to PyPi
if: github.repository == 'microsoft/mlos' && github.event_name == 'release' && github.ref_type == 'tag'
run: |
if [ -n "${{ secrets.PYPI_USERNAME }}" ]; then
docker exec --user vscode --env USER=vscode --env MAKEFLAGS=-Oline \
--env TWINE_USERNAME=${{ secrets.PYPI_USERNAME }} --env TWINE_PASSWORD=${{ secrets.PYPI_PASSWORD }} \
mlos-devcontainer make CONDA_INFO_LEVEL=-v publish-pypi
fi
- name: Cleanup the devcontainer
run: |
set -x
docker stop -t 1 mlos-devcontainer || true
docker rm --force mlos-devcontainer || true
- name: Container Registry Login
if: (github.repository == 'microsoft/mlos') && (github.ref == 'refs/heads/main' || github.ref_type == 'tag')
uses: docker/login-action@v3
with:
# This is the URL of the container registry, which is configured in Github
# Settings and currently corresponds to the mlos-core ACR.
registry: ${{ secrets.ACR_LOGINURL }}
username: ${{ secrets.ACR_USERNAME }}
# This secret is configured in Github Settings.
# It can also be obtained in a keyvault in the Azure portal alongside the
# other resources used.
password: ${{ secrets.ACR_PASSWORD }}
- name: Publish the container images
# TODO: add cleanup step to remove old images
if: (github.repository == 'microsoft/mlos') && (github.ref == 'refs/heads/main' || github.ref_type == 'tag')
timeout-minutes: 15
run: |
set -x
image_tag=''
if [ "${{ github.ref_type }}" == 'tag' ]; then
image_tag="${{ github.ref_name }}"
elif [ "${{ github.ref }}" == 'refs/heads/main' ]; then
image_tag='latest'
fi
if [ -z "$image_tag" ]; then
echo "ERROR: Unhandled event condition or ref: event=${{ github.event}}, ref=${{ github.ref }}, ref_type=${{ github.ref_type }}"
exit 1
fi
docker tag devcontainer-cli:latest ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:$image_tag
docker push ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:$image_tag
docker tag mlos-devcontainer:latest ${{ secrets.ACR_LOGINURL }}/mlos-devcontainer:$image_tag
docker push ${{ secrets.ACR_LOGINURL }}/mlos-devcontainer:$image_tag
DeployDocs:
if: github.ref == 'refs/heads/main'
needs: DevContainer
runs-on: ubuntu-latest
# Required for github-pages-deploy-action to push to the gh-pages branch.
permissions:
contents: write
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: docs
path: doc/build/html
- name: Check docs
run: |
set -x
# Check that the docs are not empty.
if [ ! -d doc/build/html ]; then
echo "ERROR: No docs found in doc/build/html" >&2
exit 1
fi
if [ ! -f doc/build/html/index.html ]; then
echo "ERROR: No index.html found in doc/build/html" >&2
exit 1
fi
- name: Deploy to GitHub pages
uses: JamesIves/github-pages-deploy-action@v4
with:
branch: gh-pages
folder: doc/build/html
clean: true