Initial commit

.github/workflows/build_container.yml

@@ -0,0 +1,47 @@
+#
+name: Create and publish a Docker image
+
+on:
+  push:
+  workflow_dispatch:
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      # 
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: docker
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Ohio Supercomputer Center
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,3 @@
+# NeSI training environment ML101 Jupyter app
+
+JupyterLab app for running the ML101 workshop on the NeSI training environment.

docker/Dockerfile

@@ -0,0 +1,39 @@
+FROM ubuntu:22.04
+
+# install system dependencies
+# TODO: move ldap-utils, libnss-ldapd, libpam-ldapd, nscd, nslcd to base image??
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        curl \
+        git \
+        ldap-utils \
+        libnss-ldapd \
+        libpam-ldapd \
+        less \
+        nano \
+        nodejs \
+        nscd \
+        nslcd \
+        python-is-python3 \
+        python3 \
+        python3-pip \
+        rsync \
+        unzip \
+        vim \
+        wget \
+        zip \
+    && rm -rf /var/lib/apt/lists/*
+
+# copy the repo source (e.g. notebooks) to the container image
+ARG ML101_HASH="de82d6c5221d79a3358f16c3441ebfc9147f6dfa"
+RUN git clone https://github.com/nesi/sklearn_tutorial.git /opt/sklearn_tutorial \
+    && cd /opt/sklearn_tutorial \
+    && git checkout $ML101_HASH \
+    && rm -rf /opt/sklearn_tutorial/.git
+
+# copy requirements.txt and install dependencies
+COPY requirements.txt /opt/sklearn_tutorial/
+RUN pip3 --no-cache-dir install -r requirements.txt \
+    && pip3 --no-cache-dir install jupyterlab \
+
+# copy cleanup script
+COPY cleanup-homes /usr/local/bin/

docker/cleanup-homes

@@ -0,0 +1,16 @@
+#!/bin/bash -e
+
+# script to cleanup all home directories of training users so they start fresh
+
+echo "This script will delete files from all training user home directories"
+echo "This will likely disrupt any training users currently running an app"
+echo "Type \"yes\" if you wish to continue: "
+read -r line
+if [ "${line}" = "yes" ]; then
+    # perform deletions
+    echo "Cleaning training user homes..."
+
+
+else
+    exit 1
+fi

docker/requirements.txt

@@ -0,0 +1,7 @@
+numpy==1.22.4
+scipy==1.8.1
+matplotlib==3.5.2
+scikit-learn==1.1.1
+jupyterlab==3.4.3
+ipywidgets==7.7.0
+pandas==1.4.2

form.yml

@@ -0,0 +1,17 @@
+---
+cluster: "my-k8s-cluster"
+
+form:
+  - cpu
+  - memory
+  - wall_time
+
+attributes:
+  cpu: 2
+  memory: 4
+  wall_time:
+    widget: number_field
+    label: "Hours"
+    min: 4
+    max: 12
+    value: 8

manifest.yml

@@ -0,0 +1,7 @@
+---
+name: ML101
+category: Interactive Apps
+subcategory: Servers
+role: batch_connect
+description: |
+  This app will launch a Jupyter Lab server for the ML101 workshop

submit.yml.erb

@@ -0,0 +1,79 @@
+<%
+   pwd_cfg = "c.ServerApp.password=u\'sha1:${SALT}:${PASSWORD_SHA1}\'"
+   host_port_cfg = "c.ServerApp.base_url=\'/node/${HOST_CFG}/${PORT_CFG}/\'"
+
+   configmap_filename = "ondemand_config.py"
+   configmap_data = "c.NotebookApp.port = 8080"
+   utility_img = "ghcr.io/nesi/training-environment-k8s-utils:v0.1.0"
+
+   user = OodSupport::User.new
+
+   services_node = Resolv.getaddress("servicesnode")
+%>
+---
+script:
+  accounting_id: "<%= account %>"
+  wall_time: "<%= wall_time.to_i * 3600 %>"
+  native:
+    container:
+      name: "jupyter"
+      image: "ghcr.io/nesi/training-environment-jupyter-ml101-app:main"
+      command: ["/bin/bash","-l","<%= staged_root %>/job_script_content.sh"]
+      working_dir: "<%= Etc.getpwnam(ENV['USER']).dir %>"
+      restart_policy: 'OnFailure'
+      env:
+        NB_UID: "<%= user.uid %>"
+        NB_USER: "<%= user.name %>"
+        NB_GID: "<%= user.group.id %>"
+        HOME: "<%= user.home %>"
+        LOG_DIR: "<%= staged_root %>"
+        SHELL: "/bin/bash"
+      port: "8080"
+      cpu: "<%= cpu %>"
+      memory: "<%= memory %>Gi"
+    mounts:
+      - type: nfs
+        name: home
+        host: <%= services_node %>
+        path: /srv/homes
+        destination_path: /home/shared
+      - type: host
+        name: nslcd-socket
+        host_type: Socket
+        path: /var/run/nslcd/socket
+        destination_path: /var/run/nslcd/socket
+      - type: host
+        name: nsswitch-conf
+        host_type: File
+        path: /etc/nsswitch.conf
+        destination_path: /etc/nsswitch.conf
+    configmap:
+      files:
+        - filename: "<%= configmap_filename %>"
+          data: |
+            c.ServerApp.ip = '0.0.0.0'
+            c.ServerApp.port = 8080
+            c.ServerApp.port_retries = 0
+            c.ServerApp.open_browser = False
+            c.ServerApp.allow_origin = '*'
+            c.ServerApp.root_dir = '<%= user.home %>/sklearn_tutorial'
+            c.ServerApp.disable_check_xsrf = True
+          mount_path: '/ood'
+    init_containers:
+    - name: "init-secret"
+      image: "<%= utility_img %>"
+      command: 
+      - "/bin/save_passwd_as_secret"
+      - "user-<%= user.name %>"
+    - name: "add-passwd-to-cfg"
+      image: "<%= utility_img %>"
+      command:
+      - "/bin/bash"
+      - "-c"
+      - "source /bin/passwd_from_secret; source /bin/create_salt_and_sha1; /bin/add_line_to_configmap \\\"<%= pwd_cfg %>\\\" <%= configmap_filename %>"
+    - name: "add-hostport-to-cfg"
+      image: "<%= utility_img %>"
+      command:
+      - "/bin/bash"
+      - "-c"
+      - "source /bin/find_host_port; /bin/add_line_to_configmap \\\"<%= host_port_cfg %>\\\" <%= configmap_filename %>"

template/script.sh.erb

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+exec &> >(tee -a "${LOG_DIR}/pod.log")
+
+# Benchmark info
+echo "TIMING - Starting main script at: $(date)"
+
+cp /ood/ondemand_config.py ./
+
+# Set working directory to home directory
+cd "${HOME}"
+
+#
+# Start Jupyter Notebook Server
+#
+
+# Benchmark info
+echo "TIMING - Starting jupyter at: $(date)"
+
+# Launch the Jupyter Notebook Server
+set -x
+rsync --ignore-existing -avz /opt/sklearn_tutorial/ ~/sklearn_tutorial/
+jupyter lab --config="/ood/ondemand_config.py"

view.html.erb

@@ -0,0 +1,7 @@
+<form action="/node/<%= host %>/<%= port %>/login" method="post" target="_blank">
+  <input type="hidden" name="password" value="<%= password %>">
+  <button class="btn btn-primary" type="submit">
+    <i class="fa fa-registered"></i> Connect to ML101 JupyterLab
+  </button>
+</form>
+