From 8663c4b79ec6739a7b0c9d9cea915e458d52b944 Mon Sep 17 00:00:00 2001 From: Dinindu Senanayake Date: Thu, 16 May 2024 10:58:53 +1200 Subject: [PATCH] populate OOD config files --- .github/workflows/build_container.yml | 50 +++++++++++++++ docker/Dockerfile | 56 +++++++++++++++++ form.yml | 17 +++++ manifest.yml | 7 +++ submit.yml.erb | 89 +++++++++++++++++++++++++++ template/script.sh.erb | 19 ++++++ view.html.erb | 6 ++ 7 files changed, 244 insertions(+) create mode 100644 .github/workflows/build_container.yml create mode 100644 docker/Dockerfile create mode 100644 form.yml create mode 100644 manifest.yml create mode 100644 submit.yml.erb create mode 100755 template/script.sh.erb create mode 100644 view.html.erb diff --git a/.github/workflows/build_container.yml b/.github/workflows/build_container.yml new file mode 100644 index 0000000..04833d6 --- /dev/null +++ b/.github/workflows/build_container.yml @@ -0,0 +1,50 @@ +# +name: Create and publish a Docker image + +on: + push: + workflow_dispatch: + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + # + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: docker + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..5cb51f2 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu:22.04 + +# run unminimise, e.g. for man pages to work +RUN yes | unminimize + +# install system dependencies +# TODO: move ldap-utils, libnss-ldapd, libpam-ldapd, nscd, nslcd to base image?? +RUN echo 'tzdata tzdata/Areas select Pacific' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/Pacific select Auckland' | debconf-set-selections \ + && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + bioawk \ + curl \ + fakeroot \ + file \ + git \ + ldap-utils \ + libnss-ldapd \ + libpam-ldapd \ + less \ + locales \ + man-db \ + nano \ + nodejs \ + nscd \ + nslcd \ + python-is-python3 \ + python3 \ + python3-pip \ + rsync \ + software-properties-common \ + tzdata \ + unzip \ + vim \ + wget \ + zip \ + openjdk-17-jdk openjdk-17-jre \ + && rm -rf /var/lib/apt/lists/* \ + && localedef -i en_NZ -c -f UTF-8 -A /usr/share/locale/locale.alias en_NZ.UTF-8 + +ENV LANG en_NZ.utf8 + +# install apptainer +RUN add-apt-repository -y ppa:apptainer/ppa \ + && apt update \ + && apt install -y apptainer-suid \ + && rm -rf /var/lib/apt/lists/* + +#install nextflow +RUN curl -s https://get.nextflow.io | bash \ + && chmod +x nextflow \ + && mv nextflow /usr/local/bin \ + && chmod a+rw /usr/local/bin/nextflow + +# install jupyterlab and nf-core +RUN pip3 --no-cache-dir install jupyterlab +RUN pip3 --no-cache-dir install nf-core diff --git a/form.yml b/form.yml new file mode 100644 index 0000000..60af4ea --- /dev/null +++ b/form.yml @@ -0,0 +1,17 @@ +--- +cluster: "my-k8s-cluster" + +form: + - cpu + - memory + - wall_time + +attributes: + cpu: 2 + memory: 4 + wall_time: + widget: number_field + label: "Hours" + min: 4 + max: 12 + value: 8 diff --git a/manifest.yml b/manifest.yml new file mode 100644 index 0000000..afe4db9 --- /dev/null +++ b/manifest.yml @@ -0,0 +1,7 @@ +--- +name: Introduction to Nextflow Workshop +category: Interactive Apps +subcategory: Servers +role: batch_connect +description: | + This app will launch a Jupyter Lab for the Introduction to Nextflow workshop diff --git a/submit.yml.erb b/submit.yml.erb new file mode 100644 index 0000000..5293ff0 --- /dev/null +++ b/submit.yml.erb @@ -0,0 +1,89 @@ +<% + pwd_cfg = "c.ServerApp.password=u\'sha1:${SALT}:${PASSWORD_SHA1}\'" + host_port_cfg = "c.ServerApp.base_url=\'/node/${HOST_CFG}/${PORT_CFG}/\'" + + configmap_filename = "ondemand_config.py" + configmap_data = "c.NotebookApp.port = 8080" + utility_img = "ghcr.io/nesi/training-environment-k8s-utils:v0.1.0" + + user = OodSupport::User.new + + services_node = Resolv.getaddress("servicesnode") +%> +--- +script: + accounting_id: "<%= account %>" + wall_time: "<%= wall_time.to_i * 3600 %>" + native: + container: + name: "nextflow" + image: "ghcr.io/nesi/training-environment-jupyter-intronextflow-app:v0.1.0" + command: ["/bin/bash","-l","<%= staged_root %>/job_script_content.sh"] + working_dir: "<%= Etc.getpwnam(ENV['USER']).dir %>" + restart_policy: 'OnFailure' + env: + NB_UID: "<%= user.uid %>" + NB_USER: "<%= user.name %>" + NB_GID: "<%= user.group.id %>" + HOME: "<%= user.home %>" + LOG_DIR: "<%= staged_root %>" + SHELL: "/bin/bash" + port: "8080" + cpu: "<%= cpu %>" + memory: "<%= memory %>Gi" + mounts: + - type: nfs + name: home + host: <%= services_node %> + path: /srv/homes + destination_path: /home/shared + - type: host + name: nslcd-socket + host_type: Socket + path: /var/run/nslcd/socket + destination_path: /var/run/nslcd/socket + - type: host + name: nsswitch-conf + host_type: File + path: /etc/nsswitch.conf + destination_path: /etc/nsswitch.conf + - type: host + name: subuid + host_type: File + path: /etc/subuid + destination_path: /etc/subuid + - type: host + name: subgid + host_type: File + path: /etc/subgid + destination_path: /etc/subgid + configmap: + files: + - filename: "<%= configmap_filename %>" + data: | + c.ServerApp.ip = '0.0.0.0' + c.ServerApp.port = 8080 + c.ServerApp.port_retries = 0 + c.ServerApp.open_browser = False + c.ServerApp.allow_origin = '*' + c.ServerApp.root_dir = '<%= user.home %>' + c.ServerApp.disable_check_xsrf = True + mount_path: '/ood' + init_containers: + - name: "init-secret" + image: "<%= utility_img %>" + command: + - "/bin/save_passwd_as_secret" + - "user-<%= user.name %>" + - name: "add-passwd-to-cfg" + image: "<%= utility_img %>" + command: + - "/bin/bash" + - "-c" + - "source /bin/passwd_from_secret; source /bin/create_salt_and_sha1; /bin/add_line_to_configmap \\\"<%= pwd_cfg %>\\\" <%= configmap_filename %>" + - name: "add-hostport-to-cfg" + image: "<%= utility_img %>" + command: + - "/bin/bash" + - "-c" + - "source /bin/find_host_port; /bin/add_line_to_configmap \\\"<%= host_port_cfg %>\\\" <%= configmap_filename %>" diff --git a/template/script.sh.erb b/template/script.sh.erb new file mode 100755 index 0000000..e7ec968 --- /dev/null +++ b/template/script.sh.erb @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# send all output to pod.log +exec &> >(tee -a "${LOG_DIR}/pod.log") + +echo "TIMING - Starting main script at: $(date)" + +# copying config file for debugging +cp /ood/ondemand_config.py ./ + +# Set working directory to home directory +cd "${HOME}" + +echo "TIMING - Starting jupyter at: $(date)" + +set -x + +# launch JupyterLab +jupyter lab --config="/ood/ondemand_config.py" diff --git a/view.html.erb b/view.html.erb new file mode 100644 index 0000000..02f5786 --- /dev/null +++ b/view.html.erb @@ -0,0 +1,6 @@ +
+ + +