Merge remote-tracking branch 'origin/main' into full-drunc-image

.dockerignore

@@ -3,6 +3,6 @@
 !manage.py
 !pyproject.toml
 !poetry.lock
-!dune_processes
+!drunc_ui
 !db
 db/*

.github/workflows/check-links.yml

@@ -16,3 +16,4 @@ jobs:
         with:
           use-quiet-mode: yes
           use-verbose-mode: yes
+          config-file: .markdown-link-check.json

.github/workflows/ci.yml

@@ -31,4 +31,11 @@ jobs:
         run: poetry install
 
       - name: Run tests
-        run: poetry run pytest
+        run: poetry run pytest --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# VSCode settings
+.vscode/settings.json

.markdown-link-check.json

@@ -0,0 +1,5 @@
+{
+    "ignorePatterns": [
+        ".*localhost.*"
+    ]
+}

.pre-commit-config.yaml

@@ -1,11 +1,12 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-merge-conflict
       - id: debug-statements
       - id: trailing-whitespace
       - id: end-of-file-fixer
+        exclude: .*/static/.*/js/.*
       - id: pretty-format-json
         args: [--autofix, --indent, '4', --no-sort]
   - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
@@ -14,21 +15,27 @@ repos:
       - id: pretty-format-yaml
         args: [--autofix, --indent, '2', --offset, '2']
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.29.2
+    rev: 0.29.4
     hooks:
       - id: check-github-workflows
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.4
+    rev: v0.6.9
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.41.0
+    rev: v0.42.0
     hooks:
       - id: markdownlint-fix
   - repo: https://github.com/codespell-project/codespell
     rev: v2.3.0
     hooks:
       - id: codespell
         args: [-L, .codespell_ignore.txt]
+        exclude: .*/static/.*/js/.*
+  - repo: https://github.com/djlint/djLint
+    rev: v1.35.2
+    hooks:
+      - id: djlint-reformat-django
+      - id: djlint-django

README.md

@@ -1,8 +1,76 @@
-# dune_processes
+<!-- markdownlint-disable MD041 -->
+[![GitHub](https://img.shields.io/github/license/ImperialCollegeLondon/drunc_ui)](https://raw.githubusercontent.com/ImperialCollegeLondon/drunc_ui/main/LICENSE)
+[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/ImperialCollegeLondon/drunc_ui/main.svg)](https://results.pre-commit.ci/latest/github/ImperialCollegeLondon/drunc_ui/main)
+[![Test and build](https://github.com/ImperialCollegeLondon/drunc_ui/actions/workflows/ci.yml/badge.svg)](https://github.com/ImperialCollegeLondon/drunc_ui/actions/workflows/ci.yml)
+[![codecov](https://codecov.io/gh/ImperialCollegeLondon/drunc_ui/graph/badge.svg?token=PG0WTYF8EY)](https://codecov.io/gh/ImperialCollegeLondon/drunc_ui)
 
-This repo defines the web app for the Dune Process Manager web interface.
+# DUNE Run Control User Interface (drunc-ui)
 
-## For developers
+This repo defines the web interface for various drunc tools. Including:
+
+- The Processes Manager
+- The Controller
+
+## Running the App
+
+_Note that this depends on a very large docker base image._
+
+To run with a demo version of the drunc process manager, run it with docker compose:
+
+```bash
+docker compose up
+```
+
+It can take a few moment for the services to boot but the application should then be
+available in the browser at <http://localhost:8000>.  Authentication is required to work
+with the application so you need to create a user account to work with:
+
+```bash
+docker compose exec app python manage.py createsuperuser
+```
+
+and follow the prompts. You should then be able to use the details you supplied to pass
+the login screen. You can use the "boot" button on the main page to create simple
+processes to experiment with. You can also do this via the command line:
+
+```bash
+docker compose exec app python scripts/talk_to_process_manager.py
+```
+
+To boot a more realistic test session:
+
+```bash
+docker compose exec drunc bash -c "source env.sh && drunc-process-manager-shell grpc://localhost:10054 boot test/config/test-session.data.xml test-session"
+```
+
+_Note that the above consumes several Gb of memory._
+
+Once booted you can interact with the root controller via:
+
+```bash
+docker compose exec drunc bash -c "source env.sh && drunc-controller-shell grpc://localhost:3333"
+```
+
+For details of working with the controller see the [drunc wiki].
+
+Take the services down with `docker compose down` or by pressing Ctrl+C in the
+corresponding terminal.
+
+[drunc wiki]: https://github.com/DUNE-DAQ/drunc/wiki/Controller
+
+## Development
+
+Working with the full functionality of the web application requires a number of services
+to be started and to work in concert. The Docker Compose stack provides the required
+services and is suitable for development and manual testing but is not suitable for
+running QA (pre-commit) tooling or unit tests. The project directory is mounted into the
+`app` service which allows the Django development server's auto-reload mechanism to
+detect changes to local files and work as expected.
+
+It is recommended that you follow the below instructions on working with poetry to run
+the project's QA tooling and Unit Tests.
+
+### Working with Poetry
 
 This is a Python application that uses [poetry](https://python-poetry.org) for packaging
 and dependency management. It also provides [pre-commit](https://pre-commit.com/) hooks
@@ -32,44 +100,58 @@ To get started:
    pre-commit install
    ```
 
-1. Run the main app (this will not receive any data from the drunc process manager):
+Pre-commit should now work as expected when making commits even without the need to have
+an active poetry shell. You can also manually run pre-commit (e.g. `pre-commit run -a`)
+and the unit tests with `pytest`. Remember you'll need to prefix these with `poetry run`
+first if you don't have an active poetry shell.
+
+#### Running the web application with Poetry
+
+You can also start the web application though at a minimum this requires the drunc
+process manager to be running. Note that drunc only works on Linux so this approach will
+not work on any other platforms. See the next section on also working with
+Kafka. Assuming you have an active poetry shell for all steps:
+
+1. Start the drunc shell:
 
    ```bash
-   python manage.py runserver
+   drunc-unified-shell --log-level debug ./data/process-manager-no-kafka.json
    ```
 
-### Running the App
-
-_Note that this depends on a very large docker base image._
+1. In another terminal, run the main app:
 
-To run this with a demo version of the drunc process manager, run it with docker compose:
+   ```bash
+   python manage.py runserver
+   ```
 
-```bash
-docker compose up -d
-```
+1. As above you'll need to create a user to get past the login page:
 
-Dummy processes can be sent to the server with the `scripts/talk_to_process_manager.py` script:
+   ```bash
+   python manage.py createsuperuser
+   ```
 
-```bash
-docker compose exec app python scripts/talk_to_process_manager.py
-```
+Note that if you boot any processes in the web application this will immediately die
+with an exit code of 255. This is because the drunc shell requires an ssh server on
+localhost in order to be able to run processes. In most cases this isn't very limiting.
 
-To boot a more realistic test session:
+#### Running the web application with Poetry and Kafka
 
-```bash
-docker compose exec drunc bash -c "source env.sh && drunc-process-manager-shell grpc://localhost:10054 boot test/config/test-session.data.xml test-session"
-```
+In the event that you want to work with the full application without using Docker
+Compose you must start the required components manually. Assuming you have an active
+poetry shell for all steps.
 
-_Note that the above consumes several Gb of memory._
+1. Start Kafka - See [Running drunc with pocket kafka].
 
-Once booted you can interact with the root controller via:
+1. Start the drunc shell:
+   `drunc-unified-shell --log-level debug ./data/process-manager-pocket-kafka.json`
 
-```bash
-docker compose exec drunc bash -c "source env.sh && drunc-controller-shell grpc://localhost:3333"
-```
+1. Start the application server:
+   `python manage.py runserver`
 
-For details of working with the controller see the [drunc wiki].
+1. Start the Kafka consumer:
+   `python manage.py kafka_consumer --debug`
 
-Take the servers down with `docker compose down`.
+From here you should be able to see broadcast messages displayed at the top of the index
+page on every refresh.
 
-[drunc wiki]: https://github.com/DUNE-DAQ/drunc/wiki/Controller
+[Running drunc with pocket kafka]: https://github.com/DUNE-DAQ/drunc/wiki/Running-drunc-with-pocket-kafka

codecov.yml

@@ -0,0 +1,12 @@
+# Don't fail CI if coverage drops
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch:
+      default:
+        informational: true
+
+ignore:
+  - drunc_ui/settings/_production.py

controller/__init__.py

@@ -0,0 +1 @@
+"""The controller app for the drunc_ui project."""

controller/admin.py

@@ -0,0 +1,3 @@
+"""Admin module for the controller app."""
+
+# Register your models here.

controller/apps.py

@@ -0,0 +1,10 @@
+"""Apps module for the controller app."""
+
+from django.apps import AppConfig
+
+
+class ControllerConfig(AppConfig):
+    """The app config for the controller app."""
+
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "controller"

controller/models.py

@@ -0,0 +1,3 @@
+"""Models module for the controller app."""
+
+# Create your models here.

controller/templates/controller/index.html

@@ -0,0 +1,4 @@
+{% extends "main/base.html" %}
+{% block title %}
+  Controller
+{% endblock title %}

controller/urls.py

@@ -0,0 +1,10 @@
+"""Urls module for the controller app."""
+
+from django.urls import path
+
+from . import views
+
+app_name = "controller"
+urlpatterns = [
+    path("", views.index, name="index"),
+]

controller/views.py

@@ -0,0 +1,11 @@
+"""Views module for the controller app."""
+
+from django.contrib.auth.decorators import login_required
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+
+
+@login_required
+def index(request: HttpRequest) -> HttpResponse:
+    """View that renders the index/home page."""
+    return render(request=request, template_name="controller/index.html")

data/README.md

@@ -0,0 +1,9 @@
+# Data files
+
+## process-manager-pocket-kafka.json
+
+Process manager configuration file for use in local development with Kafka.
+
+## process-manager-no-kafka.json
+
+Process manager configuration file for use in local development without Kafka.

data/process-manager-pocket-kafka.json

@@ -0,0 +1,13 @@
+{
+    "type": "ssh",
+    "name": "SSHProcessManager",
+    "command_address": "0.0.0.0:10054",
+    "authoriser": {
+        "type": "dummy"
+    },
+    "broadcaster": {
+        "type": "kafka",
+        "kafka_address": "127.0.0.1:30092",
+        "publish_timeout": 2
+    }
+}