Merge branch 'main' into feature/private-media

* main:
  Update the embed block to be a custom video embed that uses the design system video embed template (#62)
  Django Migration Linter Integration (#43)
  Tidy up the info/warning panel and fix title/no title output (#73)
  Introduce Functional Tests using Playwright and Behave #31

# Conflicts:
#	poetry.lock
#	pyproject.toml

.github/workflows/ci.yml

@@ -140,10 +140,62 @@ jobs:
         with:
           name: html-report
           path: htmlcov
+      - name: Lint Migrations
+        run: DJANGO_SETTINGS_MODULE=cms.settings.dev make lint-migrations
 
   docker:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Build Docker Image
         run: docker build --target web -t ons .
+
+  functional-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        browser:
+          - chromium
+    needs:
+      - lint
+      - lint-front-end
+      - compile_static
+
+    env:
+      DJANGO_SETTINGS_MODULE: cms.settings.functional_test
+      SECRET_KEY: fake_secret_key_to_run_tests # pragma: allowlist secret
+      PLAYWRIGHT_BROWSER: ${{ matrix.browser }}
+      PLAYWRIGHT_TRACES_DIR: playwright_traces
+      PLAYWRIGHT_TRACE: 'true'
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Poetry
+        run: pipx install poetry==${{ env.POETRY_VERSION }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: .python-version
+          cache: poetry
+
+      - name: Install dependencies
+        run: make install-dev
+
+      - name: Playwright Install
+        run: poetry run python -m playwright install --with-deps ${{ matrix.browser }}
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: static
+          path: cms/static_compiled/
+
+      - name: Run Functional Tests
+        run: make functional-tests
+
+      - name: Upload Failure Traces
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: playwright-traces-${{ matrix.browser }}
+          path: playwright_traces/

.gitignore

@@ -196,3 +196,6 @@ static/
 
 # bak files
 *.bak
+
+# Playwright Traces
+/tmp_traces

Makefile

@@ -33,7 +33,7 @@ format-frontend:  ## Format front-end files (CSS, JS, YAML, MD)
 	npm run format
 
 .PHONY: lint
-lint: lint-py lint-html lint-frontend ## Run all linters (python, html, front-end)
+lint: lint-py lint-html lint-frontend lint-migrations ## Run all linters (python, html, front-end, migrations)
 
 .PHONY: lint-py
 lint-py:  ## Run all Python linters (ruff/pylint/mypy).
@@ -50,6 +50,10 @@ lint-html:  ## Run HTML Linters
 lint-frontend:  ## Run front-end linters
 	npm run lint
 
+.PHONY: lint-migrations
+lint-migrations: ## Run django-migration-linter
+	poetry run python manage.py lintmigrations --quiet ignore ok
+
 .PHONY: test
 test:  ## Run the tests and check coverage.
 	poetry run coverage erase
@@ -152,3 +156,28 @@ runserver: ## Run the Django application locally
 
 .PHONY: dev-init
 dev-init: load-design-system-templates collectstatic makemigrations migrate createsuperuser ## Run the pre-run setup scripts
+
+.PHONY: functional-tests-up
+functional-tests-up:  ## Start the functional tests docker compose dependencies
+	docker compose -f functional_tests/docker-compose.yml up -d
+
+.PHONY: functional-tests-dev-up
+functional-tests-dev-up:  ## Start the functional tests docker compose dependencies and dev app
+	docker compose -f functional_tests/docker-compose-dev.yml up -d
+
+.PHONY: functional-tests-down
+functional-tests-down:  ## Stop the functional tests docker compose dependencies (and dev app if running)
+	docker compose -f functional_tests/docker-compose-dev.yml down
+
+.PHONY: functional-tests-run
+functional-tests-run: load-design-system-templates collectstatic ## Only run the functional tests (dependencies must be run separately)
+	# Run migrations to work around Django bug (#35967)
+	poetry run ./manage.py migrate --noinput --settings cms.settings.functional_test
+	poetry run behave functional_tests
+
+.PHONY: functional-tests
+functional-tests: functional-tests-up functional-tests-run functional-tests-down  ## Run the functional tests with dependencies (all in one)
+
+.PHONY: playwright-install
+playwright-install:  ## Install Playwright dependencies
+	poetry run playwright install --with-deps

README.md

@@ -28,6 +28,7 @@ The Wagtail CMS for managing and publishing content for the Office for National
     - [Front-end tooling](#front-end-tooling)
     - [Adding Python packages](#adding-python-packages)
     - [Run Tests with Coverage](#run-tests-with-coverage)
+    - [Functional Tests](#functional-tests)
     - [Linting and Formatting](#linting-and-formatting)
         - [Python](#python)
         - [Front-end](#front-end)
@@ -245,6 +246,67 @@ make test
 During tests, the `cms.settings.test` settings module is used. When running test without using `make test`, ensure this
 settings module is used.
 
+### Functional Tests
+
+Our suite of functional browser driven tests uses [Behave](https://behave.readthedocs.io/en/latest/),
+[Playwright](https://playwright.dev/python/docs/intro) and
+[Django Live Server Test Cases](https://docs.djangoproject.com/en/stable/topics/testing/tools/#liveservertestcase) to
+run BDD Cucumber feature tests against the app from a browser.
+
+#### Installation
+
+Install the Playwright dependencies (including its browser drivers) with:
+
+```shell
+make playwright-install
+```
+
+#### Run the Functional Tests
+
+You can run the tests as an all-in-one command with:
+
+```shell
+make functional-tests
+```
+
+This will start and stop the docker compose services with the relevant tests.
+
+To run the docker compose dependencies (database and redis) separately, e.g. if you want to run individual functional
+tests yourself for development, start the docker compose dependencies with:
+
+```shell
+make functional-tests-up
+```
+
+This will start the dependent services in the background, allowing you to then run the tests separately.
+
+Then once you are finished testing, stop the dependencies with:
+
+```shell
+make functional-tests-down
+```
+
+#### Showing the Tests Browser
+
+By default, the tests will run in headless mode with no visible browser window.
+
+To disable headless mode and show the browser, set `PLAYWRIGHT_HEADLESS=False` in the environment from which you are
+running the tests. In this circumstance, you will probably also find it helpful to enable "slow mo" mode, which slows
+down the automated browser interactions to make it possible to follow what the tests are doing. You can configure it
+using the `PLAYWRIGHT_SLOW_MO` environment variable, passing it a value of milliseconds by which to slow each
+interaction, e.g. `PLAYWRIGHT_SLOW_MO=1000` will cause each individual browser interaction from the tests to be delayed
+by 1 second.
+
+For example, you can run the tests with visible browser and each interaction slowed by a second by running:
+
+```shell
+PLAYWRIGHT_HEADLESS=False PLAYWRIGHT_SLOW_MO=1000 make functional-tests
+```
+
+#### Developing Functional Tests
+
+Refer to the detailed [functional tests development docs](./functional_tests/README.md)
+
 ### Linting and Formatting
 
 Various tools are used to lint and format the code in this project.
@@ -262,6 +324,18 @@ To lint the Python code, run:
 make lint
 ```
 
+#### Django Migration Linter
+
+[Django Migration Linter](https://github.com/3YOURMIND/django-migration-linter) for linting migrations files in the project.
+
+To lint the django migration files:
+
+```bash
+make lint-migrations
+```
+
+#### Format
+
 To auto-format the Python code, and correct fixable linting issues, run:
 
 ```bash

cms/core/blocks/__init__.py

@@ -1,4 +1,10 @@
-from .embeddable import DocumentBlock, DocumentsBlock, ImageBlock, ONSEmbedBlock
+from .embeddable import (
+    DocumentBlock,
+    DocumentsBlock,
+    ImageBlock,
+    ONSEmbedBlock,
+    VideoEmbedBlock,
+)
 from .headline_figures import HeadlineFiguresBlock
 from .markup import BasicTableBlock, HeadingBlock, QuoteBlock
 from .panels import PanelBlock
@@ -17,4 +23,5 @@
     "QuoteBlock",
     "RelatedContentBlock",
     "RelatedLinksBlock",
+    "VideoEmbedBlock",
 ]

cms/core/blocks/embeddable.py

@@ -1,4 +1,6 @@
+import re
 from typing import TYPE_CHECKING, ClassVar
+from urllib.parse import urlparse
 
 from django.conf import settings
 from django.core.exceptions import ValidationError
@@ -98,3 +100,90 @@ def clean(self, value: "StructValue") -> "StructValue":
     class Meta:
         icon = "code"
         template = "templates/components/streamfield/ons_embed_block.html"
+
+
+class VideoEmbedBlock(blocks.StructBlock):
+    """A video embed block."""
+
+    link_url = blocks.URLBlock(
+        help_text=_(
+            "The URL to the video hosted on YouTube or Vimeo, for example, "
+            "https://www.youtube.com/watch?v={ video ID } or https://vimeo.com/video/{ video ID }. "
+            "Used to link to the video when cookies are not enabled."
+        )
+    )
+    image = ImageChooserBlock(help_text=_("The video cover image, used when cookies are not enabled."))
+    title = blocks.CharBlock(help_text=_("The descriptive title for the video used by screen readers."))
+    link_text = blocks.CharBlock(
+        help_text=_("The text to be shown when cookies are not enabled e.g. 'Watch the {title} on Youtube'.")
+    )
+
+    def get_embed_url(self, link_url: str) -> str:
+        """Get the embed URL for the video based on the link URL."""
+        embed_url = ""
+        # Vimeo
+        if urlparse(link_url).hostname in ["www.vimeo.com", "vimeo.com", "player.vimeo.com"]:
+            url_path = urlparse(link_url).path.strip("/")
+            # Handle different Vimeo URL patterns
+            if "video/" in url_path:  # noqa: SIM108
+                # Handle https://vimeo.com/showcase/7934865/video/ID format
+                video_id = url_path.split("video/")[-1]
+            else:
+                # Handle https://player.vimeo.com/video/ID or https://vimeo.com/ID format
+                video_id = url_path.split("/")[0]
+            # Remove any query parameters from video ID
+            video_id = video_id.split("?")[0]
+            embed_url = "https://player.vimeo.com/video/" + video_id
+        # YouTube
+        elif urlparse(link_url).hostname in ["www.youtube.com", "youtube.com", "youtu.be"]:
+            url_parts = urlparse(link_url)
+            if url_parts.hostname == "youtu.be":
+                # Handle https://youtu.be/ID format
+                video_id = url_parts.path.lstrip("/")
+            elif "/v/" in url_parts.path:
+                # Handle https://www.youtube.com/v/ID format
+                video_id = url_parts.path.split("/v/")[-1]
+            else:
+                # Handle https://www.youtube.com/watch?v=ID format
+                query = dict(param.split("=") for param in url_parts.query.split("&"))
+                video_id = query.get("v", "")
+            # Remove any query parameters from video ID
+            video_id = video_id.split("?")[0]
+            embed_url = "https://www.youtube.com/embed/" + video_id
+        return embed_url
+
+    def get_context(self, value: "StreamValue", parent_context: dict | None = None) -> dict:
+        """Get the embed URL for the video based on the link URL."""
+        context: dict = super().get_context(value, parent_context=parent_context)
+        context["value"]["embed_url"] = self.get_embed_url(value["link_url"])
+        return context
+
+    def clean(self, value: "StructValue") -> "StructValue":
+        """Checks that the given embed and link urls match youtube or vimeo."""
+        errors = {}
+
+        vimeo_showcase_pattern = r"^https?://vimeo\.com/showcase/[^/]+/video/[^/]+$"
+        other_patterns = [
+            r"^https?://(?:[-\w]+\.)?youtube\.com/watch[^/]+$",
+            r"^https?://(?:[-\w]+\.)?youtube\.com/v/[^/]+$",
+            r"^https?://youtu\.be/[^/]+$",
+            r"^https?://vimeo\.com/[^/]+$",
+            r"^https?://player\.vimeo\.com/video/[^/]+$",
+        ]
+
+        # Check if the URL is a Vimeo showcase URL - do this first to avoid it clashing
+        # with the r"^https?://vimeo\.com/[^/]+$", pattern
+        if re.match(vimeo_showcase_pattern, value["link_url"]):
+            return super().clean(value)
+
+        if not any(re.match(pattern, value["link_url"]) for pattern in other_patterns):
+            errors["link_url"] = ValidationError(_("The link URL must use a valid Vimeo or YouTube video URL"))
+
+        if errors:
+            raise StructBlockValidationError(block_errors=errors)
+
+        return super().clean(value)
+
+    class Meta:
+        icon = "code"
+        template = "templates/components/streamfield/video_embed_block.html"

cms/core/blocks/panels.py

@@ -1,4 +1,5 @@
 from django.conf import settings
+from django.utils.translation import gettext_lazy as _
 from wagtail import blocks
 
 
@@ -12,20 +13,17 @@ class PanelBlock(blocks.StructBlock):
 
     variant = blocks.ChoiceBlock(
         choices=[
+            ("warn", _("Warning")),
+            ("info", _("Information")),
             ("announcement", "Announcement"),
-            ("bare", "Bare"),
-            ("branded", "Branded"),
             ("error", "Error"),
-            ("ghost", "Ghost"),
             ("success", "Success"),
-            ("warn-branded", "Warn (branded)"),
-            ("warn", "Warn"),
         ],
         default="warn",
     )
     body = blocks.RichTextBlock(features=settings.RICH_TEXT_BASIC)
-    title = blocks.CharBlock(required=False, label="Title (optional)")
+    title = blocks.CharBlock(required=False, label=_("Title (optional)"))
 
     class Meta:
-        label = "Warning or information panel"
+        label = _("Warning or information panel")
         template = "templates/components/streamfield/panel_block.html"

cms/core/blocks/stream_blocks.py

@@ -1,7 +1,6 @@
 from typing import ClassVar
 
 from wagtail.blocks import RichTextBlock, StreamBlock
-from wagtail.embeds.blocks import EmbedBlock
 from wagtail.images.blocks import ImageChooserBlock
 from wagtailmath.blocks import MathBlock
 
@@ -12,6 +11,7 @@
     PanelBlock,
     QuoteBlock,
     RelatedLinksBlock,
+    VideoEmbedBlock,
 )
 from cms.core.blocks.section_blocks import SectionBlock
 
@@ -32,7 +32,7 @@ class CoreStoryBlock(StreamBlock):
     rich_text = RichTextBlock()
     quote = QuoteBlock()
     panel = PanelBlock()
-    embed = EmbedBlock(group="Media")
+    video_embed = VideoEmbedBlock(group="Media")
     image = ImageChooserBlock(group="Media")
     documents = DocumentsBlock(group="Media")
     related_links = RelatedLinksBlock()