Merge pull request #14 from canonical/emitted-events

emitted_events util to assert a given sequence of events have been (re)/emitted

.github/workflows/quality_checks.yaml

@@ -0,0 +1,40 @@
+name: Tests
+
+on:
+  pull-request:
+    branches:
+      - main
+
+
+jobs:
+  linting:
+    name: Linting
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: Set up python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: python -m pip install tox
+      - name: Run tests
+        run: tox -vve lint
+
+  unit-test:
+    name: Unit Tests
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: Set up python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: python -m pip install tox
+      - name: Run tests
+        run: tox -vve unit

README.md

@@ -486,6 +486,47 @@ And the charm's runtime will see `self.stored_State.foo` and `.baz` as expected.
 Also, you can run assertions on it on the output side the same as any other bit of state.
 
 
+# Emitted events
+If your charm deals with deferred events, custom events, and charm libs that in turn emit their own custom events, it can be hard to examine the resulting control flow.
+In these situations it can be useful to verify that, as a result of a given juju event triggering (say, 'start'), a specific chain of deferred and custom events is emitted on the charm. The resulting state, black-box as it is, gives little insight into how exactly it was obtained. `scenario.capture_events` allows you to open a peephole and intercept any events emitted by the framework. 
+
+Usage: 
+
+```python
+from ops.charm import StartEvent, UpdateStatusEvent
+from scenario import State, DeferredEvent
+from scenario import capture_events
+with capture_events() as emitted:
+    state_out = State(deferred=[DeferredEvent('start', ...)]).trigger('update-status', ...)
+
+# deferred events get reemitted first
+assert isinstance(emitted[0], StartEvent)
+# the main juju event gets emitted next
+assert isinstance(emitted[1], UpdateStatusEvent)
+# possibly followed by a tail of all custom events that the main juju event triggered in turn
+# assert isinstance(emitted[2], MyFooEvent)
+# ... 
+```
+
+
+You can filter events by type like so:
+
+```python
+from ops.charm import StartEvent, RelationEvent
+from scenario import capture_events
+with capture_events(StartEvent, RelationEvent) as emitted:
+    # capture all `start` and `*-relation-*` events.
+    pass  
+```
+
+Passing no event types, like: `capture_events()`, is equivalent to `capture_events(EventBase)`.
+
+By default, **framework events** (`PreCommit`, `Commit`) are not considered for inclusion in the output list even if they match the instance check. You can toggle that by passing: `capture_events(include_framework=True)`.
+
+By default, **deferred events** are included in the listing if they match the instance check. You can toggle that by passing:
+`capture_events(include_deferred=True)`.
+
+
 # The virtual charm root
 Before executing the charm, Scenario writes the metadata, config, and actions `yaml`s to a temporary directory. 
 The charm will see that tempdir as its 'root'. This allows us to keep things simple when dealing with metadata that can 

pyproject.toml

@@ -1,11 +1,14 @@
 [build-system]
-requires = ["setuptools"]
+requires = [
+    "setuptools >= 35.0.2",
+    "setuptools_scm >= 2.0.0, <3"
+]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "ops-scenario"
 
-version = "2.1.3.2"
+version = "2.1.3.3"
 authors = [
     { name = "Pietro Pasotti", email = "[email protected]" }
 ]
@@ -46,4 +49,4 @@ include = '\.pyi?$'
 profile = "black"
 
 [bdist_wheel]
-universal=1
+universal = 1

scenario/__init__.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 # Copyright 2023 Canonical Ltd.
 # See LICENSE file for licensing details.
+from scenario.capture_events import capture_events
 from scenario.runtime import trigger
 from scenario.state import *

scenario/capture_events.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.
+import typing
+from contextlib import contextmanager
+from typing import ContextManager, List, Type, TypeVar
+
+from ops.framework import (
+    CommitEvent,
+    EventBase,
+    Framework,
+    Handle,
+    NoTypeError,
+    PreCommitEvent,
+)
+
+_T = TypeVar("_T", bound=EventBase)
+
+
+@contextmanager
+def capture_events(
+    *types: Type[EventBase], include_framework=False, include_deferred=True
+) -> ContextManager[List[EventBase]]:
+    """Capture all events of type `*types` (using instance checks).
+
+    Example::
+    >>> from ops.charm import StartEvent
+    >>> from scenario import Event, State
+    >>> from charm import MyCustomEvent, MyCharm  # noqa
+    >>>
+    >>> def test_my_event():
+    >>>     with capture_events(StartEvent, MyCustomEvent) as captured:
+    >>>         State().trigger("start", MyCharm, meta=MyCharm.META)
+    >>>
+    >>>     assert len(captured) == 2
+    >>>     e1, e2 = captured
+    >>>     assert isinstance(e2, MyCustomEvent)
+    >>>     assert e2.custom_attr == 'foo'
+    """
+    allowed_types = types or (EventBase,)
+
+    captured = []
+    _real_emit = Framework._emit
+    _real_reemit = Framework.reemit
+
+    def _wrapped_emit(self, evt):
+        if not include_framework and isinstance(evt, (PreCommitEvent, CommitEvent)):
+            return _real_emit(self, evt)
+
+        if isinstance(evt, allowed_types):
+            captured.append(evt)
+
+        return _real_emit(self, evt)
+
+    def _wrapped_reemit(self):
+        # Framework calls reemit() before emitting the main juju event. We intercept that call
+        # and capture all events in storage.
+
+        if not include_deferred:
+            return _real_reemit(self)
+
+        # load all notices from storage as events.
+        for event_path, observer_path, method_name in self._storage.notices():
+            event_handle = Handle.from_path(event_path)
+            try:
+                event = self.load_snapshot(event_handle)
+            except NoTypeError:
+                continue
+            event = typing.cast(EventBase, event)
+            event.deferred = False
+            self._forget(event)  # prevent tracking conflicts
+
+            if not include_framework and isinstance(
+                event, (PreCommitEvent, CommitEvent)
+            ):
+                continue
+
+            if isinstance(event, allowed_types):
+                captured.append(event)
+
+        return _real_reemit(self)
+
+    Framework._emit = _wrapped_emit  # type: ignore # noqa # ugly
+    Framework.reemit = _wrapped_reemit  # type: ignore # noqa # ugly
+
+    yield captured
+
+    Framework._emit = _real_emit  # type: ignore # noqa # ugly
+    Framework.reemit = _real_reemit  # type: ignore # noqa # ugly

scenario/scripts/main.py

@@ -21,8 +21,8 @@ def main():
     app = typer.Typer(
         name="scenario",
         help="Scenario utilities. "
-             "For docs, issues and feature requests, visit "
-             "the github repo --> https://github.com/canonical/ops-scenario",
+        "For docs, issues and feature requests, visit "
+        "the github repo --> https://github.com/canonical/ops-scenario",
         no_args_is_help=True,
         rich_markup_mode="markdown",
     )

scenario/scripts/snapshot.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 # Copyright 2023 Canonical Ltd.
 # See LICENSE file for licensing details.
-import base64
 import datetime
 import json
 import logging
@@ -14,7 +13,7 @@
 from enum import Enum
 from itertools import chain
 from pathlib import Path
-from subprocess import CalledProcessError, check_output, run
+from subprocess import run
 from textwrap import dedent
 from typing import Any, BinaryIO, Dict, Iterable, List, Optional, TextIO, Tuple, Union
 
@@ -174,7 +173,6 @@ def get_network(target: JujuUnitName, model: Optional[str], endpoint: str) -> Ne
 
     bind_addresses = []
     for raw_bind in jsn["bind-addresses"]:
-
         addresses = []
         for raw_adds in raw_bind["addresses"]:
             addresses.append(

tests/test_e2e/test_pebble.py

@@ -104,7 +104,7 @@ def callback(self: CharmBase):
                 container.push("/foo/bar/baz.txt", text, make_dirs=make_dirs)
 
             # check that nothing was changed
-            with pytest.raises(FileNotFoundError):
+            with pytest.raises((FileNotFoundError, pebble.PathError)):
                 container.pull("/foo/bar/baz.txt")
 
     td = tempfile.TemporaryDirectory()

tests/test_e2e/test_relations.py

@@ -74,7 +74,11 @@ def test_relation_events(mycharm, evt_name):
 
     mycharm._call = lambda self, evt: None
 
-    State(relations=[relation,],).trigger(
+    State(
+        relations=[
+            relation,
+        ],
+    ).trigger(
         getattr(relation, f"{evt_name}_event"),
         mycharm,
         meta={
@@ -106,7 +110,11 @@ def callback(charm: CharmBase, _):
 
     mycharm._call = callback
 
-    State(relations=[relation,],).trigger(
+    State(
+        relations=[
+            relation,
+        ],
+    ).trigger(
         getattr(relation, f"{evt_name}_event"),
         mycharm,
         meta={

tests/test_e2e/test_rubbish_events.py

@@ -46,7 +46,6 @@ def _on_event(self, e):
 @pytest.mark.parametrize("evt_name", ("rubbish", "foo", "bar", "kazoo_pebble_ready"))
 def test_rubbish_event_raises(mycharm, evt_name):
     with pytest.raises(NoObserverError):
-
         if evt_name.startswith("kazoo"):
             os.environ["SCENARIO_SKIP_CONSISTENCY_CHECKS"] = "true"
             # else it will whine about the container not being in state and meta;

tests/test_emitted_events_util.py

@@ -0,0 +1,89 @@
+import pytest
+from ops.charm import CharmBase, CharmEvents, StartEvent
+from ops.framework import CommitEvent, EventBase, EventSource, PreCommitEvent
+
+from scenario import Event, State, capture_events
+
+
+class Foo(EventBase):
+    pass
+
+
+class MyCharmEvents(CharmEvents):
+    foo = EventSource(Foo)
+
+
+class MyCharm(CharmBase):
+    META = {"name": "mycharm"}
+    on = MyCharmEvents()
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.framework.observe(self.on.start, self._on_start)
+        self.framework.observe(self.on.foo, self._on_foo)
+
+    def _on_start(self, e):
+        self.on.foo.emit()
+
+    def _on_foo(self, e):
+        pass
+
+
+def test_capture_custom_evt():
+    with capture_events(Foo) as emitted:
+        State().trigger("foo", MyCharm, meta=MyCharm.META)
+
+    assert len(emitted) == 1
+    assert isinstance(emitted[0], Foo)
+
+
+def test_capture_custom_evt_nonspecific_capture():
+    with capture_events() as emitted:
+        State().trigger("foo", MyCharm, meta=MyCharm.META)
+
+    assert len(emitted) == 1
+    assert isinstance(emitted[0], Foo)
+
+
+def test_capture_custom_evt_nonspecific_capture_include_fw_evts():
+    with capture_events(include_framework=True) as emitted:
+        State().trigger("foo", MyCharm, meta=MyCharm.META)
+
+    assert len(emitted) == 3
+    assert isinstance(emitted[0], Foo)
+    assert isinstance(emitted[1], PreCommitEvent)
+    assert isinstance(emitted[2], CommitEvent)
+
+
+def test_capture_juju_evt():
+    with capture_events() as emitted:
+        State().trigger("start", MyCharm, meta=MyCharm.META)
+
+    assert len(emitted) == 2
+    assert isinstance(emitted[0], StartEvent)
+    assert isinstance(emitted[1], Foo)
+
+
+def test_capture_deferred_evt():
+    # todo: this test should pass with ops < 2.1 as well
+    with capture_events() as emitted:
+        State(deferred=[Event("foo").deferred(handler=MyCharm._on_foo)]).trigger(
+            "start", MyCharm, meta=MyCharm.META
+        )
+
+    assert len(emitted) == 3
+    assert isinstance(emitted[0], Foo)
+    assert isinstance(emitted[1], StartEvent)
+    assert isinstance(emitted[2], Foo)
+
+
+def test_capture_no_deferred_evt():
+    # todo: this test should pass with ops < 2.1 as well
+    with capture_events(include_deferred=False) as emitted:
+        State(deferred=[Event("foo").deferred(handler=MyCharm._on_foo)]).trigger(
+            "start", MyCharm, meta=MyCharm.META
+        )
+
+    assert len(emitted) == 2
+    assert isinstance(emitted[0], StartEvent)
+    assert isinstance(emitted[1], Foo)