Track cell+test execution

nbcelltests/test.py

@@ -46,17 +46,18 @@ def assemble_code(notebook):
 
         if is_empty(cell['source']):
             skiptest = "@nbcelltests.tests_vendored.unittest.skip('empty code cell')\n" + INDENT
-        elif is_empty("".join(test_lines).replace(r"%cell", "pass")):
+        elif is_empty("".join(test_lines).replace(r"%cell", "pass # no test was supplied")):
             skiptest = "@nbcelltests.tests_vendored.unittest.skip('no test supplied')\n" + INDENT
         elif not cell_injected_into_test(test_lines):
             skiptest = "@nbcelltests.tests_vendored.unittest.skip('cell code not injected into test')\n" + INDENT
         else:
             skiptest = ""
 
-        cells.append([i, [], "%sdef test_code_cell_%d(self):\n" % (skiptest, code_cell)])
+        # TODO: Use namedtuples for these:
+        cells.append([code_cell, [], "%sdef test_code_cell_%d(self):\n" % (skiptest, code_cell)])
 
         if skiptest:
-            cells[-1][1].append(INDENT + 'pass # code cell %d\n\n' % code_cell)
+            cells[-1][1].append(INDENT + 'pass # code cell %d was skipped\n' % code_cell)
             continue
 
         for test_line in test_lines:
@@ -74,46 +75,27 @@ def assemble_code(notebook):
                 cells[-1][1].append(INDENT + test_line)
                 if not test_line[-1] == '\n':
                     cells[-1][1][-1] += '\n'
-
     return cells
 
 
 def writeout_test(fp, cells, kernel_name):
-    # base import and class
     fp.write(BASE.format(kernel_name=kernel_name))
 
-    # grab all code to write out
-    for i, code, meth in cells:
+    # write out source of cells+tests
+    fp.write(INDENT + "cells_and_tests = {")
+    for i, code, _ in cells:
+        fp.write('\n')
+        fp.write(INDENT * 2 + '%d: """\n' % i)
+        for line in code:
+            fp.write(INDENT + line)
+        fp.write(INDENT * 2 + '""",\n')
+    fp.write(INDENT + "}\n")
+
+    # write out test methods
+    for i, _, meth in cells:
         fp.write('\n')
         fp.write(INDENT + meth)
-        fp.write(INDENT * 2 + 'self.run_test("""\n')
-        to_write = []
-
-        for j, code2, _ in cells:
-            if j < i:
-                for c in code2:
-                    if(c != '\n'):
-                        # indent if necessary
-                        to_write.append(INDENT + c)
-                    else:
-                        to_write.append(c)
-
-            else:
-                break
-
-        for c in code:
-            if(c != '\n'):
-                to_write.append(INDENT + c)
-            else:
-                to_write.append(c)
-
-        if len(to_write) == 0:
-            to_write.append(INDENT + 'pass')
-
-        fp.writelines(to_write)
-        fp.write('        """)\n')
-
-    fp.write('\n')
+        fp.write(INDENT * 2 + 'self.run_test(%d)\n' % i)
 
 
 def writeout_lines_per_cell(fp, lines_per_cell, metadata):

nbcelltests/tests/_test_fail.ipynb

@@ -13,7 +13,21 @@
    "source": [
     "x = 1"
    ]
-  }  
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "tests": [
+     "%cell\n",
+     "assert x == -2, \"x should have been -2 but was %s\"%x"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "x = 2"
+   ]
+  }    
  ],
  "metadata": {
   "kernelspec": {

nbcelltests/tests/test_test.py

@@ -33,7 +33,7 @@ def _assert_x_undefined(t):
     """
     Convenience method to assert that x is not already defined in the kernel.
     """
-    t.run_test("""
+    t._run("""
     try:
         x
     except NameError:
@@ -223,7 +223,7 @@ def test_state(self):
             t.setUpClass()
         t.setUp()
         t.test_code_cell_2()
-        t.run_test("""
+        t._run("""
         assert x == 0, x
         """)
         t.tearDown()
@@ -235,7 +235,7 @@ def test_state(self):
             t.setUpClass()
         t.setUp()
         t.test_code_cell_3()
-        t.run_test("""
+        t._run("""
         assert x == 1, x
         """)
         t.tearDown()
@@ -247,7 +247,7 @@ def test_state(self):
             t.setUpClass()
         t.setUp()
         t.test_code_cell_4()
-        t.run_test("""
+        t._run("""
         assert x == 2, x
         """)
         t.tearDown()
@@ -259,7 +259,7 @@ def test_state(self):
             t.setUpClass()
         t.setUp()
         t.test_code_cell_5()
-        t.run_test("""
+        t._run("""
         assert x == 3, x
         """)
         t.tearDown()
@@ -282,7 +282,7 @@ def test_exception_in_cell_is_detected(self):
         try:
             t.test_code_cell_1()
         except Exception as e:
-            assert e.args[0].startswith("Cell execution caused an exception")
+            assert e.args[0].startswith("Running cell+test for code cell 1; execution caused an exception")
             assert e.args[0].endswith("My code does not even run")
         else:
             raise Exception("Cell should have errored out")
@@ -318,7 +318,7 @@ def test_exception_in_test_is_detected(self):
         try:
             t.test_code_cell_2()
         except Exception as e:
-            assert e.args[0].startswith("Cell execution caused an exception")
+            assert e.args[0].startswith("Running cell+test for code cell 2; execution caused an exception")
             assert e.args[0].endswith("My test is bad too")
         else:
             raise Exception("Test should have failed")
@@ -341,14 +341,30 @@ def test_failure_is_detected(self):
         try:
             t.test_code_cell_1()
         except Exception as e:
-            assert e.args[0].startswith("Cell execution caused an exception")
+            assert e.args[0].startswith("Running cell+test for code cell 1; execution caused an exception")
             assert e.args[0].endswith("x should have been -1 but was 1")
         else:
             raise Exception("Test should have failed")
-        finally:
-            t.tearDown()
+
+        t.tearDown()
+        if FORKED:
             t.tearDownClass()
 
+        if FORKED:
+            t.setUpClass()
+        t.setUp()
+        # subsequent cell should also fail
+        try:
+            t.test_code_cell_2()
+        except Exception as e:
+            assert e.args[0].startswith("Running cell+test for code cell 1; execution caused an exception")
+            assert e.args[0].endswith("x should have been -1 but was 1")
+        else:
+            raise Exception("Test should have failed at cell 1")
+
+        t.tearDown()
+        t.tearDownClass()
+
 
 class TestCellCounting(_TestCellTests):
     """Check that various things don't throw off cell+test correspondence."""

nbcelltests/tests_vendored.py

@@ -58,35 +58,91 @@
 
 
 class TestNotebookBase(unittest.TestCase):
-    # abstract - subclasses must define KERNEL_NAME
+    """Base class for representing a notebook's code cells and their
+    associated tests; can submit cells to a kernel, track which cells
+    have been executed, and ensure all necessary cells are executed in
+    order.
+
+    For instance, requesting to run test_code_cell_7,
+    test_code_cell_8, and test_code_cell_9 (in that order) will result
+    in:
+
+      1. test_code_cell_7: executes cells 1, 2, 3, 4, 5, 6, 7
+      2. test_code_cell_8: executes cell 8
+      3. test_code_cell_9: executes cell 9
+
+    The above assumes all cells+tests succeed.
+
+    If e.g. cell 3 fails, the above changes to:
+
+      1. test_code_cell_7: executes cells 1, 2; fails on 3 (with a
+                           message that cell 3 failed)
+      2. test_code_cell_8: also fails on 3
+      3. test_code_cell_9: also fails on 3
+
+    Requesting to run test_code_cell_5 and test_code_cell_3 (in that order)
+    will result in:
+
+      1. test_code_cell_5: executes cells 1, 2, 3, 4, 5 (test passes)
+      2. test_code_cell_3: execute nothing (test passes)
+
+    This is an abstract class; subclasses will supply the source of
+    code cells and their associated tests in cells_and_tests, plus
+    test methods as entry points for test runners.
+
+    Note: 'cell' used in this class refers to cell number; 'cell
+    content' typically refers to code_cell+test (depending what is
+    passed in).
+    """
+    # abstract - subclasses will define KERNEL_NAME (TODO: make
+    # actually abstract...)
 
     @classmethod
     def setUpClass(cls):
         cls.kernel = RunningKernel(cls.KERNEL_NAME)
+        cls.cells_run = set()
 
     @classmethod
     def tearDownClass(cls):
         cls.kernel.stop()
 
-    # TODO: starting a new kernel per test is expensive, and
-    # could be optimized.
-    def setUp(self):
-        self.kernel = RunningKernel(self.KERNEL_NAME)
-
-    def tearDown(self):
-        self.kernel.stop()
-
-    def run_test(self, cell_content):
-        # Start of code from nbval
+    def run_test(self, cell):
+        """
+        Run any cells preceding cell (number) that have not already been
+        run, then run cell itself.
+        """
+        # maybe do some assertions that we didn't get all messed up
+        # with missing cells etc
+        preceding_cells = set(range(1, cell))
+        for preceding_cell in sorted(set(preceding_cells) - self.cells_run):
+            self._run_cell(preceding_cell)
+        self._run_cell(cell)
+
+    def _run_cell(self, cell):
+        # convenience method
+        self._run(self.cells_and_tests[cell], "Running cell+test for code cell %d" % cell)
+        # will only add if there was no error running
+        self.cells_run.add(cell)
+
+    def _run(self, cell_content, description=''):
+        """
+        Send supplied cell_content (cell source string) to kernel and
+        check it runs without exception.
+        """
+        # Start of code from nbval (with modifications)
         # https://github.com/computationalmodelling/nbval
-        # (but note, there are evidently some modifications)
+        #
+        # Modifications:
+        #   * ? (things before 2020)
+        #   * Add description to exception messages, so it's easy to see which
+        #     cell is failing.
         msg_id = self.kernel.execute_cell_input(cell_content, allow_stdin=False)
 
         # Poll the shell channel to get a message
         try:
             self.kernel.await_reply(msg_id)
         except Empty:
-            raise Exception('Kernel timed out waiting for message!')
+            raise Exception('%s; Kernel timed out waiting for message!' % description)
 
         while True:
             # The iopub channel broadcasts a range of messages. We keep reading
@@ -97,7 +153,7 @@ def run_test(self, cell_content):
                 msg = self.kernel.get_message(stream='iopub')
 
             except Empty:
-                raise Exception('Kernel timed out waiting for message!')
+                raise Exception('%s; Kernel timed out waiting for message!' % description)
 
             # now we must handle the message by checking the type and reply
             # info and we store the output of the cell in a notebook node object
@@ -133,13 +189,13 @@ def run_test(self, cell_content):
             # traceback information.
             elif msg_type == 'error':
                 traceback = '\\n' + '\\n'.join(reply['traceback'])
-                msg = "Cell execution caused an exception"
+                msg = "%s; execution caused an exception" % description
                 raise Exception(msg + '\\n' + traceback)
 
             # any other message type is not expected
             # should this raise an error?
             else:
-                print("unhandled iopub msg:", msg_type)
+                print("%s; unhandled iopub msg:" % description, msg_type)
 
         # End of code from nbval