YAML test loader

README.rst

@@ -64,3 +64,7 @@ Use it like this:
 
 Run this file directly using python, or use nosetests/py.test to find and
 run it.
+
+You can also avoid the unittest boilerplate by writing the necessary code
+samples in a YAML file, which can then be run with the ``test_kernels.py``
+script.

ipython3.toml

@@ -0,0 +1,33 @@
+kernel_name = "python3"
+language_name = "python"
+
+code_hello_world = "print('hello, world')"
+
+complete_code_samples = [
+    "1",
+    "print('hello, world')",
+    "def f(x):\n  return x*2\n\n"
+]
+
+incomplete_code_samples = [
+    "print('''hello",
+    "def f(x):\n  x*2"
+]
+
+code_page_something = "zip?"
+code_generate_error = "raise"
+
+[[completion_samples]]
+text = "zi"
+matches = ["zip"]
+
+[[code_execute_result]]
+code = "1+1"
+result = "2"
+
+[[code_display_data]]
+code = "from IPython.display import HTML, display; display(HTML('<b>test</b>'))"
+mime = "text/html"
+[[code_display_data]]
+code = "from IPython.display import Math, display; display(Math('\\frac{1}{2}'))"
+mime = "text/latex"

ipython3.yaml

@@ -0,0 +1,24 @@
+kernel_name: python3
+language_name: python
+code_hello_world: "print('hello, world')"
+completion_samples:
+    - text: zi
+      matches:
+        - zip
+complete_code_samples:
+    - "1"
+    - "print('hello, world')"
+    - "def f(x):\n  return x*2\n\n"
+incomplete_code_samples:
+    - "print('''hello"
+    - "def f(x):\n  x*2"
+code_page_something: zip?
+code_generate_error: raise
+code_execute_result:
+    - code: 1+1
+      result: "2"
+code_display_data:
+    - code: "from IPython.display import HTML, display; display(HTML('<b>test</b>'))"
+      mime: text/html
+    - code: "from IPython.display import Math, display; display(Math('\\frac{1}{2}'))"
+      mime: text/latex

test_kernels.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+"""
+This reads multiple kernel test cases from YAML file(s), generates the unittest
+objects for each, and then runs the unittest(s).
+
+The yaml file is expected to contain one or more documents like, which
+correspond to the equivalent named fields in KernelTest
+
+    kernel_name: foo
+    language_name: bar
+    code_hello_world: print
+    completion_samples:
+        - text: foo
+          matches:
+            - fooo
+            - foooo
+        - text: bar
+          matches:
+            - barrr
+    complete_code_samples:
+     - foo
+     - bar
+    incomplete_code_samples:
+     - foo
+     - bar
+    invalid_code_samples:
+     - foo
+     - bar
+    code_page_something: foo
+"""
+
+import yaml
+import unittest
+import jupyter_kernel_test as jkt
+import argparse
+import os
+
+def load_specs(specfile):
+    """
+    Load a YAML file, and convert each of the documents within into a
+    KernelTests subclass. Returns a list of class objects.
+    """
+    test_classes = []
+    assert os.path.exists(specfile)
+    with open(specfile) as sf:
+        for spec in yaml.load_all(sf):
+            assert isinstance(spec, dict)
+            assert 'kernel_name' in spec
+            tc = type(spec['kernel_name'], (jkt.KernelTests, ), spec)
+            test_classes.append(tc)
+    return test_classes
+
+def generate_test_suite(testclasses):
+    "Generate a TestSuite class from a list of unittest classes."
+    tests = []
+    for testclass in testclasses:
+        tests.append(unittest.TestLoader().loadTestsFromTestCase(testclass))
+    return unittest.TestSuite(tests)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("specfiles", nargs="+",
+                        help="YAML files containing test specs")
+    parser.add_argument("-v", "--verbosity", default=2, type=int,
+                        help="unittest verbosity")
+    opts = parser.parse_args()
+
+    for f in opts.specfiles:
+        suite = generate_test_suite(load_specs(f))
+        unittest.TextTestRunner(verbosity=opts.verbosity).run(suite)