Generate test report using pytest

dothttp/exceptions.py

@@ -89,12 +89,12 @@ class PayloadDataNotValidException(PayloadNotValidException):
     pass
 
 
-@exception_wrapper("python/js test script compilation failed `{payload}`")
+@exception_wrapper("PreRequest Function:`{function}` failed with error: `{payload}`")
 class PreRequestScriptException(DotHttpException):
     pass
 
 
-@exception_wrapper("python/js test script compilation failed `{payload}`")
+@exception_wrapper("Python test function: `{function}` execution failed `{payload}`")
 class ScriptException(DotHttpException):
     pass
 

dothttp/script/js3py.py

@@ -199,11 +199,13 @@ def init_request_script(self):
     def pre_request_script(self):
         try:
             self._pre_request_script()
+        except PreRequestScriptException:
+            raise
         except Exception as exc:
             request_logger.error("unknown exception happened", exc_info=True)
-            raise PreRequestScriptException(payload=str(exc))
+            raise PreRequestScriptException(payload=str(exc), function="''")
 
-    def execute_test_script(self, resp):
+    def execute_test_script(self, resp) -> ScriptResult:
         if not self.client.request.test_script:
             return ScriptResult(stdout="", error="", properties={}, tests=[])
         try:
@@ -235,7 +237,7 @@ def _execute_test_script(self, resp) -> None:
                 js_template.replace("JS_CODE_REPLACE", self.client.request.test_script)
             )
         except JsException as exc:
-            raise ScriptException(payload=str(exc))
+            raise ScriptException(payload=str(exc), function="''")
         content_type = resp.headers.get("content-type", "text/plain")
         # in some cases mimetype can have charset
         # like text/plain; charset=utf-8
@@ -279,7 +281,7 @@ def __init__(self, httpdef: HttpDef, prop: PropertyProvider) -> None:
             )
             exec(byte_code, script_gloabal, self.local)
         except Exception as exc:
-            raise ScriptException(payload=str(exc))
+            raise ScriptException(payload=str(exc), function="test_script.py")
 
     def _init_request_script(self) -> None:
         # just for variables initialization
@@ -288,9 +290,13 @@ def _init_request_script(self) -> None:
                 func()
 
     def _pre_request_script(self) -> None:
-        for key, func in self.local.items():
-            if (key.startswith("pre")) and isinstance(func, types.FunctionType):
-                func()
+        for key, pre_request_func in self.local.items():
+            if (key.startswith("pre")) and isinstance(pre_request_func, types.FunctionType):
+                try:
+                    pre_request_func()
+                except Exception as exc:
+                    request_logger.error(f"pre request script failed {exc}")
+                    raise PreRequestScriptException(function=key, payload=str(exc))
 
     def _execute_test_script(self, resp: Response) -> ScriptResult:
         script_result = ScriptResult(stdout="", error="", properties={}, tests=[])
@@ -306,7 +312,7 @@ def _execute_test_script(self, resp: Response) -> ScriptResult:
                         test_result.result = func()
                         test_result.success = True
                     except BaseException as exc:
-                        test_result.error = str(exc)
+                        test_result.error = f"\n\nTest function with name `{key}` failed with error `{exc}`\n\n"
                         test_result.success = False
                         request_logger.error(f"test execution failed {exc}")
                     script_result.tests.append(test_result)

dothttp_test/__main__.py

@@ -0,0 +1,8 @@
+import pytest
+from .conftest import *
+
+if __name__ == "__main__":
+    # run pytest with the current module
+    # this will not work because of arguments
+    # use `pytest dothttp_test --directory test/extensions/commands/names.http --prefix test -v -s --html=report.html`
+    pytest.main(["dothttp_test", "-v", "-s", "--html=report.html"])

dothttp_test/conftest.py

@@ -0,0 +1,26 @@
+def pytest_addoption(parser):
+    parser.addoption(
+        "--prefix", action="store", type=str, help="prefix for httpdef name", default="*"
+    )
+    parser.addoption(
+        "--directory",
+        action="store",
+        nargs="+",
+        help="list of directories to search for httpdef files",
+        default=["."],
+    )
+    parser.addoption("--property-file", help="property file")
+    parser.addoption(
+        "--no-cookie",
+        help="cookie storage is disabled",
+        action="store_const",
+        const=True,
+    )
+    parser.addoption(
+        "--env",
+        help="environment to select in property file. properties will be enabled on FIFO",
+        nargs="+",
+        default=["*"],
+    )
+    parser.addoption("--property", help="list of property's",
+                     nargs="+", default=[])

dothttp_test/dothttp_test.py

@@ -0,0 +1,125 @@
+import os
+from collections import namedtuple
+import logging
+
+from dothttp.parse import (
+    Config,
+)
+from dothttp.parse.request_base import (
+    RequestCompiler,
+    dothttp_model,
+)
+from dotextensions.server.handlers.basic_handlers import RunHttpFileHandler
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+HttpDefTest = namedtuple(
+    "HttpDef", ["display", "file", "name", "config", "failed"])
+
+
+def pytest_generate_tests(metafunc):
+    prefix = metafunc.config.getoption("prefix")
+    directory = metafunc.config.getoption("directory")
+    # iterate through all sub directories in the directory
+    # and get all httpdef files
+    # and pass it to the test function
+    tests_to_run = []
+
+    for dir in directory:
+        if not os.path.exists(dir):
+            LOGGER.error(f"{dir} does not exists")
+            continue
+        if os.path.isfile(dir):
+            tests_to_run += extrct_tests_to_run(metafunc,
+                                                prefix, dir)
+        else:
+            for root, _dirs, files in os.walk(dir):
+                for filename in files:
+                    tests_to_run += extrct_tests_to_run(
+                        metafunc, prefix, os.path.join(root, filename))
+    metafunc.parametrize("httpdeftest", tests_to_run, ids=lambda x: x.display)
+
+
+def extrct_tests_to_run(metafunc, prefix, filename):
+    tests_to_run = []
+    if filename.endswith(".http"):
+        with open(os.path.join(filename)) as f:
+            httpdef = f.read()
+        try:
+            model = dothttp_model.model_from_str(httpdef)
+            for http in model.allhttps:
+                if prefix is not None:
+                    if prefix == "*":
+                        tests_to_run.append(
+                            HttpDefTest(
+                                file=filename,
+                                name=http.namewrap.name,
+                                display=f"{filename} - name={http.namewrap.name}",
+                                config=metafunc.config,
+                                failed=False
+                            )
+                        )
+                    elif http.namewrap and http.namewrap.name.startswith(prefix):
+                        tests_to_run.append(
+                            HttpDefTest(
+                                file=filename,
+                                name=http.namewrap.name,
+                                display=f"{filename} - name={http.namewrap.name}",
+                                config=metafunc.config,
+                                failed=False
+                            )
+                        )
+        except:
+            tests_to_run.append(
+                HttpDefTest(
+                    file=filename,
+                    name="entire_file",
+                    display=f"{filename} - name='entire_file'",
+                    config=metafunc.config,
+                    failed=True
+                )
+            )
+            LOGGER.error(f"error in parsing {filename}")
+    return tests_to_run
+
+
+def test_httpdef(httpdeftest: HttpDefTest):
+    if httpdeftest.failed:
+        assert False, "failed to parse"
+    # read below arguments from command line
+    config = Config(
+        file=httpdeftest.file,
+        target=httpdeftest.name,
+        property_file=httpdeftest.config.getoption("property_file"),
+        env=httpdeftest.config.getoption("env"),
+        properties=httpdeftest.config.getoption("property"),
+        debug=True,
+        info=True,
+        curl=False,
+        no_cookie=False,
+        format=False,
+    )
+    comp = RequestCompiler(config)
+
+    realized_http_def_content = RunHttpFileHandler.get_http_from_req(
+        comp.httpdef)
+
+    logging.warning(
+        f"realized_http_def_content {realized_http_def_content}",
+    )
+
+    resp = comp.get_response()
+
+    logging.warning(f"resp={resp}")
+
+    script_result = comp.script_execution.execute_test_script(resp)
+
+    for test in script_result.tests:
+        # at present only one failure is reported in the test.
+        # if there are multiple failures, only the first one is reported
+        logging.warning(
+            f"script_result: test={test}, test_success: {test.success}, error:{test.error}",
+        )
+        assert test.success, test.error

pyproject.toml

@@ -10,7 +10,7 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 exclude = ["test", "test.core", "test.extensions", "benchmarks"]
-packages = [{'include'='dotextensions'}, {'include'='dothttp'}, {'include'='dothttp_req'}]
+packages = [{'include'='dotextensions'}, {'include'='dothttp'}, {'include'='dothttp_req'}, {'include'='dothttp_test'}]
 repository = "https://github.com/cedric05/dothttp"
 
 [tool.poetry.urls]