Merge pull request #157 from oasis-open/issue-154

interop requirements

README.rst

@@ -166,12 +166,16 @@ Configs may also contain a "taxii" section as well, as shown below:
     {
         "taxii": {
            "max_page_size": 100
+           "interop_requirements": true
         }
     }
 
 All TAXII servers require a config, though if any of the sections specified above
 are missing, they will be filled with default values.
 
+The ``interop_requirements`` option will enforce additional requireemnts from
+the TAXII 2.1 Interoperability specification. It defaults to ``false``.
+
 We welcome contributions for other back-end plugins.
 
 Docker

docs/compatibility.rst

@@ -15,5 +15,5 @@ Its main purpose is for use in testing scenarios of STIX-based applications that
 use the `python-stix2 API <https://github.com/oasis-open/cti-python-stix2>`_.
 It has been developed in conjunction with
 `cti-taxii-client <https://github.com/oasis-open/cti-taxii-client>`_ but should
-be compatible with any TAXII client which makes HTTP requests as defined in TAXII 2.0
+be compatible with any TAXII client which makes HTTP requests as defined in TAXII 2.1
 specification.

docs/custom_backend.rst

@@ -121,7 +121,7 @@ If you need or prefer a library different from ``Flask-HTTPAuth``, you can overr
             return decorated_function
 
         def get_password():
-            return None  # Custom stuff to get password using other libraries, users_backend can go here.
+            return None  # Custom stuff to get password using other libraries, users_config can go here.
 
     # Set the default implementation to the dummy auth
     auth = dummy_auth()
@@ -133,7 +133,7 @@ If you need or prefer a library different from ``Flask-HTTPAuth``, you can overr
 How to use a different backend to control users
 -----------------------------------------------
 
-Our implementation of a users authentication system is not suitable for a production environment. Thus requiring to write custom code to handle credential authentication, sessions, etc. Most likely you will require the changes described in the section above on `How to use a different authentication library`_, plus changing the ``users_backend``.
+Our implementation of a users authentication system is not suitable for a production environment. Thus requiring to write custom code to handle credential authentication, sessions, etc. Most likely you will require the changes described in the section above on `How to use a different authentication library`_, plus changing the ``users_config``.
 
 .. code-block:: python
 
@@ -152,15 +152,15 @@ Our implementation of a users authentication system is not suitable for a produc
 
         def get_password():
             # Usage of MyCustomDBforUsers would likely happen here.
-            return something # Custom stuff to get password using other libraries, users_backend functionality.
+            return something # Custom stuff to get password using other libraries, users_config functionality.
 
     # Set the default implementation to the dummy auth
     auth = dummy_auth()
 
     db = MyCustomDBforUsers.init()  # Do some setup before attaching to application... (Imagine other steps happening here)
 
     with application_instance.app_context():
-        current_app.users_backend = db  # This will make it available inside the Flask instance in case you decide to perform changes to the internal blueprints.
+        current_app.users_config = db  # This will make it available inside the Flask instance in case you decide to perform changes to the internal blueprints.
 
     init_backend(application_instance, {...})
     application_instance.run()

medallion/__init__.py

@@ -2,11 +2,12 @@
 import logging
 import warnings
 
-from flask import Flask, Response, current_app, json
+from flask import Response, current_app, json
 from flask_httpauth import HTTPBasicAuth
 
 from .backends import base as mbe_base
-from .exceptions import BackendError, ProcessingError
+from .common import APPLICATION_INSTANCE
+from .exceptions import BackendError, InitializationError, ProcessingError
 from .version import __version__  # noqa
 from .views import MEDIA_TYPE_TAXII_V21
 
@@ -18,49 +19,32 @@
 log = logging.getLogger(__name__)
 log.addHandler(ch)
 
-application_instance = Flask(__name__)
 auth = HTTPBasicAuth()
 
 
-def load_app(config_file):
-    with open(config_file, "r") as f:
-        configuration = json.load(f)
-
-    set_config(application_instance, "users", configuration)
-    set_config(application_instance, "taxii", configuration)
-    set_config(application_instance, "backend", configuration)
-    register_blueprints(application_instance)
-
-    return application_instance
-
-
 def set_config(flask_application_instance, prop_name, config):
-    with flask_application_instance.app_context():
-        log.debug("Registering medallion {} configuration into {}".format(prop_name, current_app))
-        if prop_name == "taxii":
-            try:
-                flask_application_instance.taxii_config = config[prop_name]
-            except KeyError:
-                flask_application_instance.taxii_config = {'max_page_size': 100}
-        elif prop_name == "users":
-            try:
-                flask_application_instance.users_backend = config[prop_name]
-            except KeyError:
-                log.warning("You did not give user information in your config.")
-                log.warning("We are giving you the default user information of:")
-                log.warning("User = user")
-                log.warning("Pass = pass")
-                flask_application_instance.users_backend = {"user": "pass"}
-        elif prop_name == "backend":
-            try:
-                flask_application_instance.medallion_backend = connect_to_backend(config[prop_name])
-            except KeyError:
-                log.warning("You did not give backend information in your config.")
-                log.warning("We are giving medallion the default settings,")
-                log.warning("which includes a data file of 'default_data.json'.")
-                log.warning("Please ensure this file is in your CWD.")
-                back = {'module_class': 'MemoryBackend', 'filename': None}
-                flask_application_instance.medallion_backend = connect_to_backend(back)
+    log.debug("Registering medallion {} configuration into {}".format(prop_name, flask_application_instance))
+    if prop_name == "taxii":
+        if prop_name in config:
+            flask_application_instance.taxii_config = config[prop_name]
+        else:
+            flask_application_instance.taxii_config = {'max_page_size': 100}
+        if "interop_requirements" not in flask_application_instance.taxii_config:
+            flask_application_instance.taxii_config["interop_requirements"] = False
+    elif prop_name == "users":
+        try:
+            flask_application_instance.users_config = config[prop_name]
+        except KeyError:
+            log.warning("You did not give user information in your config.")
+            log.warning("We are giving you the default user information of:")
+            log.warning("User = user")
+            log.warning("Pass = pass")
+            flask_application_instance.users_config = {"user": "pass"}
+    elif prop_name == "backend":
+        if prop_name in config:
+            flask_application_instance.backend_config = config[prop_name]
+        else:
+            raise InitializationError("You did not give backend information in your config.", 408)
 
 
 def connect_to_backend(config_info):
@@ -125,12 +109,12 @@ def register_blueprints(flask_application_instance):
 
 @auth.get_password
 def get_pwd(username):
-    if username in current_app.users_backend:
-        return current_app.users_backend.get(username)
+    if username in current_app.users_config:
+        return current_app.users_config.get(username)
     return None
 
 
-@application_instance.errorhandler(500)
+@APPLICATION_INSTANCE.errorhandler(500)
 def handle_error(error):
     e = {
         "title": "InternalError",
@@ -144,7 +128,7 @@ def handle_error(error):
     )
 
 
-@application_instance.errorhandler(ProcessingError)
+@APPLICATION_INSTANCE.errorhandler(ProcessingError)
 def handle_processing_error(error):
     e = {
         "title": str(error.__class__.__name__),
@@ -159,7 +143,7 @@ def handle_processing_error(error):
     )
 
 
-@application_instance.errorhandler(BackendError)
+@APPLICATION_INSTANCE.errorhandler(BackendError)
 def handle_backend_error(error):
     e = {
         "title": str(error.__class__.__name__),

medallion/backends/base.py

@@ -1,3 +1,22 @@
+import logging
+from urllib.parse import urlparse
+
+from ..common import (
+    APPLICATION_INSTANCE, TaskChecker, get_application_instance_config_values
+)
+from ..exceptions import InitializationError
+
+# Module-level logger
+log = logging.getLogger(__name__)
+
+SECONDS_IN_24_HOURS = 24*60*60
+
+
+def get_api_root_name(url):
+    pr = urlparse(url)
+    return pr.path.replace("/", "")
+
+
 class BackendRegistry(type):
     __SUBCLASS_MAP = dict()
 
@@ -26,6 +45,45 @@ def iter_(mcls):
 
 class Backend(object, metaclass=BackendRegistry):
 
+    def __init__(self, **kwargs):
+        self.next = {}
+
+        interop_requirements_enforced = get_application_instance_config_values(APPLICATION_INSTANCE, "taxii", "interop_requirements")
+        if kwargs.get("run_cleanup_threads", True):
+            self.timeout = kwargs.get("session_timeout", 30)
+            checker = TaskChecker(kwargs.get("check_interval", 10), self._pop_expired_sessions)
+            checker.start()
+
+            self.status_retention = kwargs.get("status_retention", SECONDS_IN_24_HOURS)
+            if self.status_retention != -1:
+                if self.status_retention < SECONDS_IN_24_HOURS and interop_requirements_enforced:
+                    # interop MUST requirement
+                    raise InitializationError("Status retention interval must be more than 24 hours", 408)
+                status_checker = TaskChecker(kwargs.get("check_interval", 10), self._pop_old_statuses)
+                status_checker.start()
+        else:
+            if interop_requirements_enforced:
+                # interop MUST requirement
+                raise InitializationError("Status retention interval must be more than 24 hours", 408)
+
+    def _get_all_api_roots(self):
+        discovery_info = self.server_discovery()
+        return [get_api_root_name(x) for x in discovery_info["api_roots"]]
+
+    def _get_api_root_statuses(self, api_root):
+        """
+        Fill:
+            Returns the statuses of the given api root
+
+        Args:
+            api_root -
+
+        Returns:
+            list of statuses
+
+        """
+        raise NotImplementedError()
+
     def server_discovery(self):
         """
         Fill:
@@ -227,3 +285,31 @@ def get_object_versions(self, api_root, collection_id, object_id, filter_args, a
 
         """
         raise NotImplementedError()
+
+    def _pop_expired_sessions(self):
+        """
+        Fill:
+            Implement thread to remove expired get requests from request queue
+
+        Args:
+            None
+
+        Returns:
+            None
+
+        """
+        raise NotImplementedError()
+
+    def _pop_old_statuses(self):
+        """
+        Fill:
+            Implement thread to remove old request status info
+
+        Args:
+            None
+
+        Returns:
+            None
+
+        """
+        raise NotImplementedError()

medallion/backends/memory_backend.py

@@ -1,21 +1,27 @@
 import copy
 import io
 import json
+import logging
 import os
 import uuid
 
 import environ
 from six import string_types
 
 from ..common import (
-    SessionChecker, create_resource, datetime_to_float, datetime_to_string,
-    determine_spec_version, determine_version, find_att, generate_status,
-    generate_status_details, get_timestamp, iterpath
+    APPLICATION_INSTANCE, create_resource, datetime_to_float,
+    datetime_to_string, determine_spec_version, determine_version, find_att,
+    generate_status, generate_status_details,
+    get_application_instance_config_values, get_timestamp, iterpath,
+    string_to_datetime
 )
 from ..exceptions import InitializationError, ProcessingError
 from ..filters.basic_filter import BasicFilter
 from .base import Backend
 
+# Module-level logger
+log = logging.getLogger(__name__)
+
 
 def remove_hidden_field(objs):
     for obj in objs:
@@ -57,11 +63,27 @@ def __init__(self, **kwargs):
             self.collections_manifest_check()
         else:
             self.data = {}
-        self.next = {}
-        self.timeout = kwargs.get("session_timeout", 30)
+        super(MemoryBackend, self).__init__(**kwargs)
 
-        checker = SessionChecker(kwargs.get("check_interval", 10), self._pop_expired_sessions)
-        checker.start()
+    def _pop_expired_sessions(self):
+        expired_ids = []
+        boundary = datetime_to_float(get_timestamp())
+        for next_id, record in self.next.items():
+            if boundary - record["request_time"] > self.timeout:
+                expired_ids.append(next_id)
+
+        for item in expired_ids:
+            self.next.pop(item)
+
+    def _pop_old_statuses(self):
+        api_roots = self._get_all_api_roots()
+        boundary = datetime_to_float(get_timestamp())
+        for ar in api_roots:
+            statuses_of_api_root = copy.copy(self._get_api_root_statuses(ar))
+            for s in statuses_of_api_root:
+                if boundary - datetime_to_float(string_to_datetime(s["request_timestamp"])) > self.status_retention:
+                    self._get_api_root_statuses(ar).remove(s)
+                    log.info("Status {} was deleted from {} because it was older than the status retention time".format(s['id'], ar))
 
     def set_next(self, objects, args):
         u = str(uuid.uuid4())
@@ -114,16 +136,6 @@ def get_next(self, filter_args, allowed, manifest, lim):
         else:
             raise ProcessingError("The server did not understand the request or filter parameters: 'next' not valid", 400)
 
-    def _pop_expired_sessions(self):
-        expired_ids = []
-        boundary = datetime_to_float(get_timestamp())
-        for next_id, record in self.next.items():
-            if boundary - record["request_time"] > self.timeout:
-                expired_ids.append(next_id)
-
-        for item in expired_ids:
-            self.next.pop(item)
-
     def collections_manifest_check(self):
         """
         Checks collections for proper manifest, if objects are present in a collection, a manifest should be present with
@@ -212,6 +224,9 @@ def get_collections(self, api_root):
             collection.pop("manifest", None)
             collection.pop("responses", None)
             collection.pop("objects", None)
+        # interop wants results sorted by id
+        if get_application_instance_config_values(APPLICATION_INSTANCE, "taxii", "interop_requirements"):
+            collections = sorted(collections, key=lambda o: o["id"])
         return create_resource("collections", collections)
 
     def get_collection(self, api_root, collection_id):
@@ -262,6 +277,12 @@ def get_api_root_information(self, api_root):
             if "information" in api_info:
                 return api_info["information"]
 
+    def _get_api_root_statuses(self, api_root):
+        api_info = self._get(api_root)
+
+        if "status" in api_info:
+            return api_info["status"]
+
     def get_status(self, api_root, status_id):
         if api_root in self.data:
             api_info = self._get(api_root)