context: adding ContextExternalModuleLoader class

This patch adds class ContextExternalModuleLoader, which
adds ability to add custom module load callback, which
allows user to load modules from remote source etc.

Signed-off-by: Stefan Gula <[email protected]>

cffi/cdefs.h

@@ -1071,6 +1071,11 @@ typedef enum {
 LY_ERR lys_parse(struct ly_ctx *, struct ly_in *, LYS_INFORMAT, const char **, struct lys_module **);
 LY_ERR ly_ctx_new_ylpath(const char *, const char *, LYD_FORMAT, int, struct ly_ctx **);
 LY_ERR ly_ctx_get_yanglib_data(const struct ly_ctx *, struct lyd_node **, const char *, ...);
+typedef void (*ly_module_imp_data_free_clb)(void *, void *);
+typedef LY_ERR (*ly_module_imp_clb)(const char *, const char *, const char *, const char *, void *, LYS_INFORMAT *, const char **, ly_module_imp_data_free_clb *);
+void ly_ctx_set_module_imp_clb(struct ly_ctx *, ly_module_imp_clb, void *);
+extern "Python" void lypy_module_imp_data_free_clb(void *, void *);
+extern "Python" LY_ERR lypy_module_imp_clb(const char *, const char *, const char *, const char *, void *, LYS_INFORMAT *, const char **, ly_module_imp_data_free_clb *);
 
 LY_ERR lydict_insert(const struct ly_ctx *, const char *, size_t, const char **);
 LY_ERR lydict_remove(const struct ly_ctx *, const char *);

libyang/context.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: MIT
 
 import os
-from typing import IO, Any, Iterator, Optional, Union
+from typing import IO, Any, Callable, Iterator, Optional, Tuple, Union
 
 from _libyang import ffi, lib
 from .data import (
@@ -19,9 +19,176 @@
 from .util import DataType, IOType, LibyangError, c2str, data_load, str2c
 
 
+# -------------------------------------------------------------------------------------
+@ffi.def_extern(name="lypy_module_imp_data_free_clb")
+def libyang_c_module_imp_data_free_clb(cdata, user_data):
+    instance = ffi.from_handle(user_data)
+    instance.free_module_data(cdata)
+
+
+# -------------------------------------------------------------------------------------
+@ffi.def_extern(name="lypy_module_imp_clb")
+def libyang_c_module_imp_clb(
+    mod_name,
+    mod_rev,
+    submod_name,
+    submod_rev,
+    user_data,
+    fmt,
+    module_data,
+    free_module_data,
+):
+    """
+    Implements the C callback function for loading modules from any location.
+
+    :arg c_str mod_name:
+        The YANG module name
+    :arg c_str mod_rev:
+        The YANG module revision
+    :arg c_str submod_name:
+        The YANG submodule name
+    :arg c_str submod_rev:
+        The YANG submodule revision
+    :arg user_data:
+        The user data provided by user during registration. In this implementation
+        it is always considered to be handle of Python object
+    :arg fmt:
+        The output pointer where to set the format of schema
+    :arg module_data:
+        The output pointer where to set the schema data itself
+    :arg free_module_data:
+        The output pointer of callback function which will be called when the schema
+        data are no longer needed
+
+    :returns:
+        The LY_SUCCESS in case the needed YANG (sub)module schema was found
+        The LY_ENOT in case the needed YANG (sub)module schema was not found
+    """
+    fmt[0] = lib.LYS_IN_UNKNOWN
+    module_data[0] = ffi.NULL
+    free_module_data[0] = lib.lypy_module_imp_data_free_clb
+    instance = ffi.from_handle(user_data)
+    ret = instance.get_module_data(
+        c2str(mod_name), c2str(mod_rev), c2str(submod_name), c2str(submod_rev)
+    )
+    if ret is None:
+        return lib.LY_ENOT
+    in_fmt, content = ret
+    fmt[0] = schema_in_format(in_fmt)
+    module_data[0] = content
+    return lib.LY_SUCCESS
+
+
+# -------------------------------------------------------------------------------------
+class ContextExternalModuleLoader:
+    __slots__ = (
+        "_cdata",
+        "_module_data_clb",
+        "_cffi_handle",
+        "_cdata_modules",
+    )
+
+    def __init__(self, cdata) -> None:
+        self._cdata = cdata  # C type: "struct ly_ctx *"
+        self._module_data_clb = None
+        self._cffi_handle = ffi.new_handle(self)
+        self._cdata_modules = []
+
+    def free_module_data(self, cdata) -> None:
+        """
+        Gets the YANG module schema data based requirements from libyang_c_module_imp_clb
+        function and forward that request to user Python based callback function.
+        The returned data from callback function are stored within the context to make sure
+        of no memory access issues. These data a stored until the free_module_data function
+        is called directly by libyang
+
+        :arg cdata:
+            The pointer to YANG modelu schema (c_str), which shall be released from memory
+        """
+        self._cdata_modules.remove(cdata)
+
+    def get_module_data(
+        self,
+        mod_name: Optional[str],
+        mod_rev: Optional[str],
+        submod_name: Optional[str],
+        submod_rev: Optional[str],
+    ) -> Optional[Tuple[str, str]]:
+        """
+        Gets the YANG module schema data based requirements from libyang_c_module_imp_clb
+        function and forward that request to user Python based callback function.
+        The returned data from callback function are stored within the context to make sure
+        of no memory access issues. These data a stored until the free_module_data function
+        is called directly by libyang
+
+        :arg self
+            This instance on context
+        :arg mod_name:
+            The optional YANG module name
+        :arg mod_rev:
+            The optional YANG module revision
+        :arg submod_name:
+            The optional YANG submodule name
+        :arg submod_rev:
+            The optional YANG submodule revision
+
+        :returns:
+            Tuple of format string and YANG (sub)module schema
+        """
+        if self._module_data_clb is None:
+            return "", None
+        fmt_str, module_data = self._module_data_clb(
+            mod_name, mod_rev, submod_name, submod_rev
+        )
+        if module_data is None:
+            return fmt_str, None
+        module_data_c = str2c(module_data)
+        self._cdata_modules.append(module_data_c)
+        return fmt_str, module_data_c
+
+    def set_module_data_clb(
+        self,
+        clb: Optional[
+            Callable[
+                [Optional[str], Optional[str], Optional[str], Optional[str]],
+                Optional[Tuple[str, str]],
+            ]
+        ] = None,
+    ) -> None:
+        """
+        Sets the callback function, which will be called if libyang context would like to
+        load module or submodule, which is not locally available in context path(s).
+
+        :arg self
+            This instance on context
+        :arg clb:
+            The callback function. The expected arguments are:
+                mod_name: Module name
+                mod_rev: Module revision
+                submod_name: Submodule name
+                submod_rev: Submodule revision
+            The expeted return value is either:
+                tuple of:
+                    format: The string format of the loaded data
+                    data: The YANG (sub)module data as string
+                or None in case of error
+        """
+        self._module_data_clb = clb
+        if clb is None:
+            lib.ly_ctx_set_module_imp_clb(self._cdata, ffi.NULL, ffi.NULL)
+        else:
+            lib.ly_ctx_set_module_imp_clb(
+                self._cdata, lib.lypy_module_imp_clb, self._cffi_handle
+            )
+
+
 # -------------------------------------------------------------------------------------
 class Context:
-    __slots__ = ("cdata", "__dict__")
+    __slots__ = (
+        "cdata",
+        "external_module_loader",
+        "__dict__",
+    )
 
     def __init__(
         self,
@@ -37,6 +204,7 @@ def __init__(
     ):
         if cdata is not None:
             self.cdata = ffi.cast("struct ly_ctx *", cdata)
+            self.external_module_loader = ContextExternalModuleLoader(self.cdata)
             return  # already initialized
 
         options = 0
@@ -90,6 +258,7 @@ def __init__(
         )
         if not self.cdata:
             raise self.error("cannot create context")
+        self.external_module_loader = ContextExternalModuleLoader(self.cdata)
 
     def compile_schema(self):
         ret = lib.ly_ctx_compile(self.cdata)

tests/test_context.py

@@ -124,3 +124,26 @@ def test_ctx_disable_searchdirs(self):
         with Context(YANG_DIR, disable_searchdirs=True) as ctx:
             with self.assertRaises(LibyangError):
                 ctx.load_module("yolo-nodetypes")
+
+    def test_ctx_using_clb(self):
+        def get_module_valid_clb(mod_name, *_):
+            YOLO_NODETYPES_MOD_PATH = os.path.join(YANG_DIR, "yolo/yolo-nodetypes.yang")
+            self.assertEqual(mod_name, "yolo-nodetypes")
+            with open(YOLO_NODETYPES_MOD_PATH, encoding="utf-8") as f:
+                mod_str = f.read()
+            return "yang", mod_str
+
+        def get_module_invalid_clb(mod_name, *_):
+            return None
+
+        with Context(YANG_DIR, disable_searchdirs=True) as ctx:
+            with self.assertRaises(LibyangError):
+                ctx.load_module("yolo-nodetypes")
+
+            ctx.external_module_loader.set_module_data_clb(get_module_invalid_clb)
+            with self.assertRaises(LibyangError):
+                mod = ctx.load_module("yolo-nodetypes")
+
+            ctx.external_module_loader.set_module_data_clb(get_module_valid_clb)
+            mod = ctx.load_module("yolo-nodetypes")
+            self.assertIsInstance(mod, Module)