From ae31525dedc750394f2bd8bb70d9218d191f32f0 Mon Sep 17 00:00:00 2001 From: Stefan Gula Date: Mon, 12 Feb 2024 09:31:05 +0100 Subject: [PATCH] Port to libyang 3 Refactor the code to switch to libyang 3: - the cdefs.h file is updated to match the new definitions - the flags used in lyd_new_* are regrouped in a newvaloptions bitmap - the log callback management is reworked - the system-ordered lists / leaf-lists are now ordered by key (hence the unit test change) Fixes: https://github.com/CESNET/libyang-python/pull/105 Link: https://github.com/CESNET/libyang/blob/master/doc/transition_2_3.dox Signed-off-by: Stefan Gula Signed-off-by: Samuel Gauthier --- cffi/cdefs.h | 43 +++++++++++++++++++------------ cffi/source.c | 7 ++--- libyang/context.py | 15 +++++++---- libyang/data.py | 64 +++++++++++++++++++++++++++++++--------------- libyang/log.py | 21 +++++++++------ tests/test_data.py | 16 +++++++----- 6 files changed, 105 insertions(+), 61 deletions(-) diff --git a/cffi/cdefs.h b/cffi/cdefs.h index 304ae55d..e5bcc935 100644 --- a/cffi/cdefs.h +++ b/cffi/cdefs.h @@ -179,10 +179,10 @@ int ly_log_options(int); LY_LOG_LEVEL ly_log_level(LY_LOG_LEVEL); extern "Python" void lypy_log_cb(LY_LOG_LEVEL, const char *, const char *); -void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *), int); -struct ly_err_item *ly_err_first(const struct ly_ctx *); +void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *, const char *, uint64_t)); +const struct ly_err_item *ly_err_first(const struct ly_ctx *); +const struct ly_err_item *ly_err_last(const struct ly_ctx *); void ly_err_clean(struct ly_ctx *, struct ly_err_item *); -LY_VECODE ly_vecode(const struct ly_ctx *); #define LYS_UNKNOWN ... #define LYS_CONTAINER ... @@ -238,14 +238,15 @@ struct lysc_node { struct ly_err_item { LY_LOG_LEVEL level; - LY_ERR no; + LY_ERR err; LY_VECODE vecode; char *msg; - char *path; + char *data_path; + char *schema_path; + uint64_t line; char *apptag; struct ly_err_item *next; struct ly_err_item *prev; - ...; }; struct lyd_node { @@ -261,11 +262,12 @@ struct lyd_node { LY_ERR lys_set_implemented(struct lys_module *, const char **); +#define LYD_NEW_VAL_OUTPUT ... +#define LYD_NEW_VAL_BIN ... +#define LYD_NEW_VAL_CANON ... +#define LYD_NEW_META_CLEAR_DFLT ... #define LYD_NEW_PATH_UPDATE ... -#define LYD_NEW_PATH_OUTPUT ... -#define LYD_NEW_PATH_OPAQ ... -#define LYD_NEW_PATH_BIN_VALUE ... -#define LYD_NEW_PATH_CANON_VALUE ... +#define LYD_NEW_PATH_OPAQ ... LY_ERR lyd_new_path(struct lyd_node *, const struct ly_ctx *, const char *, const char *, uint32_t, struct lyd_node **); LY_ERR lyd_find_xpath(const struct lyd_node *, const char *, struct ly_set **); void lyd_unlink_siblings(struct lyd_node *node); @@ -614,6 +616,7 @@ struct lysp_node_list { }; struct lysc_type { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -641,6 +644,7 @@ struct lysp_type { struct lysp_qname { const char *str; const struct lysp_module *mod; + ...; }; struct lysp_node { @@ -682,7 +686,6 @@ struct lysc_ext { struct lysc_ext_instance *exts; struct lyplg_ext *plugin; struct lys_module *module; - uint32_t refcount; uint16_t flags; }; @@ -703,11 +706,10 @@ typedef enum { LYD_PATH_STD_NO_LAST_PRED } LYD_PATH_TYPE; -LY_ERR lyd_new_term(struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_node **); +LY_ERR lyd_new_term(struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_node **); char* lyd_path(const struct lyd_node *, LYD_PATH_TYPE, char *, size_t); LY_ERR lyd_new_inner(struct lyd_node *, const struct lys_module *, const char *, ly_bool, struct lyd_node **); -LY_ERR lyd_new_list(struct lyd_node *, const struct lys_module *, const char *, ly_bool, struct lyd_node **, ...); -LY_ERR lyd_new_list2(struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_node **); +LY_ERR lyd_new_list(struct lyd_node *, const struct lys_module *, const char *, uint32_t, struct lyd_node **node, ...); struct lyd_node_inner { union { @@ -821,6 +823,7 @@ struct lysp_restr { }; struct lysc_type_num { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -829,6 +832,7 @@ struct lysc_type_num { }; struct lysc_type_dec { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -838,6 +842,7 @@ struct lysc_type_dec { }; struct lysc_type_str { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -859,6 +864,7 @@ struct lysc_type_bitenum_item { }; struct lysc_type_enum { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -867,6 +873,7 @@ struct lysc_type_enum { }; struct lysc_type_bits { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -875,18 +882,19 @@ struct lysc_type_bits { }; struct lysc_type_leafref { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; uint32_t refcount; struct lyxp_expr *path; struct lysc_prefix *prefixes; - const struct lys_module *cur_mod; struct lysc_type *realtype; uint8_t require_instance; }; struct lysc_type_identityref { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -895,6 +903,7 @@ struct lysc_type_identityref { }; struct lysc_type_instanceid { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -903,6 +912,7 @@ struct lysc_type_instanceid { }; struct lysc_type_union { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -911,6 +921,7 @@ struct lysc_type_union { }; struct lysc_type_bin { + const char *name; struct lysc_ext_instance *exts; struct lyplg_type *plugin; LY_DATA_TYPE basetype; @@ -1053,7 +1064,7 @@ LY_ERR lyd_merge_module(struct lyd_node **, const struct lyd_node *, const struc LY_ERR lyd_new_implicit_tree(struct lyd_node *, uint32_t, struct lyd_node **); LY_ERR lyd_new_implicit_all(struct lyd_node **, const struct ly_ctx *, uint32_t, struct lyd_node **); -LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_meta **); +LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_meta **); struct ly_opaq_name { const char *name; diff --git a/cffi/source.c b/cffi/source.c index 2682dd88..b54ba0de 100644 --- a/cffi/source.c +++ b/cffi/source.c @@ -6,9 +6,6 @@ #include #include -#if (LY_VERSION_MAJOR != 2) -#error "This version of libyang bindings only works with libyang 2.x" -#endif -#if (LY_VERSION_MINOR < 37) -#error "Need at least libyang 2.37" +#if (LY_VERSION_MAJOR != 3) +#error "This version of libyang bindings only works with libyang 3.x" #endif diff --git a/libyang/context.py b/libyang/context.py index f598a225..fa5eb5bf 100644 --- a/libyang/context.py +++ b/libyang/context.py @@ -11,8 +11,8 @@ DNode, data_format, data_type, + newval_flags, parser_flags, - path_flags, validation_flags, ) from .schema import Module, SNode, schema_in_format @@ -117,8 +117,12 @@ def error(self, msg: str, *args) -> LibyangError: while err: if err.msg: msg += ": %s" % c2str(err.msg) - if err.path: - msg += ": %s" % c2str(err.path) + if err.data_path: + msg += ": Data path: %s" % c2str(err.data_path) + if err.schema_path: + msg += ": Schema path: %s" % c2str(err.schema_path) + if err.line != 0: + msg += " (line %u)" % err.line err = err.next lib.ly_err_clean(self.cdata, ffi.NULL) @@ -244,7 +248,7 @@ def create_data_path( value = str(value).lower() elif not isinstance(value, str): value = str(value) - flags = path_flags(update=update, rpc_output=rpc_output) + flags = newval_flags(update=update, rpc_output=rpc_output) dnode = ffi.new("struct lyd_node **") ret = lib.lyd_new_path( parent.cdata if parent else ffi.NULL, @@ -256,7 +260,8 @@ def create_data_path( ) dnode = dnode[0] if ret != lib.LY_SUCCESS: - if lib.ly_vecode(self.cdata) != lib.LYVE_SUCCESS: + err = lib.ly_err_last(self.cdata) + if err != ffi.NULL and err.vecode != lib.LYVE_SUCCESS: raise self.error("cannot create data path: %s", path) lib.ly_err_clean(self.cdata, ffi.NULL) if not dnode and not force_return_value: diff --git a/libyang/data.py b/libyang/data.py index 1c1dfe35..2895daaf 100644 --- a/libyang/data.py +++ b/libyang/data.py @@ -77,12 +77,30 @@ def data_format(fmt_string: str) -> int: # ------------------------------------------------------------------------------------- -def path_flags(update: bool = False, rpc_output: bool = False) -> int: +def newval_flags( + rpc_output: bool = False, + bin_value: bool = False, + canon_value: bool = False, + meta_clear_default: bool = False, + update: bool = False, + opaq: bool = False, +) -> int: + """ + Translate from booleans to newvaloptions flags. + """ flags = 0 + if rpc_output: + flags |= lib.LYD_NEW_VAL_OUTPUT + if bin_value: + flags |= lib.LYD_NEW_VAL_BIN + if canon_value: + flags |= lib.LYD_NEW_VAL_CANON + if meta_clear_default: + flags |= lib.LYD_NEW_META_CLEAR_DFLT if update: flags |= lib.LYD_NEW_PATH_UPDATE - if rpc_output: - flags |= lib.LYD_NEW_PATH_OUTPUT + if opaq: + flags |= lib.LYD_NEW_PATH_OPAQ return flags @@ -297,13 +315,14 @@ def meta_free(self, name): item = item.next def new_meta(self, name: str, value: str, clear_dflt: bool = False): + flags = newval_flags(meta_clear_default=clear_dflt) ret = lib.lyd_new_meta( ffi.NULL, self.cdata, ffi.NULL, str2c(name), str2c(value), - clear_dflt, + flags, ffi.NULL, ) if ret != lib.LY_SUCCESS: @@ -364,20 +383,15 @@ def new_path( opt_bin_value: bool = False, opt_canon_value: bool = False, ): - opt = 0 - if opt_update: - opt |= lib.LYD_NEW_PATH_UPDATE - if opt_output: - opt |= lib.LYD_NEW_PATH_OUTPUT - if opt_opaq: - opt |= lib.LYD_NEW_PATH_OPAQ - if opt_bin_value: - opt |= lib.LYD_NEW_PATH_BIN_VALUE - if opt_canon_value: - opt |= lib.LYD_NEW_PATH_CANON_VALUE - + flags = newval_flags( + update=opt_update, + rpc_output=opt_output, + opaq=opt_opaq, + bin_value=opt_bin_value, + canon_value=opt_canon_value, + ) ret = lib.lyd_new_path( - self.cdata, ffi.NULL, str2c(path), str2c(value), opt, ffi.NULL + self.cdata, ffi.NULL, str2c(path), str2c(value), flags, ffi.NULL ) if ret != lib.LY_SUCCESS: raise self.context.error("cannot get module") @@ -1003,7 +1017,10 @@ def _get_path(cdata) -> str: @DNode.register(SNode.CONTAINER) class DContainer(DNode): def create_path( - self, path: str, value: Any = None, rpc_output: bool = False + self, + path: str, + value: Any = None, + rpc_output: bool = False, ) -> Optional[DNode]: return self.context.create_data_path( path, parent=self, value=value, rpc_output=rpc_output @@ -1177,8 +1194,14 @@ def _create_leaf(_parent, module, name, value, in_rpc_output=False): value = str(value) n = ffi.new("struct lyd_node **") + flags = newval_flags(rpc_output=in_rpc_output) ret = lib.lyd_new_term( - _parent, module.cdata, str2c(name), str2c(value), in_rpc_output, n + _parent, + module.cdata, + str2c(name), + str2c(value), + flags, + n, ) if ret != lib.LY_SUCCESS: @@ -1209,11 +1232,12 @@ def _create_container(_parent, module, name, in_rpc_output=False): def _create_list(_parent, module, name, key_values, in_rpc_output=False): n = ffi.new("struct lyd_node **") + flags = newval_flags(rpc_output=in_rpc_output) ret = lib.lyd_new_list( _parent, module.cdata, str2c(name), - in_rpc_output, + flags, n, *[str2c(str(i)) for i in key_values], ) diff --git a/libyang/log.py b/libyang/log.py index 2b241157..b033ccaa 100644 --- a/libyang/log.py +++ b/libyang/log.py @@ -20,13 +20,18 @@ @ffi.def_extern(name="lypy_log_cb") -def libyang_c_logging_callback(level, msg, path): +def libyang_c_logging_callback(level, msg, data_path, schema_path, line): args = [c2str(msg)] - if path: - fmt = "%s: %s" - args.append(c2str(path)) - else: - fmt = "%s" + fmt = "%s" + if data_path: + fmt += ": %s" + args.append(c2str(data_path)) + if schema_path: + fmt += ": %s" + args.append(c2str(schema_path)) + if line != 0: + fmt += " line %u" + args.append(str(line)) LOG.log(LOG_LEVELS.get(level, logging.NOTSET), fmt, *args) @@ -51,10 +56,10 @@ def configure_logging(enable_py_logger: bool, level: int = logging.ERROR) -> Non break if enable_py_logger: lib.ly_log_options(lib.LY_LOLOG | lib.LY_LOSTORE) - lib.ly_set_log_clb(lib.lypy_log_cb, True) + lib.ly_set_log_clb(lib.lypy_log_cb) else: lib.ly_log_options(lib.LY_LOSTORE) - lib.ly_set_log_clb(ffi.NULL, False) + lib.ly_set_log_clb(ffi.NULL) configure_logging(False, logging.ERROR) diff --git a/tests/test_data.py b/tests/test_data.py index 10d9045f..a40056c0 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -132,17 +132,17 @@ def test_data_parse_config_json_without_yang_lib(self): "path": "/CESNET/libyang-python", "enabled": false }, + { + "proto": "http", + "host": "barfoo.com", + "path": "/barfoo/index.html" + }, { "proto": "http", "host": "foobar.com", "port": 8080, "path": "/index.html", "enabled": true - }, - { - "proto": "http", - "host": "barfoo.com", - "path": "/barfoo/index.html" } ], "number": [ @@ -282,7 +282,9 @@ def test_data_parse_config_xml_multi_error(self): self.assertEqual( str(cm.exception), 'failed to parse data tree: Invalid boolean value "abcd".: ' - 'List instance is missing its key "host".', + "Data path: /yolo-system:conf/url[proto='https']/enabled (line 6): " + 'List instance is missing its key "host".: ' + "Data path: /yolo-system:conf/url[proto='https'] (line 7)", ) XML_STATE = """ @@ -808,7 +810,7 @@ def test_data_to_dict_keyless_list(self): foobar.com false - + ftp github.com /CESNET/libyang-python