From 6b2460f2a5d549dfd7030d4c4aad4f1b09827936 Mon Sep 17 00:00:00 2001 From: Steve Date: Fri, 29 Sep 2023 12:42:51 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20xlate=20openconfig-system-ext:track-int?= =?UTF-8?q?erface=20when=20sending=20to=20tes=E2=80=A6=20(#31)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: xlate openconfig-system-ext:track-interface when sending to test devices * feat: remove VTY ACLs when sending configs to test * feat: reformat code * feat: reformat code * Updates to make config truncate more generic * Improvements and bug fixes * Update galaxy.yml * Added truncate config to update playbook * Fixed indentation error * feat: prune harvested configs * Updated config truncate to accept a list of key paths * feat: better format for yaml files * fix: README.md typos * feat: now compare weights at the same hierarchy level * feat: now compare weights at the same hierarchy level * Added openconfig-acl interfaces to truncate --------- Co-authored-by: Jason King --- README.md | 10 +++--- galaxy.yml | 2 +- playbooks/cml_update_lab.yml | 4 +-- playbooks/nso_dump_config.yml | 2 +- playbooks/nso_update_data.yml | 6 ++-- playbooks/show.yml | 8 ++--- plugins/filter/intf.py | 53 +++++++++++++++++++++++++--- plugins/filter/nso_oc.py | 5 ++- plugins/filter/to_even_nicer_yaml.py | 36 +++++++++++++++++++ plugins/modules/mdd_combine.py | 21 +++++++---- roles/nso/tasks/update_yaml_data.yml | 6 ++-- 11 files changed, 122 insertions(+), 31 deletions(-) create mode 100644 plugins/filter/to_even_nicer_yaml.py diff --git a/README.md b/README.md index 08c4790..1fdee49 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Cisco Model Driven Devops Ansible Collection +# Cisco Model-Driven Devops Ansible Collection This Ansible Collection is an implementation of Model-Driven DevOps and includes the following roles: * [oc](https://github.com/model-driven-devops/ansible-mdd/blob/main/roles/oc/README.md) @@ -8,14 +8,14 @@ This Ansible Collection is an implementation of Model-Driven DevOps and includes * [netbox](https://github.com/model-driven-devops/ansible-mdd/blob/main/roles/netbox/README.md) * [nso](https://github.com/model-driven-devops/ansible-mdd/blob/main/roles/nso/README.md) -## Dependancies -### Environmnet Variables +## Dependencies +### Environment Variables If using NetBox: - `NETBOX_API` - `NETBOX_TOKEN` -To use an external NSO (ie. not deployed in CML): -- `NSO_URL` Protocol, FQDN or IP address and port (ex. `http://192.168.1.100:8080`) +To use an external NSO (i.e. not deployed in CML): +- `NSO_URL` Protocol, FQDN or IP address and port (e.g. `http://192.168.1.100:8080`) - `NSO_USERNAME` - `NSO_PASSWORD` diff --git a/galaxy.yml b/galaxy.yml index 5a8e659..d79abe0 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -8,7 +8,7 @@ namespace: ciscops name: mdd # The version of the collection. Must be compatible with semantic versioning -version: 1.2.7 +version: 1.2.8 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection readme: README.md diff --git a/playbooks/cml_update_lab.yml b/playbooks/cml_update_lab.yml index c5f7bee..4640a7a 100644 --- a/playbooks/cml_update_lab.yml +++ b/playbooks/cml_update_lab.yml @@ -93,13 +93,13 @@ - name: Create topology file copy: - content: "{{ topology | to_nice_yaml(indent=2,sort_keys=False) }}" + content: "{{ topology | ciscops.mdd.to_even_nicer_yaml }}" dest: "{{ lookup('env', 'PWD') }}/files/cml_lab.yaml" run_once: yes no_log: yes - name: Create mapping file copy: - content: "{{ {'all': {'hosts': results.mappings}} | to_nice_yaml(indent=2,sort_keys=False) }}" + content: "{{ {'all': {'hosts': results.mappings}} | ciscops.mdd.to_even_nicer_yaml }}" dest: "{{ lookup('env', 'PWD') }}/{{ inventory_dir }}/cml_intf_map.yml" run_once: yes diff --git a/playbooks/nso_dump_config.yml b/playbooks/nso_dump_config.yml index 476894c..c144029 100644 --- a/playbooks/nso_dump_config.yml +++ b/playbooks/nso_dump_config.yml @@ -20,5 +20,5 @@ nso_device_config: "{ 'mdd_data': { 'config': {{ nso_device_config['config'] }} } }" - copy: - content: "{{ nso_device_config | to_nice_yaml(indent=2,sort_keys=False) }}" + content: "{{ nso_device_config | ciscops.mdd.to_even_nicer_yaml }}" dest: "{{ current_dir }}/device_config_data/{{ inventory_hostname }}.yml" diff --git a/playbooks/nso_update_data.yml b/playbooks/nso_update_data.yml index 1ce717b..c4011e3 100644 --- a/playbooks/nso_update_data.yml +++ b/playbooks/nso_update_data.yml @@ -17,10 +17,10 @@ when: ('oc' in mdd_data_types) throttle: "{{ workers }}" block: - - name: Translate and truncate interface names + - name: Translate/truncate interface names and truncate config set_fact: - mdd_data: "{{ mdd_data | ciscops.mdd.intf_xform(cml_intf_xlate) }}" - when: (cml_group is defined and cml_group in group_names) and (cml_intf_xlate is defined and cml_intf_xlate) + mdd_data: "{{ mdd_data | ciscops.mdd.config_xform(cml_intf_xlate | default(None), cml_truncate_list | default(None)) }}" + when: (cml_intf_xlate is defined and cml_intf_xlate) or (cml_truncate_list is defined and cml_truncate_list) - name: Update MDD Data ansible.builtin.include_role: diff --git a/playbooks/show.yml b/playbooks/show.yml index 69b1b0a..a65e7dd 100644 --- a/playbooks/show.yml +++ b/playbooks/show.yml @@ -4,10 +4,10 @@ roles: - ciscops.mdd.data tasks: - - name: Translate and truncate interface names + - name: Translate/truncate interface names and truncate config set_fact: - mdd_data: "{{ mdd_data | ciscops.mdd.intf_xform(cml_intf_xlate) }}" - when: (cml_group is defined and cml_group in group_names) and (cml_intf_xlate is defined and cml_intf_xlate) - + mdd_data: "{{ mdd_data | ciscops.mdd.config_xform(cml_intf_xlate | default(None), cml_truncate_list | default(None)) }}" + when: (cml_intf_xlate is defined and cml_intf_xlate) or (cml_truncate_list is defined and cml_truncate_list) + - debug: var: mdd_data \ No newline at end of file diff --git a/plugins/filter/intf.py b/plugins/filter/intf.py index f178753..4808287 100644 --- a/plugins/filter/intf.py +++ b/plugins/filter/intf.py @@ -15,7 +15,8 @@ "openconfig-system-ext:ssh-source-interface", "openconfig-network-instance:interface-id", "openconfig-network-instance:index", - "openconfig-network-instance:local-address" + "openconfig-network-instance:local-address", + "openconfig-system-ext:track-interface" ] @@ -51,20 +52,26 @@ def xlate_value(data, intf_dict): return -def intf_xlate(data, intf_dict): +def intf_xlate(data, intf_dict=None): if not data: return {} + if intf_dict is None: + return data + data_out = data.copy() xlate_value(data_out, intf_dict) return data_out -def intf_truncate(data, intf_dict): +def intf_truncate(data, intf_dict=None): if not data: return {} + if intf_dict is None: + return data + regex_list = intf_dict.keys() temp_interface_list = [] temp_stp_interface_list = [] @@ -145,12 +152,48 @@ def intf_truncate(data, intf_dict): [instance_index]["openconfig-network-instance:mpls"]["openconfig-network-instance:global"] ["openconfig-network-instance:interface-attributes"]["openconfig-network-instance:interface"]) = temp_mpls_interface_list + # Truncate openconfig-acl interfaces + if "openconfig-acl:acl" in oc_data and "openconfig-acl:interfaces" in oc_data["openconfig-acl:acl"]: + temp_acl_interface_list = [] + for interface in oc_data["openconfig-acl:acl"]["openconfig-acl:interfaces"]["openconfig-acl:interface"]: + if found_full_match(interface["openconfig-acl:id"].split(".")[0], intf_dict): + temp_acl_interface_list.append(interface) + + data_out["mdd:openconfig"]["openconfig-acl:acl"]["openconfig-acl:interfaces"]["openconfig-acl:interface"] = temp_acl_interface_list + + return data_out + + +def delete_key(data, key_list): + if isinstance(data, dict): + if key_list[0] in list(data): + if len(key_list) == 1: + del data[key_list[0]] + else: + key = key_list.pop(0) + delete_key(data[key], key_list) + + +def config_truncate(data, truncate_list=None): + """Find all values from a nested dictionary for a given key.""" + + if not data: + return {} + + if truncate_list is None: + return data + + data_out = data.copy() + + for path in truncate_list: + delete_key(data_out, path) return data_out -def intf_xform(data, intf_dict): +def config_xform(data, intf_dict=None, truncate_list=None): data = intf_truncate(data, intf_dict) data = intf_xlate(data, intf_dict) + data = config_truncate(data, truncate_list) return data @@ -160,5 +203,5 @@ def filters(self): return { 'intf_xlate': intf_xlate, 'intf_truncate': intf_truncate, - 'intf_xform': intf_xform + 'config_xform': config_xform } diff --git a/plugins/filter/nso_oc.py b/plugins/filter/nso_oc.py index 43d0b10..ce176f6 100644 --- a/plugins/filter/nso_oc.py +++ b/plugins/filter/nso_oc.py @@ -26,7 +26,10 @@ def nso_oc(config_data): } native_dict = copy.deepcopy(config_data) translation_notes = [] - main_xe.build_xe_to_oc(config_data, native_dict, oc_dict, translation_notes) + oc_dict_clean = main_xe.build_xe_to_oc(config_data, native_dict, oc_dict, translation_notes) + + if oc_dict_clean: + oc_dict = oc_dict_clean mdd_dict['mdd_data'] = { "mdd:openconfig": oc_dict['mdd:openconfig'], diff --git a/plugins/filter/to_even_nicer_yaml.py b/plugins/filter/to_even_nicer_yaml.py new file mode 100644 index 0000000..68c3bf3 --- /dev/null +++ b/plugins/filter/to_even_nicer_yaml.py @@ -0,0 +1,36 @@ +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +from ansible.utils.unsafe_proxy import AnsibleUnsafeText +import yaml + + +class MyDumper(yaml.Dumper): + def increase_indent(self, flow=False, indentless=False): + return super(MyDumper, self).increase_indent(flow, False) + + +def convert_ansible_unsafe_text_to_safe(data): + if isinstance(data, dict): + return {convert_ansible_unsafe_text_to_safe(k): convert_ansible_unsafe_text_to_safe(v) for k, v in data.items()} + elif isinstance(data, list): + return [convert_ansible_unsafe_text_to_safe(item) for item in data] + elif isinstance(data, AnsibleUnsafeText): + return str(data) + else: + return data + + +def to_even_nicer_yaml(config_data): + ansible_safe_config_data = convert_ansible_unsafe_text_to_safe(config_data) + return yaml.dump(ansible_safe_config_data, Dumper=MyDumper, default_flow_style=False, explicit_start=True, + sort_keys=False) + + +class FilterModule(object): + + def filters(self): + return { + 'to_even_nicer_yaml': to_even_nicer_yaml + } diff --git a/plugins/modules/mdd_combine.py b/plugins/modules/mdd_combine.py index fcf9198..f3bfc18 100644 --- a/plugins/modules/mdd_combine.py +++ b/plugins/modules/mdd_combine.py @@ -175,18 +175,27 @@ def _merge(result_cfgs, v, path=None, filepath=None, hierarchy_level=None, playb else: result_cfgs[k] = _merge({}, v, path + [str(k)], filepath, hierarchy_level, playbook_tags, weight) else: - # if key not there, add + # if key not there, add it if k not in result_cfgs: result_cfgs[k] = (v, filepath, playbook_tags, hierarchy_level, weight) - # if key found multiple places at same hierarchy level, error - elif k in result_cfgs and hierarchy_level == result_cfgs[k][3] and result_cfgs[k][0]: + # if key found multiple places at same hierarchy level and the new key's weight is higher, go with the highest weight. + elif k in result_cfgs and hierarchy_level == result_cfgs[k][3] and result_cfgs[k][0] and weight > \ + result_cfgs[k][4]: + result_cfgs[k] = (v, filepath, playbook_tags, hierarchy_level, weight) + # if key found multiple places at same hierarchy level and the new key's weight is lower, skip. + elif k in result_cfgs and hierarchy_level == result_cfgs[k][3] and result_cfgs[k][0] and weight < \ + result_cfgs[k][4]: + continue + # if key found multiple places at same hierarchy level and if the weight is the same, error. + elif k in result_cfgs and hierarchy_level == result_cfgs[k][3] and result_cfgs[k][0] and weight == \ + result_cfgs[k][4]: if filepath == result_cfgs[k][1]: module.fail_json( - msg="Merge Error: key {1} was found multiple times at the same hierarchy level (level: {2}) in file {3}.".format( + msg="Merge Error: key {0} was found multiple times at the same hierarchy level (level: {1}) in file {2}.".format( k, result_cfgs[k][3], filepath)) else: module.fail_json( - msg="Merge Error: key {1} was found multiple times at the same hierarchy level (level: {2}) in files {3} and {4}.".format( + msg="Merge Error: key {0} was found multiple times at the same hierarchy level (level: {1}) in files {2} and {3}.".format( k, result_cfgs[k][3], filepath, result_cfgs[k][1])) module.exit_json(changed=False, failed=True) # if key exists but weight is higher, replace @@ -378,7 +387,7 @@ def find_and_read_configs(top_dir, device_name, filespec_list, default_weight, t ) except yaml.YAMLError: module.fail_json( - msg="An error occurred loading file {1}".format(os.path.join(current_dir, filename))) + msg="An error occurred loading file {0}".format(os.path.join(current_dir, filename))) module.exit_json(changed=False, failed=True) hierarchy_level += 1 diff --git a/roles/nso/tasks/update_yaml_data.yml b/roles/nso/tasks/update_yaml_data.yml index 93ffe70..f243b74 100644 --- a/roles/nso/tasks/update_yaml_data.yml +++ b/roles/nso/tasks/update_yaml_data.yml @@ -4,7 +4,7 @@ - name: Write Config Data to Device Directory copy: - content: "---\n{{ data | to_nice_yaml(indent=2,sort_keys=False) }}" + content: "{{ data | ciscops.mdd.to_even_nicer_yaml }}" dest: "{{ mdd_device_dir }}/config-data.yml" vars: data: "{{ { 'mdd_data': { 'config': nso_device_config['config'] } } }}" @@ -17,7 +17,7 @@ - name: Write OC Data to Device Directory copy: - content: "---\n{{ data | to_nice_yaml(indent=2,sort_keys=False) }}" + content: "{{ data | ciscops.mdd.to_even_nicer_yaml }}" dest: "{{ mdd_device_dir }}/oc-{{ (item.key | ansible.builtin.split(':'))[1] }}.yml" with_dict: "{{ oc_data['mdd_data']['mdd:openconfig'] }}" vars: @@ -25,7 +25,7 @@ - name: Write Native Data to Device Directory copy: - content: "---\n{{ data | to_nice_yaml(indent=2,sort_keys=False) }}" + content: "{{ data | ciscops.mdd.to_even_nicer_yaml }}" dest: "{{ mdd_device_dir }}/config-remaining.yml" vars: data: "{{ { 'mdd_data': { 'config': oc_data['mdd_data']['config'] } } }}"