From c6933dda4d138e9f9e2c2decd7ef7576e82d0479 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Wed, 10 Jan 2024 22:17:12 -0700
Subject: [PATCH 01/62] [module_base] Refactor

- Allow for managing root_elt, elements
- Add _find_this_element_index
- Handle no existing elements
- Implement _create_target and _find_target for node
---
 plugins/module_utils/module_base.py | 45 ++++++++++++++++++++++++++---
 1 file changed, 41 insertions(+), 4 deletions(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 996ec9ae..5e903ea9 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -21,7 +21,7 @@ def get_argument_spec():
     ##############################
     # init
     #
-    def __init__(self, module, pfsense=None):
+    def __init__(self, module, pfsense=None, root=None, node=None, create_node=False):
         if pfsense is None:
             pfsense = PFSenseModule(module)
         self.module = module    # ansible module
@@ -31,9 +31,25 @@ def __init__(self, module, pfsense=None):
         self.pfsense = pfsense  # helper module
         self.apply = True       # apply configuration at the end
 
+        # xml parent of target_elt, node named by root
+        if root is not None:
+            if root == 'pfsense':
+                self.root_elt = self.pfsense.root
+            else:
+                self.root_elt = self.pfsense.get_element(root, create_node=create_node)
+        else:
+            self.root_elt = None
+        self.root = root
+
+        # List of elements named node
+        if node is not None:
+            self.elements = self.pfsense.get_elements(node)
+        else:
+            self.elememts = None
+        self.node = node
+
         self.obj = None         # dict holding target pfsense parameters
         self.target_elt = None  # xml object holding target pfsense parameters
-        self.root_elt = None    # xml parent of target_elt
 
         self.change_descr = ''
 
@@ -136,11 +152,32 @@ def _copy_and_update_target(self):
 
     def _create_target(self):
         """ create the XML target_elt """
-        raise NotImplementedError()
+        if self.node is not None:
+            return self.pfsense.new_element(self.node)
+        else:
+            raise NotImplementedError()
+
+    def _find_this_element_index(self):
+        return self.elements.index(self.target_elt)
+
+    def _find_last_element_index(self):
+        if len(self.elements):
+            return list(self.root_elt).index(self.elements[len(self.elements) - 1])
+        else:
+            return len(list(self.root_elt))
 
     def _find_target(self):
         """ find the XML target_elt """
-        raise NotImplementedError()
+        if self.node is not None:
+            result = self.root_elt.findall("{node}[descr='{descr}']".format(node=self.node, descr=self.obj['descr']))
+            if len(result) == 1:
+                return result[0]
+            elif len(result) > 1:
+                self.module.fail_json(msg='Found multiple {node}s for descr {descr}.'.format(node=self.node, descr=self.obj['descr']))
+            else:
+                return None
+        else:
+            raise NotImplementedError()
 
     @staticmethod
     def _get_params_to_remove():

From 5cf4c74b4aee2ed8d716129093e507a3e3f9b3a1 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Wed, 10 Jan 2024 22:17:50 -0700
Subject: [PATCH 02/62] [pfsense_cert] Use expanded PFSenseModuleBase

---
 plugins/modules/pfsense_cert.py | 29 +++--------------------------
 1 file changed, 3 insertions(+), 26 deletions(-)

diff --git a/plugins/modules/pfsense_cert.py b/plugins/modules/pfsense_cert.py
index 0dca97cc..bd37c837 100644
--- a/plugins/modules/pfsense_cert.py
+++ b/plugins/modules/pfsense_cert.py
@@ -207,10 +207,8 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None):
-        super(PFSenseCertModule, self).__init__(module, pfsense)
+        super(PFSenseCertModule, self).__init__(module, pfsense, root='pfsense', node='cert')
         self.name = "pfsense_cert"
-        self.root_elt = self.pfsense.root
-        self.certs = self.pfsense.get_elements('cert')
 
     ##############################
     # params processing
@@ -275,21 +273,6 @@ def _params_to_obj(self):
     ##############################
     # XML processing
     #
-    def _find_target(self):
-        result = self.root_elt.findall("cert[descr='{0}']".format(self.obj['descr']))
-        if len(result) == 1:
-            return result[0]
-        elif len(result) > 1:
-            self.module.fail_json(msg='Found multiple certificates for descr {0}.'.format(self.obj['descr']))
-        else:
-            return None
-
-    def _find_this_cert_index(self):
-        return self.certs.index(self.target_elt)
-
-    def _find_last_cert_index(self):
-        return list(self.root_elt).index(self.certs[len(self.certs) - 1])
-
     def _find_ca(self, caref):
         result = self.root_elt.findall("ca[descr='{0}']".format(caref))
         if len(result) == 1:
@@ -305,10 +288,6 @@ def _find_ca(self, caref):
             else:
                 return None
 
-    def _create_target(self):
-        """ create the XML target_elt """
-        return self.pfsense.new_element('cert')
-
     def _copy_and_add_target(self):
         """ populate the XML target_elt """
         obj = self.obj
@@ -316,9 +295,7 @@ def _copy_and_add_target(self):
         obj['refid'] = self.pfsense.uniqid()
         self.diff['after'] = obj
         self.pfsense.copy_dict_to_element(self.obj, self.target_elt)
-        self.root_elt.insert(self._find_last_cert_index(), self.target_elt)
-        # Reset certs list
-        self.certs = self.root_elt.findall('cert')
+        self.root_elt.insert(self._find_last_element_index(), self.target_elt)
 
     def _copy_and_update_target(self):
         """ update the XML target_elt """
@@ -439,7 +416,7 @@ def _pre_remove_target_elt(self):
         self.diff['after'] = {}
         if self.target_elt is not None:
             self.diff['before'] = self.pfsense.element_to_dict(self.target_elt)
-            self.certs.remove(self.target_elt)
+            self.elements.remove(self.target_elt)
         else:
             self.diff['before'] = {}
 

From 4a67cda9bb830f84ca933cc4b05652641e2efda6 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Thu, 11 Jan 2024 11:13:30 -0700
Subject: [PATCH 03/62] [pfsense_ca] Use expanded PFSenseModuleBase

---
 plugins/modules/pfsense_ca.py | 28 ++++------------------------
 1 file changed, 4 insertions(+), 24 deletions(-)

diff --git a/plugins/modules/pfsense_ca.py b/plugins/modules/pfsense_ca.py
index 9cdd1611..6795593f 100644
--- a/plugins/modules/pfsense_ca.py
+++ b/plugins/modules/pfsense_ca.py
@@ -122,10 +122,8 @@ def get_argument_spec():
         return PFSENSE_CA_ARGUMENT_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSenseCAModule, self).__init__(module, pfsense)
+        super(PFSenseCAModule, self).__init__(module, pfsense, root='pfsense', node='ca')
         self.name = "pfsense_ca"
-        self.root_elt = self.pfsense.root
-        self.cas = self.pfsense.get_elements('ca')
         self.refresh_crls = False
         self.crl = None
 
@@ -186,24 +184,6 @@ def _params_to_obj(self):
     ##############################
     # XML processing
     #
-    def _find_target(self):
-        result = self.root_elt.findall("ca[descr='{0}']".format(self.obj['descr']))
-        if len(result) == 1:
-            return result[0]
-        elif len(result) > 1:
-            self.module.fail_json(msg='Found multiple certificate authorities for name {0}.'.format(self.obj['descr']))
-        else:
-            return None
-
-    def _find_this_ca_index(self):
-        return self.cas.index(self.target_elt)
-
-    def _find_last_ca_index(self):
-        if len(self.cas):
-            return list(self.root_elt).index(self.cas[len(self.cas) - 1])
-        else:
-            return len(list(self.root_elt))
-
     def _find_crl_for_ca(self, caref):
         result = self.root_elt.findall("crl[caref='{0}']".format(caref))
         if len(result) == 1:
@@ -247,7 +227,7 @@ def _copy_and_add_target(self):
         """ populate the XML target_elt """
         self.pfsense.copy_dict_to_element(self.obj, self.target_elt)
         self.diff['after'] = self.pfsense.element_to_dict(self.target_elt)
-        self.root_elt.insert(self._find_last_ca_index(), self.target_elt)
+        self.root_elt.insert(self._find_last_element_index(), self.target_elt)
         if self.crl is not None:
             crl_elt = self.pfsense.new_element('crl')
             self.crl['caref'] = self.obj['refid']
@@ -283,7 +263,7 @@ def _copy_and_update_target(self):
                     self.crl['refid'] = self.pfsense.uniqid()
                 self.pfsense.copy_dict_to_element(self.crl, crl_elt)
                 # Add after the existing ca entry
-                self.pfsense.root.insert(self._find_this_ca_index() + 1, crl_elt)
+                self.pfsense.root.insert(self._find_this_element_index() + 1, crl_elt)
                 self.refresh_crls = True
             else:
                 before['crl'] = crl_elt.find('text').text
@@ -355,7 +335,7 @@ def _pre_remove_target_elt(self):
         if self.target_elt is not None:
             self.diff['before'] = self.pfsense.element_to_dict(self.target_elt)
             crl_elt = self._find_crl_for_ca(self.target_elt.find('refid').text)
-            self.cas.remove(self.target_elt)
+            self.elements.remove(self.target_elt)
             if crl_elt is not None:
                 self.diff['before']['crl'] = crl_elt.find('text').text
                 self.root_elt.remove(crl_elt)

From d7b929c3983402b02409962c78ca9c7b1375dffe Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Thu, 11 Jan 2024 12:17:42 -0700
Subject: [PATCH 04/62] [module_base] Allow setting root_is_exclusive; self.obj
 default to {}; _find_target only for root_is_exclusive == False

---
 plugins/module_utils/module_base.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 5e903ea9..6b2f35ea 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -21,7 +21,7 @@ def get_argument_spec():
     ##############################
     # init
     #
-    def __init__(self, module, pfsense=None, root=None, node=None, create_node=False):
+    def __init__(self, module, pfsense=None, root=None, root_is_exclusive=True, create_root=False, node=None):
         if pfsense is None:
             pfsense = PFSenseModule(module)
         self.module = module    # ansible module
@@ -35,10 +35,16 @@ def __init__(self, module, pfsense=None, root=None, node=None, create_node=False
         if root is not None:
             if root == 'pfsense':
                 self.root_elt = self.pfsense.root
+                self.root_is_exclusive = False
             else:
-                self.root_elt = self.pfsense.get_element(root, create_node=create_node)
+                self.root_elt = self.pfsense.get_element(root, create_node=create_root)
+                if root in ['system']:
+                    self.root_is_exclusive = False
+                else:
+                    self.root_is_exclusive = root_is_exclusive
         else:
             self.root_elt = None
+            self.root_is_exclusive = None
         self.root = root
 
         # List of elements named node
@@ -48,7 +54,7 @@ def __init__(self, module, pfsense=None, root=None, node=None, create_node=False
             self.elememts = None
         self.node = node
 
-        self.obj = None         # dict holding target pfsense parameters
+        self.obj = dict()       # dict holding target pfsense parameters
         self.target_elt = None  # xml object holding target pfsense parameters
 
         self.change_descr = ''
@@ -168,7 +174,7 @@ def _find_last_element_index(self):
 
     def _find_target(self):
         """ find the XML target_elt """
-        if self.node is not None:
+        if self.root_is_exclusive is False and self.node is not None:
             result = self.root_elt.findall("{node}[descr='{descr}']".format(node=self.node, descr=self.obj['descr']))
             if len(result) == 1:
                 return result[0]

From fe33ad8e22beee99ca85d6ba07ab9cd648cdef05 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Thu, 11 Jan 2024 12:19:13 -0700
Subject: [PATCH 05/62] [pfsense_alias] Use expanded PFSenseModuleBase

---
 plugins/module_utils/alias.py                 | 36 ++-----------------
 .../plugins/modules/test_pfsense_alias.py     |  6 ++--
 .../modules/test_pfsense_alias_null.py        |  2 +-
 3 files changed, 6 insertions(+), 38 deletions(-)

diff --git a/plugins/module_utils/alias.py b/plugins/module_utils/alias.py
index d9d2f66d..960904d8 100644
--- a/plugins/module_utils/alias.py
+++ b/plugins/module_utils/alias.py
@@ -37,10 +37,8 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None):
-        super(PFSenseAliasModule, self).__init__(module, pfsense)
+        super(PFSenseAliasModule, self).__init__(module, pfsense, root='aliases', node='alias')
         self.name = "pfsense_alias"
-        self.root_elt = self.pfsense.get_element('aliases')
-        self.obj = dict()
 
     ##############################
     # params processing
@@ -103,43 +101,13 @@ def _validate_params(self):
     ##############################
     # XML processing
     #
-    def _copy_and_update_target(self):
-        """ update the XML target_elt """
-        before = self.pfsense.element_to_dict(self.target_elt)
-        changed = self.pfsense.copy_dict_to_element(self.obj, self.target_elt)
-        if self._remove_deleted_params():
-            changed = True
-
-        self.diff['before'] = before
-        if changed:
-            self.diff['after'] = self.pfsense.element_to_dict(self.target_elt)
-            self.result['changed'] = True
-        else:
-            self.diff['after'] = self.obj
-
-        return (before, changed)
-
-    def _create_target(self):
-        """ create the XML target_elt """
-        target_elt = self.pfsense.new_element('alias')
-        self.diff['before'] = ''
-        self.diff['after'] = self.obj
-        self.result['changed'] = True
-        return target_elt
-
     def _find_target(self):
         """ find the XML target_elt """
-        return self.pfsense.find_alias(self.obj['name'])
+        return self.pfsense.find_alias(self.obj['name'], aliastype=self.obj.get('type'))
 
     ##############################
     # run
     #
-    def _remove(self):
-        """ delete obj """
-        self.diff['after'] = ''
-        self.diff['before'] = ''
-        super(PFSenseAliasModule, self)._remove()
-
     def _update(self):
         """ make the target pfsense reload """
         return self.pfsense.phpshell('''require_once("filter.inc");
diff --git a/tests/unit/plugins/modules/test_pfsense_alias.py b/tests/unit/plugins/modules/test_pfsense_alias.py
index fe1e2b28..ed8b88a6 100644
--- a/tests/unit/plugins/modules/test_pfsense_alias.py
+++ b/tests/unit/plugins/modules/test_pfsense_alias.py
@@ -38,7 +38,7 @@ def do_alias_creation_test(self, alias, failed=False, msg='', command=None):
         result = self.execute_module(changed=True, failed=failed, msg=msg)
 
         if not failed:
-            diff = dict(before='', after=alias)
+            diff = dict(before={}, after=alias)
             self.assertEqual(result['diff'], diff)
             self.assert_xml_elt_dict('aliases', dict(name=alias['name'], type=alias['type']), diff['after'])
             self.assertEqual(result['commands'], [command])
@@ -50,7 +50,7 @@ def do_alias_deletion_test(self, alias, command=None):
         set_module_args(self.args_from_var(alias, 'absent'))
         result = self.execute_module(changed=True)
 
-        diff = dict(before=alias, after='')
+        diff = dict(before=alias, after={})
         self.assertEqual(result['diff'], diff)
         self.assert_has_xml_tag('aliases', dict(name=alias['name'], type=alias['type']), absent=True)
         self.assertEqual(result['commands'], [command])
@@ -277,7 +277,7 @@ def test_delete_inexistent_alias(self):
         set_module_args(self.args_from_var(alias, 'absent'))
         result = self.execute_module(changed=False)
 
-        diff = dict(before='', after='')
+        diff = dict(before={}, after={})
         self.assertEqual(result['diff'], diff)
         self.assertEqual(result['commands'], [])
 
diff --git a/tests/unit/plugins/modules/test_pfsense_alias_null.py b/tests/unit/plugins/modules/test_pfsense_alias_null.py
index 5be91a7b..ad956b79 100644
--- a/tests/unit/plugins/modules/test_pfsense_alias_null.py
+++ b/tests/unit/plugins/modules/test_pfsense_alias_null.py
@@ -38,7 +38,7 @@ def do_alias_creation_test(self, alias, failed=False, msg='', command=None):
         result = self.execute_module(changed=True, failed=failed, msg=msg)
 
         if not failed:
-            diff = dict(before='', after=alias)
+            diff = dict(before={}, after=alias)
             self.assertEqual(result['diff'], diff)
             self.assert_xml_elt_dict('aliases', dict(name=alias['name'], type=alias['type']), diff['after'])
             self.assertEqual(result['commands'], [command])

From dbc449d90b0df73a48d6181c1cc3e083a3c5c752 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Thu, 11 Jan 2024 12:45:29 -0700
Subject: [PATCH 06/62] [module_base] Add key, root_is_exclusive defaults to
 True; root_is_exclusive affects _copy_and_add_target(); _get_obj_name() uses
 key

---
 plugins/module_utils/module_base.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 6b2f35ea..da91eea2 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -21,7 +21,7 @@ def get_argument_spec():
     ##############################
     # init
     #
-    def __init__(self, module, pfsense=None, root=None, root_is_exclusive=True, create_root=False, node=None):
+    def __init__(self, module, pfsense=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr'):
         if pfsense is None:
             pfsense = PFSenseModule(module)
         self.module = module    # ansible module
@@ -44,16 +44,17 @@ def __init__(self, module, pfsense=None, root=None, root_is_exclusive=True, crea
                     self.root_is_exclusive = root_is_exclusive
         else:
             self.root_elt = None
-            self.root_is_exclusive = None
+            self.root_is_exclusive = root_is_exclusive
         self.root = root
 
         # List of elements named node
         if node is not None:
             self.elements = self.pfsense.get_elements(node)
         else:
-            self.elememts = None
+            self.elements = None
         self.node = node
 
+        self.key = key          # item that identifies a target element
         self.obj = dict()       # dict holding target pfsense parameters
         self.target_elt = None  # xml object holding target pfsense parameters
 
@@ -143,7 +144,10 @@ def _copy_and_add_target(self):
         """ create the XML target_elt """
         self.pfsense.copy_dict_to_element(self.obj, self.target_elt)
         self.diff['after'] = self.obj
-        self.root_elt.append(self.target_elt)
+        if self.root_is_exclusive:
+            self.root_elt.append(self.target_elt)
+        else:
+            self.root_elt.insert(self._find_last_element_index(), self.target_elt)
 
     def _copy_and_update_target(self):
         """ update the XML target_elt """
@@ -174,12 +178,12 @@ def _find_last_element_index(self):
 
     def _find_target(self):
         """ find the XML target_elt """
-        if self.root_is_exclusive is False and self.node is not None:
-            result = self.root_elt.findall("{node}[descr='{descr}']".format(node=self.node, descr=self.obj['descr']))
+        if self.node is not None:
+            result = self.root_elt.findall("{node}[{key}='{value}']".format(node=self.node, key=self.key, value=self.obj[self.key]))
             if len(result) == 1:
                 return result[0]
             elif len(result) > 1:
-                self.module.fail_json(msg='Found multiple {node}s for descr {descr}.'.format(node=self.node, descr=self.obj['descr']))
+                self.module.fail_json(msg='Found multiple {node}s for {key} {value}.'.format(node=self.node, key=self.key, value=self.obj[self.key]))
             else:
                 return None
         else:
@@ -319,7 +323,7 @@ def _log_update(self, before):
 
     def _get_obj_name(self):
         """ return obj's name """
-        raise NotImplementedError()
+        return "'{0}'".format(self.obj[self.key])
 
     def _get_module_name(self, strip=False):
         """ return ansible module's name """

From d7ca82f879fbd40781d32f3eeaa4a019158dfaea Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Thu, 11 Jan 2024 12:46:20 -0700
Subject: [PATCH 07/62] [pfsense_interface_group] Use expanded
 PFSenseModuleBase

---
 plugins/module_utils/interface_group.py | 25 +------------------------
 1 file changed, 1 insertion(+), 24 deletions(-)

diff --git a/plugins/module_utils/interface_group.py b/plugins/module_utils/interface_group.py
index 76457e24..9cf9f255 100644
--- a/plugins/module_utils/interface_group.py
+++ b/plugins/module_utils/interface_group.py
@@ -38,11 +38,8 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None):
-        super(PFSenseInterfaceGroupModule, self).__init__(module, pfsense)
+        super(PFSenseInterfaceGroupModule, self).__init__(module, pfsense, root='ifgroups', node='ifgroupentry', key='ifname')
         self.name = "pfsense_interface_group"
-        self.obj = dict()
-
-        self.root_elt = self.pfsense.get_element('ifgroups')
 
     ##############################
     # params processing
@@ -90,26 +87,6 @@ def _validate_params(self):
     ##############################
     # XML processing
     #
-    def _create_target(self):
-        """ create the XML target_elt """
-        self.diff['before'] = ''
-        self.diff['after'] = self.obj
-        return self.pfsense.new_element('ifgroupentry')
-
-    def _find_target(self):
-        """ find the XML target_elt """
-        result = self.root_elt.findall("ifgroupentry[ifname='{0}']".format(self.obj['ifname']))
-        if len(result) == 1:
-            return result[0]
-        elif len(result) > 1:
-            self.module.fail_json(msg='Found multiple interface groups for name {0}.'.format(self.obj['ifname']))
-        else:
-            return None
-
-    def _pre_remove_target_elt(self):
-        """ processing before removing elt """
-        self.diff['before'] = self.pfsense.element_to_dict(self.target_elt)
-
     def _remove_all_rules(self, interface):
         """ delete all interface rules """
 

From f9185d9792c3a7b18371bd6e07a3acd186540964 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Thu, 11 Jan 2024 18:54:09 -0700
Subject: [PATCH 08/62] [pfsense_default_gateway] Use PFSenseModuleBase root

---
 plugins/module_utils/default_gateway.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/plugins/module_utils/default_gateway.py b/plugins/module_utils/default_gateway.py
index eeb7e58f..4fa4d3cf 100644
--- a/plugins/module_utils/default_gateway.py
+++ b/plugins/module_utils/default_gateway.py
@@ -29,11 +29,9 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None):
-        super(PFSenseDefaultGatewayModule, self).__init__(module, pfsense)
+        super(PFSenseDefaultGatewayModule, self).__init__(module, pfsense, root='gateways')
         self.name = "pfsense_default_gateway"
-        self.root_elt = self.pfsense.get_element('gateways')
         self.target_elt = self.root_elt
-        self.obj = dict()
         self.interface_elt = None
         self.read_only = False
 

From bb235e163cae39a210b35b44c245d4691800dc2d Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Thu, 11 Jan 2024 19:07:12 -0700
Subject: [PATCH 09/62] [pfsense_gateway] Use expanded PFSenseModuleBase

---
 plugins/module_utils/gateway.py | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/plugins/module_utils/gateway.py b/plugins/module_utils/gateway.py
index 922ac6a8..0761970c 100644
--- a/plugins/module_utils/gateway.py
+++ b/plugins/module_utils/gateway.py
@@ -41,17 +41,11 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None):
-        super(PFSenseGatewayModule, self).__init__(module, pfsense)
+        super(PFSenseGatewayModule, self).__init__(module, pfsense, root='gateways', create_root=True, node='gateway_item', key='name')
         self.name = "pfsense_gateway"
-        self.root_elt = self.pfsense.get_element('gateways')
-        self.obj = dict()
         self.interface_elt = None
         self.dynamic = False
 
-        if self.root_elt is None:
-            self.root_elt = self.pfsense.new_element('gateways')
-            self.pfsense.root.append(self.root_elt)
-
     ##############################
     # params processing
     #
@@ -187,14 +181,6 @@ def _validate_params(self):
     ##############################
     # XML processing
     #
-    def _create_target(self):
-        """ create the XML target_elt """
-        return self.pfsense.new_element('gateway_item')
-
-    def _find_target(self):
-        """ find the XML target_elt """
-        return self.pfsense.find_gateway_elt(self.obj['name'])
-
     @staticmethod
     def _get_params_to_remove():
         """ returns the list of params to remove if they are not set """

From 22e40a8a830aa318dd62ad111e7cf9ac46f56358 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Thu, 11 Jan 2024 19:13:02 -0700
Subject: [PATCH 10/62] [pfsense_interface_group] Drop _get_obj_name()

---
 plugins/module_utils/interface_group.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/plugins/module_utils/interface_group.py b/plugins/module_utils/interface_group.py
index 9cf9f255..43d296ed 100644
--- a/plugins/module_utils/interface_group.py
+++ b/plugins/module_utils/interface_group.py
@@ -154,10 +154,6 @@ def _update(self):
     ##############################
     # Logging
     #
-    def _get_obj_name(self):
-        """ return obj's name """
-        return "'{0}'".format(self.obj['ifname'])
-
     def _log_fields(self, before=None):
         """ generate pseudo-CLI command fields parameters to create an obj """
         values = ''

From af6a85bec966a145619d7f1d236da9d838c7b9f3 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Thu, 11 Jan 2024 20:01:57 -0700
Subject: [PATCH 11/62] [module_base] set name; allow update_php code

---
 plugins/module_utils/module_base.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index da91eea2..a2981095 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -21,11 +21,16 @@ def get_argument_spec():
     ##############################
     # init
     #
-    def __init__(self, module, pfsense=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr'):
+    def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr', update_php=None):
         if pfsense is None:
             pfsense = PFSenseModule(module)
         self.module = module    # ansible module
-        self.name = None        # ansible module name
+        if name is not None:    # ansible module name
+            self.name = name
+        elif node is not None:
+            self.name = 'pfsense_' + node
+        else:
+            self.name = None
         self.params = None      # ansible input parameters
 
         self.pfsense = pfsense  # helper module
@@ -58,6 +63,8 @@ def __init__(self, module, pfsense=None, root=None, root_is_exclusive=True, crea
         self.obj = dict()       # dict holding target pfsense parameters
         self.target_elt = None  # xml object holding target pfsense parameters
 
+        self.update_php = update_php  # php code to update configuration
+
         self.change_descr = ''
 
         self.result = {}
@@ -269,10 +276,12 @@ def _pre_update():
         """ tasks to run before making config changes """
         return ('', '', '')
 
-    @staticmethod
-    def _update():
+    def _update(self):
         """ make the target pfsense reload """
-        return ('', '', '')
+        if self.update_php is not None:
+            return self.pfsense.phpshell(self.update_php)
+        else:
+            return ('', '', '')
 
     def run(self, params):
         """ process input params to add/update/delete """

From 2397e58922db752739ec4b39811093aa17b3e26c Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Thu, 11 Jan 2024 20:02:49 -0700
Subject: [PATCH 12/62] [pfsense_alias] auto set name; drop _find_target,
 _update

---
 plugins/module_utils/alias.py | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/plugins/module_utils/alias.py b/plugins/module_utils/alias.py
index 960904d8..a215e1bf 100644
--- a/plugins/module_utils/alias.py
+++ b/plugins/module_utils/alias.py
@@ -24,6 +24,11 @@
     ["type", "urltable_ports", ["updatefreq"]],
 ]
 
+ALIAS_PHP_COMMAND_SET = """
+require_once("filter.inc");
+if (filter_configure() == 0) { clear_subsystem_dirty('aliases'); }
+"""
+
 
 class PFSenseAliasModule(PFSenseModuleBase):
     """ module managing pfsense aliases """
@@ -37,8 +42,7 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None):
-        super(PFSenseAliasModule, self).__init__(module, pfsense, root='aliases', node='alias')
-        self.name = "pfsense_alias"
+        super(PFSenseAliasModule, self).__init__(module, pfsense, root='aliases', node='alias', key='name', update_php=ALIAS_PHP_COMMAND_SET)
 
     ##############################
     # params processing
@@ -98,21 +102,6 @@ def _validate_params(self):
                 if detail.startswith('|') or detail.endswith('|'):
                     self.module.fail_json(msg='Vertical bars (|) at start or end of descriptions not allowed')
 
-    ##############################
-    # XML processing
-    #
-    def _find_target(self):
-        """ find the XML target_elt """
-        return self.pfsense.find_alias(self.obj['name'], aliastype=self.obj.get('type'))
-
-    ##############################
-    # run
-    #
-    def _update(self):
-        """ make the target pfsense reload """
-        return self.pfsense.phpshell('''require_once("filter.inc");
-if (filter_configure() == 0) { clear_subsystem_dirty('aliases'); }''')
-
     ##############################
     # Logging
     #

From e9370d40108dde3d808926a44495f4a9685b8c94 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Fri, 12 Jan 2024 08:43:19 -0700
Subject: [PATCH 13/62] [module_base] Implement basic _params_to_obj()

---
 plugins/module_utils/module_base.py | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index a2981095..2218cbf7 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -21,10 +21,13 @@ def get_argument_spec():
     ##############################
     # init
     #
-    def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr', update_php=None):
+    def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr', update_php=None,
+                 map_param_if=None, param_force=None):
         if pfsense is None:
             pfsense = PFSenseModule(module)
         self.module = module    # ansible module
+        self.argument_spec = module.argument_spec  # Allow for being overrriden for use with aggregate
+
         if name is not None:    # ansible module name
             self.name = name
         elif node is not None:
@@ -61,6 +64,8 @@ def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive
 
         self.key = key          # item that identifies a target element
         self.obj = dict()       # dict holding target pfsense parameters
+        self.map_param_if = map_param_if  # rules for mapping parameters
+        self.param_force = param_force    # parameters that are forced to be present
         self.target_elt = None  # xml object holding target pfsense parameters
 
         self.update_php = update_php  # php code to update configuration
@@ -106,10 +111,24 @@ def _get_ansible_param_bool(self, obj, name, fname=None, force=False, value='yes
         elif force:
             obj[fname] = value_false
 
-    @staticmethod
-    def _params_to_obj():
+    def _params_to_obj(self):
         """ return a dict from module params """
-        raise NotImplementedError()
+        obj = dict()
+        obj[self.key] = self.params[self.key]
+        if self.params.get('state') == 'present':
+            for param in [n for n in self.argument_spec.keys() if n != 'state']:
+                force = False
+                if param in self.param_force:
+                    force = True
+                self._get_ansible_param(obj, param, force=force)
+
+        for map_param, map_value, map_tuple in self.map_param_if:
+            if self.params.get(map_param) == map_value and map_tuple[0] in obj:
+                if map_tuple[1] not in obj:
+                    obj[map_tuple[1]] = obj[map_tuple[0]]
+                del obj[map_tuple[0]]
+
+        return obj
 
     @staticmethod
     def _validate_params():

From fd5818977003d72ce5a62c9ac3df52b7ef67dce1 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Fri, 12 Jan 2024 13:02:14 -0700
Subject: [PATCH 14/62] [checks] Add name to check_name output for clarity with
 aggregate

---
 plugins/module_utils/__impl/checks.py              | 3 ++-
 tests/unit/plugins/modules/test_pfsense_gateway.py | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/plugins/module_utils/__impl/checks.py b/plugins/module_utils/__impl/checks.py
index f6beebac..5088f1fe 100644
--- a/plugins/module_utils/__impl/checks.py
+++ b/plugins/module_utils/__impl/checks.py
@@ -15,7 +15,8 @@ def check_name(self, name, objtype):
 
     msg = None
     if len(name) >= 32 or len(re.findall(r'(^_*$|^\d*$|[^a-zA-Z0-9_])', name)) > 0:
-        msg = "The {0} name must be less than 32 characters long, may not consist of only numbers, may not consist of only underscores, ".format(objtype)
+        msg = "The {0} name '{1}' must be less than 32 characters long, may not consist of only numbers, may not consist of only underscores, ".format(
+            objtype, name)
         msg += "and may only contain the following characters: a-z, A-Z, 0-9, _"
     elif name in ["port", "pass"]:
         msg = "The {0} name must not be either of the reserved words 'port' or 'pass'".format(objtype)
diff --git a/tests/unit/plugins/modules/test_pfsense_gateway.py b/tests/unit/plugins/modules/test_pfsense_gateway.py
index 6e8ef439..916077a4 100644
--- a/tests/unit/plugins/modules/test_pfsense_gateway.py
+++ b/tests/unit/plugins/modules/test_pfsense_gateway.py
@@ -82,8 +82,8 @@ def test_gateway_create_in_vip(self):
     def test_gateway_create_invalid_name(self):
         """ test """
         obj = dict(name='___', interface='lan', gateway='192.168.1.1')
-        msg = 'The gateway name must be less than 32 characters long, may not consist of only numbers, '
-        msg += 'may not consist of only underscores, and may only contain the following characters: a-z, A-Z, 0-9, _'
+        msg = "The gateway name '___' must be less than 32 characters long, may not consist of only numbers, "
+        msg += "may not consist of only underscores, and may only contain the following characters: a-z, A-Z, 0-9, _"
         self.do_module_test(obj, msg=msg, failed=True)
 
     def test_gateway_create_invalid_interface(self):

From 2d1dd71ba9cd77da465234d854279f50111d91f0 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Fri, 12 Jan 2024 13:20:03 -0700
Subject: [PATCH 15/62] [pfsense_alias] Refactor _params_to_obj() and add url

- Add url parameter, deprecate address for urltable/urltable_ports
- Use map_param_if and param_force
---
 .../fragments/pfsense_alias-add-url.yml       |  2 +
 plugins/module_utils/alias.py                 | 55 ++++++++++++-------
 plugins/modules/pfsense_aggregate.py          | 11 +++-
 plugins/modules/pfsense_alias.py              |  9 ++-
 .../modules/fixtures/pfsense_alias_config.xml |  1 -
 .../pfsense_ipsec_aggregate_config.xml        |  1 -
 .../plugins/modules/test_pfsense_aggregate.py | 18 +++---
 .../plugins/modules/test_pfsense_alias.py     | 52 ++++++++++++------
 8 files changed, 96 insertions(+), 53 deletions(-)
 create mode 100644 changelogs/fragments/pfsense_alias-add-url.yml

diff --git a/changelogs/fragments/pfsense_alias-add-url.yml b/changelogs/fragments/pfsense_alias-add-url.yml
new file mode 100644
index 00000000..c285d0a7
--- /dev/null
+++ b/changelogs/fragments/pfsense_alias-add-url.yml
@@ -0,0 +1,2 @@
+minor_changes:
+ - pfsense_alias - Add `url` parameter and deprecate using `address` for `urltable` and `urltable_ports` types.
diff --git a/plugins/module_utils/alias.py b/plugins/module_utils/alias.py
index a215e1bf..fe35427c 100644
--- a/plugins/module_utils/alias.py
+++ b/plugins/module_utils/alias.py
@@ -13,15 +13,33 @@
     state=dict(default='present', choices=['present', 'absent']),
     type=dict(default=None, required=False, choices=['host', 'network', 'port', 'urltable', 'urltable_ports']),
     address=dict(default=None, required=False, type='str'),
+    url=dict(default=None, required=False, type='str'),
     descr=dict(default=None, required=False, type='str'),
     detail=dict(default=None, required=False, type='str'),
     updatefreq=dict(default=None, required=False, type='int'),
 )
 
+ALIAS_PARAM_FORCE = ['descr', 'detail']
+
+ALIAS_MUTUALLY_EXCLUSIVE = [
+    ('address', 'url'),
+]
+
 ALIAS_REQUIRED_IF = [
-    ["state", "present", ["type", "address"]],
+    ["state", "present", ["type"]],
+    ["type", "host", ["address"]],
+    ["type", "network", ["address"]],
+    ["type", "port", ["address"]],
     ["type", "urltable", ["updatefreq"]],
     ["type", "urltable_ports", ["updatefreq"]],
+    # When "address" deprecation period is over
+    # ["type", "urltable", ["updatefreq", "url"]],
+    # ["type", "urltable_ports", ["updatefreq", "url"]],
+]
+
+ALIAS_MAP_PARAM_IF = [
+    ["type", "urltable", ("address", "url")],
+    ["type", "urltable_ports", ("address", "url")],
 ]
 
 ALIAS_PHP_COMMAND_SET = """
@@ -33,6 +51,10 @@
 class PFSenseAliasModule(PFSenseModuleBase):
     """ module managing pfsense aliases """
 
+    ##############################
+    # unit tests
+    #
+    # Must be class method for unit test usage
     @staticmethod
     def get_argument_spec():
         """ return argument spec """
@@ -42,26 +64,14 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None):
-        super(PFSenseAliasModule, self).__init__(module, pfsense, root='aliases', node='alias', key='name', update_php=ALIAS_PHP_COMMAND_SET)
+        super(PFSenseAliasModule, self).__init__(module, pfsense, root='aliases', node='alias', key='name', update_php=ALIAS_PHP_COMMAND_SET,
+                                                 map_param_if=ALIAS_MAP_PARAM_IF, param_force=ALIAS_PARAM_FORCE)
+        # Override for use with aggregate
+        self.argument_spec = ALIAS_ARGUMENT_SPEC
 
     ##############################
     # params processing
     #
-    def _params_to_obj(self):
-        """ return dict from module params """
-        obj = dict()
-        obj['name'] = self.params['name']
-        if self.params['state'] == 'present':
-            obj['type'] = self.params['type']
-            obj['address'] = self.params['address']
-            obj['descr'] = self.params['descr']
-            obj['detail'] = self.params['detail']
-            if obj['type'] == 'urltable' or obj['type'] == 'urltable_ports':
-                obj['url'] = self.params['address']
-                obj['updatefreq'] = str(self.params['updatefreq'])
-
-        return obj
-
     def _validate_params(self):
         """ do some extra checks on input parameters """
         params = self.params
@@ -93,9 +103,12 @@ def _validate_params(self):
 
             # check details count
             details = params['detail'].split('||') if params['detail'] is not None else []
-            addresses = params['address'].split(' ')
-            if len(details) > len(addresses):
-                self.module.fail_json(msg='Too many details in relation to addresses')
+            if params['address'] is not None:
+                addresses = params['address'].split(' ')
+                if len(details) > len(addresses):
+                    self.module.fail_json(msg='Too many details in relation to addresses')
+                if params['type'] in ['urltable', 'urltable_ports']:
+                    self.module.warn('Use of "address" with {type} is depracated, please use "url" instead'.format(type=params['type']))
 
             # pfSense GUI rule
             for detail in details:
@@ -115,12 +128,14 @@ def _log_fields(self, before=None):
         if before is None:
             values += self.format_cli_field(self.obj, 'type')
             values += self.format_cli_field(self.obj, 'address')
+            values += self.format_cli_field(self.obj, 'url')
             values += self.format_cli_field(self.obj, 'updatefreq')
             values += self.format_cli_field(self.obj, 'descr')
             values += self.format_cli_field(self.obj, 'detail')
         else:
             values += self.format_updated_cli_field(self.obj, before, 'type', add_comma=(values))
             values += self.format_updated_cli_field(self.obj, before, 'address', add_comma=(values))
+            values += self.format_updated_cli_field(self.obj, before, 'url', add_comma=(values))
             values += self.format_updated_cli_field(self.obj, before, 'updatefreq', add_comma=(values))
             values += self.format_updated_cli_field(self.obj, before, 'descr', add_comma=(values))
             values += self.format_updated_cli_field(self.obj, before, 'detail', add_comma=(values))
diff --git a/plugins/modules/pfsense_aggregate.py b/plugins/modules/pfsense_aggregate.py
index 33547211..3ef4c343 100644
--- a/plugins/modules/pfsense_aggregate.py
+++ b/plugins/modules/pfsense_aggregate.py
@@ -43,7 +43,11 @@
         default: null
         type: str
       address:
-        description: The address of the alias. Use a space separator for multiple values
+        description: The address of the alias for `host`, `network` or `port` types. Use a space separator for multiple values
+        default: null
+        type: str
+      url:
+        description: The URL of the alias for `urltable` or `urltable_ports` types. Use a space separator for multiple values
         default: null
         type: str
       descr:
@@ -609,7 +613,7 @@
 """
 
 from ansible_collections.pfsensible.core.plugins.module_utils.pfsense import PFSenseModule
-from ansible_collections.pfsensible.core.plugins.module_utils.alias import PFSenseAliasModule, ALIAS_ARGUMENT_SPEC, ALIAS_REQUIRED_IF
+from ansible_collections.pfsensible.core.plugins.module_utils.alias import PFSenseAliasModule, ALIAS_ARGUMENT_SPEC, ALIAS_MUTUALLY_EXCLUSIVE, ALIAS_REQUIRED_IF
 from ansible_collections.pfsensible.core.plugins.module_utils.interface import (
     PFSenseInterfaceModule,
     INTERFACE_ARGUMENT_SPEC,
@@ -1091,7 +1095,8 @@ def commit_changes(self):
 
 def main():
     argument_spec = dict(
-        aggregated_aliases=dict(type='list', elements='dict', options=ALIAS_ARGUMENT_SPEC, required_if=ALIAS_REQUIRED_IF),
+        aggregated_aliases=dict(
+            type='list', elements='dict', options=ALIAS_ARGUMENT_SPEC, mutually_exclusive=ALIAS_MUTUALLY_EXCLUSIVE, required_if=ALIAS_REQUIRED_IF),
         aggregated_interfaces=dict(
             type='list', elements='dict',
             options=INTERFACE_ARGUMENT_SPEC, required_if=INTERFACE_REQUIRED_IF, mutually_exclusive=INTERFACE_MUTUALLY_EXCLUSIVE),
diff --git a/plugins/modules/pfsense_alias.py b/plugins/modules/pfsense_alias.py
index c472f69b..2a486ad0 100644
--- a/plugins/modules/pfsense_alias.py
+++ b/plugins/modules/pfsense_alias.py
@@ -37,7 +37,11 @@
     default: null
     type: str
   address:
-    description: The address of the alias. Use a space separator for multiple values
+    description: The address of the alias for `host`, `network` or `port` types. Use a space separator for multiple values
+    default: null
+    type: str
+  url:
+    description: The URL of the alias for `urltable` or `urltable_ports` types. Use a space separator for multiple values
     default: null
     type: str
   descr:
@@ -81,12 +85,13 @@
 """
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible_collections.pfsensible.core.plugins.module_utils.alias import PFSenseAliasModule, ALIAS_ARGUMENT_SPEC, ALIAS_REQUIRED_IF
+from ansible_collections.pfsensible.core.plugins.module_utils.alias import PFSenseAliasModule, ALIAS_ARGUMENT_SPEC, ALIAS_MUTUALLY_EXCLUSIVE, ALIAS_REQUIRED_IF
 
 
 def main():
     module = AnsibleModule(
         argument_spec=ALIAS_ARGUMENT_SPEC,
+        mutually_exclusive=ALIAS_MUTUALLY_EXCLUSIVE,
         required_if=ALIAS_REQUIRED_IF,
         supports_check_mode=True)
 
diff --git a/tests/unit/plugins/modules/fixtures/pfsense_alias_config.xml b/tests/unit/plugins/modules/fixtures/pfsense_alias_config.xml
index 94e4eff1..b8ef3e9e 100644
--- a/tests/unit/plugins/modules/fixtures/pfsense_alias_config.xml
+++ b/tests/unit/plugins/modules/fixtures/pfsense_alias_config.xml
@@ -1376,7 +1376,6 @@
 			<address>172.16.1.3</address>
 			</alias>
 		<alias>
-			<address>http://www.acme-corp.com</address>
 			<type>urltable</type>
 			<updatefreq>10</updatefreq>
 			<url>http://www.acme-corp.com</url>
diff --git a/tests/unit/plugins/modules/fixtures/pfsense_ipsec_aggregate_config.xml b/tests/unit/plugins/modules/fixtures/pfsense_ipsec_aggregate_config.xml
index b8d226e5..aff3649d 100644
--- a/tests/unit/plugins/modules/fixtures/pfsense_ipsec_aggregate_config.xml
+++ b/tests/unit/plugins/modules/fixtures/pfsense_ipsec_aggregate_config.xml
@@ -1634,7 +1634,6 @@
 			<address>172.16.1.3</address>
 			</alias>
 		<alias>
-			<address>http://www.acme-corp.com</address>
 			<type>urltable</type>
 			<updatefreq>10</updatefreq>
 			<url>http://www.acme-corp.com</url>
diff --git a/tests/unit/plugins/modules/test_pfsense_aggregate.py b/tests/unit/plugins/modules/test_pfsense_aggregate.py
index 63732525..943b2c38 100644
--- a/tests/unit/plugins/modules/test_pfsense_aggregate.py
+++ b/tests/unit/plugins/modules/test_pfsense_aggregate.py
@@ -154,9 +154,9 @@ def test_aggregate_aliases(self):
         set_module_args(args)
         result = self.execute_module(changed=True)
         result_aliases = []
-        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7'")
-        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6'")
-        result_aliases.append("update alias 'port_ssh' set address='2222', descr=none, detail=none")
+        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7', descr='', detail=''")
+        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6', descr='', detail=''")
+        result_aliases.append("update alias 'port_ssh' set address='2222'")
         result_aliases.append("delete alias 'port_http'")
 
         self.assertEqual(result['result_aliases'], result_aliases)
@@ -183,9 +183,9 @@ def test_aggregate_aliases_checkmode(self):
         set_module_args(args)
         result = self.execute_module(changed=True)
         result_aliases = []
-        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7'")
-        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6'")
-        result_aliases.append("update alias 'port_ssh' set address='2222', descr=none, detail=none")
+        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7', descr='', detail=''")
+        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6', descr='', detail=''")
+        result_aliases.append("update alias 'port_ssh' set address='2222'")
         result_aliases.append("delete alias 'port_http'")
 
         self.assertEqual(result['result_aliases'], result_aliases)
@@ -207,9 +207,9 @@ def test_aggregate_aliases_purge(self):
         set_module_args(args)
         result = self.execute_module(changed=True)
         result_aliases = []
-        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7'")
-        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6'")
-        result_aliases.append("update alias 'port_ssh' set address='2222', descr=none, detail=none")
+        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7', descr='', detail=''")
+        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6', descr='', detail=''")
+        result_aliases.append("update alias 'port_ssh' set address='2222'")
         result_aliases.append("delete alias 'port_http'")
         result_aliases.append("delete alias 'port_dns'")
 
diff --git a/tests/unit/plugins/modules/test_pfsense_alias.py b/tests/unit/plugins/modules/test_pfsense_alias.py
index ed8b88a6..5fbeac6e 100644
--- a/tests/unit/plugins/modules/test_pfsense_alias.py
+++ b/tests/unit/plugins/modules/test_pfsense_alias.py
@@ -32,13 +32,18 @@ def __init__(self, *args, **kwargs):
     # First we run the module
     # Then, we check return values
     # Finally, we check the xml
-    def do_alias_creation_test(self, alias, failed=False, msg='', command=None):
+    def do_alias_creation_test(self, alias, set_after=None, unset_after=None, failed=False, msg='', command=None):
         """ test creation of a new alias """
         set_module_args(self.args_from_var(alias))
         result = self.execute_module(changed=True, failed=failed, msg=msg)
 
         if not failed:
             diff = dict(before={}, after=alias)
+            if set_after is not None:
+                diff['after'].update(set_after)
+            if unset_after is not None:
+                for n in unset_after:
+                    del diff['after'][n]
             self.assertEqual(result['diff'], diff)
             self.assert_xml_elt_dict('aliases', dict(name=alias['name'], type=alias['type']), diff['after'])
             self.assertEqual(result['commands'], [command])
@@ -76,7 +81,10 @@ def do_alias_update_field(self, alias, set_after=None, command=None, **kwargs):
         if set_after is not None:
             diff['after'].update(set_after)
         self.assertEqual(result['diff'], diff)
-        self.assert_xml_elt_value('aliases', dict(name=alias['name'], type=alias['type']), 'address', diff['after']['address'])
+        if alias['type'] in ['host', 'port', 'network']:
+            self.assert_xml_elt_value('aliases', dict(name=alias['name'], type=alias['type']), 'address', diff['after']['address'])
+        else:
+            self.assert_xml_elt_value('aliases', dict(name=alias['name'], type=alias['type']), 'url', diff['after']['url'])
         self.assertEqual(result['commands'], [command])
 
     ##############
@@ -181,49 +189,59 @@ def test_network_update_descr(self):
     def test_urltable_create(self):
         """ test creation of a new urltable alias """
         alias = dict(name='acme_table', address='http://www.acme.com', descr='', type='urltable', updatefreq='10', detail='')
-        alias['url'] = alias['address']
-        command = "create alias 'acme_table', type='urltable', address='http://www.acme.com', updatefreq='10', descr='', detail=''"
+        command = "create alias 'acme_table', type='urltable', url='http://www.acme.com', updatefreq='10', descr='', detail=''"
+        self.do_alias_creation_test(alias, command=command, set_after=dict(url='http://www.acme.com'), unset_after=['address'])
+
+    def test_urltable_create_url(self):
+        """ test creation of a new urltable alias """
+        alias = dict(name='acme_table', url='http://www.acme.com', descr='', type='urltable', updatefreq='10', detail='')
+        command = "create alias 'acme_table', type='urltable', url='http://www.acme.com', updatefreq='10', descr='', detail=''"
         self.do_alias_creation_test(alias, command=command)
 
+    def test_urltable_create_exclusive(self):
+        """ test creattion of a urltable alias with both address and url - fails """
+        alias = dict(
+            name='acme_corp', address='http://www.acme-corp.com', url='http://www.acme-corp.com', descr='', type='urltable', updatefreq='10', detail='')
+        self.do_alias_creation_test(alias, failed=True, msg='parameters are mutually exclusive: address|url')
+
     def test_urltable_delete(self):
         """ test deletion of a urltable alias """
         alias = dict(
-            name='acme_corp', address='http://www.acme-corp.com', url='http://www.acme-corp.com', descr='', type='urltable', updatefreq='10', detail='')
+            name='acme_corp', url='http://www.acme-corp.com', descr='', type='urltable', updatefreq='10', detail='')
         command = "delete alias 'acme_corp'"
         self.do_alias_deletion_test(alias, command=command)
 
     def test_urltable_update_noop(self):
         """ test not updating a urltable alias """
         alias = dict(
-            name='acme_corp', address='http://www.acme-corp.com', url='http://www.acme-corp.com', descr='', type='urltable', updatefreq='10', detail='')
+            name='acme_corp', url='http://www.acme-corp.com', descr='', type='urltable', updatefreq='10', detail='')
         self.do_alias_update_noop_test(alias)
 
     def test_urltable_update_url(self):
-        """ test updating address of a urltable alias """
+        """ test updating url of a urltable alias """
         alias = dict(
-            name='acme_corp', address='http://www.acme-corp.com', url='http://www.acme-corp.com', descr='', type='urltable', updatefreq='10', detail='')
-        command = "update alias 'acme_corp' set address='http://www.new-acme-corp.com'"
-        self.do_alias_update_field(alias, address='http://www.new-acme-corp.com', set_after=dict(url='http://www.new-acme-corp.com'), command=command)
+            name='acme_corp', url='http://www.acme-corp.com', descr='', type='urltable', updatefreq='10', detail='')
+        command = "update alias 'acme_corp' set url='http://www.new-acme-corp.com'"
+        self.do_alias_update_field(alias, url='http://www.new-acme-corp.com', set_after=dict(url='http://www.new-acme-corp.com'), command=command)
 
     def test_urltable_update_descr(self):
         """ test updating descr of a urltable alias """
         alias = dict(
-            name='acme_corp', address='http://www.acme-corp.com', url='http://www.acme-corp.com', descr='', type='urltable', updatefreq='10', detail='')
+            name='acme_corp', url='http://www.acme-corp.com', descr='', type='urltable', updatefreq='10', detail='')
         command = "update alias 'acme_corp' set descr='acme corp urls'"
         self.do_alias_update_field(alias, descr='acme corp urls', command=command)
 
     def test_urltable_update_freq(self):
         """ test updating updatefreq of a urltable alias """
         alias = dict(
-            name='acme_corp', address='http://www.acme-corp.com', url='http://www.acme-corp.com', descr='', type='urltable', updatefreq='10', detail='')
+            name='acme_corp', url='http://www.acme-corp.com', descr='', type='urltable', updatefreq='10', detail='')
         command = "update alias 'acme_corp' set updatefreq='20'"
         self.do_alias_update_field(alias, updatefreq='20', command=command)
 
     def test_urltable_ports_create(self):
         """ test creation of a new urltable_ports alias """
-        alias = dict(name='acme_table', address='http://www.acme.com', descr='', type='urltable_ports', updatefreq='10', detail='')
-        alias['url'] = alias['address']
-        command = "create alias 'acme_table', type='urltable_ports', address='http://www.acme.com', updatefreq='10', descr='', detail=''"
+        alias = dict(name='acme_table', url='http://www.acme.com', descr='', type='urltable_ports', updatefreq='10', detail='')
+        command = "create alias 'acme_table', type='urltable_ports', url='http://www.acme.com', updatefreq='10', descr='', detail=''"
         self.do_alias_creation_test(alias, command=command)
 
     ##############
@@ -237,7 +255,7 @@ def test_create_alias_duplicate(self):
     def test_create_alias_invalid_name(self):
         """ test creation of a new alias with invalid name """
         alias = dict(name='ads-ervers', address='10.0.0.1 10.0.0.2', type='host')
-        msg = "The alias name must be less than 32 characters long, may not consist of only numbers, may not consist of only underscores, "
+        msg = "The alias name 'ads-ervers' must be less than 32 characters long, may not consist of only numbers, may not consist of only underscores, "
         msg += "and may only contain the following characters: a-z, A-Z, 0-9, _"
         self.do_alias_creation_test(alias, failed=True, msg=msg)
 
@@ -259,7 +277,7 @@ def test_create_alias_without_type(self):
     def test_create_alias_without_address(self):
         """ test creation of a new host alias without address """
         alias = dict(name='adservers', type='host')
-        self.do_alias_creation_test(alias, failed=True, msg='state is present but all of the following are missing: address')
+        self.do_alias_creation_test(alias, failed=True, msg='type is host but all of the following are missing: address')
 
     def test_create_alias_invalid_details(self):
         """ test creation of a new host alias with invalid details """

From db0f3527885fe2427f9d64d0386aefe7debdc06d Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Fri, 12 Jan 2024 18:02:38 -0700
Subject: [PATCH 16/62] [module_base] Basic _log_fields() implementation

---
 plugins/module_utils/module_base.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 2218cbf7..fa5ce377 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -336,7 +336,14 @@ def _log_delete(self):
 
     def _log_fields(self, before=None):
         """ generate pseudo-CLI command fields parameters to create an obj """
-        raise NotImplementedError()
+        values = ''
+        if before is None:
+            for param in [n for n in self.argument_spec.keys() if n != 'state' and n != self.key]:
+                values += self.format_cli_field(self.obj, param)
+        else:
+            for param in [n for n in self.argument_spec.keys() if n != 'state' and n != self.key]:
+                values += self.format_updated_cli_field(self.obj, before, param, add_comma=(values))
+        return values
 
     @staticmethod
     def _log_fields_delete():

From 254360b990b2ebf16f6db76b36e77d290845ec6e Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Fri, 12 Jan 2024 18:04:00 -0700
Subject: [PATCH 17/62] [pfsense_alias] Use PFSenseModuleBase _get_obj_name and
 _log_fields

Remove unneeded test for type presence, don't mark it default
---
 plugins/module_utils/alias.py                 | 42 +++----------------
 plugins/modules/pfsense_aggregate.py          |  1 -
 plugins/modules/pfsense_alias.py              |  1 -
 .../plugins/modules/test_pfsense_alias.py     |  6 +--
 4 files changed, 9 insertions(+), 41 deletions(-)

diff --git a/plugins/module_utils/alias.py b/plugins/module_utils/alias.py
index fe35427c..28dcc7d4 100644
--- a/plugins/module_utils/alias.py
+++ b/plugins/module_utils/alias.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright: (c) 2018, Orion Poplawski <orion@nwra.com>
+# Copyright: (c) 2018-2024, Orion Poplawski <orion@nwra.com>
 # Copyright: (c) 2018, Frederic Bor <frederic.bor@wanadoo.fr>
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
@@ -11,7 +11,7 @@
 ALIAS_ARGUMENT_SPEC = dict(
     name=dict(required=True, type='str'),
     state=dict(default='present', choices=['present', 'absent']),
-    type=dict(default=None, required=False, choices=['host', 'network', 'port', 'urltable', 'urltable_ports']),
+    type=dict(required=False, choices=['host', 'network', 'port', 'urltable', 'urltable_ports']),
     address=dict(default=None, required=False, type='str'),
     url=dict(default=None, required=False, type='str'),
     descr=dict(default=None, required=False, type='str'),
@@ -87,26 +87,22 @@ def _validate_params(self):
                     if params['type'] != alias_elt.find('type').text:
                         self.module.fail_json(msg='An alias with this name and a different type already exists: \'{0}\''.format(params['name']))
 
+            # Aliases cannot have the same name as an interface description
             if self.pfsense.get_interface_by_display_name(params['name']) is not None:
                 self.module.fail_json(msg='An interface description with this name already exists: \'{0}\''.format(params['name']))
 
-            missings = ['type']
-            for param, value in params.items():
-                if param in missings and value is not None and value != '':
-                    missings.remove(param)
-            if missings:
-                self.module.fail_json(msg='state is present but all of the following are missing: ' + ','.join(missings))
-
             # updatefreq is for urltable only
             if params['updatefreq'] is not None and params['type'] != 'urltable' and params['type'] != 'urltable_ports':
                 self.module.fail_json(msg='updatefreq is only valid with type urltable or urltable_ports')
 
-            # check details count
             details = params['detail'].split('||') if params['detail'] is not None else []
             if params['address'] is not None:
+                # check details count
                 addresses = params['address'].split(' ')
                 if len(details) > len(addresses):
                     self.module.fail_json(msg='Too many details in relation to addresses')
+
+                # warn if address is used with urltable to urltable_ports
                 if params['type'] in ['urltable', 'urltable_ports']:
                     self.module.warn('Use of "address" with {type} is depracated, please use "url" instead'.format(type=params['type']))
 
@@ -114,29 +110,3 @@ def _validate_params(self):
             for detail in details:
                 if detail.startswith('|') or detail.endswith('|'):
                     self.module.fail_json(msg='Vertical bars (|) at start or end of descriptions not allowed')
-
-    ##############################
-    # Logging
-    #
-    def _get_obj_name(self):
-        """ return obj's name """
-        return "'" + self.obj['name'] + "'"
-
-    def _log_fields(self, before=None):
-        """ generate pseudo-CLI command fields parameters to create an obj """
-        values = ''
-        if before is None:
-            values += self.format_cli_field(self.obj, 'type')
-            values += self.format_cli_field(self.obj, 'address')
-            values += self.format_cli_field(self.obj, 'url')
-            values += self.format_cli_field(self.obj, 'updatefreq')
-            values += self.format_cli_field(self.obj, 'descr')
-            values += self.format_cli_field(self.obj, 'detail')
-        else:
-            values += self.format_updated_cli_field(self.obj, before, 'type', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'address', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'url', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'updatefreq', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'descr', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'detail', add_comma=(values))
-        return values
diff --git a/plugins/modules/pfsense_aggregate.py b/plugins/modules/pfsense_aggregate.py
index 3ef4c343..f6bd936a 100644
--- a/plugins/modules/pfsense_aggregate.py
+++ b/plugins/modules/pfsense_aggregate.py
@@ -40,7 +40,6 @@
       type:
         description: The type of the alias
         choices: [ "host", "network", "port", "urltable", "urltable_ports" ]
-        default: null
         type: str
       address:
         description: The address of the alias for `host`, `network` or `port` types. Use a space separator for multiple values
diff --git a/plugins/modules/pfsense_alias.py b/plugins/modules/pfsense_alias.py
index 2a486ad0..5c6306a5 100644
--- a/plugins/modules/pfsense_alias.py
+++ b/plugins/modules/pfsense_alias.py
@@ -34,7 +34,6 @@
   type:
     description: The type of the alias
     choices: [ "host", "network", "port", "urltable", "urltable_ports" ]
-    default: null
     type: str
   address:
     description: The address of the alias for `host`, `network` or `port` types. Use a space separator for multiple values
diff --git a/tests/unit/plugins/modules/test_pfsense_alias.py b/tests/unit/plugins/modules/test_pfsense_alias.py
index 5fbeac6e..e1576245 100644
--- a/tests/unit/plugins/modules/test_pfsense_alias.py
+++ b/tests/unit/plugins/modules/test_pfsense_alias.py
@@ -189,13 +189,13 @@ def test_network_update_descr(self):
     def test_urltable_create(self):
         """ test creation of a new urltable alias """
         alias = dict(name='acme_table', address='http://www.acme.com', descr='', type='urltable', updatefreq='10', detail='')
-        command = "create alias 'acme_table', type='urltable', url='http://www.acme.com', updatefreq='10', descr='', detail=''"
+        command = "create alias 'acme_table', type='urltable', url='http://www.acme.com', descr='', detail='', updatefreq='10'"
         self.do_alias_creation_test(alias, command=command, set_after=dict(url='http://www.acme.com'), unset_after=['address'])
 
     def test_urltable_create_url(self):
         """ test creation of a new urltable alias """
         alias = dict(name='acme_table', url='http://www.acme.com', descr='', type='urltable', updatefreq='10', detail='')
-        command = "create alias 'acme_table', type='urltable', url='http://www.acme.com', updatefreq='10', descr='', detail=''"
+        command = "create alias 'acme_table', type='urltable', url='http://www.acme.com', descr='', detail='', updatefreq='10'"
         self.do_alias_creation_test(alias, command=command)
 
     def test_urltable_create_exclusive(self):
@@ -241,7 +241,7 @@ def test_urltable_update_freq(self):
     def test_urltable_ports_create(self):
         """ test creation of a new urltable_ports alias """
         alias = dict(name='acme_table', url='http://www.acme.com', descr='', type='urltable_ports', updatefreq='10', detail='')
-        command = "create alias 'acme_table', type='urltable_ports', url='http://www.acme.com', updatefreq='10', descr='', detail=''"
+        command = "create alias 'acme_table', type='urltable_ports', url='http://www.acme.com', descr='', detail='', updatefreq='10'"
         self.do_alias_creation_test(alias, command=command)
 
     ##############

From ad5ecac5d3214e30e6e7636a2db6bc3496de5191 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sat, 13 Jan 2024 18:26:49 -0700
Subject: [PATCH 18/62] [generate_module] Initial checkin

---
 misc/generate_module.py   | 221 ++++++++++++++++++++++++++++++++++++++
 misc/pfsense_module.py.j2 | 123 +++++++++++++++++++++
 2 files changed, 344 insertions(+)
 create mode 100755 misc/generate_module.py
 create mode 100644 misc/pfsense_module.py.j2

diff --git a/misc/generate_module.py b/misc/generate_module.py
new file mode 100755
index 00000000..6d711145
--- /dev/null
+++ b/misc/generate_module.py
@@ -0,0 +1,221 @@
+#!/usr/bin/python3
+
+import argparse
+import datetime
+import jinja2
+import lxml.etree as ET
+import lxml.html
+from paramiko import SSHClient
+import re
+import requests
+from scp import SCPClient
+import sys
+from urllib.parse import urlparse
+
+parser = argparse.ArgumentParser(description='Generate a pfsensible module.')
+parser.add_argument('--url', help='The URL to scrape')
+parser.add_argument('--urlfile', help='A local file copy of the URL to scrape')
+parser.add_argument('--user', default='admin', help='The user to connect as')
+parser.add_argument('--password', default='changeme', help='The password of user')
+parser.add_argument('--module_name', help='The name of the module to generate - defaults to being based on the url')
+parser.add_argument('--item_min', default='item_min', help='The name of the minimally configured item to search for in config.xml')
+parser.add_argument('--item_full', default='item_full',
+                    help='The name of the fully configured item to search for in config.xml, will be used for exmaples in the documentation')
+
+args = parser.parse_args()
+
+if args.url is not None:
+    parsed_uri = urlparse(args.url)
+
+    # Login using just the base URL
+    login_url = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
+
+    # Collect to host for later use to scp config.xml
+    host = f'{parsed_uri.netloc}'
+
+    # Construct a likely module name from the URL
+    if args.module_name is None:
+        module_name = re.sub(r'^/(?:firewall_)?(.*)(?:_edit)\.php.*$', r'pfsense_\1', parsed_uri.path)
+        module_name = re.sub(r'ses$', 's', module_name)
+    else:
+        module_name = args.module_name
+
+    # We likely don't have a valid certificate
+    requests.packages.urllib3.disable_warnings()
+
+    # Start our session (need cookies for login)
+    client = requests.Session()
+
+    # Retrieve the CSRF token first
+    r = client.get(login_url, verify=False)
+    csrf = re.search(".*name='__csrf_magic' value=\"([^\"]+)\".*", r.text, flags=re.MULTILINE).group(1)
+
+    # Login to the web interface
+    login_data = dict(login='Login', usernamefld=args.user, passwordfld=args.password, __csrf_magic=csrf)
+    r = client.post(login_url, data=login_data, verify=False)
+    csrf = re.search(".*name='__csrf_magic' value=\"([^\"]+)\".*", r.text, flags=re.MULTILINE).group(1)
+
+    # Retrieve the configuration web page and parse it
+    r = client.get(args.url, verify=False)
+    html = lxml.html.fromstring(r.text)
+
+elif args.urlfile is not None:
+    # Use a cached copy of the web page - get rid of this?  Need to specify host and module name
+    html = lxml.html.parse(args.urlfile)
+    host = '192.168.100.2'
+    module_name = 'pfsense_nat_1to1'
+
+else:
+    sys.exit('You must specify one of --url or --urlfile')
+
+# The web page should have a single form
+if len(html.forms) != 1:
+    sys.exit(f'Found {len(html.forms)} forms instead of a single one!')
+
+# Collected parameters from the web form
+params = dict()
+
+# Collect the input elements
+for input in html.forms[0].inputs:
+    # Skip internal items
+    if input.name == '__csrf_magic':
+        continue
+
+    param = dict()
+    print(f'attrib={input.attrib}')
+    if isinstance(input, lxml.html.InputElement):
+        print(f'input name={input.name} id={input.get("id")} type={input.type} value={input.value} '
+              'text={input.text} title={input.get("title")} tail={input.tail}')
+        if input.type == 'checkbox':
+            param['type'] = 'bool'
+            param['value'] = input.attrib['value']
+            param['example'] = 'true'
+        elif input.type == 'text':
+            param['type'] = 'str'
+        # Text sometimes is after the input element inside the enclosing <label>
+        if input.tail:
+            param['description'] = input.tail.strip()
+    elif isinstance(input, lxml.html.SelectElement):
+        print(f'select name={input.name} value={input.value} value_options={input.value_options} multiple={input.multiple}')
+        if input.value_options is not None:
+            param['choices'] = input.value_options
+            if input.multiple:
+                param['type'] = 'list'
+            else:
+                param['type'] = 'str'
+            param['multiple'] = input.multiple
+    params[input.name] = param
+
+# Debug
+print(f'Web paramters: {params.keys()}')
+
+# Collect the /cf/conf/config.xml file
+ssh = SSHClient()
+ssh.load_system_host_keys()
+ssh.connect(host, username='root', password=args.password)
+scp = SCPClient(ssh.get_transport())
+scp.get('/cf/conf/config.xml')
+scp.close()
+
+# Parse the config.xml file
+root = ET.parse('config.xml').getroot()
+
+# Search for any element with our target text, make sure we found only one
+xpath = f'.//*[.="{args.item_min}"]'
+key_elts = root.findall(xpath)
+if len(key_elts) > 1:
+    sys.exit(f'Found {len(key_elts)} items with path "{xpath}"')
+elif len(key_elts) == 0:
+    sys.exit(f'Cannot find minimally configured item with path "{xpath}"')
+else:
+    key_elt = key_elts[0]
+
+# This element should be the key for the items
+module_key = key_elt.tag
+
+# Key is handled separately from other parameters so remove it
+del params[module_key]
+
+# The full node configuration element will be the parent
+node_elt = key_elt.find('..')
+module_node = node_elt.tag
+
+# The "root" for this type of element is above that
+root_elt = node_elt.find('..')
+module_root = root_elt.tag
+
+# Debug
+print('item_min:\t' + ET.tostring(node_elt).decode())
+
+# Let's use our node and key as a check
+full_elt = root.find(f'.//{module_node}[{module_key}="{args.item_full}"]')
+if full_elt is None:
+    sys.exit(f'Cannot find fully configured item with path ".//{module_node}[{module_key}="{args.item_full}"]"')
+
+# Debug
+print('item_full:\t' + ET.tostring(full_elt).decode())
+
+# Collect the items for comparison with web elements and example values
+params_full = dict()
+for elt in full_elt:
+    if elt.tag == '':
+        continue
+    params_full[elt.tag] = elt.text
+
+
+print('')
+params_web_only = list(set(params.keys()) - set(params_full.keys()))
+print('Web parameters not in xml: ' + str(params_web_only))
+
+# Cleanup extra web parameters
+for param in params_web_only:
+    # See if the items are numbered, likely maps to an unnumbered XML tag
+    newp = re.sub(r'0$', '', param)
+    if newp != param:
+        if newp in params_full.keys():
+            print(f'Renaming {param} to {newp}')
+            params[newp] = params.pop(param)
+            continue
+
+    # Common renamings
+    for f, t in [('dst', 'destination'), ('src', 'source')]:
+        if param == f and t in params_full.keys():
+            print(f'Renaming {f} to {t}')
+            params[t] = params.pop(f)
+            break
+    else:
+        # Otherwise, drop - probably just used to construct the final elements
+        if param in params:
+            print(f'Removing {param}')
+            del params[param]
+
+# Create some sample descriptions and example values
+for name, param in params.items():
+    if 'description' not in param or param['description'] == '':
+        param['description'] = f'The {name} of the {module_node}'
+    if 'example' not in param or param['example'] == '':
+        if name in params_full and params_full[name] != '' and params_full[name] is not None:
+            param['example'] = params_full[name]
+
+# Template variables
+context = dict(
+    module_name=module_name,
+    module_root=module_root,
+    module_node=module_node,
+    module_key=module_key,
+    params=params,
+    author_name='Orion Poplawski',
+    author_handle='opoplawski',
+    author_email='orion@nwra.com',
+    year=datetime.date.today().year,
+)
+
+# Render our module!
+environment = jinja2.Environment(loader=jinja2.FileSystemLoader("misc/"), trim_blocks=True, keep_trailing_newline=True)
+template = environment.get_template("pfsense_module.py.j2")
+
+# Todo - prompt for overwrite
+print(f'Writing plugins/modules/{module_name}.py with {context}')
+f = open(f'plugins/modules/{module_name}.py', 'w')
+f.write(template.render(context))
+f.close()
diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
new file mode 100644
index 00000000..832c9255
--- /dev/null
+++ b/misc/pfsense_module.py.j2
@@ -0,0 +1,123 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright: (c) {{ year }}, {{ author_name }} <{{ author_email }}>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+---
+module: {{ module_name }}
+
+short_description: Manage pfSense {{ module_node }}s
+
+version_added: "0.6.0"
+
+description:
+  - Manage pfSense {{ module_node }}s
+
+options:
+  name:
+    description: The name of the {{ module_node }}
+    required: true
+    type: str
+  state:
+    description: State in which to leave the {{ module_node }}
+    choices: [ "present", "absent" ]
+    type: str
+{% for param in params %}
+  {{ param }}:
+    description: {{ params[param]['description'] | default('') }}
+{% if 'choices' in params[param] %}
+    choices: {{ params[param]['choices'] | default('') }}
+{% endif %}
+    type: {{ params[param]['type'] | default('') }}
+{% endfor %}
+
+author: {{ author_name }} (@{{ author_handle }})
+'''
+
+EXAMPLES = r'''
+- name: Add myitem {{ module_node }}
+  pfsensible.core.{{ module_name }}:
+    name: myitem
+{% for name, param in params.items() %}
+{% if param['example'] is defined %}
+    {{ name }}: {{ param['example'] }}
+{% else %}
+    {{ name }}: {% if param['type'] == 'bool' %}false{% elif param['type'] == 'list' %}{% if 'choices' in param %}['{{ param['choices'][0:1] | join("', '") }}']{% else %}{{ param['choices'][0] }}{% endif %}{% elif param['type'] == 'str' %}{{ param['choices'][0] if 'choices' in param else '' }}{% endif %}
+
+{% endif %}
+{% endfor %}
+    state: present
+
+- name: Remove myitem {{ module_node }}
+  pfsensible.core.{{ module_name }}:
+    name: myitem
+    state: absent
+'''
+RETURN = r'''
+commands:
+    description: the set of commands that would be pushed to the remote device (if pfSense had a CLI)
+    returned: always
+    type: list
+    sample: ["create {{ module_node }} 'myitem'", "update {{ module_node }} 'myitem' set ...", "delete {{ module_node }} 'myitem'"]
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible_collections.pfsensible.core.plugins.module_utils.module_base import PFSenseModuleBase
+
+# Change to name of module, extend for needed parameters
+{{ module_node | upper() }}_ARGUMENT_SPEC = {
+    'name': {'required': True, 'type': 'str'},
+    'state': {
+        'default': 'present',
+        'choices': ['present', 'absent']
+    },
+{% for param in params %}
+    '{{ param }}': {
+{% if 'choices' in params[param] %}
+        'choices': {{ params[param]['choices'] }},
+{% endif %}
+        'type': '{{ params[param]['type'] | default('') }}',
+    },
+{% endfor %}
+}
+
+
+class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
+    """ module managing pfsense {{ module_node }} """
+
+    def __init__(self, module, pfsense=None):
+        super(PFSenseMODULEModule, self).__init__(module, pfsense, root='{{ module_root }}', node='{{ module_node }}')
+
+    ##############################
+    # params processing
+    #
+    def _validate_params(self):
+        """ do some extra checks on input parameters """
+        params = self.params
+
+        # check name
+        self.pfsense.check_name(params['{{ module_key }}'], '<TYPE>')
+
+        if params['state'] == 'present':
+            #  ... more checks, e.g.:
+            if int(params['timeout']) < 1:
+                self.module.fail_json(msg='timeout {0} must be greater than 1'.format(params['timeout']))
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec={{ module_node | upper() }}_ARGUMENT_SPEC,
+        supports_check_mode=True)
+
+    pfmodule = PFSense{{ module_node | capitalize() }}Module(module)
+    pfmodule.run(module.params)
+    pfmodule.commit_changes()
+
+
+if __name__ == '__main__':
+    main()

From 998a62989f2c93f116c978be9ed00828d751b2df Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sat, 13 Jan 2024 19:36:54 -0700
Subject: [PATCH 19/62] [generate_module] Add author handling

---
 misc/generate_module.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/misc/generate_module.py b/misc/generate_module.py
index 6d711145..f3f6d800 100755
--- a/misc/generate_module.py
+++ b/misc/generate_module.py
@@ -2,6 +2,7 @@
 
 import argparse
 import datetime
+import git
 import jinja2
 import lxml.etree as ET
 import lxml.html
@@ -12,11 +13,18 @@
 import sys
 from urllib.parse import urlparse
 
+gitconfig = git.GitConfigParser()
+author_name = gitconfig.get_value('user', 'name')
+author_email = gitconfig.get_value('user', 'email')
+
 parser = argparse.ArgumentParser(description='Generate a pfsensible module.')
 parser.add_argument('--url', help='The URL to scrape')
 parser.add_argument('--urlfile', help='A local file copy of the URL to scrape')
 parser.add_argument('--user', default='admin', help='The user to connect as')
 parser.add_argument('--password', default='changeme', help='The password of user')
+parser.add_argument('--author_name', default=author_name, help='The full name of the module author')
+parser.add_argument('--author_email', default=author_email, help='The email address of the module author')
+parser.add_argument('--author_handle', default='', help='The github handle of the module author')
 parser.add_argument('--module_name', help='The name of the module to generate - defaults to being based on the url')
 parser.add_argument('--item_min', default='item_min', help='The name of the minimally configured item to search for in config.xml')
 parser.add_argument('--item_full', default='item_full',
@@ -204,9 +212,9 @@
     module_node=module_node,
     module_key=module_key,
     params=params,
-    author_name='Orion Poplawski',
-    author_handle='opoplawski',
-    author_email='orion@nwra.com',
+    author_name=args.author_name,
+    author_email=args.author_email,
+    author_handle=args.author_handle,
     year=datetime.date.today().year,
 )
 

From 8ed3cb6d9401c92ba1d391ae7b445c1e941e0cc1 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sat, 13 Jan 2024 22:40:23 -0700
Subject: [PATCH 20/62] [generate_module] Better name generation; Do not
 overwrite module file; Detect "type"; Scrape more web help text

---
 misc/generate_module.py   | 156 ++++++++++++++++++++++++++------------
 misc/pfsense_module.py.j2 |   9 +++
 2 files changed, 115 insertions(+), 50 deletions(-)

diff --git a/misc/generate_module.py b/misc/generate_module.py
index f3f6d800..98459fe3 100755
--- a/misc/generate_module.py
+++ b/misc/generate_module.py
@@ -6,6 +6,7 @@
 import jinja2
 import lxml.etree as ET
 import lxml.html
+import os
 from paramiko import SSHClient
 import re
 import requests
@@ -43,8 +44,12 @@
 
     # Construct a likely module name from the URL
     if args.module_name is None:
-        module_name = re.sub(r'^/(?:firewall_)?(.*)(?:_edit)\.php.*$', r'pfsense_\1', parsed_uri.path)
-        module_name = re.sub(r'ses$', 's', module_name)
+        module_name = re.sub(r'^/(?:firewall_|system_)?(.*)(?:_edit)?\.php.*$', r'pfsense_\1', parsed_uri.path)
+        module_name_singular = re.sub(r'ses$', 's', module_name)
+        if module_name_singular != module_name:
+            module_name = module_name_singular
+        else:
+            module_name = re.sub(r's$', '', module_name)
     else:
         module_name = args.module_name
 
@@ -66,6 +71,9 @@
     # Retrieve the configuration web page and parse it
     r = client.get(args.url, verify=False)
     html = lxml.html.fromstring(r.text)
+    f = open(f'{module_name}.html', 'w')
+    f.write(r.text)
+    f.close()
 
 elif args.urlfile is not None:
     # Use a cached copy of the web page - get rid of this?  Need to specify host and module name
@@ -76,47 +84,6 @@
 else:
     sys.exit('You must specify one of --url or --urlfile')
 
-# The web page should have a single form
-if len(html.forms) != 1:
-    sys.exit(f'Found {len(html.forms)} forms instead of a single one!')
-
-# Collected parameters from the web form
-params = dict()
-
-# Collect the input elements
-for input in html.forms[0].inputs:
-    # Skip internal items
-    if input.name == '__csrf_magic':
-        continue
-
-    param = dict()
-    print(f'attrib={input.attrib}')
-    if isinstance(input, lxml.html.InputElement):
-        print(f'input name={input.name} id={input.get("id")} type={input.type} value={input.value} '
-              'text={input.text} title={input.get("title")} tail={input.tail}')
-        if input.type == 'checkbox':
-            param['type'] = 'bool'
-            param['value'] = input.attrib['value']
-            param['example'] = 'true'
-        elif input.type == 'text':
-            param['type'] = 'str'
-        # Text sometimes is after the input element inside the enclosing <label>
-        if input.tail:
-            param['description'] = input.tail.strip()
-    elif isinstance(input, lxml.html.SelectElement):
-        print(f'select name={input.name} value={input.value} value_options={input.value_options} multiple={input.multiple}')
-        if input.value_options is not None:
-            param['choices'] = input.value_options
-            if input.multiple:
-                param['type'] = 'list'
-            else:
-                param['type'] = 'str'
-            param['multiple'] = input.multiple
-    params[input.name] = param
-
-# Debug
-print(f'Web paramters: {params.keys()}')
-
 # Collect the /cf/conf/config.xml file
 ssh = SSHClient()
 ssh.load_system_host_keys()
@@ -141,9 +108,6 @@
 # This element should be the key for the items
 module_key = key_elt.tag
 
-# Key is handled separately from other parameters so remove it
-del params[module_key]
-
 # The full node configuration element will be the parent
 node_elt = key_elt.find('..')
 module_node = node_elt.tag
@@ -170,8 +134,83 @@
         continue
     params_full[elt.tag] = elt.text
 
+if 'type' in params_full:
+    module_type = params_full['type']
+else:
+    module_type = None
 
 print('')
+
+# The web page should have a single form
+if len(html.forms) != 1:
+    sys.exit(f'Found {len(html.forms)} forms instead of a single one!')
+
+# Collected parameters from the web form
+params = dict()
+
+# Collect the input elements
+for input in html.forms[0].inputs:
+    # Skip internal items
+    if input.name == '__csrf_magic':
+        continue
+
+    param = dict(description='')
+    print(f'attrib={input.attrib}')
+    if isinstance(input, lxml.html.InputElement):
+        print(f'input name={input.name} id={input.get("id")} type={input.type} value={input.value} '
+              f'text={input.text} title={input.get("title")} tail={input.tail}')
+
+        if input.type == 'checkbox':
+            param['type'] = 'bool'
+            param['value'] = input.attrib['value'].strip()
+            param['example'] = 'true'
+        elif input.type == 'text':
+            param['type'] = 'str'
+        # Text sometimes is after the input element inside the enclosing <label>
+        if input.tail:
+            param['description'] = input.tail.strip()
+    elif isinstance(input, lxml.html.SelectElement):
+        print(f'select name={input.name} value={input.value} value_options={input.value_options} multiple={input.multiple}')
+        if input.value_options is not None:
+            param['choices'] = input.value_options
+            if input.multiple:
+                param['type'] = 'list'
+            else:
+                param['type'] = 'str'
+            param['multiple'] = input.multiple
+
+#  <div class="form-group">
+#    <label class="col-sm-2 control-label">
+#       <span class="element-required">Hostname or IP address</span>
+#    </label>
+#    <div class="col-sm-10">
+#      <input class="form-control" name="ldap_host" id="ldap_host" type="text">
+#      <span class="help-block">NOTE: When using SSL/TLS or STARTTLS, this hostname MUST match a Subject Alternative Name (SAN) or the Common Name (CN) of the LDAP server SSL/TLS Certificate.</span>
+#    </div>
+#  </div> 
+
+    div1 = input.find('..')
+    div2 = div1.find('..')
+    if div2.tag == 'div' and div2.attrib['class'] == 'form-group':
+        descr_elt = div2.find('*span')
+        if descr_elt.text:
+            print(f'Found descr_elt {descr_elt.tag} {descr_elt.text}')
+            param['description'] += f'{descr_elt.text.strip()} of the {module_node}'
+        help_elt = div1.find('span[@class="help-block"]')
+        if help_elt is not None:
+            print(f'help_elt text {help_elt.text.strip()}')
+            param['description'] += f'. {help_elt.text.strip()}'
+
+    params[input.name] = param
+
+# Key is handled separately from other parameters so remove it
+# TODO - keep the description, etc?
+del params[module_key]
+
+# Debug
+print(f'Web paramters: {params.keys()}')
+
+# Consistency
 params_web_only = list(set(params.keys()) - set(params_full.keys()))
 print('Web parameters not in xml: ' + str(params_web_only))
 
@@ -180,14 +219,22 @@
     # See if the items are numbered, likely maps to an unnumbered XML tag
     newp = re.sub(r'0$', '', param)
     if newp != param:
-        if newp in params_full.keys():
+        if newp in params_full:
+            print(f'Renaming {param} to {newp}')
+            params[newp] = params.pop(param)
+            continue
+
+    # See if the items are prefixed by a type, likely maps to un-prefixed XML tag
+    newp = re.sub(f'^{module_type}_', '', param)
+    if newp != param:
+        if newp in params_full and newp not in params:
             print(f'Renaming {param} to {newp}')
             params[newp] = params.pop(param)
             continue
 
     # Common renamings
     for f, t in [('dst', 'destination'), ('src', 'source')]:
-        if param == f and t in params_full.keys():
+        if param == f and t in params_full:
             print(f'Renaming {f} to {t}')
             params[t] = params.pop(f)
             break
@@ -197,6 +244,12 @@
             print(f'Removing {param}')
             del params[param]
 
+print('')
+params_xml_only = list(set(params_full.keys()) - set(params.keys()) - {module_key, 'refid'})
+print('XML parameters not in web: ' + str(params_xml_only))
+for param in params_xml_only:
+    params[param] = dict(type='str', example=params_full[param])
+
 # Create some sample descriptions and example values
 for name, param in params.items():
     if 'description' not in param or param['description'] == '':
@@ -223,7 +276,10 @@
 template = environment.get_template("pfsense_module.py.j2")
 
 # Todo - prompt for overwrite
-print(f'Writing plugins/modules/{module_name}.py with {context}')
-f = open(f'plugins/modules/{module_name}.py', 'w')
+filename = f'plugins/modules/{module_name}.py'
+if os.path.isfile(filename):
+    sys.exit(f'{filename} already exists!')
+print(f'Writing {filename} with {context}')
+f = open(f'{filename}', 'w')
 f.write(template.render(context))
 f.close()
diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index 832c9255..d1fcb65e 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -90,6 +90,15 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
     """ module managing pfsense {{ module_node }} """
 
+    ##############################
+    # unit tests
+    #
+    # Must be class method for unit test usage
+    @staticmethod
+    def get_argument_spec():
+        """ return argument spec """
+        return {{ module_node | upper() }}_ARGUMENT_SPEC
+
     def __init__(self, module, pfsense=None):
         super(PFSenseMODULEModule, self).__init__(module, pfsense, root='{{ module_root }}', node='{{ module_node }}')
 

From b2c7775d6f7e0142b76b49382bac46dafa60a1b4 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sat, 13 Jan 2024 22:41:10 -0700
Subject: [PATCH 21/62] [module_base] Add have_refid, create_default

---
 plugins/module_utils/module_base.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index fa5ce377..4c455ac8 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -13,6 +13,10 @@
 class PFSenseModuleBase(object):
     """ class providing base services for pfSense modules """
 
+    ##############################
+    # unit tests
+    #
+    # Must be class method for unit test usage
     @staticmethod
     def get_argument_spec():
         """ return argument spec """
@@ -22,7 +26,7 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr', update_php=None,
-                 map_param_if=None, param_force=None):
+                 map_param_if=None, param_force=None, have_refid=False, create_default=None):
         if pfsense is None:
             pfsense = PFSenseModule(module)
         self.module = module    # ansible module
@@ -66,6 +70,8 @@ def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive
         self.obj = dict()       # dict holding target pfsense parameters
         self.map_param_if = map_param_if  # rules for mapping parameters
         self.param_force = param_force    # parameters that are forced to be present
+        self.create_default = create_default  # default values for a created target
+        self.have_refid = have_refid      # if the element has a refid item
         self.target_elt = None  # xml object holding target pfsense parameters
 
         self.update_php = update_php  # php code to update configuration
@@ -189,7 +195,12 @@ def _copy_and_update_target(self):
     def _create_target(self):
         """ create the XML target_elt """
         if self.node is not None:
-            return self.pfsense.new_element(self.node)
+            elt = self.pfsense.new_element(self.node)
+            if self.have_refid:
+                elt.append(self.pfsense.new_element('refid', text=self.pfsense.uniqid()))
+            if self.create_default is not None:
+                self.pfsense.copy_dict_to_element(self.create_default, elt)
+            return elt
         else:
             raise NotImplementedError()
 

From 2d1ee75c707888cea160f242d20fbcdc655da76c Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sat, 13 Jan 2024 22:43:24 -0700
Subject: [PATCH 22/62] [pfsense_authserver_ldap] Use have_refid,
 create_default more PFSenseModuleBase methods

---
 plugins/modules/pfsense_authserver_ldap.py    | 45 +++++--------------
 .../modules/test_pfsense_authserver_ldap.py   |  4 +-
 2 files changed, 13 insertions(+), 36 deletions(-)

diff --git a/plugins/modules/pfsense_authserver_ldap.py b/plugins/modules/pfsense_authserver_ldap.py
index 067abcf0..4de33cf8 100644
--- a/plugins/modules/pfsense_authserver_ldap.py
+++ b/plugins/modules/pfsense_authserver_ldap.py
@@ -113,7 +113,7 @@
     description: Shell Authentication Group DN (pfsense-CE >=2.5.0, pfsense-PLUS >=21.2)
     type: str
   ldap_allow_unauthenticated:
-    description: Allow unauthenticated bind (pfsense-CE >=2.5.0, pfsense-PLUS >=21.2)
+    description: Allow unauthenticated bind (pfsense-CE >=2.5.0, pfsense-PLUS >=21.2). Defaults to true.
     type: bool
 """
 
@@ -185,20 +185,26 @@
     'ldap_allow_unauthenticated': {'required': False, 'type': 'bool'},
 }
 
+AUTHSERVER_LDAP_CREATE_DEFAULT = dict(
+    ldap_allow_unauthenticated=None
+)
+
 
 class PFSenseAuthserverLDAPModule(PFSenseModuleBase):
     """ module managing pfsense LDAP authentication """
 
+    ##############################
+    # unit tests
+    #
+    # Must be class method for unit test usage
     @staticmethod
     def get_argument_spec():
         """ return argument spec """
         return PFSENSE_AUTHSERVER_LDAP_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSenseAuthserverLDAPModule, self).__init__(module, pfsense)
-        self.name = "pfsense_authserver_ldap"
-        self.root_elt = self.pfsense.get_element('system')
-        self.authservers = self.root_elt.findall('authserver')
+        super(PFSenseAuthserverLDAPModule, self).__init__(module, pfsense, name='pfsense_authserver_ldap', root='system', node='authserver', key='name',
+                                                          have_refid=True, create_default=AUTHSERVER_LDAP_CREATE_DEFAULT)
 
     ##############################
     # params processing
@@ -214,7 +220,6 @@ def _params_to_obj(self):
         params = self.params
 
         obj = dict()
-        self.obj = obj
 
         obj['name'] = params['name']
         if params['state'] == 'present':
@@ -285,34 +290,6 @@ def _find_target(self):
         else:
             return None
 
-    def _find_this_index(self):
-        return self.authservers.index(self.target_elt)
-
-    def _find_last_index(self):
-        if self.authservers:
-            return list(self.root_elt).index(self.authservers[len(self.authservers) - 1])
-        else:
-            return 0
-
-    def _create_target(self):
-        """ create the XML target_elt """
-        elt = self.pfsense.new_element('authserver')
-        elt.append(self.pfsense.new_element('ldap_allow_unauthenticated', text=None))
-        elt.append(self.pfsense.new_element('refid', text=self.pfsense.uniqid()))
-        return elt
-
-    ##############################
-    # Logging
-    #
-    def _get_obj_name(self):
-        """ return obj's name """
-        return "'{0}'".format(self.obj['name'])
-
-    def _log_fields(self, before=None):
-        """ generate pseudo-CLI command fields parameters to create an obj """
-        values = ''
-        return values
-
 
 def main():
     module = AnsibleModule(
diff --git a/tests/unit/plugins/modules/test_pfsense_authserver_ldap.py b/tests/unit/plugins/modules/test_pfsense_authserver_ldap.py
index da286ac6..b77403f9 100644
--- a/tests/unit/plugins/modules/test_pfsense_authserver_ldap.py
+++ b/tests/unit/plugins/modules/test_pfsense_authserver_ldap.py
@@ -70,7 +70,7 @@ def check_target_elt(self, obj, target_elt):
     def test_authserver_create(self):
         """ test creation of a new authserver """
         obj = dict(name='authserver1', host='ldap.example.com', transport='tcp', scope='one', authcn='CN=Users')
-        self.do_module_test(obj, command="create authserver_ldap 'authserver1'")
+        self.do_module_test(obj, command="create authserver_ldap 'authserver1', host='ldap.example.com'")
 
     def test_authserver_delete(self):
         """ test deletion of a authserver """
@@ -85,7 +85,7 @@ def test_authserver_update_noop(self):
     def test_authserver_update_host(self):
         """ test updating host of a authserver """
         obj = dict(name='DELLDAP', ldap_timeout=5, host='ldap2.blah.com', transport='tcp', scope='one', authcn='CN=Users')
-        self.do_module_test(obj, command="update authserver_ldap 'DELLDAP' set ")
+        self.do_module_test(obj, command="update authserver_ldap 'DELLDAP' set host='ldap2.blah.com', ldap_pam_groupdn=none, ldap_allow_unauthenticated=none")
 
     ##############
     # misc

From 1beca476a2f42b001e97a33060dd12b9117d3af4 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sun, 14 Jan 2024 09:41:02 -0700
Subject: [PATCH 23/62] [pfsense_module] Add _REQUIRED_IF

---
 misc/pfsense_module.py.j2 | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index d1fcb65e..6033a27b 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -25,6 +25,7 @@ options:
     type: str
   state:
     description: State in which to leave the {{ module_node }}
+    default: present
     choices: [ "present", "absent" ]
     type: str
 {% for param in params %}
@@ -71,6 +72,8 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 
 # Change to name of module, extend for needed parameters
 {{ module_node | upper() }}_ARGUMENT_SPEC = {
+    # Only name should be required here - othewise you cannot remove an item with just 'name'
+    # Required arguments for creation should be note in {{ module_node | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
     'name': {'required': True, 'type': 'str'},
     'state': {
         'default': 'present',
@@ -86,6 +89,9 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 {% endfor %}
 }
 
+{{ module_node | upper() }}_REQUIRED_IF = {
+    ['state', 'present', []],
+}
 
 class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
     """ module managing pfsense {{ module_node }} """
@@ -121,6 +127,7 @@ class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
 def main():
     module = AnsibleModule(
         argument_spec={{ module_node | upper() }}_ARGUMENT_SPEC,
+        required_if={{ module_node | upper() }}_REQUIRED_IF,
         supports_check_mode=True)
 
     pfmodule = PFSense{{ module_node | capitalize() }}Module(module)

From 5c52c2d49aa797d7495cafd898f782dad4f1b010 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sun, 14 Jan 2024 22:07:36 -0700
Subject: [PATCH 24/62] Add *.swp to .gitignore

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index 2eae9cdf..3c87c6ec 100644
--- a/.gitignore
+++ b/.gitignore
@@ -108,3 +108,6 @@ venv.bak/
 
 # ansible-galaxy package
 *.tar.gz
+
+# vi
+*.swp

From 723bf26e54c7d9bf853761a0369a51eb47b008b6 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sun, 14 Jan 2024 22:11:07 -0700
Subject: [PATCH 25/62] [generate_module] Refactor

- Drop .py
- Store files in tmpdir
- Option to keep tmpfile, overwrite output
- Use '-' in options
- Handle number and password inputs
- Collect min, step, and placeholder attributes
- Template fixes
- Collect info in params_full and merge
---
 misc/{generate_module.py => generate_module} | 71 ++++++++++++++------
 misc/pfsense_module.py.j2                    | 21 +++---
 tests/sanity/ignore-2.14.txt                 |  2 +
 tests/sanity/ignore-2.15.txt                 |  2 +
 tests/sanity/ignore-2.16.txt                 |  2 +
 5 files changed, 68 insertions(+), 30 deletions(-)
 rename misc/{generate_module.py => generate_module} (79%)

diff --git a/misc/generate_module.py b/misc/generate_module
similarity index 79%
rename from misc/generate_module.py
rename to misc/generate_module
index 98459fe3..c0534752 100755
--- a/misc/generate_module.py
+++ b/misc/generate_module
@@ -1,5 +1,8 @@
 #!/usr/bin/python3
 
+# Copyright: (c) 2024, Orion Poplawski <orion@nwra.com>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
 import argparse
 import datetime
 import git
@@ -12,6 +15,7 @@
 import requests
 from scp import SCPClient
 import sys
+import tempfile
 from urllib.parse import urlparse
 
 gitconfig = git.GitConfigParser()
@@ -23,16 +27,21 @@
 parser.add_argument('--urlfile', help='A local file copy of the URL to scrape')
 parser.add_argument('--user', default='admin', help='The user to connect as')
 parser.add_argument('--password', default='changeme', help='The password of user')
-parser.add_argument('--author_name', default=author_name, help='The full name of the module author')
-parser.add_argument('--author_email', default=author_email, help='The email address of the module author')
-parser.add_argument('--author_handle', default='', help='The github handle of the module author')
-parser.add_argument('--module_name', help='The name of the module to generate - defaults to being based on the url')
-parser.add_argument('--item_min', default='item_min', help='The name of the minimally configured item to search for in config.xml')
-parser.add_argument('--item_full', default='item_full',
+parser.add_argument('--author-name', default=author_name, help='The full name of the module author')
+parser.add_argument('--author-email', default=author_email, help='The email address of the module author')
+parser.add_argument('--author-handle', default='', help='The github handle of the module author')
+parser.add_argument('--module-name', help='The name of the module to generate - defaults to being based on the url')
+parser.add_argument('--item-min', default='item_min', help='The name of the minimally configured item to search for in config.xml')
+parser.add_argument('--item-full', default='item_full',
                     help='The name of the fully configured item to search for in config.xml, will be used for exmaples in the documentation')
+parser.add_argument('--force', action=argparse.BooleanOptionalAction, help='Force overwriting the output file if it exists')
+parser.add_argument('--keep', action=argparse.BooleanOptionalAction, help='Keep the downloaded files in the temporary directory')
 
 args = parser.parse_args()
 
+# Temporary directory for files
+tmpdir = tempfile.TemporaryDirectory(prefix='pfgenmod-', delete=(not args.keep))
+
 if args.url is not None:
     parsed_uri = urlparse(args.url)
 
@@ -44,7 +53,7 @@
 
     # Construct a likely module name from the URL
     if args.module_name is None:
-        module_name = re.sub(r'^/(?:firewall_|system_)?(.*)(?:_edit)?\.php.*$', r'pfsense_\1', parsed_uri.path)
+        module_name = re.sub(r'^/(?:firewall_|system_)?(.*?)(?:_edit)?\.php.*$', r'pfsense_\1', parsed_uri.path)
         module_name_singular = re.sub(r'ses$', 's', module_name)
         if module_name_singular != module_name:
             module_name = module_name_singular
@@ -71,9 +80,12 @@
     # Retrieve the configuration web page and parse it
     r = client.get(args.url, verify=False)
     html = lxml.html.fromstring(r.text)
-    f = open(f'{module_name}.html', 'w')
-    f.write(r.text)
-    f.close()
+
+    # Save it if asked to keep files
+    if args.keep:
+        f = open(f'{tmpdir.name}/{module_name}.html', 'w')
+        f.write(r.text)
+        f.close()
 
 elif args.urlfile is not None:
     # Use a cached copy of the web page - get rid of this?  Need to specify host and module name
@@ -89,11 +101,11 @@
 ssh.load_system_host_keys()
 ssh.connect(host, username='root', password=args.password)
 scp = SCPClient(ssh.get_transport())
-scp.get('/cf/conf/config.xml')
+scp.get('/cf/conf/config.xml', f'{tmpdir.name}')
 scp.close()
 
 # Parse the config.xml file
-root = ET.parse('config.xml').getroot()
+root = ET.parse(f'{tmpdir.name}/config.xml').getroot()
 
 # Search for any element with our target text, make sure we found only one
 xpath = f'.//*[.="{args.item_min}"]'
@@ -132,10 +144,19 @@
 for elt in full_elt:
     if elt.tag == '':
         continue
-    params_full[elt.tag] = elt.text
+    params_full[elt.tag] = dict()
+    addr_elt = elt.find('address')
+    if addr_elt is not None:
+        params_full[elt.tag]['example'] = addr_elt.text
+        params_full[elt.tag]['address'] = True
+    elif elt.text is not None:
+        params_full[elt.tag]['type'] = 'str'
+        params_full[elt.tag]['example'] = elt.text.strip()
+    # else:
+        # Likely a bool?
 
 if 'type' in params_full:
-    module_type = params_full['type']
+    module_type = params_full['type']['example']
 else:
     module_type = None
 
@@ -164,8 +185,17 @@
             param['type'] = 'bool'
             param['value'] = input.attrib['value'].strip()
             param['example'] = 'true'
+        elif input.type == 'number':
+            param['type'] = 'int'
+        elif input.type == 'password':
+            param['type'] = 'str'
+            param['password'] = True  # TODO - set nolog
         elif input.type == 'text':
             param['type'] = 'str'
+        # TOOD - handle placeholder as 'default' value - description? create_default? example?
+        for attr in ['min', 'placeholder', 'step']:
+            if attr in input.attrib:
+                param[attr] = input.attrib[attr]
         # Text sometimes is after the input element inside the enclosing <label>
         if input.tail:
             param['description'] = input.tail.strip()
@@ -247,16 +277,18 @@
 print('')
 params_xml_only = list(set(params_full.keys()) - set(params.keys()) - {module_key, 'refid'})
 print('XML parameters not in web: ' + str(params_xml_only))
+
 for param in params_xml_only:
-    params[param] = dict(type='str', example=params_full[param])
+    params[param] = params_full[param]
 
-# Create some sample descriptions and example values
+# Create some sample descriptions
 for name, param in params.items():
+    # TODO - wrap long descriptions
     if 'description' not in param or param['description'] == '':
         param['description'] = f'The {name} of the {module_node}'
     if 'example' not in param or param['example'] == '':
-        if name in params_full and params_full[name] != '' and params_full[name] is not None:
-            param['example'] = params_full[name]
+        if name in params_full and 'example' in params_full[name]:
+            param['example'] = params_full[name]['example']
 
 # Template variables
 context = dict(
@@ -275,9 +307,8 @@
 environment = jinja2.Environment(loader=jinja2.FileSystemLoader("misc/"), trim_blocks=True, keep_trailing_newline=True)
 template = environment.get_template("pfsense_module.py.j2")
 
-# Todo - prompt for overwrite
 filename = f'plugins/modules/{module_name}.py'
-if os.path.isfile(filename):
+if os.path.isfile(filename) and not args.force:
     sys.exit(f'{filename} already exists!')
 print(f'Writing {filename} with {context}')
 f = open(f'{filename}', 'w')
diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index 6033a27b..1988c3a9 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -25,16 +25,16 @@ options:
     type: str
   state:
     description: State in which to leave the {{ module_node }}
+    choices: ['present', 'absent']
     default: present
-    choices: [ "present", "absent" ]
     type: str
-{% for param in params %}
-  {{ param }}:
-    description: {{ params[param]['description'] | default('') }}
-{% if 'choices' in params[param] %}
-    choices: {{ params[param]['choices'] | default('') }}
+{% for name, param in params.items() %}
+  {{ name }}:
+    description: {{ "'" if ':' in param['description'] else '' }}{{ param['description'] | default('') }}{{ "'" if ':' in param['description'] else '' }}
+{% if 'choices' in param %}
+    choices: {{ param['choices'] }}
 {% endif %}
-    type: {{ params[param]['type'] | default('') }}
+    type: {{ param['type'] | default('') }}
 {% endfor %}
 
 author: {{ author_name }} (@{{ author_handle }})
@@ -89,9 +89,10 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 {% endfor %}
 }
 
-{{ module_node | upper() }}_REQUIRED_IF = {
+{{ module_node | upper() }}_REQUIRED_IF = [
     ['state', 'present', []],
-}
+]
+
 
 class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
     """ module managing pfsense {{ module_node }} """
@@ -106,7 +107,7 @@ class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
         return {{ module_node | upper() }}_ARGUMENT_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSenseMODULEModule, self).__init__(module, pfsense, root='{{ module_root }}', node='{{ module_node }}')
+        super(PFSense{{ module_node | capitalize() }}Module, self).__init__(module, pfsense, root='{{ module_root }}', node='{{ module_node }}')
 
     ##############################
     # params processing
diff --git a/tests/sanity/ignore-2.14.txt b/tests/sanity/ignore-2.14.txt
index b7549f46..7cf6fe94 100644
--- a/tests/sanity/ignore-2.14.txt
+++ b/tests/sanity/ignore-2.14.txt
@@ -1,3 +1,5 @@
+misc/generate_module shebang # This is not a module
+misc/pfsense_module.py.j2 shebang # This is not a module
 plugins/modules/pfsense_cert.py validate-modules:no-log-needed # Argument 'keylen' is not sensitive
 plugins/modules/pfsense_dhcp_static.py validate-modules:no-log-needed # Arguments 'ddnsdomainkeyname' and 'ddnsdomainkeyalgorithm' are not sensitive
 plugins/modules/pfsense_ipsec.py validate-modules:no-log-needed # Argument 'rekey_time' is not sensitive
diff --git a/tests/sanity/ignore-2.15.txt b/tests/sanity/ignore-2.15.txt
index b7549f46..7cf6fe94 100644
--- a/tests/sanity/ignore-2.15.txt
+++ b/tests/sanity/ignore-2.15.txt
@@ -1,3 +1,5 @@
+misc/generate_module shebang # This is not a module
+misc/pfsense_module.py.j2 shebang # This is not a module
 plugins/modules/pfsense_cert.py validate-modules:no-log-needed # Argument 'keylen' is not sensitive
 plugins/modules/pfsense_dhcp_static.py validate-modules:no-log-needed # Arguments 'ddnsdomainkeyname' and 'ddnsdomainkeyalgorithm' are not sensitive
 plugins/modules/pfsense_ipsec.py validate-modules:no-log-needed # Argument 'rekey_time' is not sensitive
diff --git a/tests/sanity/ignore-2.16.txt b/tests/sanity/ignore-2.16.txt
index b7549f46..7cf6fe94 100644
--- a/tests/sanity/ignore-2.16.txt
+++ b/tests/sanity/ignore-2.16.txt
@@ -1,3 +1,5 @@
+misc/generate_module shebang # This is not a module
+misc/pfsense_module.py.j2 shebang # This is not a module
 plugins/modules/pfsense_cert.py validate-modules:no-log-needed # Argument 'keylen' is not sensitive
 plugins/modules/pfsense_dhcp_static.py validate-modules:no-log-needed # Arguments 'ddnsdomainkeyname' and 'ddnsdomainkeyalgorithm' are not sensitive
 plugins/modules/pfsense_ipsec.py validate-modules:no-log-needed # Argument 'rekey_time' is not sensitive

From 8c5953ba12beb595ba488eb5674f7e26bc052105 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sun, 14 Jan 2024 22:12:20 -0700
Subject: [PATCH 26/62] [pfsense_authserver_ldap] Update copyright

---
 plugins/modules/pfsense_authserver_ldap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/modules/pfsense_authserver_ldap.py b/plugins/modules/pfsense_authserver_ldap.py
index 4de33cf8..08a6cd45 100644
--- a/plugins/modules/pfsense_authserver_ldap.py
+++ b/plugins/modules/pfsense_authserver_ldap.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-# Copyright: (c) 2018-2020, Orion Poplawski <orion@nwra.com>
+# Copyright: (c) 2018-2024, Orion Poplawski <orion@nwra.com>
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 from __future__ import absolute_import, division, print_function

From 031b2f776136b021353e8c6da0b069984fcf7c4f Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 08:35:34 -0700
Subject: [PATCH 27/62] [generate_module] Add --type-suffix option

---
 misc/generate_module | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index c0534752..6889108e 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -3,6 +3,11 @@
 # Copyright: (c) 2024, Orion Poplawski <orion@nwra.com>
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
+# TODO:
+#   - collect .php file and scan source for update and validation code
+#   - validation for addresses
+#   - bool type determination and paramter list
+
 import argparse
 import datetime
 import git
@@ -31,6 +36,7 @@ parser.add_argument('--author-name', default=author_name, help='The full name of
 parser.add_argument('--author-email', default=author_email, help='The email address of the module author')
 parser.add_argument('--author-handle', default='', help='The github handle of the module author')
 parser.add_argument('--module-name', help='The name of the module to generate - defaults to being based on the url')
+parser.add_argument('--type-suffix', const=True, default=False, nargs='?', help='Suffix the module name with the item type', )
 parser.add_argument('--item-min', default='item_min', help='The name of the minimally configured item to search for in config.xml')
 parser.add_argument('--item-full', default='item_full',
                     help='The name of the fully configured item to search for in config.xml, will be used for exmaples in the documentation')
@@ -81,12 +87,6 @@ if args.url is not None:
     r = client.get(args.url, verify=False)
     html = lxml.html.fromstring(r.text)
 
-    # Save it if asked to keep files
-    if args.keep:
-        f = open(f'{tmpdir.name}/{module_name}.html', 'w')
-        f.write(r.text)
-        f.close()
-
 elif args.urlfile is not None:
     # Use a cached copy of the web page - get rid of this?  Need to specify host and module name
     html = lxml.html.parse(args.urlfile)
@@ -157,6 +157,10 @@ for elt in full_elt:
 
 if 'type' in params_full:
     module_type = params_full['type']['example']
+    if args.type_suffix is True:
+        module_name += f'_{module_type}'
+    if isinstance(args.type_suffix, str):
+        module_name += f'_{args.type_suffix}'
 else:
     module_type = None
 
@@ -314,3 +318,9 @@ print(f'Writing {filename} with {context}')
 f = open(f'{filename}', 'w')
 f.write(template.render(context))
 f.close()
+
+# Save the scraped web page if asked to keep files
+if args.keep:
+    f = open(f'{tmpdir.name}/{module_name}.html', 'w')
+    f.write(r.text)
+    f.close()

From 8a667b263e81aad5e12a4654d1a1466aee06cd99 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 09:43:47 -0700
Subject: [PATCH 28/62] [generate_module] Mark items required if they have
 "element-required" class, add to REQUIRED_IF; Add key to PFSenseModuleBase
 init

---
 misc/generate_module      | 8 ++++++--
 misc/pfsense_module.py.j2 | 9 +++++++--
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index 6889108e..da82342c 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -8,6 +8,7 @@
 #   - validation for addresses
 #   - bool type determination and paramter list
 
+from ansible.plugins.filter.core import dict_to_list_of_dict_key_value_elements
 import argparse
 import datetime
 import git
@@ -230,6 +231,8 @@ for input in html.forms[0].inputs:
         if descr_elt.text:
             print(f'Found descr_elt {descr_elt.tag} {descr_elt.text}')
             param['description'] += f'{descr_elt.text.strip()} of the {module_node}'
+        if 'class' in descr_elt.attrib and descr_elt.attrib['class'] == 'element-required':
+            param['required'] = True
         help_elt = div1.find('span[@class="help-block"]')
         if help_elt is not None:
             print(f'help_elt text {help_elt.text.strip()}')
@@ -308,8 +311,9 @@ context = dict(
 )
 
 # Render our module!
-environment = jinja2.Environment(loader=jinja2.FileSystemLoader("misc/"), trim_blocks=True, keep_trailing_newline=True)
-template = environment.get_template("pfsense_module.py.j2")
+jenv = jinja2.Environment(loader=jinja2.FileSystemLoader("misc/"), trim_blocks=True, keep_trailing_newline=True)
+jenv.filters['dict2items'] = dict_to_list_of_dict_key_value_elements
+template = jenv.get_template("pfsense_module.py.j2")
 
 filename = f'plugins/modules/{module_name}.py'
 if os.path.isfile(filename) and not args.force:
diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index 1988c3a9..0d741a15 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -90,7 +90,12 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 }
 
 {{ module_node | upper() }}_REQUIRED_IF = [
-    ['state', 'present', []],
+{% if module_type is none %}
+    ['state', 'present', ['{{ params | dict2items | selectattr('value.required', 'defined') | select('key') | join("', '") }}]],
+{% else %}
+    ['state', 'present', ['type']],
+    ['type', '{{ params['type']['example'] }}', ['{{ params | dict2items | rejectattr('key', 'equalto', 'type') | selectattr('value.required', 'defined') | map(attribute='key') | join("', '") }}']],
+{% endif %}
 ]
 
 
@@ -107,7 +112,7 @@ class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
         return {{ module_node | upper() }}_ARGUMENT_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSense{{ module_node | capitalize() }}Module, self).__init__(module, pfsense, root='{{ module_root }}', node='{{ module_node }}')
+        super(PFSense{{ module_node | capitalize() }}Module, self).__init__(module, pfsense, root='{{ module_root }}', node='{{ module_node }}', key='{{ module_key }}')
 
     ##############################
     # params processing

From f91f037fe69473f9b0e7632c16e033c4a41ab039 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 12:06:05 -0700
Subject: [PATCH 29/62] [generate_module] Scan php code

---
 misc/generate_module      | 35 +++++++++++++++++++++++++++++++++--
 misc/pfsense_module.py.j2 | 17 ++++++++++++++++-
 2 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index da82342c..c0b3eac0 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -4,7 +4,6 @@
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
 # TODO:
-#   - collect .php file and scan source for update and validation code
 #   - validation for addresses
 #   - bool type determination and paramter list
 
@@ -55,9 +54,12 @@ if args.url is not None:
     # Login using just the base URL
     login_url = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
 
-    # Collect to host for later use to scp config.xml
+    # Collect host for later use to scp config.xml
     host = f'{parsed_uri.netloc}'
 
+    # Collect phpfile to scp later
+    phpfile = re.sub(r'^/(.*\.php).*$', r'\1', parsed_uri.path)
+
     # Construct a likely module name from the URL
     if args.module_name is None:
         module_name = re.sub(r'^/(?:firewall_|system_)?(.*?)(?:_edit)?\.php.*$', r'pfsense_\1', parsed_uri.path)
@@ -103,6 +105,8 @@ ssh.load_system_host_keys()
 ssh.connect(host, username='root', password=args.password)
 scp = SCPClient(ssh.get_transport())
 scp.get('/cf/conf/config.xml', f'{tmpdir.name}')
+if phpfile is not None:
+    scp.get(f'/usr/local/www/{phpfile}', f'{tmpdir.name}')
 scp.close()
 
 # Parse the config.xml file
@@ -167,6 +171,30 @@ else:
 
 print('')
 
+# Parse the php file
+if phpfile is not None:
+    php_requires = ''
+    php_save = ''
+    php_subsystem = ''
+    found_save = False
+    with open(f'{tmpdir.name}/{phpfile}', 'r') as f:
+        for line in f:
+             if re.match(r'require_once', line):
+                 php_requires += line
+                 continue
+             if re.search(r'if \(\$_POST\[\'save\']', line):
+                 found_save = True
+                 continue
+             if found_save:
+                 if re.match(r'}', line):
+                     found_save = False
+                 else:
+                     php_save += re.sub(r'\t', '    ', re.sub(r'^\t', '', line))
+                     subsystem_search = re.search(r'subsystem_dirty\(\'(.*)\'\)', line)
+                     if subsystem_search:
+                         php_subsystem = subsystem_search.group(1)
+
+
 # The web page should have a single form
 if len(html.forms) != 1:
     sys.exit(f'Found {len(html.forms)} forms instead of a single one!')
@@ -307,6 +335,9 @@ context = dict(
     author_name=args.author_name,
     author_email=args.author_email,
     author_handle=args.author_handle,
+    php_requires=php_requires,
+    php_save=php_save,
+    php_subsystem=php_subsystem,
     year=datetime.date.today().year,
 )
 
diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index 0d741a15..f4d59bde 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -98,6 +98,13 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 {% endif %}
 ]
 
+{% if 'filter.inc' in php_requires %}
+{{ module_node | upper() }}_PHP_COMMAND_SET = r'''
+require_once("filter.inc");
+if (filter_configure() == 0) { clear_subsystem_dirty('{{ php_subsystem }}'); }
+'''
+
+{% endif %}
 
 class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
     """ module managing pfsense {{ module_node }} """
@@ -112,7 +119,7 @@ class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
         return {{ module_node | upper() }}_ARGUMENT_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSense{{ module_node | capitalize() }}Module, self).__init__(module, pfsense, root='{{ module_root }}', node='{{ module_node }}', key='{{ module_key }}')
+        super(PFSense{{ module_node | capitalize() }}Module, self).__init__(module, pfsense, root='{{ module_root }}', node='{{ module_node }}', key='{{ module_key }}'{{ ', update_php=' ~ module_node | upper() ~ '_PHP_COMMAND_SET' if 'filter.inc' in php_requires else '' }})
 
     ##############################
     # params processing
@@ -130,6 +137,14 @@ class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
                 self.module.fail_json(msg='timeout {0} must be greater than 1'.format(params['timeout']))
 
 
+# TODO - review this for clues for input validation.  Search for functions in the below require_once files in /etc and /usr/local/pfSense/include
+PHP_VALIDATION = r'''
+{{ php_requires }}
+
+{{ php_save }}
+'''
+
+
 def main():
     module = AnsibleModule(
         argument_spec={{ module_node | upper() }}_ARGUMENT_SPEC,

From 814955a8a64b4785343bc2fd7e19f595323f197c Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 14:57:13 -0700
Subject: [PATCH 30/62] [module_base] Store refid in obj; Use diff["after"] for
 logging comparison instead of obj, handle bool better

---
 plugins/module_utils/module_base.py                      | 9 +++++++--
 .../unit/plugins/modules/test_pfsense_authserver_ldap.py | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 4c455ac8..2970bcef 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -197,7 +197,9 @@ def _create_target(self):
         if self.node is not None:
             elt = self.pfsense.new_element(self.node)
             if self.have_refid:
-                elt.append(self.pfsense.new_element('refid', text=self.pfsense.uniqid()))
+                # Store in obj so that we can refer to it later if needed
+                self.obj['refid'] = self.pfsense.uniqid()
+                elt.append(self.pfsense.new_element('refid', text=self.obj['refid']))
             if self.create_default is not None:
                 self.pfsense.copy_dict_to_element(self.create_default, elt)
             return elt
@@ -353,7 +355,10 @@ def _log_fields(self, before=None):
                 values += self.format_cli_field(self.obj, param)
         else:
             for param in [n for n in self.argument_spec.keys() if n != 'state' and n != self.key]:
-                values += self.format_updated_cli_field(self.obj, before, param, add_comma=(values))
+                if self.argument_spec[param].get('type') == 'bool':
+                    values += self.format_updated_cli_field(self.diff['after'], before, param, fvalue=self.fvalue_bool, add_comma=(values))
+                else:
+                    values += self.format_updated_cli_field(self.diff['after'], before, param, add_comma=(values))
         return values
 
     @staticmethod
diff --git a/tests/unit/plugins/modules/test_pfsense_authserver_ldap.py b/tests/unit/plugins/modules/test_pfsense_authserver_ldap.py
index b77403f9..36648dd7 100644
--- a/tests/unit/plugins/modules/test_pfsense_authserver_ldap.py
+++ b/tests/unit/plugins/modules/test_pfsense_authserver_ldap.py
@@ -85,7 +85,7 @@ def test_authserver_update_noop(self):
     def test_authserver_update_host(self):
         """ test updating host of a authserver """
         obj = dict(name='DELLDAP', ldap_timeout=5, host='ldap2.blah.com', transport='tcp', scope='one', authcn='CN=Users')
-        self.do_module_test(obj, command="update authserver_ldap 'DELLDAP' set host='ldap2.blah.com', ldap_pam_groupdn=none, ldap_allow_unauthenticated=none")
+        self.do_module_test(obj, command="update authserver_ldap 'DELLDAP' set host='ldap2.blah.com'")
 
     ##############
     # misc

From cd2b18342ec23f50c9c8b6487de6a86711eb907b Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 15:05:42 -0700
Subject: [PATCH 31/62] [pfsense_ca] Use PFSenseModuleBase _create_target,
 logging; Drop pfSense < 2.5.0 support; Fix certificate in tests to not have
 extra newlines

---
 plugins/modules/pfsense_ca.py                 | 43 +++++--------------
 .../modules/fixtures/pfsense_ca_config.xml    |  2 +-
 tests/unit/plugins/modules/test_pfsense_ca.py | 39 ++++++++---------
 3 files changed, 30 insertions(+), 54 deletions(-)

diff --git a/plugins/modules/pfsense_ca.py b/plugins/modules/pfsense_ca.py
index 6795593f..70fe3576 100644
--- a/plugins/modules/pfsense_ca.py
+++ b/plugins/modules/pfsense_ca.py
@@ -112,6 +112,13 @@
     serial=dict(type='int'),
 )
 
+# These are default but not enforced values
+CA_CREATE_DEFAULT = dict(
+    randomserial='disabled',
+    serial='0',
+    trust='disabled',
+)
+
 
 class PFSenseCAModule(PFSenseModuleBase):
     """ module managing pfsense certificate authorities """
@@ -122,7 +129,7 @@ def get_argument_spec():
         return PFSENSE_CA_ARGUMENT_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSenseCAModule, self).__init__(module, pfsense, root='pfsense', node='ca')
+        super(PFSenseCAModule, self).__init__(module, pfsense, root='pfsense', node='ca', have_refid=True, create_default=CA_CREATE_DEFAULT)
         self.name = "pfsense_ca"
         self.refresh_crls = False
         self.crl = None
@@ -162,8 +169,6 @@ def _params_to_obj(self):
         params = self.params
 
         obj = dict()
-        self.obj = obj
-
         obj['descr'] = params['name']
         if params['state'] == 'present':
             if 'certificate' in params and params['certificate'] is not None:
@@ -211,18 +216,6 @@ def _find_crl_by_refid(self, crlrefid):
         else:
             return None
 
-    def _create_target(self):
-        """ create the XML target_elt """
-        elt = self.pfsense.new_element('ca')
-        # We need this later in _copy_and_add_target()
-        self.obj['refid'] = self.pfsense.uniqid()
-        elt.append(self.pfsense.new_element('refid', text=self.obj['refid']))
-        # These are default but not enforced values
-        elt.append(self.pfsense.new_element('randomserial', text='disabled'))
-        elt.append(self.pfsense.new_element('serial', text='0'))
-        elt.append(self.pfsense.new_element('trust', text='disabled'))
-        return elt
-
     def _copy_and_add_target(self):
         """ populate the XML target_elt """
         self.pfsense.copy_dict_to_element(self.obj, self.target_elt)
@@ -282,18 +275,6 @@ def _copy_and_update_target(self):
 
         return (before, changed)
 
-    ##############################
-    # Logging
-    #
-    def _get_obj_name(self):
-        """ return obj's name """
-        return self.obj['descr']
-
-    def _log_fields(self, before=None):
-        """ generate pseudo-CLI command fields parameters to create an obj """
-        values = ''
-        return values
-
     ##############################
     # run
     #
@@ -315,20 +296,16 @@ def _update(self):
             crl_stdout = ''
             crl_stderr = ''
             if self.refresh_crls:
-                if self.pfsense.is_at_least_2_5_0():
-                    ipsec_configure = 'ipsec_configure'
-                else:
-                    ipsec_configure = 'vpn_ipsec_configure'
                 (dummy, crl_stdout, crl_stderr) = self.pfsense.phpshell("""
                     require_once("openvpn.inc");
                     openvpn_refresh_crls();
                     require_once("vpn.inc");
-                    {0}();""".format(ipsec_configure))
+                    ipsec_configure();""")
                 return (dummy, stdout + crl_stdout, stderr + crl_stderr)
 
             return (dummy, stdout + crl_stdout, stderr + crl_stderr)
         else:
-            return (None, '', '')
+            return ('', '', '')
 
     def _pre_remove_target_elt(self):
         self.diff['after'] = {}
diff --git a/tests/unit/plugins/modules/fixtures/pfsense_ca_config.xml b/tests/unit/plugins/modules/fixtures/pfsense_ca_config.xml
index 21a1e3d3..32980ee0 100644
--- a/tests/unit/plugins/modules/fixtures/pfsense_ca_config.xml
+++ b/tests/unit/plugins/modules/fixtures/pfsense_ca_config.xml
@@ -288,7 +288,7 @@
                 <descr><![CDATA[testdel]]></descr>
                 <trust>enabled</trust>
                 <randomserial>enabled</randomserial>
-                <crt>LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlFQ0RDQ0F2Q2dBd0lCQWdJSUZqRk9oczFuTXpRd0RRWUpLb1pJaHZjTkFRRUxCUUF3WERFVE1CRUdBMVVFDQpBeE1LYjNCbGJuWndiaTFqWVRFTE1Ba0dBMVVFQmhNQ1ZWTXhFVEFQQmdOVkJBZ1RDRU52Ykc5eVlXUnZNUkF3DQpEZ1lEVlFRSEV3ZENiM1ZzWkdWeU1STXdFUVlEVlFRS0V3cHdabE5sYm5OcFlteGxNQjRYRFRJeU1ESXhOREExDQpNRGd6TVZvWERUTXlNREl4TWpBMU1EZ3pNVm93WERFVE1CRUdBMVVFQXhNS2IzQmxiblp3YmkxallURUxNQWtHDQpBMVVFQmhNQ1ZWTXhFVEFQQmdOVkJBZ1RDRU52Ykc5eVlXUnZNUkF3RGdZRFZRUUhFd2RDYjNWc1pHVnlNUk13DQpFUVlEVlFRS0V3cHdabE5sYm5OcFlteGxNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDDQpBUUVBbXN2aUpNRTFFVGVkNGZPdGJrSHBGM2Q5ZU0rNjQwOFhQbmE4dEpHZEJxM1VBQ3hFem9hQktSdDJ5MWN0DQo2elFEZTVGRjRBQXZ0VjF1Y1pwc2w1bzREUy9JR1NibjZkM1lNaytqOGpBUTNFbXpSOEdPb2huZ2YxUTlBWEM2DQpvaDRyQlA1c1g0WTh1WThrSjNZclg1cVRwRlk1S0hMVTFBb1BleVE3eXlNWkhMb2t0OW5jK0ZGWnd3VTdSQ0dTDQpjTkxaaVZ4Q1FRSzVwOGs5bUE4Ymd4bHFZa2YwbUF5Qk53OU1BZlBVY1VrcUY2UDBnV1BIbElySFovdWhnN2RVDQorMjJhb2NLVUVOaXY5bXFhK0I2Y1VnTFRGVDZzMFZTRXNYL2RBZWg2MllMZ2ZtWEpnNmROSFFJK01nNlNrZWxwDQprOVZSVGVqaUVUSUVWOEpnZHYyTjdSU201d0lEQVFBQm80SE5NSUhLTUIwR0ExVWREZ1FXQkJSazVvQS8wcWEyDQpLUHdnb1hKcUtNdCtBb0tKZ1RDQmpRWURWUjBqQklHRk1JR0NnQlJrNW9BLzBxYTJLUHdnb1hKcUtNdCtBb0tKDQpnYUZncEY0d1hERVRNQkVHQTFVRUF4TUtiM0JsYm5ad2JpMWpZVEVMTUFrR0ExVUVCaE1DVlZNeEVUQVBCZ05WDQpCQWdUQ0VOdmJHOXlZV1J2TVJBd0RnWURWUVFIRXdkQ2IzVnNaR1Z5TVJNd0VRWURWUVFLRXdwd1psTmxibk5wDQpZbXhsZ2dnV01VNkd6V2N6TkRBTUJnTlZIUk1FQlRBREFRSC9NQXNHQTFVZER3UUVBd0lCQmpBTkJna3Foa2lHDQo5dzBCQVFzRkFBT0NBUUVBVUg5S0NkbUpkb0FKbFUwd0JKSFl4akxyS2xsUFk2T05ienI1SmJoQ002OUh4eFlODQpCa2lpbXd1N09mRmFGZkZDT25NSjhvcStKVGxjMG9vREoxM2xCdHRONkdybnZrUTNQMXdZYkNFTmJuaWxPYVVCDQpUSXJpSHl0TkRRYW91TmEvS1dzN0ZhdW9iY3RCbDF3OWF0b0hac041b2VoVDNyQVR2MUNDQXRqcGFUSklmSlIzDQowSVFPWWtlNG9ZNkRrSXdIcDJ2UFBtb29HZ0l0YlR3M1UrRTQxWVplN3FDbUUvN3pMVFNaa0lNMmx4NnpENDZqDQpEZjRyZ044TVVMNnhpd09MbzlyQUp5ckRNM2JEeTJ1QjY0QkVzRFFMa2huUE92ZWtETjQ1NnV6TmpYS0E3VnE4DQpoMS9nekRaSURpK1dYQ1lBY2JnTGhaVkJxdG42MnVtRnBNUkl1dz09DQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tDQo=</crt>
+                <crt>LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVDRENDQXZDZ0F3SUJBZ0lJRmpGT2hzMW5NelF3RFFZSktvWklodmNOQVFFTEJRQXdYREVUTUJFR0ExVUUKQXhNS2IzQmxiblp3YmkxallURUxNQWtHQTFVRUJoTUNWVk14RVRBUEJnTlZCQWdUQ0VOdmJHOXlZV1J2TVJBdwpEZ1lEVlFRSEV3ZENiM1ZzWkdWeU1STXdFUVlEVlFRS0V3cHdabE5sYm5OcFlteGxNQjRYRFRJeU1ESXhOREExCk1EZ3pNVm9YRFRNeU1ESXhNakExTURnek1Wb3dYREVUTUJFR0ExVUVBeE1LYjNCbGJuWndiaTFqWVRFTE1Ba0cKQTFVRUJoTUNWVk14RVRBUEJnTlZCQWdUQ0VOdmJHOXlZV1J2TVJBd0RnWURWUVFIRXdkQ2IzVnNaR1Z5TVJNdwpFUVlEVlFRS0V3cHdabE5sYm5OcFlteGxNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDCkFRRUFtc3ZpSk1FMUVUZWQ0Zk90YmtIcEYzZDllTSs2NDA4WFBuYTh0SkdkQnEzVUFDeEV6b2FCS1J0MnkxY3QKNnpRRGU1RkY0QUF2dFYxdWNacHNsNW80RFMvSUdTYm42ZDNZTWsrajhqQVEzRW16UjhHT29obmdmMVE5QVhDNgpvaDRyQlA1c1g0WTh1WThrSjNZclg1cVRwRlk1S0hMVTFBb1BleVE3eXlNWkhMb2t0OW5jK0ZGWnd3VTdSQ0dTCmNOTFppVnhDUVFLNXA4azltQThiZ3hscVlrZjBtQXlCTnc5TUFmUFVjVWtxRjZQMGdXUEhsSXJIWi91aGc3ZFUKKzIyYW9jS1VFTml2OW1xYStCNmNVZ0xURlQ2czBWU0VzWC9kQWVoNjJZTGdmbVhKZzZkTkhRSStNZzZTa2VscAprOVZSVGVqaUVUSUVWOEpnZHYyTjdSU201d0lEQVFBQm80SE5NSUhLTUIwR0ExVWREZ1FXQkJSazVvQS8wcWEyCktQd2dvWEpxS010K0FvS0pnVENCalFZRFZSMGpCSUdGTUlHQ2dCUms1b0EvMHFhMktQd2dvWEpxS010K0FvS0oKZ2FGZ3BGNHdYREVUTUJFR0ExVUVBeE1LYjNCbGJuWndiaTFqWVRFTE1Ba0dBMVVFQmhNQ1ZWTXhFVEFQQmdOVgpCQWdUQ0VOdmJHOXlZV1J2TVJBd0RnWURWUVFIRXdkQ2IzVnNaR1Z5TVJNd0VRWURWUVFLRXdwd1psTmxibk5wCllteGxnZ2dXTVU2R3pXY3pOREFNQmdOVkhSTUVCVEFEQVFIL01Bc0dBMVVkRHdRRUF3SUJCakFOQmdrcWhraUcKOXcwQkFRc0ZBQU9DQVFFQVVIOUtDZG1KZG9BSmxVMHdCSkhZeGpMcktsbFBZNk9OYnpyNUpiaENNNjlIeHhZTgpCa2lpbXd1N09mRmFGZkZDT25NSjhvcStKVGxjMG9vREoxM2xCdHRONkdybnZrUTNQMXdZYkNFTmJuaWxPYVVCClRJcmlIeXRORFFhb3VOYS9LV3M3RmF1b2JjdEJsMXc5YXRvSFpzTjVvZWhUM3JBVHYxQ0NBdGpwYVRKSWZKUjMKMElRT1lrZTRvWTZEa0l3SHAydlBQbW9vR2dJdGJUdzNVK0U0MVlaZTdxQ21FLzd6TFRTWmtJTTJseDZ6RDQ2agpEZjRyZ044TVVMNnhpd09MbzlyQUp5ckRNM2JEeTJ1QjY0QkVzRFFMa2huUE92ZWtETjQ1NnV6TmpYS0E3VnE4CmgxL2d6RFpJRGkrV1hDWUFjYmdMaFpWQnF0bjYydW1GcE1SSXV3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=</crt>
                 <prv>LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tDQpNSUlFdlFJQkFEQU5CZ2txaGtpRzl3MEJBUUVGQUFTQ0JLY3dnZ1NqQWdFQUFvSUJBUUNheStJa3dUVVJONTNoDQo4NjF1UWVrWGQzMTR6N3JqVHhjK2RyeTBrWjBHcmRRQUxFVE9ob0VwRzNiTFZ5M3JOQU43a1VYZ0FDKzFYVzV4DQptbXlYbWpnTkw4Z1pKdWZwM2RneVQ2UHlNQkRjU2JOSHdZNmlHZUIvVkQwQmNMcWlIaXNFL214ZmhqeTVqeVFuDQpkaXRmbXBPa1Zqa29jdFRVQ2c5N0pEdkxJeGtjdWlTMzJkejRVVm5EQlR0RUlaSncwdG1KWEVKQkFybW55VDJZDQpEeHVER1dwaVIvU1lESUUzRDB3Qjg5UnhTU29Yby9TQlk4ZVVpc2RuKzZHRHQxVDdiWnFod3BRUTJLLzJhcHI0DQpIcHhTQXRNVlBxelJWSVN4ZjkwQjZIclpndUIrWmNtRHAwMGRBajR5RHBLUjZXbVQxVkZONk9JUk1nUlh3bUIyDQovWTN0RktibkFnTUJBQUVDZ2dFQUNYN0FIR2tOakVUUkZtOFFFRmRTcVBIWGJIV3hqUWZvOFJmdmMxUUxRY0dmDQo0M0xUdGFkaWZOY0dibXFta21yYVc5WUpaemdidFJCS0dnWFM2Mm0yVG5qRDJXY2RpcWJsQUJFS2lXeVJYREhaDQpJV21xQ2g5ME9kczg4cjJyZFE1TXJUMitBQTRINDRuNE9jTngzYWRwcndicThxUTRrZGtjSWYyUy9WN2x4M0U3DQpLWVNyVnNZT3hUS0dwQVl4MzVtU3oyckRadXFhcVhyRGtwSTYxa2tFVDluaGJzQStVZDE5aXZrZ3J1WlU3bVp3DQpmRzQ0V3JDeEhSNjdiYW1LQ3VPei81SnlKNUt5US9VaFkzak9lczkxZXRpeHJXUkRpaXRCVVIwc01kMXZuQ2cyDQpTSkpjQnAvYXRwbWNjMGY1OVZFSDRGOTNqY01jR3V5cE1mQmkwNExFd1FLQmdRRE1BUSsyVk5aQWJ5aVM0cWZqDQpXelE0UWl6MDNSdmUybVhaNHc1aEdKSmk0QnkvdmI4K2xoOE9acGFWSFBjRTVya0tBbExEaFRwUEl0dTFwNWZzDQpic05MTHpJYkIwUmVBdGUxc3JGOU9SRHp0VThKblhrNDEwcnFMbmdBZmhVTGUvSUttNjBtUG4vWWU0UnhVSVI3DQp2TjFNemJXMnlYdHJCSlZ5VG1jbFRkRDZ5UUtCZ1FEQ1FCa1B2NnFpYXZXaG1IRnBiZ3QvMklpSUlBUk5iSUJjDQpmVnVUYVFJbXlOZlVSVzlUVG10UHFiYXRuOGhaWFFaL1ZCbWdaMklPOCtCMXBFYW5zYjZCS2xJMXFUMFdMUzR4DQpaV0hLQ3hEdVFiMXNUYmNRMnd3NW5jbk50c1dlWlkxUlV1dlpYcVQwZFU4WUIwb2FsdGVYWTNVbDJrUWJuWUcvDQpINUpJeWJPOEx3S0JnUURKNE1zQnJoYVBrUERmMm5nMW4wMmYxcXpTYS9SbXBrMWdQemM5a3FsYU8xbDN6WGZ4DQpvWEYrT0xzUE9LaWlLd2cyQlhLTmxjdk1BRHpZR005WTQ0dFRYWk1CK0VFSm4xcURyaC9DUWJTcTEyTXRxcTRKDQpOOVFreG5OdVdWYk9GSXZEUDZjclQzSUljc0x2dDdSREZ2VVFTZ2xtcHlBQkdYb2lzYitVeE5ybk1RS0JnR09QDQpZc2oxbmNsOU5NUk1VK1NMcUkwd09Gbzh2cmZJSXNwRTNnamh5MTZCbGsyUUFRMGJwbGpBVFljVDNDWWhUZEU1DQpFNkZwRzVNNllCTXJ6YUxwc1JDVzFtZjJnLzYzelhNMzJUVXJFdFJyRVdGUE84TUI0blF0Y1Y5a2pFa3hGNHRWDQpDdGp3YjI5MEtNUFNDS00wS08vSTVDUXdpTFAydUtkeTBSRkpnRHUxQW9HQUxqSTdGZDl0ckFrY1dSenMyVHpRDQptaDRTZWxHRDFvdFAxRXpFa1hYVlZwK1lMNXlxOWM3V3hsa09RY1lFVmd1N2huNVFncnBRUFZPTFVmbkJoajdXDQo1MC9IVmR6V21wSXg5NUlqZXFDZklBV1U3N0I5cmJUR2hvWWMzbTdJcEdObzl2WlhHYWgrc2JHY3BEK1phV3UzDQp1Q25pTnJpZEhORGgzWHZQVFZkRTRlVT0NCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0NCg==</prv>
                 <serial>1</serial>
         </ca>
diff --git a/tests/unit/plugins/modules/test_pfsense_ca.py b/tests/unit/plugins/modules/test_pfsense_ca.py
index a65047ae..c2bf1418 100644
--- a/tests/unit/plugins/modules/test_pfsense_ca.py
+++ b/tests/unit/plugins/modules/test_pfsense_ca.py
@@ -14,20 +14,19 @@
 from .pfsense_module import TestPFSenseModule
 
 CERTIFICATE = (
-    "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlFQ0RDQ0F2Q2dBd0lCQWdJSUZqRk9oczFuTXpRd0RRWUpLb1pJaHZjTkFRRUxCUUF3WERFVE1CRUdBMVVFDQpBeE1LYjNCbGJuWndiaTFqWVRF"
-    "TE1Ba0dBMVVFQmhNQ1ZWTXhFVEFQQmdOVkJBZ1RDRU52Ykc5eVlXUnZNUkF3DQpEZ1lEVlFRSEV3ZENiM1ZzWkdWeU1STXdFUVlEVlFRS0V3cHdabE5sYm5OcFlteGxNQjRYRFRJeU1ESXhOREExDQpN"
-    "RGd6TVZvWERUTXlNREl4TWpBMU1EZ3pNVm93WERFVE1CRUdBMVVFQXhNS2IzQmxiblp3YmkxallURUxNQWtHDQpBMVVFQmhNQ1ZWTXhFVEFQQmdOVkJBZ1RDRU52Ykc5eVlXUnZNUkF3RGdZRFZRUUhF"
-    "d2RDYjNWc1pHVnlNUk13DQpFUVlEVlFRS0V3cHdabE5sYm5OcFlteGxNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDDQpBUUVBbXN2aUpNRTFFVGVkNGZPdGJrSHBGM2Q5ZU0r"
-    "NjQwOFhQbmE4dEpHZEJxM1VBQ3hFem9hQktSdDJ5MWN0DQo2elFEZTVGRjRBQXZ0VjF1Y1pwc2w1bzREUy9JR1NibjZkM1lNaytqOGpBUTNFbXpSOEdPb2huZ2YxUTlBWEM2DQpvaDRyQlA1c1g0WTh1"
-    "WThrSjNZclg1cVRwRlk1S0hMVTFBb1BleVE3eXlNWkhMb2t0OW5jK0ZGWnd3VTdSQ0dTDQpjTkxaaVZ4Q1FRSzVwOGs5bUE4Ymd4bHFZa2YwbUF5Qk53OU1BZlBVY1VrcUY2UDBnV1BIbElySFovdWhn"
-    "N2RVDQorMjJhb2NLVUVOaXY5bXFhK0I2Y1VnTFRGVDZzMFZTRXNYL2RBZWg2MllMZ2ZtWEpnNmROSFFJK01nNlNrZWxwDQprOVZSVGVqaUVUSUVWOEpnZHYyTjdSU201d0lEQVFBQm80SE5NSUhLTUIw"
-    "R0ExVWREZ1FXQkJSazVvQS8wcWEyDQpLUHdnb1hKcUtNdCtBb0tKZ1RDQmpRWURWUjBqQklHRk1JR0NnQlJrNW9BLzBxYTJLUHdnb1hKcUtNdCtBb0tKDQpnYUZncEY0d1hERVRNQkVHQTFVRUF4TUti"
-    "M0JsYm5ad2JpMWpZVEVMTUFrR0ExVUVCaE1DVlZNeEVUQVBCZ05WDQpCQWdUQ0VOdmJHOXlZV1J2TVJBd0RnWURWUVFIRXdkQ2IzVnNaR1Z5TVJNd0VRWURWUVFLRXdwd1psTmxibk5wDQpZbXhsZ2dn"
-    "V01VNkd6V2N6TkRBTUJnTlZIUk1FQlRBREFRSC9NQXNHQTFVZER3UUVBd0lCQmpBTkJna3Foa2lHDQo5dzBCQVFzRkFBT0NBUUVBVUg5S0NkbUpkb0FKbFUwd0JKSFl4akxyS2xsUFk2T05ienI1SmJo"
-    "Q002OUh4eFlODQpCa2lpbXd1N09mRmFGZkZDT25NSjhvcStKVGxjMG9vREoxM2xCdHRONkdybnZrUTNQMXdZYkNFTmJuaWxPYVVCDQpUSXJpSHl0TkRRYW91TmEvS1dzN0ZhdW9iY3RCbDF3OWF0b0ha"
-    "c041b2VoVDNyQVR2MUNDQXRqcGFUSklmSlIzDQowSVFPWWtlNG9ZNkRrSXdIcDJ2UFBtb29HZ0l0YlR3M1UrRTQxWVplN3FDbUUvN3pMVFNaa0lNMmx4NnpENDZqDQpEZjRyZ044TVVMNnhpd09Mbzly"
-    "QUp5ckRNM2JEeTJ1QjY0QkVzRFFMa2huUE92ZWtETjQ1NnV6TmpYS0E3VnE4DQpoMS9nekRaSURpK1dYQ1lBY2JnTGhaVkJxdG42MnVtRnBNUkl1dz09DQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t"
-    "DQo=")
+    "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVDRENDQXZDZ0F3SUJBZ0lJRmpGT2hzMW5NelF3RFFZSktvWklodmNOQVFFTEJRQXdYREVUTUJFR0ExVUUKQXhNS2IzQmxiblp3YmkxallURUxN"
+    "QWtHQTFVRUJoTUNWVk14RVRBUEJnTlZCQWdUQ0VOdmJHOXlZV1J2TVJBdwpEZ1lEVlFRSEV3ZENiM1ZzWkdWeU1STXdFUVlEVlFRS0V3cHdabE5sYm5OcFlteGxNQjRYRFRJeU1ESXhOREExCk1EZ3pN"
+    "Vm9YRFRNeU1ESXhNakExTURnek1Wb3dYREVUTUJFR0ExVUVBeE1LYjNCbGJuWndiaTFqWVRFTE1Ba0cKQTFVRUJoTUNWVk14RVRBUEJnTlZCQWdUQ0VOdmJHOXlZV1J2TVJBd0RnWURWUVFIRXdkQ2Iz"
+    "VnNaR1Z5TVJNdwpFUVlEVlFRS0V3cHdabE5sYm5OcFlteGxNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDCkFRRUFtc3ZpSk1FMUVUZWQ0Zk90YmtIcEYzZDllTSs2NDA4WFBu"
+    "YTh0SkdkQnEzVUFDeEV6b2FCS1J0MnkxY3QKNnpRRGU1RkY0QUF2dFYxdWNacHNsNW80RFMvSUdTYm42ZDNZTWsrajhqQVEzRW16UjhHT29obmdmMVE5QVhDNgpvaDRyQlA1c1g0WTh1WThrSjNZclg1"
+    "cVRwRlk1S0hMVTFBb1BleVE3eXlNWkhMb2t0OW5jK0ZGWnd3VTdSQ0dTCmNOTFppVnhDUVFLNXA4azltQThiZ3hscVlrZjBtQXlCTnc5TUFmUFVjVWtxRjZQMGdXUEhsSXJIWi91aGc3ZFUKKzIyYW9j"
+    "S1VFTml2OW1xYStCNmNVZ0xURlQ2czBWU0VzWC9kQWVoNjJZTGdmbVhKZzZkTkhRSStNZzZTa2VscAprOVZSVGVqaUVUSUVWOEpnZHYyTjdSU201d0lEQVFBQm80SE5NSUhLTUIwR0ExVWREZ1FXQkJS"
+    "azVvQS8wcWEyCktQd2dvWEpxS010K0FvS0pnVENCalFZRFZSMGpCSUdGTUlHQ2dCUms1b0EvMHFhMktQd2dvWEpxS010K0FvS0oKZ2FGZ3BGNHdYREVUTUJFR0ExVUVBeE1LYjNCbGJuWndiaTFqWVRF"
+    "TE1Ba0dBMVVFQmhNQ1ZWTXhFVEFQQmdOVgpCQWdUQ0VOdmJHOXlZV1J2TVJBd0RnWURWUVFIRXdkQ2IzVnNaR1Z5TVJNd0VRWURWUVFLRXdwd1psTmxibk5wCllteGxnZ2dXTVU2R3pXY3pOREFNQmdO"
+    "VkhSTUVCVEFEQVFIL01Bc0dBMVVkRHdRRUF3SUJCakFOQmdrcWhraUcKOXcwQkFRc0ZBQU9DQVFFQVVIOUtDZG1KZG9BSmxVMHdCSkhZeGpMcktsbFBZNk9OYnpyNUpiaENNNjlIeHhZTgpCa2lpbXd1"
+    "N09mRmFGZkZDT25NSjhvcStKVGxjMG9vREoxM2xCdHRONkdybnZrUTNQMXdZYkNFTmJuaWxPYVVCClRJcmlIeXRORFFhb3VOYS9LV3M3RmF1b2JjdEJsMXc5YXRvSFpzTjVvZWhUM3JBVHYxQ0NBdGpw"
+    "YVRKSWZKUjMKMElRT1lrZTRvWTZEa0l3SHAydlBQbW9vR2dJdGJUdzNVK0U0MVlaZTdxQ21FLzd6TFRTWmtJTTJseDZ6RDQ2agpEZjRyZ044TVVMNnhpd09MbzlyQUp5ckRNM2JEeTJ1QjY0QkVzRFFM"
+    "a2huUE92ZWtETjQ1NnV6TmpYS0E3VnE4CmgxL2d6RFpJRGkrV1hDWUFjYmdMaFpWQnF0bjYydW1GcE1SSXV3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=")
 CRL1 = (
     "LS0tLS1CRUdJTiBYNTA5IENSTC0tLS0tCk1JSUNkRENDQVZ3Q0FRRXdEUVlKS29aSWh2Y05BUUVGQlFBd1hERVRNQkVHQTFVRUF4TUtiM0JsYm5ad2JpMWoKWVRFTE1Ba0dBMVVFQmhNQ1ZWTXhFVEFQ"
     "QmdOVkJBZ1RDRU52Ykc5eVlXUnZNUkF3RGdZRFZRUUhFd2RDYjNWcwpaR1Z5TVJNd0VRWURWUVFLRXdwd1psTmxibk5wWW14bEZ3MHlNakF5TVRrd05UVXhNRFphRncwME9UQTNNRFl3Ck5UVXhNRFph"
@@ -97,22 +96,22 @@ def check_target_elt(self, obj, target_elt):
     def test_ca_create(self):
         """ test creation of a new ca """
         obj = dict(name='ca1', certificate=CERTIFICATE)
-        self.do_module_test(obj, command='create ca ca1')
+        self.do_module_test(obj, command="create ca 'ca1'")
 
     def test_ca_add_crl(self):
         """ test adding a CRL """
         obj = dict(name='ca1', certificate=CERTIFICATE, crl=CRL1)
-        self.do_module_test(obj, command='create ca ca1')
+        self.do_module_test(obj, command="create ca 'ca1'")
 
     def test_ca_change_crl(self):
         """ test adding a CRL """
         obj = dict(name='ca1', certificate=CERTIFICATE, crl=CRL2)
-        self.do_module_test(obj, command='create ca ca1')
+        self.do_module_test(obj, command="create ca 'ca1'")
 
     def test_ca_delete(self):
         """ test deletion of a ca """
         obj = dict(name='testdel')
-        self.do_module_test(obj, command='delete ca testdel', delete=True)
+        self.do_module_test(obj, command="delete ca 'testdel'", delete=True)
 
     def test_ca_update_noop(self):
         """ test not updating a ca """
@@ -122,12 +121,12 @@ def test_ca_update_noop(self):
     def test_ca_update_serial(self):
         """ test updating serial of a ca """
         obj = dict(name='testdel', certificate=CERTIFICATE, serial=10)
-        self.do_module_test(obj, command='update ca testdel set ')
+        self.do_module_test(obj, command="update ca 'testdel' set serial='10'")
 
     def test_ca_update_trust(self):
         """ test updating trust of a ca """
         obj = dict(name='testdel', certificate=CERTIFICATE, trust=False)
-        self.do_module_test(obj, command='update ca testdel set ')
+        self.do_module_test(obj, command="update ca 'testdel' set ")
 
     ##############
     # misc

From 5f1be15ec8ca91a2f6dafbb3638a9bcc6b588b03 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 15:26:45 -0700
Subject: [PATCH 32/62] [module_base] Initial support for bool_values

---
 plugins/module_utils/module_base.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 2970bcef..55951a8e 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -26,7 +26,7 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr', update_php=None,
-                 map_param_if=None, param_force=None, have_refid=False, create_default=None):
+                 map_param_if=None, param_force=None, bool_values=None, have_refid=False, create_default=None):
         if pfsense is None:
             pfsense = PFSenseModule(module)
         self.module = module    # ansible module
@@ -70,6 +70,7 @@ def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive
         self.obj = dict()       # dict holding target pfsense parameters
         self.map_param_if = map_param_if  # rules for mapping parameters
         self.param_force = param_force    # parameters that are forced to be present
+        self.bool_values = bool_values    # boolean values for arguments
         self.create_default = create_default  # default values for a created target
         self.have_refid = have_refid      # if the element has a refid item
         self.target_elt = None  # xml object holding target pfsense parameters
@@ -126,7 +127,13 @@ def _params_to_obj(self):
                 force = False
                 if param in self.param_force:
                     force = True
-                self._get_ansible_param(obj, param, force=force)
+                if self.argument_spec[param].get('type') == 'bool':
+                    if param in self.bool_values:
+                        self._get_ansible_param_bool(obj, param, value=self.bool_values[param][1], value_false=self.bool_values[param][0], force=force)
+                    else:
+                        self._get_ansible_param_bool(obj, param, force=force)
+                else:
+                    self._get_ansible_param(obj, param, force=force)
 
         for map_param, map_value, map_tuple in self.map_param_if:
             if self.params.get(map_param) == map_value and map_tuple[0] in obj:

From 665648dd025b638fc0c1154be9854efca28bc607 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 15:27:09 -0700
Subject: [PATCH 33/62] [pfsense_ca] Define CA_BOOL_VALUES

---
 plugins/modules/pfsense_ca.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/plugins/modules/pfsense_ca.py b/plugins/modules/pfsense_ca.py
index 70fe3576..ff20e014 100644
--- a/plugins/modules/pfsense_ca.py
+++ b/plugins/modules/pfsense_ca.py
@@ -119,6 +119,12 @@
     trust='disabled',
 )
 
+# Booleans that map to different values
+CA_BOOL_VALUES = dict(
+    randomserial=('disabled', 'enabled'),
+    trust=('disabled', 'enabled'),
+)
+
 
 class PFSenseCAModule(PFSenseModuleBase):
     """ module managing pfsense certificate authorities """
@@ -129,7 +135,8 @@ def get_argument_spec():
         return PFSENSE_CA_ARGUMENT_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSenseCAModule, self).__init__(module, pfsense, root='pfsense', node='ca', have_refid=True, create_default=CA_CREATE_DEFAULT)
+        super(PFSenseCAModule, self).__init__(module, pfsense, root='pfsense', node='ca', have_refid=True, create_default=CA_CREATE_DEFAULT,
+                                              bool_values=CA_BOOL_VALUES)
         self.name = "pfsense_ca"
         self.refresh_crls = False
         self.crl = None
@@ -180,8 +187,9 @@ def _params_to_obj(self):
                 self._get_ansible_param(self.crl, 'crlname', fname='descr', force=True, force_value=obj['descr'] + ' CRL')
                 self._get_ansible_param(self.crl, 'crlrefid', fname='refid')
 
-        self._get_ansible_param_bool(obj, 'trust', value='enabled', value_false='disabled')
-        self._get_ansible_param_bool(obj, 'randomserial', value='enabled', value_false='disabled')
+        for arg in CA_BOOL_VALUES:
+            self._get_ansible_param_bool(obj, arg, value=CA_BOOL_VALUES[arg][1], value_false=CA_BOOL_VALUES[arg][0])
+
         self._get_ansible_param(obj, 'serial')
 
         return obj

From 8f20ba23e850a1bcae60448361ec6b583184acc3 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 16:21:43 -0700
Subject: [PATCH 34/62] [misc/pfsense_module.py.j2] fix - handle module_type
 better

---
 misc/pfsense_module.py.j2 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index f4d59bde..b49b8189 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -90,11 +90,11 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 }
 
 {{ module_node | upper() }}_REQUIRED_IF = [
-{% if module_type is none %}
-    ['state', 'present', ['{{ params | dict2items | selectattr('value.required', 'defined') | select('key') | join("', '") }}]],
-{% else %}
+{% if module_type %}
     ['state', 'present', ['type']],
     ['type', '{{ params['type']['example'] }}', ['{{ params | dict2items | rejectattr('key', 'equalto', 'type') | selectattr('value.required', 'defined') | map(attribute='key') | join("', '") }}']],
+{% else %}
+    ['state', 'present', ['{{ params | dict2items | selectattr('value.required', 'defined') | select('key') | join("', '") }}']],
 {% endif %}
 ]
 

From 8ad79282e1a610fab3bdad3a5df900518f35d327 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 16:55:47 -0700
Subject: [PATCH 35/62] [pfsense_module.py.j2] Generate compact ARGUMENT_SPEC
 as well; fix for REQUIRED_IF

---
 misc/pfsense_module.py.j2 | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index b49b8189..5091b3f9 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -89,12 +89,23 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 {% endfor %}
 }
 
+# Compact style
+{{ module_node | upper() }}_ARGUMENT_SPEC = dict(
+    # Only name should be required here - othewise you cannot remove an item with just 'name'
+    # Required arguments for creation should be note in {{ module_node | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
+    name=dict(required=True, typ='str'),
+    state=dict(default='present', choices=['present', 'absent']),
+{% for param in params %}
+    {{ param }}=dict(type='{{ params[param]['type'] | default('') }}'{% if 'choices' in params[param] %}, choices={{ params[param]['choices'] }},{% endif %}),
+{% endfor %}
+)
+
 {{ module_node | upper() }}_REQUIRED_IF = [
 {% if module_type %}
     ['state', 'present', ['type']],
     ['type', '{{ params['type']['example'] }}', ['{{ params | dict2items | rejectattr('key', 'equalto', 'type') | selectattr('value.required', 'defined') | map(attribute='key') | join("', '") }}']],
 {% else %}
-    ['state', 'present', ['{{ params | dict2items | selectattr('value.required', 'defined') | select('key') | join("', '") }}']],
+    ['state', 'present', ['{{ params | dict2items | selectattr('value.required', 'defined') | select(attribue='key') | join("', '") }}']],
 {% endif %}
 ]
 

From 633ef2e11c0190f9515ad552052663804c6a5846 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 16:56:44 -0700
Subject: [PATCH 36/62] [generate_module] Handle web pages with different input
 methods that do not create different types of elements

---
 misc/generate_module | 41 ++++++++++++++++++++++++++++-------------
 1 file changed, 28 insertions(+), 13 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index c0b3eac0..390b92d4 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -36,17 +36,19 @@ parser.add_argument('--author-name', default=author_name, help='The full name of
 parser.add_argument('--author-email', default=author_email, help='The email address of the module author')
 parser.add_argument('--author-handle', default='', help='The github handle of the module author')
 parser.add_argument('--module-name', help='The name of the module to generate - defaults to being based on the url')
+parser.add_argument('--type-param', default='type', help='The name of the parameter for selecting different types of elements', )
 parser.add_argument('--type-suffix', const=True, default=False, nargs='?', help='Suffix the module name with the item type', )
 parser.add_argument('--item-min', default='item_min', help='The name of the minimally configured item to search for in config.xml')
 parser.add_argument('--item-full', default='item_full',
                     help='The name of the fully configured item to search for in config.xml, will be used for exmaples in the documentation')
 parser.add_argument('--force', action=argparse.BooleanOptionalAction, help='Force overwriting the output file if it exists')
-parser.add_argument('--keep', action=argparse.BooleanOptionalAction, help='Keep the downloaded files in the temporary directory')
+parser.add_argument('--keep-tmpdir', action=argparse.BooleanOptionalAction, help='Keep the downloaded files in the temporary directory')
+parser.add_argument('--keep-params', action=argparse.BooleanOptionalAction, help='Keep parameters from the web interface not found in the XML')
 
 args = parser.parse_args()
 
 # Temporary directory for files
-tmpdir = tempfile.TemporaryDirectory(prefix='pfgenmod-', delete=(not args.keep))
+tmpdir = tempfile.TemporaryDirectory(prefix='pfgenmod-', delete=(not args.keep_tmpdir))
 
 if args.url is not None:
     parsed_uri = urlparse(args.url)
@@ -160,15 +162,6 @@ for elt in full_elt:
     # else:
         # Likely a bool?
 
-if 'type' in params_full:
-    module_type = params_full['type']['example']
-    if args.type_suffix is True:
-        module_name += f'_{module_type}'
-    if isinstance(args.type_suffix, str):
-        module_name += f'_{args.type_suffix}'
-else:
-    module_type = None
-
 print('')
 
 # Parse the php file
@@ -234,6 +227,8 @@ for input in html.forms[0].inputs:
             param['description'] = input.tail.strip()
     elif isinstance(input, lxml.html.SelectElement):
         print(f'select name={input.name} value={input.value} value_options={input.value_options} multiple={input.multiple}')
+        if input.attrib.get('class') == 'form-control' and input.attrib.get('data-toggle') == 'collapse':
+            type_param = input.name
         if input.value_options is not None:
             param['choices'] = input.value_options
             if input.multiple:
@@ -275,6 +270,26 @@ del params[module_key]
 # Debug
 print(f'Web paramters: {params.keys()}')
 
+# Determine if the form produces different types of items
+if type_param in params_full:
+    # The type of item is recorded in the item configuration
+    module_type = params_full[type_param]['example']
+    if args.type_suffix is True:
+        module_name += f'_{module_type}'
+    if isinstance(args.type_suffix, str):
+        module_name += f'_{args.type_suffix}'
+elif type_param in params:
+    # The type of item is purely a fuction of the web form
+    if args.type_suffix not in params[type_param]['choices']:
+        sys.exit(f"Detected item type parameter '{type_param}' with choices {params[type_param]['choices']}.  You must set --type-suffix to one of these.")
+    else:
+        module_type = args.type_suffix
+        module_name += f'_{args.type_suffix}'
+        params_full[type_param] = params[type_param]
+        args.keep_params = True
+else:
+    module_type = None
+
 # Consistency
 params_web_only = list(set(params.keys()) - set(params_full.keys()))
 print('Web parameters not in xml: ' + str(params_web_only))
@@ -305,7 +320,7 @@ for param in params_web_only:
             break
     else:
         # Otherwise, drop - probably just used to construct the final elements
-        if param in params:
+        if param in params and not args.keep_params:
             print(f'Removing {param}')
             del params[param]
 
@@ -355,7 +370,7 @@ f.write(template.render(context))
 f.close()
 
 # Save the scraped web page if asked to keep files
-if args.keep:
+if args.keep_tmpdir:
     f = open(f'{tmpdir.name}/{module_name}.html', 'w')
     f.write(r.text)
     f.close()

From 295ae99e55b6a0f122c24945bfffe0f21fb2e6f6 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 17:56:39 -0700
Subject: [PATCH 37/62] [module_base] Use root_elt.findall() instead of
 get_elements; Reset self.elements in _copy_and_add_target - needed by some
 modules like user

---
 plugins/module_utils/module_base.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 55951a8e..56e898f4 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -61,7 +61,7 @@ def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive
 
         # List of elements named node
         if node is not None:
-            self.elements = self.pfsense.get_elements(node)
+            self.elements = self.root_elt.findall(node)
         else:
             self.elements = None
         self.node = node
@@ -187,6 +187,8 @@ def _copy_and_add_target(self):
             self.root_elt.append(self.target_elt)
         else:
             self.root_elt.insert(self._find_last_element_index(), self.target_elt)
+            # Reset elements list
+            self.elements = self.root_elt.findall(self.node)
 
     def _copy_and_update_target(self):
         """ update the XML target_elt """

From 1df96cd6afb6f43338b36a1287250b3bcb4535d3 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 15 Jan 2024 17:57:22 -0700
Subject: [PATCH 38/62] [pfsense_user] Use updated PFSenseModuleBase

---
 plugins/modules/pfsense_user.py               | 96 ++++++-------------
 .../unit/plugins/modules/test_pfsense_user.py |  2 +-
 2 files changed, 29 insertions(+), 69 deletions(-)

diff --git a/plugins/modules/pfsense_user.py b/plugins/modules/pfsense_user.py
index 6ca12837..85b344c3 100644
--- a/plugins/modules/pfsense_user.py
+++ b/plugins/modules/pfsense_user.py
@@ -1,17 +1,13 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 
-# Copyright: (c) 2019-2020, Orion Poplawski <orion@nwra.com>
+# Copyright: (c) 2019-2024, Orion Poplawski <orion@nwra.com>
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
-from __future__ import absolute_import, division, print_function
+from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-ANSIBLE_METADATA = {'metadata_version': '1.1',
-                    'status': ['preview'],
-                    'supported_by': 'community'}
-
-DOCUMENTATION = """
+DOCUMENTATION = r'''
 ---
 module: pfsense_user
 version_added: 0.1.0
@@ -35,8 +31,7 @@
     description: Description of the user
     type: str
   scope:
-    description: Scope of the user ('user' is a normal user, use 'system' for 'admin' user).
-    default: user
+    description: Scope of the user ('user' is a normal user, use 'system' for 'admin' user). Defaults to `user`.
     choices: [ "user", "system" ]
     type: str
   uid:
@@ -60,9 +55,9 @@
   authorizedkeys:
     description: Contents of ~/.ssh/authorized_keys.  Can be base64 encoded.
     type: str
-"""
+'''
 
-EXAMPLES = """
+EXAMPLES = r'''
 - name: Add operator user
   pfsense_user:
     name: operator
@@ -75,11 +70,11 @@
   pfsense_user:
     name: operator
     state: absent
-"""
+'''
 
-RETURN = """
+RETURN = r'''
 
-"""
+'''
 
 import base64
 import re
@@ -91,7 +86,7 @@
     name=dict(required=True, type='str'),
     state=dict(type='str', default='present', choices=['present', 'absent']),
     descr=dict(type='str'),
-    scope=dict(type='str', default='user', choices=['user', 'system']),
+    scope=dict(type='str', choices=['user', 'system']),
     uid=dict(type='str'),
     password=dict(type='str', no_log=True),
     groups=dict(type='list', elements='str'),
@@ -99,6 +94,10 @@
     authorizedkeys=dict(type='str'),
 )
 
+USER_CREATE_DEFAULT = dict(
+    scope='user',
+)
+
 USER_PHP_COMMAND_PREFIX = """
 require_once('auth.inc');
 init_config_arr(array('system', 'user'));
@@ -134,16 +133,17 @@
 class PFSenseUserModule(PFSenseModuleBase):
     """ module managing pfsense users """
 
+    ##############################
+    # unit tests
+    #
+    # Must be class method for unit test usage
     @staticmethod
     def get_argument_spec():
         """ return argument spec """
         return USER_ARGUMENT_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSenseUserModule, self).__init__(module, pfsense)
-        self.name = "pfsense_user"
-        self.root_elt = self.pfsense.get_element('system')
-        self.users = self.root_elt.findall('user')
+        super(PFSenseUserModule, self).__init__(module, pfsense, root='system', node='user', key='name', create_default=USER_CREATE_DEFAULT)
         self.groups = self.root_elt.findall('group')
         self.mod_groups = []
 
@@ -183,35 +183,15 @@ def _params_to_obj(self):
     ##############################
     # XML processing
     #
-    def _find_target(self):
-        result = self.root_elt.findall("user[name='{0}']".format(self.obj['name']))
-        if len(result) == 1:
-            return result[0]
-        elif len(result) > 1:
-            self.module.fail_json(msg='Found multiple users for name {0}.'.format(self.obj['name']))
-        else:
-            return None
-
     def _find_group(self, name):
-        result = self.root_elt.findall("group[name='{0}']".format(name))
-        if len(result) == 1:
-            return result[0]
-        elif len(result) > 1:
-            self.module.fail_json(msg='Found multiple groups for name {0}.'.format(name))
-        else:
-            return None
+        return self.pfsense.find_elt('group', name, search_field='name', root_elt=self.root_elt)
 
     def _find_groups_for_uid(self, uid):
-        groups = []
-        for group_elt in self.root_elt.findall("group[member='{0}']".format(uid)):
-            groups.append(group_elt.find('name').text)
-        return groups
-
-    def _find_this_user_index(self):
-        return self.users.index(self.target_elt)
-
-    def _find_last_user_index(self):
-        return list(self.root_elt).index(self.users[len(self.users) - 1])
+        groups = self.pfsense.find_elt_xpath("group[member='{0}']".format(uid), root_elt=self.root_elt, multiple_ok=True)
+        if groups is not None:
+            return groups
+        else:
+            return []
 
     def _nextuid(self):
         nextuid_elt = self.root_elt.find('nextuid')
@@ -225,10 +205,6 @@ def _format_diff_priv(self, priv):
         else:
             return priv
 
-    def _create_target(self):
-        """ create the XML target_elt """
-        return self.pfsense.new_element('user')
-
     def _copy_and_add_target(self):
         """ populate the XML target_elt """
         obj = self.obj
@@ -240,9 +216,9 @@ def _copy_and_add_target(self):
         self.diff['after'] = obj
         self.pfsense.copy_dict_to_element(self.obj, self.target_elt)
         self._update_groups()
-        self.root_elt.insert(self._find_last_user_index(), self.target_elt)
+        self.root_elt.insert(self._find_last_element_index(), self.target_elt)
         # Reset users list
-        self.users = self.root_elt.findall('user')
+        self.elements = self.root_elt.findall(self.node)
 
     def _copy_and_update_target(self):
         """ update the XML target_elt """
@@ -302,28 +278,12 @@ def _update_groups(self):
 
         return changed
 
-    ##############################
-    # Logging
-    #
-    def _get_obj_name(self):
-        """ return obj's name """
-        return "'" + self.obj['name'] + "'"
-
-    def _log_fields(self, before=None):
-        """ generate pseudo-CLI command fields parameters to create an obj """
-        values = ''
-        if before is None:
-            values += self.format_cli_field(self.params, 'descr')
-        else:
-            values += self.format_updated_cli_field(self.obj, before, 'descr', add_comma=(values))
-        return values
-
     ##############################
     # run
     #
     def _update(self):
         if self.params['state'] == 'present':
-            return self.pfsense.phpshell(USER_PHP_COMMAND_SET.format(idx=self._find_this_user_index(), mod_groups=self.mod_groups))
+            return self.pfsense.phpshell(USER_PHP_COMMAND_SET.format(idx=self._find_this_element_index(), mod_groups=self.mod_groups))
         else:
             return self.pfsense.phpshell(USER_PHP_COMMAND_DEL.format(name=self.obj['name'], uid=self.obj['uid'], mod_groups=self.mod_groups))
 
diff --git a/tests/unit/plugins/modules/test_pfsense_user.py b/tests/unit/plugins/modules/test_pfsense_user.py
index 535df690..21699f70 100644
--- a/tests/unit/plugins/modules/test_pfsense_user.py
+++ b/tests/unit/plugins/modules/test_pfsense_user.py
@@ -58,7 +58,7 @@ def check_target_elt(self, obj, target_elt):
     def test_user_create(self):
         """ test creation of a new user """
         obj = dict(name='user1', descr='User One', password='$2b$12$D2jkq4Iut3ODUBN0BCrDk.bV3J5N.MrY5YEnGvTXwxeNBkyxjbbtW')
-        self.do_module_test(obj, command="create user 'user1', descr='User One'")
+        self.do_module_test(obj, command="create user 'user1', descr='User One', uid='2001'")
 
     def test_user_delete(self):
         """ test deletion of a user """

From 906919187a06af497672dc18b0d8b9d19b773a13 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Tue, 16 Jan 2024 06:42:14 -0700
Subject: [PATCH 39/62] [generate_module] Handle textarea; type_param ->
 args.type_param

---
 misc/generate_module | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index 390b92d4..21988079 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -228,7 +228,7 @@ for input in html.forms[0].inputs:
     elif isinstance(input, lxml.html.SelectElement):
         print(f'select name={input.name} value={input.value} value_options={input.value_options} multiple={input.multiple}')
         if input.attrib.get('class') == 'form-control' and input.attrib.get('data-toggle') == 'collapse':
-            type_param = input.name
+            args.type_param = input.name
         if input.value_options is not None:
             param['choices'] = input.value_options
             if input.multiple:
@@ -236,6 +236,8 @@ for input in html.forms[0].inputs:
             else:
                 param['type'] = 'str'
             param['multiple'] = input.multiple
+    elif input.tag == 'textarea':
+        param['type'] = 'str'
 
 #  <div class="form-group">
 #    <label class="col-sm-2 control-label">
@@ -271,21 +273,21 @@ del params[module_key]
 print(f'Web paramters: {params.keys()}')
 
 # Determine if the form produces different types of items
-if type_param in params_full:
+if args.type_param in params_full:
     # The type of item is recorded in the item configuration
-    module_type = params_full[type_param]['example']
+    module_type = params_full[args.type_param]['example']
     if args.type_suffix is True:
         module_name += f'_{module_type}'
     if isinstance(args.type_suffix, str):
         module_name += f'_{args.type_suffix}'
-elif type_param in params:
+elif args.type_param in params:
     # The type of item is purely a fuction of the web form
-    if args.type_suffix not in params[type_param]['choices']:
-        sys.exit(f"Detected item type parameter '{type_param}' with choices {params[type_param]['choices']}.  You must set --type-suffix to one of these.")
+    if args.type_suffix not in params[args.type_param]['choices']:
+        sys.exit(f"Detected item type parameter '{args.type_param}' with choices {params[args.type_param]['choices']}.  You must set --type-suffix to one of these.")
     else:
         module_type = args.type_suffix
         module_name += f'_{args.type_suffix}'
-        params_full[type_param] = params[type_param]
+        params_full[args.type_param] = params[args.type_param]
         args.keep_params = True
 else:
     module_type = None

From 42692a6c93848fc171ba50ec17e734d689b3c639 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Tue, 16 Jan 2024 19:04:29 -0700
Subject: [PATCH 40/62] [module_base] Initialize instance vars; Add arg_route
 for parsing and validating - implement _validate_params

---
 plugins/module_utils/module_base.py           | 64 ++++++++++++++-----
 .../plugins/modules/test_pfsense_aggregate.py | 12 ++--
 2 files changed, 53 insertions(+), 23 deletions(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 56e898f4..92ffd7e2 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -26,7 +26,7 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr', update_php=None,
-                 map_param_if=None, param_force=None, bool_values=None, have_refid=False, create_default=None):
+                 arg_route=None, map_param_if=None, param_force=None, bool_values=None, have_refid=False, create_default=None):
         if pfsense is None:
             pfsense = PFSenseModule(module)
         self.module = module    # ansible module
@@ -68,9 +68,22 @@ def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive
 
         self.key = key          # item that identifies a target element
         self.obj = dict()       # dict holding target pfsense parameters
-        self.map_param_if = map_param_if  # rules for mapping parameters
-        self.param_force = param_force    # parameters that are forced to be present
-        self.bool_values = bool_values    # boolean values for arguments
+        if arg_route is not None:
+            self.arg_route = arg_route        # routing for argument handling
+        else:
+            self.arg_route = dict()
+        if map_param_if is not None:
+            self.map_param_if = map_param_if  # rules for mapping parameters
+        else:
+            self.map_param_if = list()
+        if param_force is not None:
+            self.param_force = param_force  # parameters that are forced to be present
+        else:
+            self.param_force = list()
+        if bool_values is not None:
+            self.bool_values = bool_values    # boolean values for arguments
+        else:
+            self.bool_values = dict()
         self.create_default = create_default  # default values for a created target
         self.have_refid = have_refid      # if the element has a refid item
         self.target_elt = None  # xml object holding target pfsense parameters
@@ -119,15 +132,20 @@ def _get_ansible_param_bool(self, obj, name, fname=None, force=False, value='yes
             obj[fname] = value_false
 
     def _params_to_obj(self):
-        """ return a dict from module params """
+        """ return a dict from module params that sets self.obj """
         obj = dict()
-        obj[self.key] = self.params[self.key]
-        if self.params.get('state') == 'present':
-            for param in [n for n in self.argument_spec.keys() if n != 'state']:
+        # Not all modules have 'state', treat them like they did
+        if self.params.get('state', 'present') == 'present':
+            # Ansible sets unspecied parameters to None, skip them and 'state'
+            for param in [p for p in self.params if p != 'state' and self.params[p] is not None]:
                 force = False
                 if param in self.param_force:
                     force = True
-                if self.argument_spec[param].get('type') == 'bool':
+
+                # If we have defined a parser for this arg, use it
+                if param in self.arg_route and 'parse' in self.arg_route[param]:
+                    self.arg_route[param]['parse'](param, self.params, obj)
+                elif self.argument_spec[param].get('type') == 'bool':
                     if param in self.bool_values:
                         self._get_ansible_param_bool(obj, param, value=self.bool_values[param][1], value_false=self.bool_values[param][0], force=force)
                     else:
@@ -135,18 +153,30 @@ def _params_to_obj(self):
                 else:
                     self._get_ansible_param(obj, param, force=force)
 
-        for map_param, map_value, map_tuple in self.map_param_if:
-            if self.params.get(map_param) == map_value and map_tuple[0] in obj:
-                if map_tuple[1] not in obj:
-                    obj[map_tuple[1]] = obj[map_tuple[0]]
-                del obj[map_tuple[0]]
+            # Handle renaming of parameters
+            for map_param, map_value, map_tuple in self.map_param_if:
+                if self.params.get(map_param) == map_value and map_tuple[0] in obj:
+                    if map_tuple[1] not in obj:
+                        obj[map_tuple[1]] = obj[map_tuple[0]]
+                    del obj[map_tuple[0]]
+        else:
+            # Just use the key to remove items
+            obj[self.key] = self.params[self.key]
 
         return obj
 
-    @staticmethod
-    def _validate_params():
+    ##############################
+    # params processing
+    #
+    def _validate_params(self):
         """ do some extra checks on input parameters """
-        pass
+        params = self.params
+        # Not all modules have 'state', treat them like they did
+        if self.params.get('state', 'present') == 'present':
+            # Ansible sets unspecied parameters to None, skip them
+            for param in [p for p in self.params if self.params[p] is not None]:
+                if param in self.arg_route and 'validate' in self.arg_route[param]:
+                    self.arg_route[param]['validate'](params[param])
 
     def _deprecated_params(self):
         """ return deprecated params """
diff --git a/tests/unit/plugins/modules/test_pfsense_aggregate.py b/tests/unit/plugins/modules/test_pfsense_aggregate.py
index 943b2c38..bcfa3961 100644
--- a/tests/unit/plugins/modules/test_pfsense_aggregate.py
+++ b/tests/unit/plugins/modules/test_pfsense_aggregate.py
@@ -154,8 +154,8 @@ def test_aggregate_aliases(self):
         set_module_args(args)
         result = self.execute_module(changed=True)
         result_aliases = []
-        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7', descr='', detail=''")
-        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6', descr='', detail=''")
+        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7'")
+        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6'")
         result_aliases.append("update alias 'port_ssh' set address='2222'")
         result_aliases.append("delete alias 'port_http'")
 
@@ -183,8 +183,8 @@ def test_aggregate_aliases_checkmode(self):
         set_module_args(args)
         result = self.execute_module(changed=True)
         result_aliases = []
-        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7', descr='', detail=''")
-        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6', descr='', detail=''")
+        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7'")
+        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6'")
         result_aliases.append("update alias 'port_ssh' set address='2222'")
         result_aliases.append("delete alias 'port_http'")
 
@@ -207,8 +207,8 @@ def test_aggregate_aliases_purge(self):
         set_module_args(args)
         result = self.execute_module(changed=True)
         result_aliases = []
-        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7', descr='', detail=''")
-        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6', descr='', detail=''")
+        result_aliases.append("create alias 'one_host', type='host', address='10.9.8.7'")
+        result_aliases.append("create alias 'another_host', type='host', address='10.9.8.6'")
         result_aliases.append("update alias 'port_ssh' set address='2222'")
         result_aliases.append("delete alias 'port_http'")
         result_aliases.append("delete alias 'port_dns'")

From edb2e85d0ca8764a1d9b6627b7c28b146100a2e4 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Tue, 16 Jan 2024 20:31:19 -0700
Subject: [PATCH 41/62] [module_base] Add map_param; Use ValueError to handle
 errors in validate methods

---
 plugins/module_utils/module_base.py | 36 ++++++++++++++++++++++-------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 92ffd7e2..55cd2d74 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -26,11 +26,14 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr', update_php=None,
-                 arg_route=None, map_param_if=None, param_force=None, bool_values=None, have_refid=False, create_default=None):
+                 arg_route=None, map_param=None, map_param_if=None, param_force=None, bool_values=None, have_refid=False, create_default=None):
+        self.module = module         # ansible module
+        self.argument_spec = module.argument_spec  # Allow for being overriden for use with aggregate
+
+        # pfSense helper module
         if pfsense is None:
             pfsense = PFSenseModule(module)
-        self.module = module    # ansible module
-        self.argument_spec = module.argument_spec  # Allow for being overrriden for use with aggregate
+        self.pfsense = pfsense
 
         if name is not None:    # ansible module name
             self.name = name
@@ -38,9 +41,7 @@ def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive
             self.name = 'pfsense_' + node
         else:
             self.name = None
-        self.params = None      # ansible input parameters
 
-        self.pfsense = pfsense  # helper module
         self.apply = True       # apply configuration at the end
 
         # xml parent of target_elt, node named by root
@@ -72,10 +73,19 @@ def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive
             self.arg_route = arg_route        # routing for argument handling
         else:
             self.arg_route = dict()
+
+        # rules for mapping parameters
+        if map_param is not None:
+            self.map_param = map_param
+        else:
+            self.map_param = list()
+
+        # conditional rules for mapping parameters
         if map_param_if is not None:
-            self.map_param_if = map_param_if  # rules for mapping parameters
+            self.map_param_if = map_param_if
         else:
             self.map_param_if = list()
+
         if param_force is not None:
             self.param_force = param_force  # parameters that are forced to be present
         else:
@@ -144,7 +154,7 @@ def _params_to_obj(self):
 
                 # If we have defined a parser for this arg, use it
                 if param in self.arg_route and 'parse' in self.arg_route[param]:
-                    self.arg_route[param]['parse'](param, self.params, obj)
+                    self.arg_route[param]['parse'](self, param, self.params, obj)
                 elif self.argument_spec[param].get('type') == 'bool':
                     if param in self.bool_values:
                         self._get_ansible_param_bool(obj, param, value=self.bool_values[param][1], value_false=self.bool_values[param][0], force=force)
@@ -154,6 +164,12 @@ def _params_to_obj(self):
                     self._get_ansible_param(obj, param, force=force)
 
             # Handle renaming of parameters
+            for p, o in self.map_param:
+                if self.params.get(p) is not None:
+                    obj[o] = obj[p]
+                    del obj[p]
+
+            # Handle conditional renaming of parameters
             for map_param, map_value, map_tuple in self.map_param_if:
                 if self.params.get(map_param) == map_value and map_tuple[0] in obj:
                     if map_tuple[1] not in obj:
@@ -176,7 +192,10 @@ def _validate_params(self):
             # Ansible sets unspecied parameters to None, skip them
             for param in [p for p in self.params if self.params[p] is not None]:
                 if param in self.arg_route and 'validate' in self.arg_route[param]:
-                    self.arg_route[param]['validate'](params[param])
+                    try:
+                        self.arg_route[param]['validate'](self, params[param])
+                    except ValueError as e:
+                        self.module.fail_json(msg=str(e))
 
     def _deprecated_params(self):
         """ return deprecated params """
@@ -354,6 +373,7 @@ def _update(self):
         else:
             return ('', '', '')
 
+    # We take params here for use with pfsense_aggregate and the test framework
     def run(self, params):
         """ process input params to add/update/delete """
         self.params = params

From 13b0e3fd3b3b15162a3982f224511e8b8357c2bb Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Tue, 16 Jan 2024 20:57:28 -0700
Subject: [PATCH 42/62] [pfsense_user] Use PFSenseModuleBase parsing and
 validation

---
 plugins/modules/pfsense_user.py | 60 ++++++++++++++-------------------
 1 file changed, 25 insertions(+), 35 deletions(-)

diff --git a/plugins/modules/pfsense_user.py b/plugins/modules/pfsense_user.py
index 85b344c3..1258726b 100644
--- a/plugins/modules/pfsense_user.py
+++ b/plugins/modules/pfsense_user.py
@@ -53,7 +53,7 @@
     type: list
     elements: str
   authorizedkeys:
-    description: Contents of ~/.ssh/authorized_keys.  Can be base64 encoded.
+    description: Authorized SSH Keys of the user. Can be base64 encoded.
     type: str
 '''
 
@@ -98,6 +98,27 @@
     scope='user',
 )
 
+USER_MAP_PARAM = [
+    ('password', 'bcrypt-hash'),
+]
+
+
+def p2o_ssh_pub_key(self, name, params, obj):
+    # Allow ssh keys to be clear or base64 encoded
+    if 'ssh-' in params[name]:
+        obj[name] = base64.b64encode(params[name].encode()).decode()
+
+
+def validate_password(self, password):
+    if not re.match(r'\$2[aby]\$', str(password)):
+        raise ValueError('Password (%s) does not appear to be a bcrypt hash' % (password))
+
+
+USER_ARG_ROUTE = dict(
+    authorizedkeys=dict(parse=p2o_ssh_pub_key),
+    password=dict(validate=validate_password),
+)
+
 USER_PHP_COMMAND_PREFIX = """
 require_once('auth.inc');
 init_config_arr(array('system', 'user'));
@@ -143,43 +164,11 @@ def get_argument_spec():
         return USER_ARGUMENT_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSenseUserModule, self).__init__(module, pfsense, root='system', node='user', key='name', create_default=USER_CREATE_DEFAULT)
+        super(PFSenseUserModule, self).__init__(module, pfsense, root='system', node='user', key='name',
+                                                arg_route=USER_ARG_ROUTE, map_param=USER_MAP_PARAM, create_default=USER_CREATE_DEFAULT)
         self.groups = self.root_elt.findall('group')
         self.mod_groups = []
 
-    ##############################
-    # params processing
-    #
-    def _validate_params(self):
-        """ do some extra checks on input parameters """
-        params = self.params
-        if 'password' in params and params['password'] is not None:
-            password = params['password']
-            if re.match(r'\$2[aby]\$', str(password)):
-                params['bcrypt-hash'] = password
-            else:
-                self.module.fail_json(msg='Password (%s) does not appear to be a bcrypt hash' % password)
-            del params['password']
-
-    def _params_to_obj(self):
-        """ return a dict from module params """
-        params = self.params
-
-        obj = dict()
-        self.obj = obj
-
-        obj['name'] = params['name']
-        if params['state'] == 'present':
-            for option in ['authorizedkeys', 'descr', 'scope', 'uid', 'bcrypt-hash', 'groups', 'priv']:
-                if option in params and params[option] is not None:
-                    obj[option] = params[option]
-
-            # Allow authorizedkeys to be clear or base64 encoded
-            if 'authorizedkeys' in obj and 'ssh-' in obj['authorizedkeys']:
-                obj['authorizedkeys'] = base64.b64encode(obj['authorizedkeys'].encode()).decode()
-
-        return obj
-
     ##############################
     # XML processing
     #
@@ -315,6 +304,7 @@ def main():
         supports_check_mode=True)
 
     pfmodule = PFSenseUserModule(module)
+    # Pass params for testing framework
     pfmodule.run(module.params)
     pfmodule.commit_changes()
 

From 71e9a5f00b03643f3a318341abe41c9b4f8dca66 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Tue, 16 Jan 2024 22:24:48 -0700
Subject: [PATCH 43/62] [generate_module] Add _CREATE_DEFAULT, _MAP_PARAM,
 _ARG_ROUTE, drop _validate_params

---
 misc/generate_module      |  4 ++-
 misc/pfsense_module.py.j2 | 63 +++++++++++++++++++++++----------------
 2 files changed, 40 insertions(+), 27 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index 21988079..56aeddce 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -328,7 +328,8 @@ for param in params_web_only:
 
 print('')
 params_xml_only = list(set(params_full.keys()) - set(params.keys()) - {module_key, 'refid'})
-print('XML parameters not in web: ' + str(params_xml_only))
+print(f'XML parameters not in web: {params_xml_only}\n')
+print(f'You may need to use {module_node.upper()}_MAP_PARAMS')
 
 for param in params_xml_only:
     params[param] = params_full[param]
@@ -349,6 +350,7 @@ context = dict(
     module_node=module_node,
     module_key=module_key,
     params=params,
+    params_xml_only=params_xml_only,
     author_name=args.author_name,
     author_email=args.author_email,
     author_handle=args.author_handle,
diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index 5091b3f9..ea5acf73 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -76,6 +76,7 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
     # Required arguments for creation should be note in {{ module_node | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
     'name': {'required': True, 'type': 'str'},
     'state': {
+        'type': 'str',
         'default': 'present',
         'choices': ['present', 'absent']
     },
@@ -94,12 +95,20 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
     # Only name should be required here - othewise you cannot remove an item with just 'name'
     # Required arguments for creation should be note in {{ module_node | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
     name=dict(required=True, typ='str'),
-    state=dict(default='present', choices=['present', 'absent']),
+    state=dict(type='str', default='present', choices=['present', 'absent']),
 {% for param in params %}
     {{ param }}=dict(type='{{ params[param]['type'] | default('') }}'{% if 'choices' in params[param] %}, choices={{ params[param]['choices'] }},{% endif %}),
 {% endfor %}
 )
 
+# TODO - check for validity - what are default values when creating a new {{ module_node }}
+{{ module_node | upper() }}_CREATE_DEFAULT = dict(
+{% for param in params_xml_only %}
+    {{ param }}='{{ params[param]['example'] | default('VALUE') }}',
+{% endfor %}
+)
+
+# TODO - check for validity - what parameters are actually required when creating a new {{ module_node }}?
 {{ module_node | upper() }}_REQUIRED_IF = [
 {% if module_type %}
     ['state', 'present', ['type']],
@@ -109,6 +118,29 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 {% endif %}
 ]
 
+{% if params_xml_only %}
+# TODO - check this for validity and matching module argument
+{{ module_node | upper() }}_MAP_PARAM = [
+{% for param in params_xml_only %}
+    ('ARG', '{{ param }}'),
+{% endfor %}
+]
+
+# TODO - review this for clues for input validation.  Search for functions in the below require_once files in /etc and /usr/local/pfSense/include
+PHP_VALIDATION = r'''
+{{ php_requires }}
+
+{{ php_save }}
+'''
+
+# TODO - add validation and parsing methods for parameters that require it
+{{ module_node | upper() }}_ARG_ROUTE = dict(
+# TODO - these are just examples
+    authorizedkeys=dict(parse=p2o_ssh_pub_key),
+    password=dict(validate=validate_password),
+)
+
+{% endif %}
 {% if 'filter.inc' in php_requires %}
 {{ module_node | upper() }}_PHP_COMMAND_SET = r'''
 require_once("filter.inc");
@@ -118,7 +150,7 @@ if (filter_configure() == 0) { clear_subsystem_dirty('{{ php_subsystem }}'); }
 {% endif %}
 
 class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
-    """ module managing pfsense {{ module_node }} """
+    """ module managing pfsense {{ module_node }}s """
 
     ##############################
     # unit tests
@@ -130,30 +162,8 @@ class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
         return {{ module_node | upper() }}_ARGUMENT_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSense{{ module_node | capitalize() }}Module, self).__init__(module, pfsense, root='{{ module_root }}', node='{{ module_node }}', key='{{ module_key }}'{{ ', update_php=' ~ module_node | upper() ~ '_PHP_COMMAND_SET' if 'filter.inc' in php_requires else '' }})
-
-    ##############################
-    # params processing
-    #
-    def _validate_params(self):
-        """ do some extra checks on input parameters """
-        params = self.params
-
-        # check name
-        self.pfsense.check_name(params['{{ module_key }}'], '<TYPE>')
-
-        if params['state'] == 'present':
-            #  ... more checks, e.g.:
-            if int(params['timeout']) < 1:
-                self.module.fail_json(msg='timeout {0} must be greater than 1'.format(params['timeout']))
-
-
-# TODO - review this for clues for input validation.  Search for functions in the below require_once files in /etc and /usr/local/pfSense/include
-PHP_VALIDATION = r'''
-{{ php_requires }}
-
-{{ php_save }}
-'''
+        super(PFSense{{ module_node | capitalize() }}Module, self).__init__(module, pfsense, root='{{ module_root }}', node='{{ module_node }}', key='{{ module_key }}'{{ ', update_php=' ~ module_node | upper() ~ '_PHP_COMMAND_SET' if 'filter.inc' in php_requires else '' }},
++                                                arg_route={{ module_node | capitalize() }}_ARG_ROUTE, map_param={{ module_node | capitalize() }}_MAP_PARAM, create_default={{ module_node | capitalize() }}_CREATE_DEFAULT)
 
 
 def main():
@@ -163,6 +173,7 @@ def main():
         supports_check_mode=True)
 
     pfmodule = PFSense{{ module_node | capitalize() }}Module(module)
+    # Pass params for testing framework
     pfmodule.run(module.params)
     pfmodule.commit_changes()
 

From 4378b7aa7ab8f85d457cfa29113109454b73cfb8 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Wed, 17 Jan 2024 11:25:19 -0700
Subject: [PATCH 44/62] [generate_module] First stab at pkg_edit paths

---
 misc/generate_module | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index 56aeddce..0689de3b 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -5,7 +5,9 @@
 
 # TODO:
 #   - validation for addresses
-#   - bool type determination and paramter list
+#   - bool type determination and parameter list
+#   - generate version_added
+#   - detect packages
 
 from ansible.plugins.filter.core import dict_to_list_of_dict_key_value_elements
 import argparse
@@ -64,7 +66,12 @@ if args.url is not None:
 
     # Construct a likely module name from the URL
     if args.module_name is None:
-        module_name = re.sub(r'^/(?:firewall_|system_)?(.*?)(?:_edit)?\.php.*$', r'pfsense_\1', parsed_uri.path)
+        # See if this pkg_edit URL first
+        pkg_module_name = re.sub(r'^/pkg_edit\.php\?xml=(.*?)\.xml.*$', r'pfsense_\1', parsed_uri.path)
+        if pkg_module_name != parsed_uri_path:
+            module_name = pkg_module_name
+        else:
+            module_name = re.sub(r'^/(?:firewall_|system_)?(.*?)(?:_edit)?\.php.*$', r'pfsense_\1', parsed_uri.path)
         module_name_singular = re.sub(r'ses$', 's', module_name)
         if module_name_singular != module_name:
             module_name = module_name_singular

From 40c86069f28b30bf663ad1bbbdb709c36cc1e821 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Wed, 17 Jan 2024 12:54:03 -0700
Subject: [PATCH 45/62] [generate_module] Start on package handling

---
 misc/generate_module                | 34 ++++++++---
 misc/pfsense_module.py.j2           | 88 ++++++++++++++++-------------
 plugins/module_utils/module_base.py |  9 ++-
 3 files changed, 81 insertions(+), 50 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index 0689de3b..7d58ad91 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -21,6 +21,7 @@ from paramiko import SSHClient
 import re
 import requests
 from scp import SCPClient
+import shutil
 import sys
 import tempfile
 from urllib.parse import urlparse
@@ -28,6 +29,8 @@ from urllib.parse import urlparse
 gitconfig = git.GitConfigParser()
 author_name = gitconfig.get_value('user', 'name')
 author_email = gitconfig.get_value('user', 'email')
+package = 'core'
+is_package = False
 
 parser = argparse.ArgumentParser(description='Generate a pfsensible module.')
 parser.add_argument('--url', help='The URL to scrape')
@@ -50,7 +53,7 @@ parser.add_argument('--keep-params', action=argparse.BooleanOptionalAction, help
 args = parser.parse_args()
 
 # Temporary directory for files
-tmpdir = tempfile.TemporaryDirectory(prefix='pfgenmod-', delete=(not args.keep_tmpdir))
+tmpdir = tempfile.TemporaryDirectory(prefix='pfgenmod-')
 
 if args.url is not None:
     parsed_uri = urlparse(args.url)
@@ -67,11 +70,12 @@ if args.url is not None:
     # Construct a likely module name from the URL
     if args.module_name is None:
         # See if this pkg_edit URL first
-        pkg_module_name = re.sub(r'^/pkg_edit\.php\?xml=(.*?)\.xml.*$', r'pfsense_\1', parsed_uri.path)
-        if pkg_module_name != parsed_uri_path:
-            module_name = pkg_module_name
+        if re.match(r'/pkg', parsed_uri.path):
+            module_name = re.sub(r'^xml=(.*?)\.xml.*$', r'\1', parsed_uri.query)
+            package = module_name
+            is_package = True
         else:
-            module_name = re.sub(r'^/(?:firewall_|system_)?(.*?)(?:_edit)?\.php.*$', r'pfsense_\1', parsed_uri.path)
+            module_name = re.sub(r'^/(?:firewall_|system_)?(.*?)(?:_edit)?\.php.*$', r'\1', parsed_uri.path)
         module_name_singular = re.sub(r'ses$', 's', module_name)
         if module_name_singular != module_name:
             module_name = module_name_singular
@@ -103,7 +107,7 @@ elif args.urlfile is not None:
     # Use a cached copy of the web page - get rid of this?  Need to specify host and module name
     html = lxml.html.parse(args.urlfile)
     host = '192.168.100.2'
-    module_name = 'pfsense_nat_1to1'
+    module_name = 'nat_1to1'
 
 else:
     sys.exit('You must specify one of --url or --urlfile')
@@ -116,6 +120,9 @@ scp = SCPClient(ssh.get_transport())
 scp.get('/cf/conf/config.xml', f'{tmpdir.name}')
 if phpfile is not None:
     scp.get(f'/usr/local/www/{phpfile}', f'{tmpdir.name}')
+if is_package:
+    scp.get(f'/usr/local/pkg/{package}.inc', f'{tmpdir.name}')
+    scp.get(f'/usr/local/pkg/{package}.xml', f'{tmpdir.name}')
 scp.close()
 
 # Parse the config.xml file
@@ -199,6 +206,8 @@ if phpfile is not None:
 if len(html.forms) != 1:
     sys.exit(f'Found {len(html.forms)} forms instead of a single one!')
 
+# TODO - For packages we could parse /usr/local/pkg/{package}.xml instead
+
 # Collected parameters from the web form
 params = dict()
 
@@ -243,6 +252,8 @@ for input in html.forms[0].inputs:
             else:
                 param['type'] = 'str'
             param['multiple'] = input.multiple
+        if input.value is not None:
+            param['default'] = input.value
     elif input.tag == 'textarea':
         param['type'] = 'str'
 
@@ -262,7 +273,7 @@ for input in html.forms[0].inputs:
         descr_elt = div2.find('*span')
         if descr_elt.text:
             print(f'Found descr_elt {descr_elt.tag} {descr_elt.text}')
-            param['description'] += f'{descr_elt.text.strip()} of the {module_node}'
+            param['description'] += f'{descr_elt.text.strip()} of the {module_name}'
         if 'class' in descr_elt.attrib and descr_elt.attrib['class'] == 'element-required':
             param['required'] = True
         help_elt = div1.find('span[@class="help-block"]')
@@ -336,7 +347,8 @@ for param in params_web_only:
 print('')
 params_xml_only = list(set(params_full.keys()) - set(params.keys()) - {module_key, 'refid'})
 print(f'XML parameters not in web: {params_xml_only}\n')
-print(f'You may need to use {module_node.upper()}_MAP_PARAMS')
+if len(params_xml_only) > 0:
+    print(f'You may need to use {module_node.upper()}_MAP_PARAMS')
 
 for param in params_xml_only:
     params[param] = params_full[param]
@@ -358,6 +370,8 @@ context = dict(
     module_key=module_key,
     params=params,
     params_xml_only=params_xml_only,
+    is_package=is_package,
+    package=package,
     author_name=args.author_name,
     author_email=args.author_email,
     author_handle=args.author_handle,
@@ -372,7 +386,7 @@ jenv = jinja2.Environment(loader=jinja2.FileSystemLoader("misc/"), trim_blocks=T
 jenv.filters['dict2items'] = dict_to_list_of_dict_key_value_elements
 template = jenv.get_template("pfsense_module.py.j2")
 
-filename = f'plugins/modules/{module_name}.py'
+filename = f'plugins/modules/pfsense_{module_name}.py'
 if os.path.isfile(filename) and not args.force:
     sys.exit(f'{filename} already exists!')
 print(f'Writing {filename} with {context}')
@@ -385,3 +399,5 @@ if args.keep_tmpdir:
     f = open(f'{tmpdir.name}/{module_name}.html', 'w')
     f.write(r.text)
     f.close()
+    shutil.copytree(tmpdir.name, f'/tmp/{module_name}', dirs_exist_ok=True)
+    print(f'Keeping /tmp/{module_name}')
diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index ea5acf73..71340238 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -9,28 +9,31 @@ __metaclass__ = type
 
 DOCUMENTATION = r'''
 ---
-module: {{ module_name }}
+module: pfsense_{{ module_name }}
 
-short_description: Manage pfSense {{ module_node }}s
+short_description: Manage pfSense {{ module_name }}s
 
 version_added: "0.6.0"
 
 description:
-  - Manage pfSense {{ module_node }}s
+  - Manage pfSense {{ module_name }}s
 
 options:
   name:
-    description: The name of the {{ module_node }}
+    description: The name of the {{ module_name }}
     required: true
     type: str
   state:
-    description: State in which to leave the {{ module_node }}
+    description: State in which to leave the {{ module_name }}
     choices: ['present', 'absent']
     default: present
     type: str
 {% for name, param in params.items() %}
   {{ name }}:
     description: {{ "'" if ':' in param['description'] else '' }}{{ param['description'] | default('') }}{{ "'" if ':' in param['description'] else '' }}
+{% if 'default' in param %}
+    default: {{ param['default'] }}
+{% endif %}
 {% if 'choices' in param %}
     choices: {{ param['choices'] }}
 {% endif %}
@@ -41,8 +44,8 @@ author: {{ author_name }} (@{{ author_handle }})
 '''
 
 EXAMPLES = r'''
-- name: Add myitem {{ module_node }}
-  pfsensible.core.{{ module_name }}:
+- name: Add myitem {{ module_name }}
+  pfsensible.{{ package }}.pfsense_{{ module_name }}:
     name: myitem
 {% for name, param in params.items() %}
 {% if param['example'] is defined %}
@@ -54,8 +57,8 @@ EXAMPLES = r'''
 {% endfor %}
     state: present
 
-- name: Remove myitem {{ module_node }}
-  pfsensible.core.{{ module_name }}:
+- name: Remove myitem {{ module_name }}
+  pfsensible.{{ package }}.pfsense_{{ module_name }}:
     name: myitem
     state: absent
 '''
@@ -64,16 +67,16 @@ commands:
     description: the set of commands that would be pushed to the remote device (if pfSense had a CLI)
     returned: always
     type: list
-    sample: ["create {{ module_node }} 'myitem'", "update {{ module_node }} 'myitem' set ...", "delete {{ module_node }} 'myitem'"]
+    sample: ["create {{ module_name }} 'myitem'", "update {{ module_name }} 'myitem' set ...", "delete {{ module_name }} 'myitem'"]
 '''
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible_collections.pfsensible.core.plugins.module_utils.module_base import PFSenseModuleBase
 
 # Change to name of module, extend for needed parameters
-{{ module_node | upper() }}_ARGUMENT_SPEC = {
+{{ module_name | upper() }}_ARGUMENT_SPEC = {
     # Only name should be required here - othewise you cannot remove an item with just 'name'
-    # Required arguments for creation should be note in {{ module_node | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
+    # Required arguments for creation should be noted in {{ module_name | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
     'name': {'required': True, 'type': 'str'},
     'state': {
         'type': 'str',
@@ -91,36 +94,29 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 }
 
 # Compact style
-{{ module_node | upper() }}_ARGUMENT_SPEC = dict(
+{{ module_name | upper() }}_ARGUMENT_SPEC = dict(
     # Only name should be required here - othewise you cannot remove an item with just 'name'
-    # Required arguments for creation should be note in {{ module_node | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
-    name=dict(required=True, typ='str'),
+    # Required arguments for creation should be noted in {{ module_name | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
+    name=dict(required=True, type='str'),
     state=dict(type='str', default='present', choices=['present', 'absent']),
 {% for param in params %}
     {{ param }}=dict(type='{{ params[param]['type'] | default('') }}'{% if 'choices' in params[param] %}, choices={{ params[param]['choices'] }},{% endif %}),
 {% endfor %}
 )
 
-# TODO - check for validity - what are default values when creating a new {{ module_node }}
-{{ module_node | upper() }}_CREATE_DEFAULT = dict(
-{% for param in params_xml_only %}
-    {{ param }}='{{ params[param]['example'] | default('VALUE') }}',
-{% endfor %}
-)
-
-# TODO - check for validity - what parameters are actually required when creating a new {{ module_node }}?
-{{ module_node | upper() }}_REQUIRED_IF = [
+# TODO - check for validity - what parameters are actually required when creating a new {{ module_name }}?
+{{ module_name | upper() }}_REQUIRED_IF = [
 {% if module_type %}
     ['state', 'present', ['type']],
-    ['type', '{{ params['type']['example'] }}', ['{{ params | dict2items | rejectattr('key', 'equalto', 'type') | selectattr('value.required', 'defined') | map(attribute='key') | join("', '") }}']],
+    ['type', '{{ params['type']['example'] }}', ['{{ params | dict2items | rejectattr('key', 'equalto', 'type') | selectattr('value.required', 'defined') | rejectattr('value.default', 'defined') | map(attribute='key') | join("', '") }}']],
 {% else %}
-    ['state', 'present', ['{{ params | dict2items | selectattr('value.required', 'defined') | select(attribue='key') | join("', '") }}']],
+    ['state', 'present', ['{{ params | dict2items | selectattr('value.required', 'defined') | rejectattr('value.default', 'defined') | map(attribute='key') | join("', '") }}']],
 {% endif %}
 ]
 
 {% if params_xml_only %}
 # TODO - check this for validity and matching module argument
-{{ module_node | upper() }}_MAP_PARAM = [
+{{ module_name | upper() }}_MAP_PARAM = [
 {% for param in params_xml_only %}
     ('ARG', '{{ param }}'),
 {% endfor %}
@@ -134,23 +130,39 @@ PHP_VALIDATION = r'''
 '''
 
 # TODO - add validation and parsing methods for parameters that require it
-{{ module_node | upper() }}_ARG_ROUTE = dict(
+{{ module_name | upper() }}_ARG_ROUTE = dict(
 # TODO - these are just examples
     authorizedkeys=dict(parse=p2o_ssh_pub_key),
     password=dict(validate=validate_password),
 )
 
 {% endif %}
-{% if 'filter.inc' in php_requires %}
-{{ module_node | upper() }}_PHP_COMMAND_SET = r'''
+# TODO - check for validity - what are default values when creating a new {{ module_name }}
+{{ module_name | upper() }}_CREATE_DEFAULT = dict(
+{% for item in params | dict2items | selectattr('value.default', 'defined') %}
+    {{ item.key }}='{{ item.value.default | default('VALUE') }}',
+{% endfor %}
+{% for param in params_xml_only %}
+    {{ param }}='{{ params[param]['example'] | default('VALUE') }}',
+{% endfor %}
+)
+
+{% if is_package %}
+{{ module_name | upper() }}_PHP_COMMAND_SET = r'''
+require_once("{{ package }}.inc");
+{{ package }}_sync_package();
+'''
+
+{% elif 'filter.inc' in php_requires %}
+{{ module_name | upper() }}_PHP_COMMAND_SET = r'''
 require_once("filter.inc");
 if (filter_configure() == 0) { clear_subsystem_dirty('{{ php_subsystem }}'); }
 '''
 
 {% endif %}
 
-class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
-    """ module managing pfsense {{ module_node }}s """
+class PFSense{{ module_name | capitalize() }}Module(PFSenseModuleBase):
+    """ module managing pfsense {{ module_name }}s """
 
     ##############################
     # unit tests
@@ -159,20 +171,20 @@ class PFSense{{ module_node | capitalize() }}Module(PFSenseModuleBase):
     @staticmethod
     def get_argument_spec():
         """ return argument spec """
-        return {{ module_node | upper() }}_ARGUMENT_SPEC
+        return {{ module_name | upper() }}_ARGUMENT_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSense{{ module_node | capitalize() }}Module, self).__init__(module, pfsense, root='{{ module_root }}', node='{{ module_node }}', key='{{ module_key }}'{{ ', update_php=' ~ module_node | upper() ~ '_PHP_COMMAND_SET' if 'filter.inc' in php_requires else '' }},
-+                                                arg_route={{ module_node | capitalize() }}_ARG_ROUTE, map_param={{ module_node | capitalize() }}_MAP_PARAM, create_default={{ module_node | capitalize() }}_CREATE_DEFAULT)
+        super(PFSense{{ module_name | capitalize() }}Module, self).__init__(module, pfsense, {{ 'package=\'' ~ package ~ '\', ' if is_package else ''}}root='{{ module_root }}', {{ 'is_package=True, ' if is_package else '' }}node='{{ module_node }}', key='{{ module_key }}'{{ ', update_php=' ~ module_name | upper() ~ '_PHP_COMMAND_SET' if 'filter.inc' in php_requires else '' }},
+                                                arg_route={{ module_name | upper() }}_ARG_ROUTE, map_param={{ module_name | upper() }}_MAP_PARAM, create_default={{ module_name | upper() }}_CREATE_DEFAULT)
 
 
 def main():
     module = AnsibleModule(
-        argument_spec={{ module_node | upper() }}_ARGUMENT_SPEC,
-        required_if={{ module_node | upper() }}_REQUIRED_IF,
+        argument_spec={{ module_name | upper() }}_ARGUMENT_SPEC,
+        required_if={{ module_name | upper() }}_REQUIRED_IF,
         supports_check_mode=True)
 
-    pfmodule = PFSense{{ module_node | capitalize() }}Module(module)
+    pfmodule = PFSense{{ module_name | capitalize() }}Module(module)
     # Pass params for testing framework
     pfmodule.run(module.params)
     pfmodule.commit_changes()
diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 55cd2d74..04882335 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -25,8 +25,8 @@ def get_argument_spec():
     ##############################
     # init
     #
-    def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr', update_php=None,
-                 arg_route=None, map_param=None, map_param_if=None, param_force=None, bool_values=None, have_refid=False, create_default=None):
+    def __init__(self, module, pfsense=None, package=None, name=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr',
+                 update_php=None, arg_route=None, map_param=None, map_param_if=None, param_force=None, bool_values=None, have_refid=False, create_default=None):
         self.module = module         # ansible module
         self.argument_spec = module.argument_spec  # Allow for being overriden for use with aggregate
 
@@ -50,7 +50,10 @@ def __init__(self, module, pfsense=None, name=None, root=None, root_is_exclusive
                 self.root_elt = self.pfsense.root
                 self.root_is_exclusive = False
             else:
-                self.root_elt = self.pfsense.get_element(root, create_node=create_root)
+                if package is None:
+                    self.root_elt = self.pfsense.get_element(root, create_node=create_root)
+                else:
+                    self.root_elt = self.pfsense.get_element(root, root_elt=self.pfsense.root.find('installedpackages'))
                 if root in ['system']:
                     self.root_is_exclusive = False
                 else:

From 61503bdcddd17978d80f18b869219aa27edcab9a Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Thu, 18 Jan 2024 21:00:59 -0700
Subject: [PATCH 46/62] [generate_module] Enhancements

- Add --name-param, and default to key
- Strip 'manage' from url for module name
- Detect lists in XML
- Better list doc/example
---
 misc/generate_module      | 30 ++++++++++++++++++++------
 misc/pfsense_module.py.j2 | 45 ++++++++++++++++++++++-----------------
 2 files changed, 49 insertions(+), 26 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index 7d58ad91..c54bed04 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -41,6 +41,7 @@ parser.add_argument('--author-name', default=author_name, help='The full name of
 parser.add_argument('--author-email', default=author_email, help='The email address of the module author')
 parser.add_argument('--author-handle', default='', help='The github handle of the module author')
 parser.add_argument('--module-name', help='The name of the module to generate - defaults to being based on the url')
+parser.add_argument('--name-param', help='The name of the primary module parameter - defaults to the key, but often "name" is used instead of "descr"')
 parser.add_argument('--type-param', default='type', help='The name of the parameter for selecting different types of elements', )
 parser.add_argument('--type-suffix', const=True, default=False, nargs='?', help='Suffix the module name with the item type', )
 parser.add_argument('--item-min', default='item_min', help='The name of the minimally configured item to search for in config.xml')
@@ -75,7 +76,7 @@ if args.url is not None:
             package = module_name
             is_package = True
         else:
-            module_name = re.sub(r'^/(?:firewall_|system_)?(.*?)(?:_edit)?\.php.*$', r'\1', parsed_uri.path)
+            module_name = re.sub(r'^/(?:firewall_|system_)?(.*?)(?:_edit|manager)?\.php.*$', r'\1', parsed_uri.path)
         module_name_singular = re.sub(r'ses$', 's', module_name)
         if module_name_singular != module_name:
             module_name = module_name_singular
@@ -140,6 +141,10 @@ else:
 
 # This element should be the key for the items
 module_key = key_elt.tag
+if args.name_param:
+   name_param = args.name_param
+else:
+   name_param = module_key
 
 # The full node configuration element will be the parent
 node_elt = key_elt.find('..')
@@ -165,16 +170,26 @@ params_full = dict()
 for elt in full_elt:
     if elt.tag == '':
         continue
-    params_full[elt.tag] = dict()
+    param = dict()
     addr_elt = elt.find('address')
     if addr_elt is not None:
-        params_full[elt.tag]['example'] = addr_elt.text
-        params_full[elt.tag]['address'] = True
+        param['example'] = addr_elt.text
+        param['address'] = True
     elif elt.text is not None:
-        params_full[elt.tag]['type'] = 'str'
-        params_full[elt.tag]['example'] = elt.text.strip()
+        if elt.tag in params_full:
+            # Copy example and possibly other values from previous copy
+            param = params_full[elt.tag]
+            # If we have already need one of these, then it is a list
+            param['type'] = 'list'
+            # TODO - can we determine the type?
+            param['elements'] = 'str'
+            param['example2'] = elt.text.strip()
+        else:
+            param['type'] = 'str'
+            param['example'] = elt.text.strip()
     # else:
         # Likely a bool?
+    params_full[elt.tag] = param
 
 print('')
 
@@ -285,7 +300,7 @@ for input in html.forms[0].inputs:
 
 # Key is handled separately from other parameters so remove it
 # TODO - keep the description, etc?
-del params[module_key]
+params.pop(module_key, None)
 
 # Debug
 print(f'Web paramters: {params.keys()}')
@@ -370,6 +385,7 @@ context = dict(
     module_key=module_key,
     params=params,
     params_xml_only=params_xml_only,
+    name_param=name_param,
     is_package=is_package,
     package=package,
     author_name=args.author_name,
diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index 71340238..031781ff 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -13,31 +13,31 @@ module: pfsense_{{ module_name }}
 
 short_description: Manage pfSense {{ module_name }}s
 
-version_added: "0.6.0"
+version_added: "0.7.0"
 
 description:
-  - Manage pfSense {{ module_name }}s
+  - Manage pfSense {{ module_name }}s.{{ ' This requires the pfSense ' ~ package ~ ' package to be installed.' if is_package else '' }}
 
 options:
-  name:
-    description: The name of the {{ module_name }}
+  {{ name_param }}:
+    description: The {{ name_param }} of the {{ module_name }}.
     required: true
     type: str
   state:
-    description: State in which to leave the {{ module_name }}
-    choices: ['present', 'absent']
+    description: State in which to leave the {{ module_name }}.
     default: present
+    choices: ['present', 'absent']
     type: str
 {% for name, param in params.items() %}
   {{ name }}:
     description: {{ "'" if ':' in param['description'] else '' }}{{ param['description'] | default('') }}{{ "'" if ':' in param['description'] else '' }}
-{% if 'default' in param %}
-    default: {{ param['default'] }}
-{% endif %}
 {% if 'choices' in param %}
     choices: {{ param['choices'] }}
 {% endif %}
     type: {{ param['type'] | default('') }}
+{% if param['type'] == 'list' %}
+    elements: 'str'
+{% endif %}
 {% endfor %}
 
 author: {{ author_name }} (@{{ author_handle }})
@@ -46,12 +46,18 @@ author: {{ author_name }} (@{{ author_handle }})
 EXAMPLES = r'''
 - name: Add myitem {{ module_name }}
   pfsensible.{{ package }}.pfsense_{{ module_name }}:
-    name: myitem
+    {{ name_param }}: myitem
 {% for name, param in params.items() %}
 {% if param['example'] is defined %}
+{% if param['type'] == 'list' %}
+    {{ name }}:
+      - {{ param['example'] }}
+      - {{ param['example2'] | default('another item') }}
+{% else %}
     {{ name }}: {{ param['example'] }}
+{% endif %}
 {% else %}
-    {{ name }}: {% if param['type'] == 'bool' %}false{% elif param['type'] == 'list' %}{% if 'choices' in param %}['{{ param['choices'][0:1] | join("', '") }}']{% else %}{{ param['choices'][0] }}{% endif %}{% elif param['type'] == 'str' %}{{ param['choices'][0] if 'choices' in param else '' }}{% endif %}
+    {{ name }}: {% if param['type'] == 'bool' %}false{% elif param['type'] == 'list' %}{% if 'choices' in param %}['{{ param['choices'][0:1] | join("', '") }}']{% else %}['item']{% endif %}{% elif param['type'] == 'str' %}{{ param['choices'][0] if 'choices' in param else '' }}{% endif %}
 
 {% endif %}
 {% endfor %}
@@ -59,7 +65,7 @@ EXAMPLES = r'''
 
 - name: Remove myitem {{ module_name }}
   pfsensible.{{ package }}.pfsense_{{ module_name }}:
-    name: myitem
+    {{ name_param }}: myitem
     state: absent
 '''
 RETURN = r'''
@@ -73,11 +79,12 @@ commands:
 from ansible.module_utils.basic import AnsibleModule
 from ansible_collections.pfsensible.core.plugins.module_utils.module_base import PFSenseModuleBase
 
-# Change to name of module, extend for needed parameters
+# TODO -Change to name of module, extend for needed parameters
+# TODO -Keep either this or the next compact version of {{ module_name | upper() }}_ARGUMENT_SPEC
 {{ module_name | upper() }}_ARGUMENT_SPEC = {
-    # Only name should be required here - othewise you cannot remove an item with just 'name'
+    # Only {{ name_param }} should be required here - othewise you cannot remove an item with just '{{ name_param }}'
     # Required arguments for creation should be noted in {{ module_name | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
-    'name': {'required': True, 'type': 'str'},
+    '{{ name_param }}': {'required': True, 'type': 'str'},
     'state': {
         'type': 'str',
         'default': 'present',
@@ -95,9 +102,9 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 
 # Compact style
 {{ module_name | upper() }}_ARGUMENT_SPEC = dict(
-    # Only name should be required here - othewise you cannot remove an item with just 'name'
+    # Only {{ name_param }} should be required here - othewise you cannot remove an item with just '{{ name_param }}'
     # Required arguments for creation should be noted in {{ module_name | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
-    name=dict(required=True, type='str'),
+    {{ name_param }}=dict(required=True, type='str'),
     state=dict(type='str', default='present', choices=['present', 'absent']),
 {% for param in params %}
     {{ param }}=dict(type='{{ params[param]['type'] | default('') }}'{% if 'choices' in params[param] %}, choices={{ params[param]['choices'] }},{% endif %}),
@@ -174,8 +181,8 @@ class PFSense{{ module_name | capitalize() }}Module(PFSenseModuleBase):
         return {{ module_name | upper() }}_ARGUMENT_SPEC
 
     def __init__(self, module, pfsense=None):
-        super(PFSense{{ module_name | capitalize() }}Module, self).__init__(module, pfsense, {{ 'package=\'' ~ package ~ '\', ' if is_package else ''}}root='{{ module_root }}', {{ 'is_package=True, ' if is_package else '' }}node='{{ module_node }}', key='{{ module_key }}'{{ ', update_php=' ~ module_name | upper() ~ '_PHP_COMMAND_SET' if 'filter.inc' in php_requires else '' }},
-                                                arg_route={{ module_name | upper() }}_ARG_ROUTE, map_param={{ module_name | upper() }}_MAP_PARAM, create_default={{ module_name | upper() }}_CREATE_DEFAULT)
+        super(PFSense{{ module_name | capitalize() }}Module, self).__init__(module, pfsense, {{ 'package=\'' ~ package ~ '\', ' if is_package else ''}}root='{{ module_root }}', node='{{ module_node }}', key='{{ module_key }}'{{ ', update_php=' ~ module_name | upper() ~ '_PHP_COMMAND_SET' if 'filter.inc' in php_requires else '' }},
+                                                arg_route={{ module_name | upper() }}_ARG_ROUTE{% if params_xml_only %}, map_param={{ module_name | upper() }}_MAP_PARAM{% endif %}, create_default={{ module_name | upper() }}_CREATE_DEFAULT)
 
 
 def main():

From 59fab4b970ba3b7a32b7c6f1b0c29fc53975f6f6 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Wed, 17 Jan 2024 21:21:21 -0700
Subject: [PATCH 47/62] [pfsense_shellcmd] Add module

---
 plugins/modules/pfsense_shellcmd.py | 139 ++++++++++++++++++++++++++++
 1 file changed, 139 insertions(+)
 create mode 100644 plugins/modules/pfsense_shellcmd.py

diff --git a/plugins/modules/pfsense_shellcmd.py b/plugins/modules/pfsense_shellcmd.py
new file mode 100644
index 00000000..4649cce5
--- /dev/null
+++ b/plugins/modules/pfsense_shellcmd.py
@@ -0,0 +1,139 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright: (c) 2024, Orion Poplawski <orion@nwra.com>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+---
+module: pfsense_shellcmd
+
+short_description: Manage pfSense shellcmds
+
+version_added: "0.7.0"
+
+description:
+  - Manage pfSense shellcmds. This requires the pfSense shellcmd package to be installed.
+
+options:
+  description:
+    description: The description of the shellcmd.
+    required: true
+    type: str
+  state:
+    description: State in which to leave the shellcmd.
+    default: present
+    choices: ['present', 'absent']
+    type: str
+  cmd:
+    description: The command to run.
+    type: str
+  cmdtype:
+    description: Type of the shell command, defaults to `shellcmd`. There can only be one `afterfilterchangeshellcmd` command.  If there is an existing one, it
+      will be replaced.
+    choices: ['shellcmd', 'earlyshellcmd', 'afterfilterchangeshellcmd', 'disabled']
+    type: str
+
+author: Orion Poplawski (@opoplawski)
+'''
+
+EXAMPLES = r'''
+- name: Add myitem shellcmd
+  pfsensible.core.pfsense_shellcmd:
+    description: myitem
+    cmd: echo hi
+    cmdtype: shellcmd
+    state: present
+
+- name: Remove myitem shellcmd
+  pfsensible.core.pfsense_shellcmd:
+    description: myitem
+    state: absent
+'''
+RETURN = r'''
+commands:
+    description: the set of commands that would be pushed to the remote device (if pfSense had a CLI)
+    returned: always
+    type: list
+    sample: ["create shellcmd 'myitem'", "update shellcmd 'myitem' set ...", "delete shellcmd 'myitem'"]
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible_collections.pfsensible.core.plugins.module_utils.module_base import PFSenseModuleBase
+
+# Compact style
+SHELLCMD_ARGUMENT_SPEC = dict(
+    # Only description should be required here - othewise you cannot remove an item with just 'description'
+    # Required arguments for creation should be noted in SHELLCMD_REQUIRED_IF = ['state', 'present', ...] below
+    description=dict(required=True, type='str'),
+    state=dict(type='str', default='present', choices=['present', 'absent']),
+    cmd=dict(type='str'),
+    cmdtype=dict(type='str', choices=['shellcmd', 'earlyshellcmd', 'afterfilterchangeshellcmd', 'disabled'],),
+)
+
+SHELLCMD_REQUIRED_IF = [
+    ['state', 'present', ['cmd']],
+]
+
+# default values when creating a new shellcmd
+SHELLCMD_CREATE_DEFAULT = dict(
+    cmdtype='shellcmd',
+)
+
+SHELLCMD_PHP_COMMAND_SET = r'''
+require_once("shellcmd.inc");
+shellcmd_sync_package();
+'''
+
+
+class PFSenseShellcmdModule(PFSenseModuleBase):
+    """ module managing pfsense shellcmds """
+
+    ##############################
+    # unit tests
+    #
+    # Must be class method for unit test usage
+    @staticmethod
+    def get_argument_spec():
+        """ return argument spec """
+        return SHELLCMD_ARGUMENT_SPEC
+
+    def __init__(self, module, pfsense=None):
+        super(PFSenseShellcmdModule, self).__init__(module, pfsense, package='shellcmd', root='shellcmdsettings', node='config', key='description',
+                                                    update_php=SHELLCMD_PHP_COMMAND_SET, create_default=SHELLCMD_CREATE_DEFAULT)
+
+    ##############################
+    # XML processing
+    #
+    def _find_target(self):
+        """ find the XML target_elt """
+        # There can be only one 'afterfilterchangeshellcmd' shellcmd
+        if self.params['cmdtype'] == 'afterfilterchangeshellcmd':
+            result = self.root_elt.findall("{node}[{key}='{value}']".format(node=self.node, key='cmdtype', value='afterfilterchangeshellcmd'))
+        else:
+            result = self.root_elt.findall("{node}[{key}='{value}']".format(node=self.node, key=self.key, value=self.obj[self.key]))
+        if len(result) == 1:
+            return result[0]
+        elif len(result) > 1:
+            self.module.fail_json(msg='Found multiple {node}s for {key} {value}.'.format(node=self.node, key=self.key, value=self.obj[self.key]))
+        else:
+            return None
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=SHELLCMD_ARGUMENT_SPEC,
+        required_if=SHELLCMD_REQUIRED_IF,
+        supports_check_mode=True)
+
+    pfmodule = PFSenseShellcmdModule(module)
+    # Pass params for testing framework
+    pfmodule.run(module.params)
+    pfmodule.commit_changes()
+
+
+if __name__ == '__main__':
+    main()

From ec59b801e5249fc2e242d6420d21a6e9dc41850b Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Fri, 19 Jan 2024 21:19:51 -0700
Subject: [PATCH 48/62] [generate_module] Enforce "." at end of descriptions

---
 misc/generate_module      | 15 +++++++++++----
 misc/pfsense_module.py.j2 |  2 +-
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index c54bed04..cae9a04d 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -223,6 +223,12 @@ if len(html.forms) != 1:
 
 # TODO - For packages we could parse /usr/local/pkg/{package}.xml instead
 
+# Make sure a string has a trailing period
+def enforce_period(s):
+    if len(s) > 0 and s[-1] != '.':
+        s += '.'
+    return s
+
 # Collected parameters from the web form
 params = dict()
 
@@ -255,7 +261,7 @@ for input in html.forms[0].inputs:
                 param[attr] = input.attrib[attr]
         # Text sometimes is after the input element inside the enclosing <label>
         if input.tail:
-            param['description'] = input.tail.strip()
+            param['description'] = enforce_period(input.tail.strip())
     elif isinstance(input, lxml.html.SelectElement):
         print(f'select name={input.name} value={input.value} value_options={input.value_options} multiple={input.multiple}')
         if input.attrib.get('class') == 'form-control' and input.attrib.get('data-toggle') == 'collapse':
@@ -288,13 +294,14 @@ for input in html.forms[0].inputs:
         descr_elt = div2.find('*span')
         if descr_elt.text:
             print(f'Found descr_elt {descr_elt.tag} {descr_elt.text}')
-            param['description'] += f'{descr_elt.text.strip()} of the {module_name}'
+            param['description'] += f'{descr_elt.text.strip()} of the {module_name}.'
         if 'class' in descr_elt.attrib and descr_elt.attrib['class'] == 'element-required':
             param['required'] = True
         help_elt = div1.find('span[@class="help-block"]')
         if help_elt is not None:
             print(f'help_elt text {help_elt.text.strip()}')
-            param['description'] += f'. {help_elt.text.strip()}'
+            descr = enforce_period(help_elt.text.strip())
+            param['description'] += f' {descr}'
 
     params[input.name] = param
 
@@ -372,7 +379,7 @@ for param in params_xml_only:
 for name, param in params.items():
     # TODO - wrap long descriptions
     if 'description' not in param or param['description'] == '':
-        param['description'] = f'The {name} of the {module_node}'
+        param['description'] = f'The {name} of the {module_node}.'
     if 'example' not in param or param['example'] == '':
         if name in params_full and 'example' in params_full[name]:
             param['example'] = params_full[name]['example']
diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index 031781ff..274815c6 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -70,7 +70,7 @@ EXAMPLES = r'''
 '''
 RETURN = r'''
 commands:
-    description: the set of commands that would be pushed to the remote device (if pfSense had a CLI)
+    description: the set of commands that would be pushed to the remote device (if pfSense had a CLI).
     returned: always
     type: list
     sample: ["create {{ module_name }} 'myitem'", "update {{ module_name }} 'myitem' set ...", "delete {{ module_name }} 'myitem'"]

From 396c0c6e9f13114b80cd3cb254a1d8983ac8cd96 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sat, 20 Jan 2024 19:31:42 -0700
Subject: [PATCH 49/62] [module_base] Add default arg_route for interface;
 Process all parameters again

---
 plugins/module_utils/arg_route.py   | 15 +++++++++++++++
 plugins/module_utils/module_base.py | 17 ++++++++++++-----
 plugins/modules/pfsense_user.py     |  2 +-
 3 files changed, 28 insertions(+), 6 deletions(-)
 create mode 100644 plugins/module_utils/arg_route.py

diff --git a/plugins/module_utils/arg_route.py b/plugins/module_utils/arg_route.py
new file mode 100644
index 00000000..b60dd631
--- /dev/null
+++ b/plugins/module_utils/arg_route.py
@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+
+# Copyright: (c) 2024, Orion Poplawski <orion@nwra.com>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+
+def p2o_interface(self, name, params, obj):
+    obj[name] = self.pfsense.parse_interface(params[name], with_virtual=True)
+
+
+def p2o_interface_without_virtual(self, name, params, obj):
+    obj[name] = self.pfsense.parse_interface(params[name], with_virtual=False)
diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 04882335..bf0072f2 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -8,6 +8,11 @@
 __metaclass__ = type
 
 from ansible_collections.pfsensible.core.plugins.module_utils.pfsense import PFSenseModule
+from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import p2o_interface
+
+BASE_ARG_ROUTE = dict(
+    interface=dict(parse=p2o_interface,),
+)
 
 
 class PFSenseModuleBase(object):
@@ -45,6 +50,7 @@ def __init__(self, module, pfsense=None, package=None, name=None, root=None, roo
         self.apply = True       # apply configuration at the end
 
         # xml parent of target_elt, node named by root
+        # TODO - handle paths with creation - e.g.  <nat> <outbound>
         if root is not None:
             if root == 'pfsense':
                 self.root_elt = self.pfsense.root
@@ -72,10 +78,11 @@ def __init__(self, module, pfsense=None, package=None, name=None, root=None, roo
 
         self.key = key          # item that identifies a target element
         self.obj = dict()       # dict holding target pfsense parameters
+
+        # routing for argument handling
+        self.arg_route = BASE_ARG_ROUTE
         if arg_route is not None:
-            self.arg_route = arg_route        # routing for argument handling
-        else:
-            self.arg_route = dict()
+            self.arg_route.update(arg_route)
 
         # rules for mapping parameters
         if map_param is not None:
@@ -149,8 +156,8 @@ def _params_to_obj(self):
         obj = dict()
         # Not all modules have 'state', treat them like they did
         if self.params.get('state', 'present') == 'present':
-            # Ansible sets unspecied parameters to None, skip them and 'state'
-            for param in [p for p in self.params if p != 'state' and self.params[p] is not None]:
+            # Skip 'state', but otherwise process all parameters.  Ansible sets unspecified parameters to None.
+            for param in [p for p in self.params if p != 'state']:
                 force = False
                 if param in self.param_force:
                     force = True
diff --git a/plugins/modules/pfsense_user.py b/plugins/modules/pfsense_user.py
index 1258726b..01a552e4 100644
--- a/plugins/modules/pfsense_user.py
+++ b/plugins/modules/pfsense_user.py
@@ -105,7 +105,7 @@
 
 def p2o_ssh_pub_key(self, name, params, obj):
     # Allow ssh keys to be clear or base64 encoded
-    if 'ssh-' in params[name]:
+    if params[name] is not None and 'ssh-' in params[name]:
         obj[name] = base64.b64encode(params[name].encode()).decode()
 
 

From a12ed88faecf84fe27eabc8a84f7fc4ebaf2daae Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Fri, 19 Jan 2024 22:26:35 -0700
Subject: [PATCH 50/62] [generate_module] Support for standard interface
 parser; stub in validate

---
 misc/generate_module      | 27 ++++++++++++++++++++++-----
 misc/pfsense_module.py.j2 | 12 +++++++++++-
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index cae9a04d..0de8d042 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -10,6 +10,7 @@
 #   - detect packages
 
 from ansible.plugins.filter.core import dict_to_list_of_dict_key_value_elements
+from ansible.plugins.filter.mathstuff import unique
 import argparse
 import datetime
 import git
@@ -31,6 +32,7 @@ author_name = gitconfig.get_value('user', 'name')
 author_email = gitconfig.get_value('user', 'email')
 package = 'core'
 is_package = False
+args_imports = []
 
 parser = argparse.ArgumentParser(description='Generate a pfsensible module.')
 parser.add_argument('--url', help='The URL to scrape')
@@ -266,15 +268,26 @@ for input in html.forms[0].inputs:
         print(f'select name={input.name} value={input.value} value_options={input.value_options} multiple={input.multiple}')
         if input.attrib.get('class') == 'form-control' and input.attrib.get('data-toggle') == 'collapse':
             args.type_param = input.name
+
         if input.value_options is not None:
-            param['choices'] = input.value_options
-            if input.multiple:
-                param['type'] = 'list'
-            else:
+            if input.name == 'interface':
                 param['type'] = 'str'
-            param['multiple'] = input.multiple
+                # By default, interfaces will be parsed allowing virtual interfaces.  If not allowed we need a different parser.
+                if len(set(input.value_options).intersection(['enc0', 'openvpn'])) == 0:
+                    param['parse'] = 'p2o_interface_without_virtual'
+                    if 'p2o_interface_without_virtual' not in args_imports:
+                        args_imports.append('p2o_interface_without_virtual')
+            else:
+                param['choices'] = input.value_options
+                if input.multiple:
+                    param['type'] = 'list'
+                else:
+                    param['type'] = 'str'
+                param['multiple'] = input.multiple
+
         if input.value is not None:
             param['default'] = input.value
+
     elif input.tag == 'textarea':
         param['type'] = 'str'
 
@@ -383,6 +396,8 @@ for name, param in params.items():
     if 'example' not in param or param['example'] == '':
         if name in params_full and 'example' in params_full[name]:
             param['example'] = params_full[name]['example']
+    if 'default' in param:
+        param['description'] += f' Defaults to {param["default"]}.'
 
 # Template variables
 context = dict(
@@ -393,6 +408,7 @@ context = dict(
     params=params,
     params_xml_only=params_xml_only,
     name_param=name_param,
+    args_imports=args_imports,
     is_package=is_package,
     package=package,
     author_name=args.author_name,
@@ -407,6 +423,7 @@ context = dict(
 # Render our module!
 jenv = jinja2.Environment(loader=jinja2.FileSystemLoader("misc/"), trim_blocks=True, keep_trailing_newline=True)
 jenv.filters['dict2items'] = dict_to_list_of_dict_key_value_elements
+jenv.filters['unique'] = unique
 template = jenv.get_template("pfsense_module.py.j2")
 
 filename = f'plugins/modules/pfsense_{module_name}.py'
diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index 274815c6..88a07d68 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -78,6 +78,9 @@ commands:
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible_collections.pfsensible.core.plugins.module_utils.module_base import PFSenseModuleBase
+{% if args_imports %}
+from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import {{ args_imports | sort | join(', ') }}
+{% endif %}
 
 # TODO -Change to name of module, extend for needed parameters
 # TODO -Keep either this or the next compact version of {{ module_name | upper() }}_ARGUMENT_SPEC
@@ -129,6 +132,7 @@ from ansible_collections.pfsensible.core.plugins.module_utils.module_base import
 {% endfor %}
 ]
 
+{% endif %}
 # TODO - review this for clues for input validation.  Search for functions in the below require_once files in /etc and /usr/local/pfSense/include
 PHP_VALIDATION = r'''
 {{ php_requires }}
@@ -138,12 +142,18 @@ PHP_VALIDATION = r'''
 
 # TODO - add validation and parsing methods for parameters that require it
 {{ module_name | upper() }}_ARG_ROUTE = dict(
+{% set param_items = ((params | dict2items | selectattr('value.parse', 'defined') | list) + (params | dict2items | selectattr('value.validate', 'defined')) | list) | unique %}
+{% if param_items %}
+{% for param_item in param_items %}
+    {{ param_item.key }}=dict({% if param_item.value.parse is defined %}parse={{ param_item.value.parse }},{% endif %}{% if param_item.value.validate is defined %}validate={{ param_item.value.validate }},{% endif %}),
+{% endfor %}
+{% else %}
 # TODO - these are just examples
     authorizedkeys=dict(parse=p2o_ssh_pub_key),
     password=dict(validate=validate_password),
+{% endif %}
 )
 
-{% endif %}
 # TODO - check for validity - what are default values when creating a new {{ module_name }}
 {{ module_name | upper() }}_CREATE_DEFAULT = dict(
 {% for item in params | dict2items | selectattr('value.default', 'defined') %}

From c9c2ee33406557443984961d33746cf90bf5d9a3 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sat, 20 Jan 2024 19:33:08 -0700
Subject: [PATCH 51/62] [pfsense_alias] Use CREATE_DEFAULT for detail/descr

---
 plugins/module_utils/alias.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/plugins/module_utils/alias.py b/plugins/module_utils/alias.py
index 28dcc7d4..b6554f7b 100644
--- a/plugins/module_utils/alias.py
+++ b/plugins/module_utils/alias.py
@@ -19,8 +19,6 @@
     updatefreq=dict(default=None, required=False, type='int'),
 )
 
-ALIAS_PARAM_FORCE = ['descr', 'detail']
-
 ALIAS_MUTUALLY_EXCLUSIVE = [
     ('address', 'url'),
 ]
@@ -42,6 +40,11 @@
     ["type", "urltable_ports", ("address", "url")],
 ]
 
+ALIAS_CREATE_DEFAULT = dict(
+    descr='',
+    detail='',
+)
+
 ALIAS_PHP_COMMAND_SET = """
 require_once("filter.inc");
 if (filter_configure() == 0) { clear_subsystem_dirty('aliases'); }
@@ -65,7 +68,7 @@ def get_argument_spec():
     #
     def __init__(self, module, pfsense=None):
         super(PFSenseAliasModule, self).__init__(module, pfsense, root='aliases', node='alias', key='name', update_php=ALIAS_PHP_COMMAND_SET,
-                                                 map_param_if=ALIAS_MAP_PARAM_IF, param_force=ALIAS_PARAM_FORCE)
+                                                 map_param_if=ALIAS_MAP_PARAM_IF, create_default=ALIAS_CREATE_DEFAULT)
         # Override for use with aggregate
         self.argument_spec = ALIAS_ARGUMENT_SPEC
 

From 6e73cd78bf8bb1b554d17ee5d2eef98e0e327c3f Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sat, 20 Jan 2024 20:24:12 -0700
Subject: [PATCH 52/62] [pfsense_nat_outbound] Use mutually_exclusive; Use
 arg_route; Add some tests

---
 plugins/module_utils/nat_outbound.py          | 68 ++++++++++---------
 plugins/modules/pfsense_nat_outbound.py       |  8 ++-
 .../modules/test_pfsense_nat_outbound.py      | 12 ++++
 3 files changed, 54 insertions(+), 34 deletions(-)

diff --git a/plugins/module_utils/nat_outbound.py b/plugins/module_utils/nat_outbound.py
index cd40353b..5389f12f 100644
--- a/plugins/module_utils/nat_outbound.py
+++ b/plugins/module_utils/nat_outbound.py
@@ -33,11 +33,43 @@
     before=dict(required=False, type='str'),
 )
 
+NAT_OUTBOUND_MUTUALLY_EXCLUSIVE = [
+    ('after', 'before'),
+]
+
 NAT_OUTBOUND_REQUIRED_IF = [
     ["state", "present", ["interface", "source", "destination"]]
 ]
 
 
+def p2o_after(self, name, params, obj):
+    self.after = params[name]
+
+
+def p2o_before(self, name, params, obj):
+    self.before = params[name]
+
+
+def p2o_ipprotocol(self, name, params, obj):
+    # IPv4+6 is marked by the absense of an ipprotocol element
+    if params[name] != 'inet46':
+        self.obj[name] = params[name]
+
+
+def p2o_protocol(self, name, params, obj):
+    # 'any' is marked by the absense of a protocol element
+    if params[name] != 'any':
+        self.obj[name] = params[name]
+
+
+NAT_OUTBOUND_ARG_ROUTE = dict(
+    after=dict(parse=p2o_after),
+    before=dict(parse=p2o_before),
+    ipprotocol=dict(parse=p2o_ipprotocol),
+    protocol=dict(parse=p2o_protocol),
+)
+
+
 class PFSenseNatOutboundModule(PFSenseModuleBase):
     """ module managing pfsense NAT rules """
 
@@ -50,9 +82,8 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None):
-        super(PFSenseNatOutboundModule, self).__init__(module, pfsense)
+        super(PFSenseNatOutboundModule, self).__init__(module, pfsense, arg_route=NAT_OUTBOUND_ARG_ROUTE)
         self.name = "pfsense_nat_outbound"
-        self.obj = dict()
 
         self.after = None
         self.before = None
@@ -74,32 +105,9 @@ def __init__(self, module, pfsense=None):
     def _params_to_obj(self):
         """ return a dict from module params """
 
-        obj = dict()
-        obj['descr'] = self.params['descr']
+        obj = super(PFSenseNatOutboundModule, self)._params_to_obj()
         params = self.params
         if params['state'] == 'present':
-            obj['sourceport'] = ''
-            obj['interface'] = self.pfsense.parse_interface(params['interface'])
-            self._get_ansible_param(obj, 'ipprotocol')
-            if obj['ipprotocol'] == 'inet46':
-                del obj['ipprotocol']
-            self._get_ansible_param(obj, 'protocol')
-            if obj['protocol'] == 'any':
-                del obj['protocol']
-            self._get_ansible_param(obj, 'poolopts')
-            self._get_ansible_param(obj, 'source_hash_key')
-            self._get_ansible_param(obj, 'natport')
-            self._get_ansible_param_bool(obj, 'disabled', value='')
-            self._get_ansible_param_bool(obj, 'nonat', value='')
-            self._get_ansible_param_bool(obj, 'staticnatport', value='')
-            self._get_ansible_param_bool(obj, 'nosync', value='')
-
-            if 'after' in params and params['after'] is not None:
-                self.after = params['after']
-
-            if 'before' in params and params['before'] is not None:
-                self.before = params['before']
-
             self._parse_address(obj, 'source', 'sourceport', True, 'network')
             self._parse_address(obj, 'destination', 'dstport', False, 'network')
             if params['invert']:
@@ -224,9 +232,7 @@ def _parse_translated_address(self, obj):
     def _validate_params(self):
         """ do some extra checks on input parameters """
 
-        if self.params.get('after') and self.params.get('before'):
-            self.module.fail_json(msg='Cannot specify both after and before')
-        elif self.params.get('after'):
+        if self.params.get('after'):
             if self.params['after'] == self.params['descr']:
                 self.module.fail_json(msg='Cannot specify the current rule in after')
         elif self.params.get('before'):
@@ -381,10 +387,6 @@ def _update(self):
     ##############################
     # Logging
     #
-    def _get_obj_name(self):
-        """ return obj's name """
-        return "'{0}'".format(self.obj['descr'])
-
     @staticmethod
     def fvalue_protocol(value):
         """ boolean value formatting function """
diff --git a/plugins/modules/pfsense_nat_outbound.py b/plugins/modules/pfsense_nat_outbound.py
index 9586de96..da5ec890 100644
--- a/plugins/modules/pfsense_nat_outbound.py
+++ b/plugins/modules/pfsense_nat_outbound.py
@@ -123,12 +123,18 @@
 """
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible_collections.pfsensible.core.plugins.module_utils.nat_outbound import PFSenseNatOutboundModule, NAT_OUTBOUND_ARGUMENT_SPEC, NAT_OUTBOUND_REQUIRED_IF
+from ansible_collections.pfsensible.core.plugins.module_utils.nat_outbound import (
+    PFSenseNatOutboundModule,
+    NAT_OUTBOUND_ARGUMENT_SPEC,
+    NAT_OUTBOUND_MUTUALLY_EXCLUSIVE,
+    NAT_OUTBOUND_REQUIRED_IF,
+)
 
 
 def main():
     module = AnsibleModule(
         argument_spec=NAT_OUTBOUND_ARGUMENT_SPEC,
+        mutually_exclusive=NAT_OUTBOUND_MUTUALLY_EXCLUSIVE,
         required_if=NAT_OUTBOUND_REQUIRED_IF,
         supports_check_mode=True)
 
diff --git a/tests/unit/plugins/modules/test_pfsense_nat_outbound.py b/tests/unit/plugins/modules/test_pfsense_nat_outbound.py
index 7ac92dc8..48e4d8b9 100644
--- a/tests/unit/plugins/modules/test_pfsense_nat_outbound.py
+++ b/tests/unit/plugins/modules/test_pfsense_nat_outbound.py
@@ -197,6 +197,18 @@ def test_nat_outbound_create_networks(self):
         command = "create nat_outbound 'https-source-rewriting', interface='lan', source='1.2.3.4/24', destination='2.3.4.5/24:443'"
         self.do_module_test(obj, command=command, target_idx=3)
 
+    def test_nat_outbound_ipprotocol(self):
+        """ test """
+        obj = dict(descr='https-source-rewriting', interface='lan', ipprotocol='inet', source='any', destination='1.2.3.4:443')
+        command = "create nat_outbound 'https-source-rewriting', interface='lan', ipprotocol='inet', source='any', destination='1.2.3.4:443'"
+        self.do_module_test(obj, command=command, target_idx=3)
+
+    def test_nat_outbound_protocol(self):
+        """ test """
+        obj = dict(descr='https-source-rewriting', interface='lan', protocol='tcp', source='any', destination='1.2.3.4:443')
+        command = "create nat_outbound 'https-source-rewriting', interface='lan', protocol='tcp', source='any', destination='1.2.3.4:443'"
+        self.do_module_test(obj, command=command, target_idx=3)
+
     def test_nat_outbound_create_networks_invert(self):
         """ test """
         obj = dict(descr='https-source-rewriting', interface='lan', source='1.2.3.4/24', destination='2.3.4.5/24:443', invert=True)

From c6cb4f1d299cc2e3348a6813a34880e8afb50809 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sat, 20 Jan 2024 20:43:05 -0700
Subject: [PATCH 53/62] [pfsense_ipsec] Start using expanded PFSenseModuleBase;
 Start dropping support for older pfsense

---
 plugins/module_utils/ipsec.py                 | 263 ++++++++----------
 .../modules/fixtures/pfsense_ipsec_config.xml |   2 +-
 .../plugins/modules/test_pfsense_ipsec.py     |  63 +----
 .../modules/test_pfsense_ipsec_aggregate.py   |  24 +-
 4 files changed, 132 insertions(+), 220 deletions(-)

diff --git a/plugins/module_utils/ipsec.py b/plugins/module_utils/ipsec.py
index b225dabf..00c6edf6 100644
--- a/plugins/module_utils/ipsec.py
+++ b/plugins/module_utils/ipsec.py
@@ -76,6 +76,36 @@
     ["peerid_type", "keyid tag", ["peerid_data"]],
 ]
 
+# Booleans that map to different values
+IPSEC_BOOL_VALUES = dict(
+    gw_duplicates=(None, ''),
+)
+
+IPSEC_MAP_PARAM = [
+    ('preshared_key', 'pre-shared-key'),
+    ('remote_gateway', 'remote-gateway'),
+]
+
+IPSEC_CREATE_DEFAULT = dict(
+    rand_time=None,
+    reauth_time=None,
+    rekey_time=None,
+)
+
+
+def p2o_ipsec_interface(self, name, params, obj):
+    # Valid interfaces are physical, virtual IPs, and gateway groups
+    # TODO - handle gateway groups
+    if params[name].lower().startswith('vip:'):
+        obj[name] = self.pfsense.get_virtual_ip_interface(params[name][4:])
+    else:
+        obj[name] = self.pfsense.parse_interface(params[name], with_virtual=False)
+
+
+IPSEC_ARG_ROUTE = dict(
+    interface=dict(parse=p2o_ipsec_interface,),
+)
+
 
 class PFSenseIpsecModule(PFSenseModuleBase):
     """ module managing pfsense ipsec tunnels phase 1 options """
@@ -89,9 +119,11 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None):
-        super(PFSenseIpsecModule, self).__init__(module, pfsense)
+        super(PFSenseIpsecModule, self).__init__(module, pfsense, arg_route=IPSEC_ARG_ROUTE, bool_values=IPSEC_BOOL_VALUES, map_param=IPSEC_MAP_PARAM,
+                                                 create_default=IPSEC_CREATE_DEFAULT)
+        # Override for use with aggregate
+        self.argument_spec = IPSEC_ARGUMENT_SPEC
         self.name = "pfsense_ipsec"
-        self.obj = dict()
         self.apply = True
 
         self.root_elt = self.pfsense.ipsec
@@ -162,36 +194,12 @@ def _remove_phases2(self):
     def _params_to_obj(self):
         """ return an ipsec dict from module params """
 
+        ipsec = super(PFSenseIpsecModule, self)._params_to_obj()
         params = self.params
         self.apply = params['apply']
-
-        ipsec = dict()
-        ipsec['descr'] = params['descr']
+        ipsec.pop('apply', None)
 
         if params['state'] == 'present':
-            # Valid interfaces are physical, virtual IPs, and gateway groups
-            # TODO - handle gateway groups
-            if params['interface'].lower().startswith('vip:'):
-                ipsec['interface'] = self.pfsense.get_virtual_ip_interface(params['interface'][4:])
-            else:
-                ipsec['interface'] = self.pfsense.parse_interface(params['interface'], with_virtual=False)
-            ipsec['iketype'] = params['iketype']
-
-            if params.get('mode') is not None:
-                ipsec['mode'] = params['mode']
-            ipsec['nat_traversal'] = params['nat_traversal']
-
-            ipsec['protocol'] = params['protocol']
-            ipsec['remote-gateway'] = params['remote_gateway']
-            if params.get('disabled'):
-                ipsec['disabled'] = None
-
-            ipsec['myid_type'] = params['myid_type']
-            ipsec['myid_data'] = params['myid_data']
-            ipsec['peerid_type'] = params['peerid_type']
-            ipsec['peerid_data'] = params['peerid_data']
-
-            ipsec['authentication_method'] = params['authentication_method']
             if params['authentication_method'] == 'rsasig':
                 ca_elt = self.pfsense.find_ca_elt(params['certificate_authority'])
                 if ca_elt is None:
@@ -206,9 +214,6 @@ def _params_to_obj(self):
             else:
                 ipsec['caref'] = ''
                 ipsec['certref'] = ''
-                ipsec['pre-shared-key'] = params['preshared_key']
-
-            ipsec['lifetime'] = str(params['lifetime'])
 
             if params.get('disable_rekey'):
                 ipsec['rekey_enable'] = ''
@@ -216,29 +221,10 @@ def _params_to_obj(self):
             if params.get('enable_dpd'):
                 ipsec['dpd_delay'] = str(params['dpd_delay'])
                 ipsec['dpd_maxfail'] = str(params['dpd_maxfail'])
+                del ipsec['enable_dpd']
 
             if params.get('disable_reauth'):
                 ipsec['reauth_enable'] = ''
-            if params.get('splitconn'):
-                ipsec['splitconn'] = ''
-            if params.get('mobike'):
-                ipsec['mobike'] = params['mobike']
-
-            if self.pfsense.is_at_least_2_5_0():
-                self._get_ansible_param_bool(ipsec, 'gw_duplicates', value='')
-                self._get_ansible_param(ipsec, 'nattport')
-                self._get_ansible_param(ipsec, 'rekey_time', force=True)
-                self._get_ansible_param(ipsec, 'reauth_time', force=True)
-                self._get_ansible_param(ipsec, 'rand_time', force=True)
-            else:
-                self._get_ansible_param(ipsec, 'margintime', force=True)
-
-            if self.pfsense.is_at_least_2_5_2():
-                self._get_ansible_param(ipsec, 'startaction')
-                self._get_ansible_param(ipsec, 'closeaction')
-            else:
-                if params.get('responderonly'):
-                    ipsec['responderonly'] = params['responderonly']
 
         return ipsec
 
@@ -267,14 +253,10 @@ def _validate_params(self):
         if params['state'] == 'absent':
             return
 
-        if self.pfsense.is_at_least_2_5_0():
-            if params.get('lifetime') is not None:
-                if (params.get('rekey_time') is not None and params.get('rekey_time') >= params.get('lifetime') or
-                        params.get('reauth_time') is not None and params.get('reauth_time') >= params.get('lifetime')):
-                    self.module.fail_json(msg='Life Time must be larger than Rekey Time and Reauth Time.')
-        else:
-            if params.get('disable_rekey') is None:
-                params['disable_rekey'] = False
+        if params.get('lifetime') is not None:
+            if (params.get('rekey_time') is not None and params.get('rekey_time') >= params.get('lifetime') or
+                    params.get('reauth_time') is not None and params.get('reauth_time') >= params.get('lifetime')):
+                self.module.fail_json(msg='Life Time must be larger than Rekey Time and Reauth Time.')
 
         for ipsec_elt in self.root_elt:
             if ipsec_elt.tag != 'phase1':
@@ -322,118 +304,99 @@ def _update(self):
     ##############################
     # Logging
     #
-    def _get_obj_name(self):
-        """ return obj's name """
-        return "'" + self.obj['descr'] + "'"
-
     def _log_fields(self, before=None):
         """ generate pseudo-CLI command fields parameters to create an obj """
         values = ''
         if before is None:
             values += self.format_cli_field(self.params, 'disabled', fvalue=self.fvalue_bool)
-            values += self.format_cli_field(self.obj, 'iketype')
-            if self.obj['iketype'] != 'ikev2':
-                values += self.format_cli_field(self.obj, 'mode')
+            values += self.format_cli_field(self.diff['after'], 'iketype')
+            if self.diff['after']['iketype'] != 'ikev2':
+                values += self.format_cli_field(self.diff['after'], 'mode')
 
-            values += self.format_cli_field(self.obj, 'protocol')
+            values += self.format_cli_field(self.diff['after'], 'protocol')
             values += self.format_cli_field(self.params, 'interface')
-            values += self.format_cli_field(self.obj, 'remote-gateway', fname='remote_gateway')
-            values += self.format_cli_field(self.obj, 'nattport')
-            values += self.format_cli_field(self.obj, 'authentication_method')
-            if self.obj['authentication_method'] == 'rsasig':
+            values += self.format_cli_field(self.diff['after'], 'remote-gateway', fname='remote_gateway')
+            values += self.format_cli_field(self.diff['after'], 'nattport')
+            values += self.format_cli_field(self.diff['after'], 'authentication_method')
+            if self.diff['after']['authentication_method'] == 'rsasig':
                 values += self.format_cli_field(self.params, 'certificate')
                 values += self.format_cli_field(self.params, 'certificate_authority')
             else:
-                values += self.format_cli_field(self.obj, 'pre-shared-key', fname='preshared_key')
+                values += self.format_cli_field(self.diff['after'], 'pre-shared-key', fname='preshared_key')
 
             id_types = ['address', 'fqdn', 'user_fqdn', 'asn1dn', 'keyid tag', 'dyn_dns']
-            values += self.format_cli_field(self.obj, 'myid_type')
-            if self.obj['myid_type'] in id_types:
-                values += self.format_cli_field(self.obj, 'myid_data')
-
-            values += self.format_cli_field(self.obj, 'peerid_type')
-            if self.obj['peerid_type'] in id_types:
-                values += self.format_cli_field(self.obj, 'peerid_data')
-
-            values += self.format_cli_field(self.obj, 'lifetime')
-            values += self.format_cli_field(self.obj, 'rekey_time')
-            values += self.format_cli_field(self.obj, 'reauth_time')
-            values += self.format_cli_field(self.obj, 'rand_time')
-
-            if not self.pfsense.is_at_least_2_5_0():
-                values += self.format_cli_field(self.params, 'disable_rekey', fvalue=self.fvalue_bool)
-                if not self.params['disable_rekey']:
-                    values += self.format_cli_field(self.obj, 'margintime')
-
-            if self.obj['iketype'] == 'ikev2':
-                values += self.format_cli_field(self.obj, 'reauth_enable', fname='disable_reauth', fvalue=self.fvalue_bool)
-                values += self.format_cli_field(self.obj, 'mobike')
-                values += self.format_cli_field(self.obj, 'splitconn', fvalue=self.fvalue_bool)
-
-            values += self.format_cli_field(self.obj, 'gw_duplicates', fvalue=self.fvalue_bool)
-
-            if self.pfsense.is_at_least_2_5_2():
-                values += self.format_cli_field(self.params, 'startaction')
-                values += self.format_cli_field(self.params, 'closeaction')
-            else:
-                values += self.format_cli_field(self.params, 'responderonly', fvalue=self.fvalue_bool)
-            values += self.format_cli_field(self.obj, 'nat_traversal')
+            values += self.format_cli_field(self.diff['after'], 'myid_type')
+            if self.diff['after']['myid_type'] in id_types:
+                values += self.format_cli_field(self.diff['after'], 'myid_data')
+
+            values += self.format_cli_field(self.diff['after'], 'peerid_type')
+            if self.diff['after']['peerid_type'] in id_types:
+                values += self.format_cli_field(self.diff['after'], 'peerid_data')
+
+            values += self.format_cli_field(self.diff['after'], 'lifetime')
+            values += self.format_cli_field(self.diff['after'], 'rekey_time')
+            values += self.format_cli_field(self.diff['after'], 'reauth_time')
+            values += self.format_cli_field(self.diff['after'], 'rand_time')
+
+            if self.diff['after']['iketype'] == 'ikev2':
+                values += self.format_cli_field(self.diff['after'], 'reauth_enable', fname='disable_reauth', fvalue=self.fvalue_bool)
+                values += self.format_cli_field(self.diff['after'], 'mobike')
+                values += self.format_cli_field(self.diff['after'], 'splitconn', fvalue=self.fvalue_bool)
+
+            values += self.format_cli_field(self.diff['after'], 'gw_duplicates', fvalue=self.fvalue_bool)
+
+            values += self.format_cli_field(self.params, 'startaction')
+            values += self.format_cli_field(self.params, 'closeaction')
+            values += self.format_cli_field(self.diff['after'], 'nat_traversal')
 
             values += self.format_cli_field(self.params, 'enable_dpd', fvalue=self.fvalue_bool)
             if self.params['enable_dpd']:
-                values += self.format_cli_field(self.obj, 'dpd_delay')
-                values += self.format_cli_field(self.obj, 'dpd_maxfail')
+                values += self.format_cli_field(self.diff['after'], 'dpd_delay')
+                values += self.format_cli_field(self.diff['after'], 'dpd_maxfail')
         else:
-            values += self.format_updated_cli_field(self.obj, before, 'disabled', add_comma=(values), fvalue=self.fvalue_bool)
-            values += self.format_updated_cli_field(self.obj, before, 'iketype', add_comma=(values))
-            if self.obj['iketype'] != 'ikev2':
-                values += self.format_updated_cli_field(self.obj, before, 'mode', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'protocol', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'interface', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'remote-gateway', add_comma=(values), fname='remote_gateway')
-            values += self.format_updated_cli_field(self.obj, before, 'nattport', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'authentication_method', add_comma=(values))
-            if self.obj['authentication_method'] == 'rsasig':
+            values += self.format_updated_cli_field(self.diff['after'], before, 'disabled', add_comma=(values), fvalue=self.fvalue_bool)
+            values += self.format_updated_cli_field(self.diff['after'], before, 'iketype', add_comma=(values))
+            if self.diff['after']['iketype'] != 'ikev2':
+                values += self.format_updated_cli_field(self.diff['after'], before, 'mode', add_comma=(values))
+            values += self.format_updated_cli_field(self.diff['after'], before, 'protocol', add_comma=(values))
+            values += self.format_updated_cli_field(self.diff['after'], before, 'interface', add_comma=(values))
+            values += self.format_updated_cli_field(self.diff['after'], before, 'remote-gateway', add_comma=(values), fname='remote_gateway')
+            values += self.format_updated_cli_field(self.diff['after'], before, 'nattport', add_comma=(values))
+            values += self.format_updated_cli_field(self.diff['after'], before, 'authentication_method', add_comma=(values))
+            if self.diff['after']['authentication_method'] == 'rsasig':
                 values += self.format_updated_cli_field(self.params, before, 'certificate', add_comma=(values))
                 values += self.format_updated_cli_field(self.params, before, 'certificate_authority', add_comma=(values))
             else:
-                values += self.format_updated_cli_field(self.obj, before, 'pre-shared-key', add_comma=(values), fname='preshared_key')
-            values += self.format_updated_cli_field(self.obj, before, 'myid_type', add_comma=(values))
+                values += self.format_updated_cli_field(self.diff['after'], before, 'pre-shared-key', add_comma=(values), fname='preshared_key')
+            values += self.format_updated_cli_field(self.diff['after'], before, 'myid_type', add_comma=(values))
             id_types = ['address', 'fqdn', 'user_fqdn', 'asn1dn', 'keyid tag', 'dyn_dns']
-            if self.obj['myid_type'] in id_types:
-                values += self.format_updated_cli_field(self.obj, before, 'myid_data', add_comma=(values))
-
-            values += self.format_updated_cli_field(self.obj, before, 'peerid_type', add_comma=(values))
-            if self.obj['peerid_type'] in id_types:
-                values += self.format_updated_cli_field(self.obj, before, 'peerid_data', add_comma=(values))
-
-            values += self.format_updated_cli_field(self.obj, before, 'lifetime', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'rekey_time', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'reauth_time', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'rand_time', add_comma=(values))
-
-            if not self.pfsense.is_at_least_2_5_0():
-                values += self.format_updated_cli_field(self.obj, before, 'disable_rekey', add_comma=(values), fvalue=self.fvalue_bool)
-                if not self.params['disable_rekey']:
-                    values += self.format_updated_cli_field(self.obj, before, 'margintime', add_comma=(values))
-
-            if self.obj['iketype'] == 'ikev2':
-                values += self.format_updated_cli_field(self.obj, before, 'reauth_enable', add_comma=(values), fname='disable_reauth', fvalue=self.fvalue_bool)
-                values += self.format_updated_cli_field(self.obj, before, 'mobike', add_comma=(values))
-                values += self.format_updated_cli_field(self.obj, before, 'splitconn', add_comma=(values), fvalue=self.fvalue_bool)
-
-            values += self.format_updated_cli_field(self.obj, before, 'gw_duplicates', add_comma=(values), fvalue=self.fvalue_bool)
-
-            if self.pfsense.is_at_least_2_5_2():
-                values += self.format_updated_cli_field(self.obj, before, 'startaction', add_comma=(values))
-                values += self.format_updated_cli_field(self.obj, before, 'closeaction', add_comma=(values))
-            else:
-                values += self.format_updated_cli_field(self.obj, before, 'responderonly', add_comma=(values), fvalue=self.fvalue_bool)
-            values += self.format_updated_cli_field(self.obj, before, 'nat_traversal', add_comma=(values))
-            values += self.format_updated_cli_field(self.obj, before, 'enable_dpd', add_comma=(values), fvalue=self.fvalue_bool)
+            if self.diff['after']['myid_type'] in id_types:
+                values += self.format_updated_cli_field(self.diff['after'], before, 'myid_data', add_comma=(values))
+
+            values += self.format_updated_cli_field(self.diff['after'], before, 'peerid_type', add_comma=(values))
+            if self.diff['after']['peerid_type'] in id_types:
+                values += self.format_updated_cli_field(self.diff['after'], before, 'peerid_data', add_comma=(values))
+
+            values += self.format_updated_cli_field(self.diff['after'], before, 'lifetime', add_comma=(values))
+            values += self.format_updated_cli_field(self.diff['after'], before, 'rekey_time', add_comma=(values))
+            values += self.format_updated_cli_field(self.diff['after'], before, 'reauth_time', add_comma=(values))
+            values += self.format_updated_cli_field(self.diff['after'], before, 'rand_time', add_comma=(values))
+
+            if self.diff['after']['iketype'] == 'ikev2':
+                values += self.format_updated_cli_field(self.diff['after'], before, 'reauth_enable', add_comma=(values), fname='disable_reauth',
+                                                        fvalue=self.fvalue_bool)
+                values += self.format_updated_cli_field(self.diff['after'], before, 'mobike', add_comma=(values))
+                values += self.format_updated_cli_field(self.diff['after'], before, 'splitconn', add_comma=(values), fvalue=self.fvalue_bool)
+
+            values += self.format_updated_cli_field(self.diff['after'], before, 'gw_duplicates', add_comma=(values), fvalue=self.fvalue_bool)
+
+            values += self.format_updated_cli_field(self.diff['after'], before, 'startaction', add_comma=(values))
+            values += self.format_updated_cli_field(self.diff['after'], before, 'closeaction', add_comma=(values))
+            values += self.format_updated_cli_field(self.diff['after'], before, 'nat_traversal', add_comma=(values))
+            values += self.format_updated_cli_field(self.diff['after'], before, 'enable_dpd', add_comma=(values), fvalue=self.fvalue_bool)
             if self.params['enable_dpd']:
-                values += self.format_updated_cli_field(self.obj, before, 'dpd_delay', add_comma=(values))
-                values += self.format_updated_cli_field(self.obj, before, 'dpd_maxfail', add_comma=(values))
+                values += self.format_updated_cli_field(self.diff['after'], before, 'dpd_delay', add_comma=(values))
+                values += self.format_updated_cli_field(self.diff['after'], before, 'dpd_maxfail', add_comma=(values))
         return values
 
     def _get_ref_names(self, before):
diff --git a/tests/unit/plugins/modules/fixtures/pfsense_ipsec_config.xml b/tests/unit/plugins/modules/fixtures/pfsense_ipsec_config.xml
index 5f520cea..12186704 100644
--- a/tests/unit/plugins/modules/fixtures/pfsense_ipsec_config.xml
+++ b/tests/unit/plugins/modules/fixtures/pfsense_ipsec_config.xml
@@ -1307,7 +1307,7 @@
 	                        </item>
 	                </encryption>
 	                <lifetime>28800</lifetime>
-	    	    <rekey_time></rekey_time>
+	    	    <rekey_time>25000</rekey_time>
 	    	    <reauth_time></reauth_time>
 	    	    <rand_time></rand_time>
 	                <pre-shared-key>1234</pre-shared-key>
diff --git a/tests/unit/plugins/modules/test_pfsense_ipsec.py b/tests/unit/plugins/modules/test_pfsense_ipsec.py
index d4dacd65..a6de1e32 100644
--- a/tests/unit/plugins/modules/test_pfsense_ipsec.py
+++ b/tests/unit/plugins/modules/test_pfsense_ipsec.py
@@ -25,8 +25,6 @@ def __init__(self, *args, **kwargs):
 
     def get_config_file(self):
         """ get config file """
-        if self.get_version.return_value.startswith("2.4."):
-            return '2.4/pfsense_ipsec_config.xml'
 
         return 'pfsense_ipsec_config.xml'
 
@@ -67,22 +65,13 @@ def check_target_elt(self, obj, target_elt):
         else:
             self.assert_not_find_xml_elt(target_elt, 'disabled')
 
-        if self.get_version.return_value == "2.4.4":
-            if obj.get('disable_rekey'):
-                self.assert_xml_elt_is_none_or_empty(target_elt, 'disable_rekey')
-                self.assert_not_find_xml_elt(target_elt, 'margintime')
-            else:
-                self.assert_not_find_xml_elt(target_elt, 'disable_rekey')
-                if obj.get('margintime'):
-                    self.assert_xml_elt_equal(target_elt, 'margintime', obj['margintime'])
-                else:
-                    self.assert_xml_elt_is_none_or_empty(target_elt, 'margintime')
-        else:
-            self.check_param_bool(obj, target_elt, 'gw_duplicates')
-            self.check_param_equal_or_not_find(obj, target_elt, 'nattport')
-            self.check_param_equal(obj, target_elt, 'rekey_time')
-            self.check_param_equal(obj, target_elt, 'reauth_time')
-            self.check_param_equal(obj, target_elt, 'rand_time')
+        self.check_param_bool(obj, target_elt, 'gw_duplicates')
+        self.check_param_equal_or_not_find(obj, target_elt, 'nattport')
+        for param in ['rand_time', 'reauth_time', 'rekey_time']:
+            if obj.get(param):
+                self.check_param_equal(obj, target_elt, 'rekey_time')
+                self.check_param_equal(obj, target_elt, 'reauth_time')
+                self.check_param_equal(obj, target_elt, 'rand_time')
 
         # Added in 2.5.2
         if obj.get('startaction'):
@@ -168,23 +157,11 @@ def check_target_elt(self, obj, target_elt):
             self.assert_xml_elt_equal(target_elt, 'lifetime', '28800')
 
     def strip_commands(self, commands):
-        if self.get_version.return_value.startswith("2.4."):
-            commands = commands.replace("nattport='4501', ", "")
-            commands = commands.replace("rekey_time='', ", "")
-            commands = commands.replace("reauth_time='', ", "")
-            commands = commands.replace("rand_time='', ", "")
-        else:
-            commands = commands.replace("margintime='', ", "")
-            commands = commands.replace("disable_rekey=False, ", "")
+        commands = commands.replace("margintime='', ", "")
+        commands = commands.replace("disable_rekey=False, ", "")
 
         return commands
 
-    def strip_params(self, params):
-        if self.get_version.return_value.startswith("2.4."):
-            params.pop('nattport', None)
-            params.pop('gw_duplicates', None)
-        return params
-
     ##############
     # tests
     #
@@ -208,7 +185,6 @@ def test_ipsec_create_ikev1(self):
         command = (
             "create ipsec 'new_tunnel', iketype='ikev1', mode='main', protocol='inet', interface='lan_100', remote_gateway='1.2.3.4', "
             "authentication_method='pre_shared_key', preshared_key='1234', myid_type='myaddress', peerid_type='peeraddress', lifetime='28800', "
-            "rekey_time='', reauth_time='', rand_time='', "
             "disable_rekey=False, margintime='', startaction='none', closeaction='none', nat_traversal='on', enable_dpd=True, dpd_delay='10', dpd_maxfail='5'")
         self.do_module_test(ipsec, command=command)
 
@@ -220,7 +196,7 @@ def test_ipsec_create_vip_descr(self):
         command = (
             "create ipsec 'new_tunnel', iketype='ikev1', mode='main', protocol='inet', interface='vip:WAN CARP', remote_gateway='1.2.3.4', "
             "authentication_method='pre_shared_key', preshared_key='1234', myid_type='myaddress', peerid_type='peeraddress', lifetime='28800', "
-            "rekey_time='', reauth_time='', rand_time='', disable_rekey=False, margintime='', startaction='start', closeaction='start', "
+            "disable_rekey=False, margintime='', startaction='start', closeaction='start', "
             "nat_traversal='on', enable_dpd=True, dpd_delay='10', dpd_maxfail='5'")
         self.do_module_test(ipsec, command=command)
 
@@ -232,7 +208,6 @@ def test_ipsec_create_vip_subnet(self):
         command = (
             "create ipsec 'new_tunnel', iketype='ikev1', mode='main', protocol='inet', interface='vip:151.25.19.11', remote_gateway='1.2.3.4', "
             "authentication_method='pre_shared_key', preshared_key='1234', myid_type='myaddress', peerid_type='peeraddress', lifetime='28800', "
-            "rekey_time='', reauth_time='', rand_time='', "
             "disable_rekey=False, margintime='', startaction='trap', closeaction='trap', nat_traversal='on', enable_dpd=True, dpd_delay='10', dpd_maxfail='5'")
         self.do_module_test(ipsec, command=command)
 
@@ -244,7 +219,6 @@ def test_ipsec_create_auto(self):
         command = (
             "create ipsec 'new_tunnel', iketype='auto', mode='main', protocol='inet', interface='lan_100', remote_gateway='1.2.3.4', "
             "authentication_method='pre_shared_key', preshared_key='1234', myid_type='myaddress', peerid_type='peeraddress', lifetime='28800', "
-            "rekey_time='', reauth_time='', rand_time='', "
             "disable_rekey=False, margintime='', startaction='', closeaction='', nat_traversal='on', enable_dpd=True, dpd_delay='10', dpd_maxfail='5'")
         self.do_module_test(ipsec, command=command)
 
@@ -261,23 +235,6 @@ def test_ipsec_update_noop(self):
             authentication_method='pre_shared_key', preshared_key='1234')
         self.do_module_test(ipsec, changed=False)
 
-    def test_ipsec_update_2_5_0(self):
-        """ test updating 2_5_0 fields ipsec """
-        ipsec = dict(
-            descr='test_tunnel', interface='lan_100', remote_gateway='1.2.4.8', iketype='ikev2',
-            nattport=4501, gw_duplicates=True, rekey_time=2500, reauth_time=2600, rand_time=2700,
-            authentication_method='pre_shared_key', preshared_key='1234')
-        command = "update ipsec 'test_tunnel' set nattport='4501', rekey_time='2500', reauth_time='2600', rand_time='2700', gw_duplicates=True"
-        self.do_module_test(ipsec, command=command)
-
-    def test_ipsec_update_remove_2_5_0(self):
-        """ test updating 2_5_0 fields ipsec """
-        ipsec = dict(
-            descr='test_tunnel_2_5_0', interface='lan_100', remote_gateway='1.2.4.16', iketype='ikev2',
-            authentication_method='pre_shared_key', preshared_key='1234')
-        command = "update ipsec 'test_tunnel_2_5_0' set nattport=none, rekey_time='', reauth_time='', rand_time='', gw_duplicates=False"
-        self.do_module_test(ipsec, command=command)
-
     def test_ipsec_update_ike(self):
         """ test updating ike """
         ipsec = dict(
diff --git a/tests/unit/plugins/modules/test_pfsense_ipsec_aggregate.py b/tests/unit/plugins/modules/test_pfsense_ipsec_aggregate.py
index 4a628182..687147ea 100644
--- a/tests/unit/plugins/modules/test_pfsense_ipsec_aggregate.py
+++ b/tests/unit/plugins/modules/test_pfsense_ipsec_aggregate.py
@@ -26,8 +26,6 @@ def __init__(self, *args, **kwargs):
 
     def get_config_file(self):
         """ get config file """
-        if self.get_version.return_value.startswith("2.4."):
-            return '2.4/pfsense_ipsec_aggregate_config.xml'
 
         return 'pfsense_ipsec_aggregate_config.xml'
 
@@ -72,14 +70,8 @@ def assert_not_find_ipsec(self, ipsec):
     def strip_commands(self, commands):
         """ remove old or new parameters """
         def strip_command(command):
-            if self.get_version.return_value.startswith("2.4."):
-                command = command.replace(", prf='sha256'", "")
-                command = command.replace("rekey_time='', ", "")
-                command = command.replace("reauth_time='', ", "")
-                command = command.replace("rand_time='', ", "")
-            else:
-                command = command.replace("margintime='', ", "")
-                command = command.replace("disable_rekey=False, ", "")
+            command = command.replace("margintime='', ", "")
+            command = command.replace("disable_rekey=False, ", "")
             return command
 
         if isinstance(commands, str):
@@ -117,13 +109,13 @@ def test_ipsec_aggregate_ipsecs(self, pfsense_version):
         result_ipsecs.append(
             "create ipsec 't1', iketype='ikev2', protocol='inet', interface='wan', remote_gateway='1.3.3.1', authentication_method='pre_shared_key', "
             "preshared_key='azerty123', myid_type='myaddress', peerid_type='peeraddress', lifetime='28800', "
-            "rekey_time='', reauth_time='', rand_time='', disable_rekey=False, margintime='', "
+            "disable_rekey=False, margintime='', "
             "mobike='off', startaction='', closeaction='', nat_traversal='on', enable_dpd=True, dpd_delay='10', dpd_maxfail='5'"
         )
         result_ipsecs.append(
             "create ipsec 't2', iketype='ikev2', protocol='inet', interface='wan', remote_gateway='1.3.3.2', authentication_method='pre_shared_key', "
             "preshared_key='qwerty123', myid_type='myaddress', peerid_type='peeraddress', lifetime='28800', "
-            "rekey_time='', reauth_time='', rand_time='', disable_rekey=False, margintime='', "
+            "disable_rekey=False, margintime='', "
             "mobike='off', startaction='', closeaction='', nat_traversal='on', enable_dpd=True, dpd_delay='10', dpd_maxfail='5'"
         )
         result_ipsecs.append("delete ipsec 'test_tunnel2'")
@@ -153,13 +145,13 @@ def test_ipsec_aggregate_ipsecs_purge(self, pfsense_version):
         result_ipsecs.append(
             "create ipsec 't1', iketype='ikev2', protocol='inet', interface='wan', remote_gateway='1.3.3.1', authentication_method='pre_shared_key', "
             "preshared_key='azerty123', myid_type='myaddress', peerid_type='peeraddress', lifetime='28800', "
-            "rekey_time='', reauth_time='', rand_time='', disable_rekey=False, margintime='', "
+            "disable_rekey=False, margintime='', "
             "mobike='off', startaction='', closeaction='', nat_traversal='on', enable_dpd=True, dpd_delay='10', dpd_maxfail='5'"
         )
         result_ipsecs.append(
             "create ipsec 't2', iketype='ikev2', protocol='inet', interface='wan', remote_gateway='1.3.3.2', authentication_method='pre_shared_key', "
             "preshared_key='qwerty123', myid_type='myaddress', peerid_type='peeraddress', lifetime='28800', "
-            "rekey_time='', reauth_time='', rand_time='', disable_rekey=False, margintime='', "
+            "disable_rekey=False, margintime='', "
             "mobike='off', startaction='', closeaction='', nat_traversal='on', enable_dpd=True, dpd_delay='10', dpd_maxfail='5'"
         )
         result_ipsecs.append("delete ipsec 'test_tunnel'")
@@ -172,7 +164,7 @@ def test_ipsec_aggregate_ipsecs_purge(self, pfsense_version):
         self.assert_not_find_ipsec('test_tunnel')
         self.assert_not_find_ipsec('test_tunnel2')
 
-    @parameterized.expand([["2.4.4"], ["2.5.0"], ["2.5.2"]])
+    @parameterized.expand([["2.5.2"]])
     def test_ipsec_aggregate_proposals(self, pfsense_version):
         """ test creation of a some proposals """
         self.get_version.return_value = pfsense_version
@@ -197,7 +189,7 @@ def test_ipsec_aggregate_proposals(self, pfsense_version):
 
         self.assertEqual(result['result_ipsec_proposals'], result_ipsec_proposals)
 
-    @parameterized.expand([["2.4.4"], ["2.5.0"], ["2.5.2"]])
+    @parameterized.expand([["2.5.2"]])
     def test_ipsec_aggregate_proposals_purge(self, pfsense_version):
         """ test creation of a some proposals with purge """
         self.get_version.return_value = pfsense_version

From 95cd1674787daddeaa403edbddc2600faa206cd8 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sat, 20 Jan 2024 22:04:29 -0700
Subject: [PATCH 54/62] [module_base] Bugfixes and enhancements

- Handle root paths
- Make diff['after'] reflect _remove_deleted_params
- Allow map_param to move items into nested dictionaries
- Fix handling lack of 'state'
- Fix bool_style
- Allow passing obj to params_to_obj
- Remove bool params from obj, unset bools from parameters
- Add fail message for not finding package
---
 plugins/module_utils/module_base.py | 82 +++++++++++++++++++++++------
 1 file changed, 66 insertions(+), 16 deletions(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index bf0072f2..1a182835 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -15,6 +15,36 @@
 )
 
 
+# Merge two nested dictionaries, combining instead of overwriting elements
+def merge_dicts(a: dict, b: dict, path=None):
+    if path is None:
+        path = []
+    for key in b:
+        if key in a:
+            if isinstance(a[key], dict) and isinstance(b[key], dict):
+                merge_dicts(a[key], b[key], path + [str(key)])
+            else:
+                a[key] = b[key]
+        else:
+            a[key] = b[key]
+    return a
+
+
+# Move a key in dict to a new one, allowing the use '/' to specify nested dict location
+def move_dict_key(obj, src, dst):
+    item = None
+    for n in reversed(dst.split('/')):
+        if item is None:
+            item = dict()
+            item[n] = obj[src]
+        else:
+            parent = dict()
+            parent[n] = item
+            item = parent
+    merge_dicts(obj, item)
+    del obj[src]
+
+
 class PFSenseModuleBase(object):
     """ class providing base services for pfSense modules """
 
@@ -31,7 +61,8 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None, package=None, name=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr',
-                 update_php=None, arg_route=None, map_param=None, map_param_if=None, param_force=None, bool_values=None, have_refid=False, create_default=None):
+                 update_php=None, arg_route=None, map_param=None, map_param_if=None, param_force=None, bool_style=None, bool_values=None, have_refid=False,
+                 create_default=None):
         self.module = module         # ansible module
         self.argument_spec = module.argument_spec  # Allow for being overriden for use with aggregate
 
@@ -56,10 +87,16 @@ def __init__(self, module, pfsense=None, package=None, name=None, root=None, roo
                 self.root_elt = self.pfsense.root
                 self.root_is_exclusive = False
             else:
-                if package is None:
-                    self.root_elt = self.pfsense.get_element(root, create_node=create_root)
-                else:
+                if package is not None:
                     self.root_elt = self.pfsense.get_element(root, root_elt=self.pfsense.root.find('installedpackages'))
+                    if self.root_elt is None:
+                        self.module.fail_json(msg='Unable to find configuration for the package {package}.  Are you sure that it is installed?'.format(package=package))
+                else:
+                    root_elt = self.pfsense.root
+                    for this in root.split('/'):
+                        root_elt = self.pfsense.get_element(this, root_elt=root_elt, create_node=create_root)
+                    self.root_elt = root_elt
+
                 if root in ['system']:
                     self.root_is_exclusive = False
                 else:
@@ -100,8 +137,9 @@ def __init__(self, module, pfsense=None, package=None, name=None, root=None, roo
             self.param_force = param_force  # parameters that are forced to be present
         else:
             self.param_force = list()
+        self.bool_style = bool_style         # default boolean value style for arguments
         if bool_values is not None:
-            self.bool_values = bool_values    # boolean values for arguments
+            self.bool_values = bool_values    # boolean values for specific arguments
         else:
             self.bool_values = dict()
         self.create_default = create_default  # default values for a created target
@@ -148,12 +186,15 @@ def _get_ansible_param_bool(self, obj, name, fname=None, force=False, value='yes
                 obj[fname] = value_false
             elif force:
                 obj[fname] = None
+            elif value_false is None and fname in obj:
+                del obj[fname]
         elif force:
             obj[fname] = value_false
 
-    def _params_to_obj(self):
+    def _params_to_obj(self, obj=None):
         """ return a dict from module params that sets self.obj """
-        obj = dict()
+        if obj is None:
+            obj = dict()
         # Not all modules have 'state', treat them like they did
         if self.params.get('state', 'present') == 'present':
             # Skip 'state', but otherwise process all parameters.  Ansible sets unspecified parameters to None.
@@ -168,6 +209,8 @@ def _params_to_obj(self):
                 elif self.argument_spec[param].get('type') == 'bool':
                     if param in self.bool_values:
                         self._get_ansible_param_bool(obj, param, value=self.bool_values[param][1], value_false=self.bool_values[param][0], force=force)
+                    elif self.bool_style == 'absent/present':
+                        self._get_ansible_param_bool(obj, param, value='', force=force)
                     else:
                         self._get_ansible_param_bool(obj, param, force=force)
                 else:
@@ -176,15 +219,16 @@ def _params_to_obj(self):
             # Handle renaming of parameters
             for p, o in self.map_param:
                 if self.params.get(p) is not None:
-                    obj[o] = obj[p]
-                    del obj[p]
+                    move_dict_key(obj, p, o)
 
             # Handle conditional renaming of parameters
             for map_param, map_value, map_tuple in self.map_param_if:
                 if self.params.get(map_param) == map_value and map_tuple[0] in obj:
+                    # Do not overwrite destination if it exists
                     if map_tuple[1] not in obj:
-                        obj[map_tuple[1]] = obj[map_tuple[0]]
-                    del obj[map_tuple[0]]
+                        move_dict_key(obj, map_tuple[0], map_tuple[1])
+                    else:
+                        del obj[map_tuple[0]]
         else:
             # Just use the key to remove items
             obj[self.key] = self.params[self.key]
@@ -254,9 +298,9 @@ def _copy_and_update_target(self):
         before = self.pfsense.element_to_dict(self.target_elt)
         self.diff['before'] = before
         changed = self.pfsense.copy_dict_to_element(self.obj, self.target_elt)
-        self.diff['after'] = self.pfsense.element_to_dict(self.target_elt)
         if self._remove_deleted_params():
             changed = True
+        self.diff['after'] = self.pfsense.element_to_dict(self.target_elt)
 
         return (before, changed)
 
@@ -296,10 +340,16 @@ def _find_target(self):
         else:
             raise NotImplementedError()
 
-    @staticmethod
-    def _get_params_to_remove():
+    def _get_params_to_remove(self):
         """ returns the list of params to remove if they are not set """
-        return []
+        to_remove = []
+        # We need to remove any unset booleans that are "None" when unset
+        for param in [n for n in self.argument_spec.keys() if self.argument_spec[n].get('type') == 'bool']:
+            if param in self.bool_values and self.bool_values[param][0] is None:
+                to_remove.append(param)
+            elif self.bool_style == 'absent/present':
+                to_remove.append(param)
+        return to_remove
 
     def _remove_deleted_params(self):
         """ Remove from target_elt a few deleted params """
@@ -396,7 +446,7 @@ def run(self, params):
         if self.target_elt is None:
             self.target_elt = self._find_target()
 
-        if params['state'] == 'absent':
+        if params.get('state', None) == 'absent':
             self._remove()
         else:
             self._add()

From efdb97aed02c9fb6107faed40ffd9fea318182e9 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sat, 20 Jan 2024 22:07:05 -0700
Subject: [PATCH 55/62] [pfsense_nat_outbound] Refactor

- Use bool_values, set root
- targetip -> target for pfSense 2.7.1 pf+ 23.09
- Add diff support
---
 plugins/module_utils/nat_outbound.py          | 52 ++++++++++---------
 .../modules/fixtures/pfsense_nat_outbound.xml |  9 ++--
 .../modules/test_pfsense_nat_outbound.py      | 13 ++---
 3 files changed, 36 insertions(+), 38 deletions(-)

diff --git a/plugins/module_utils/nat_outbound.py b/plugins/module_utils/nat_outbound.py
index 5389f12f..fd3252c7 100644
--- a/plugins/module_utils/nat_outbound.py
+++ b/plugins/module_utils/nat_outbound.py
@@ -9,6 +9,7 @@
 from ansible_collections.pfsensible.core.plugins.module_utils.module_base import PFSenseModuleBase
 from string import hexdigits
 from hashlib import md5
+import re
 import sys
 
 NAT_OUTBOUND_ARGUMENT_SPEC = dict(
@@ -41,6 +42,14 @@
     ["state", "present", ["interface", "source", "destination"]]
 ]
 
+# Booleans that map to different values
+NAT_OUTBOUND_BOOL_VALUES = dict(
+    disabled=(None, ''),
+    staticnatport=(None, ''),
+    nonat=(None, ''),
+    nosync=(None, ''),
+)
+
 
 def p2o_after(self, name, params, obj):
     self.after = params[name]
@@ -82,23 +91,14 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None):
-        super(PFSenseNatOutboundModule, self).__init__(module, pfsense, arg_route=NAT_OUTBOUND_ARG_ROUTE)
+        super(PFSenseNatOutboundModule, self).__init__(module, pfsense, root='nat/outbound', create_root=True, arg_route=NAT_OUTBOUND_ARG_ROUTE,
+                                                       bool_values=NAT_OUTBOUND_BOOL_VALUES)
         self.name = "pfsense_nat_outbound"
 
         self.after = None
         self.before = None
         self.position_changed = False
 
-        nat_elt = self.pfsense.get_element('nat')
-        if nat_elt is None:
-            nat_elt = self.pfsense.new_element('nat')
-            self.pfsense.root.append(nat_elt)
-
-        self.root_elt = nat_elt.find('outbound')
-        if self.root_elt is None:
-            self.root_elt = self.pfsense.new_element('outbound')
-            nat_elt.append(self.root_elt)
-
     ##############################
     # params processing
     #
@@ -194,8 +194,7 @@ def _parse_ports(self, obj, ports, field_port, param):
     def _parse_translated_address(self, obj):
         """ validate param address field and returns it as a dict """
         obj['target'] = ''
-        obj['targetip'] = ''
-        obj['targetip_subnet'] = ''
+        obj['target_subnet'] = ''
 
         if self.params.get('address') is None or self.params['address'] == '':
             return
@@ -211,21 +210,22 @@ def _parse_translated_address(self, obj):
         if address is not None and address != '':
             if self.pfsense.is_virtual_ip(address):
                 obj['target'] = address
+                obj['target_subnet'] = None
             elif self.pfsense.find_alias(address, 'host') is not None or self.pfsense.find_alias(address, 'network') is not None:
                 obj['target'] = address
                 if obj['poolopts'] != '' and not obj['poolopts'].startswith('round-robin'):
                     self.module.fail_json(msg='Only Round Robin pool options may be chosen when selecting an alias.')
+                obj['target_subnet'] = '32'
             elif self.pfsense.is_ipv4_address(address):
-                obj['target'] = 'other-subnet'
-                obj['targetip'] = address
-                obj['targetip_subnet'] = '32'
+                obj['target'] = address
+                obj['target_subnet'] = '32'
             else:
                 (addr, part) = self.pfsense.parse_ip_network(address, False, False)
                 if addr is None:
                     self.module.fail_json(msg='Cannot parse address %s, not IP or alias' % (address))
-                obj['target'] = 'other-subnet'
-                obj['targetip'] = addr
-                obj['targetip_subnet'] = str(part)
+                obj['target'] = addr
+                obj['target_subnet'] = str(part)
+            del obj['address']
 
         self._parse_ports(obj, ports, 'natport', param)
 
@@ -250,12 +250,15 @@ def _validate_params(self):
     def _copy_and_add_target(self):
         """ create the XML target_elt """
         self.pfsense.copy_dict_to_element(self.obj, self.target_elt)
+        self.diff['after'] = self.obj
         self._insert(self.target_elt)
 
     def _copy_and_update_target(self):
         """ update the XML target_elt """
         before = self.pfsense.element_to_dict(self.target_elt)
+        self.diff['before'] = before
         changed = self.pfsense.copy_dict_to_element(self.obj, self.target_elt)
+        self.diff['after'] = self.pfsense.element_to_dict(self.target_elt)
         if self._remove_deleted_params():
             changed = True
 
@@ -471,11 +474,12 @@ def _obj_to_log_fields(self, rule):
         res['destination'] = self._obj_address_to_log_field(rule, 'destination', 'network', 'dstport')
         res['interface'] = self.pfsense.get_interface_display_name(rule['interface'])
 
-        if rule['target'] == 'other-subnet':
-            res['address'] = rule['targetip'] + '/' + rule['targetip_subnet']
-        else:
-            res['address'] = rule['target']
-        if 'natport' in rule and rule['natport'] != '':
+        if rule.get('target', '') != '':
+            if re.match(r'[a-zA-Z]', rule['target']):
+                res['address'] = rule['target']
+            else:
+                res['address'] = rule['target'] + '/' + rule['target_subnet']
+        if rule.get('natport', '') != '':
             res['address'] += ':'
             res['address'] += rule['natport'].replace(':', '-')
         return res
diff --git a/tests/unit/plugins/modules/fixtures/pfsense_nat_outbound.xml b/tests/unit/plugins/modules/fixtures/pfsense_nat_outbound.xml
index 4df7b021..c04a687d 100644
--- a/tests/unit/plugins/modules/fixtures/pfsense_nat_outbound.xml
+++ b/tests/unit/plugins/modules/fixtures/pfsense_nat_outbound.xml
@@ -193,8 +193,7 @@
 				<sourceport></sourceport>
 				<descr>one rule</descr>
 				<target></target>
-				<targetip></targetip>
-				<targetip_subnet></targetip_subnet>
+				<target_subnet></target_subnet>
 				<interface>wan</interface>
 				<poolopts></poolopts>
 				<source_hash_key></source_hash_key>
@@ -217,8 +216,7 @@
 				<sourceport></sourceport>
 				<descr>another rule</descr>
 				<target></target>
-				<targetip></targetip>
-				<targetip_subnet></targetip_subnet>
+				<target_subnet></target_subnet>
 				<interface>wan</interface>
 				<poolopts></poolopts>
 				<source_hash_key></source_hash_key>
@@ -241,8 +239,7 @@
 				<sourceport></sourceport>
 				<descr>third rule</descr>
 				<target></target>
-				<targetip></targetip>
-				<targetip_subnet></targetip_subnet>
+				<target_subnet></target_subnet>
 				<interface>wan</interface>
 				<poolopts></poolopts>
 				<source_hash_key></source_hash_key>
diff --git a/tests/unit/plugins/modules/test_pfsense_nat_outbound.py b/tests/unit/plugins/modules/test_pfsense_nat_outbound.py
index 48e4d8b9..42056e3f 100644
--- a/tests/unit/plugins/modules/test_pfsense_nat_outbound.py
+++ b/tests/unit/plugins/modules/test_pfsense_nat_outbound.py
@@ -91,18 +91,15 @@ def check_target_addr(self, params, target_elt):
         """ test the addresses definition """
         if 'address' not in params or params['address'] == '':
             self.assert_xml_elt_is_none_or_empty(target_elt, 'target')
-            self.assert_xml_elt_is_none_or_empty(target_elt, 'targetip')
-            self.assert_xml_elt_is_none_or_empty(target_elt, 'targetip_subnet')
+            self.assert_xml_elt_is_none_or_empty(target_elt, 'target_subnet')
             self.assert_not_find_xml_elt(target_elt, 'natport')
         elif params['address'] == '4.5.6.7:888-999':
-            self.assert_xml_elt_equal(target_elt, 'target', 'other-subnet')
-            self.assert_xml_elt_equal(target_elt, 'targetip', '4.5.6.7')
-            self.assert_xml_elt_equal(target_elt, 'targetip_subnet', '32')
+            self.assert_xml_elt_equal(target_elt, 'target', '4.5.6.7')
+            self.assert_xml_elt_equal(target_elt, 'target_subnet', '32')
             self.assert_xml_elt_equal(target_elt, 'natport', '888:999')
         elif params['address'] == '4.5.6.7/24:888-999':
-            self.assert_xml_elt_equal(target_elt, 'target', 'other-subnet')
-            self.assert_xml_elt_equal(target_elt, 'targetip', '4.5.6.0')
-            self.assert_xml_elt_equal(target_elt, 'targetip_subnet', '24')
+            self.assert_xml_elt_equal(target_elt, 'target', '4.5.6.0')
+            self.assert_xml_elt_equal(target_elt, 'target_subnet', '24')
             self.assert_xml_elt_equal(target_elt, 'natport', '888:999')
 
     @staticmethod

From 91032dc9656a4ac9a86c580bd7f9334f3596491c Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Sun, 21 Jan 2024 15:40:17 -0700
Subject: [PATCH 56/62] [args_route] Add NOTE, p2o_strip

---
 plugins/module_utils/arg_route.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/plugins/module_utils/arg_route.py b/plugins/module_utils/arg_route.py
index b60dd631..a60bbb77 100644
--- a/plugins/module_utils/arg_route.py
+++ b/plugins/module_utils/arg_route.py
@@ -7,9 +7,17 @@
 __metaclass__ = type
 
 
+# NOTE - Ansible sets unsepecified parameters to None, so these functions
+# must be able to handle the case where params[name] is None
+
 def p2o_interface(self, name, params, obj):
     obj[name] = self.pfsense.parse_interface(params[name], with_virtual=True)
 
 
 def p2o_interface_without_virtual(self, name, params, obj):
     obj[name] = self.pfsense.parse_interface(params[name], with_virtual=False)
+
+
+def p2o_strip(self, name, params, obj):
+    if params[name] is not None:
+        obj[name] = params[name].strip()

From 011c8d525012c370ebf7b97ebc0ab44e8ef12230 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 22 Jan 2024 21:41:57 -0700
Subject: [PATCH 57/62] [module_config_base] Add initial base module for
 configuration modules

---
 plugins/module_utils/module_config_base.py | 48 ++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 plugins/module_utils/module_config_base.py

diff --git a/plugins/module_utils/module_config_base.py b/plugins/module_utils/module_config_base.py
new file mode 100644
index 00000000..354ee6cb
--- /dev/null
+++ b/plugins/module_utils/module_config_base.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+# Copyright: (c) 2019, Frederic Bor <frederic.bor@wanadoo.fr>
+# Copyright: (c) 2024, Orion Poplawski <orion@nwra.com>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import re
+from ansible_collections.pfsensible.core.plugins.module_utils.module_base import PFSenseModuleBase, merge_dicts
+
+
+class PFSenseModuleConfigBase(PFSenseModuleBase):
+    """ class for implementing pfSense modules that manage a set of configuration settings """
+
+    ##############################
+    # init
+    #
+    def __init__(self, module, pfsense=None, package=None, name=None, root=None, root_is_exclusive=True, create_root=False, node=None, key='descr',
+                 update_php=None, arg_route=None, map_param=None, map_param_if=None, param_force=None, bool_style=None, bool_values=None, have_refid=False,
+                 create_default=None):
+        super(PFSenseModuleConfigBase, self).__init__(module, pfsense=pfsense, package=package, name=name, root=root, root_is_exclusive=True, create_root=False,
+                                                      update_php=update_php, arg_route=arg_route, map_param=map_param, map_param_if=map_param_if,
+                                                      param_force=param_force, bool_style=bool_style, bool_values=bool_values, create_default=create_default)
+
+    ##############################
+    # params processing
+    #
+    def _params_to_obj(self):
+        """ return a dict from module params """
+        obj = self.pfsense.element_to_dict(self.root_elt)
+        merge_dicts(obj, super(PFSenseModuleConfigBase, self)._params_to_obj(obj=obj))
+        return obj
+
+    ##############################
+    # XML processing
+    #
+    def _find_target(self):
+        """ find the XML target_elt """
+        return self.root_elt
+
+    ##############################
+    # Logging
+    #
+    def _get_obj_name(self):
+        """ return obj's name """
+        return re.sub(r'pfsense_', '', self.name)

From 20c43a88cd78dae8c23ce5cbd4cc30c2718db998 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 22 Jan 2024 21:42:30 -0700
Subject: [PATCH 58/62] [pfsense_setup] Use PFSenseConfigBaseModule; Drop pre
 2.5.0 support

---
 plugins/modules/pfsense_setup.py              | 260 +++++++-----------
 .../fixtures/pfsense_setup_config.xml         |   2 +-
 .../plugins/modules/test_pfsense_setup.py     |   8 +-
 3 files changed, 101 insertions(+), 169 deletions(-)

diff --git a/plugins/modules/pfsense_setup.py b/plugins/modules/pfsense_setup.py
index 772b1496..15603541 100644
--- a/plugins/modules/pfsense_setup.py
+++ b/plugins/modules/pfsense_setup.py
@@ -95,7 +95,7 @@
     required: false
     type: str
   shellauth:
-    description: Use Authentication Server for Shell Authentication (pfsense-CE >=2.5.0, pfsense-PLUS >=21.2). Default is false.
+    description: Use Authentication Server for Shell Authentication. Default is false.
     type: bool
   dashboardcolumns:
     description: Dashboard columns
@@ -172,11 +172,10 @@
 """
 
 import re
-from copy import deepcopy
 from os import listdir
 from os.path import isfile, join
 from ansible.module_utils.basic import AnsibleModule
-from ansible_collections.pfsensible.core.plugins.module_utils.module_base import PFSenseModuleBase
+from ansible_collections.pfsensible.core.plugins.module_utils.module_config_base import PFSenseModuleConfigBase
 
 
 SETUP_ARGUMENT_SPEC = dict(
@@ -222,7 +221,67 @@
 )
 
 
-class PFSenseSetupModule(PFSenseModuleBase):
+def p2o_dnslocalhost(self, name, params, obj):
+    if params[name] is not None:
+        if str(params.get(name)).lower() in ['', 'false']:
+            obj[name] = ''
+        elif str(params.get(name)).lower() in ['remote', 'true']:
+            obj[name] = 'remote'
+        elif params.get(name).lower() == 'local':
+            obj[name] = 'local'
+
+
+def p2o_webguicss(self, name, params, obj):
+    if params[name] is not None:
+        # Add .css suffix if not present
+        if params[name][-4:] != '.css':
+            obj[name] = params[name] + '.css'
+        else:
+            obj[name] = params[name]
+
+
+def validate_webguicss(self, webguicss):
+    """ check css style """
+    path = '/usr/local/www/css/'
+    themes = [f for f in listdir(path) if isfile(join(path, f)) and f.endswith('.css') and f.find('login') == -1 and f.find('logo') == -1]
+    themes = map(lambda x: x.replace('.css', ''), themes)
+    if webguicss.rstrip('.css') not in themes:
+        raise ValueError("The submitted theme '%s' could not be found. Pick a different theme." % webguicss)
+
+
+SETUP_ARG_ROUTE = dict(
+    dnslocalhost=dict(parse=p2o_dnslocalhost),
+    webguicss=dict(parse=p2o_webguicss, validate=validate_webguicss),
+)
+
+# Booleans that map to different values
+SETUP_BOOL_VALUES = dict(
+    webguifixedmenu=(None, 'fixed'),
+)
+
+SETUP_MAP_PARAM = [
+    ('authmode', 'webgui/authmode'),
+    ('dashboardavailablewidgetspanel', 'webgui/dashboardavailablewidgetspanel'),
+    ('dashboardcolumns', 'webgui/dashboardcolumns'),
+    ('disablealiaspopupdetail', 'webgui/disablealiaspopupdetail'),
+    ('interfacessort', 'webgui/interfacessort'),
+    ('logincss', 'webgui/logincss'),
+    ('loginshowhost', 'webgui/loginshowhost'),
+    ('requirestatefilter', 'webgui/requirestatefilter'),
+    ('roworderdragging', 'webgui/roworderdragging'),
+    ('session_timeout', 'webgui/session_timeout'),
+    ('shellauth', 'webgui/shellauth'),
+    ('statusmonitoringsettingspanel', 'webgui/statusmonitoringsettingspanel'),
+    ('systemlogsfilterpanel', 'webgui/systemlogsfilterpanel'),
+    ('systemlogsmanagelogpanel', 'webgui/systemlogsmanagelogpanel'),
+    ('webguicss', 'webgui/webguicss'),
+    ('webguifixedmenu', 'webgui/webguifixedmenu'),
+    ('webguihostnamemenu', 'webgui/webguihostnamemenu'),
+    ('webguileftcolumnhyper', 'webgui/webguileftcolumnhyper'),
+]
+
+
+class PFSenseSetupModule(PFSenseModuleConfigBase):
     """ module managing pfsense routes """
 
     @staticmethod
@@ -234,12 +293,8 @@ def get_argument_spec():
     # init
     #
     def __init__(self, module, pfsense=None):
-        super(PFSenseSetupModule, self).__init__(module, pfsense)
-        self.name = "pfsense_setup"
-        self.root_elt = self.pfsense.get_element('system')
-        self.obj = dict()
-        self.before = None
-        self.before_elt = None
+        super(PFSenseSetupModule, self).__init__(module, pfsense, name='pfsense_setup', root='system', arg_route=SETUP_ARG_ROUTE, bool_style='absent/present',
+                                                 bool_values=SETUP_BOOL_VALUES, map_param=SETUP_MAP_PARAM)
         self.route_cmds = list()
         self.params_to_delete = list()
 
@@ -254,10 +309,13 @@ def _dns_params_to_obj(self, params, obj):
         idx = 0
         if params.get('dns_addresses') is not None:
             dns_addresses = params['dns_addresses'].split()
+            del obj['dns_addresses']
         if params.get('dns_hostnames') is not None:
             dns_hostnames = params['dns_hostnames'].split()
+            del obj['dns_hostnames']
         if params.get('dns_gateways') is not None:
             dns_gateways = params['dns_gateways'].split()
+            del obj['dns_gateways']
 
         if dns_addresses is not None:
             # set the servers
@@ -265,14 +323,12 @@ def _dns_params_to_obj(self, params, obj):
 
             # set the names & gateways
             for address in dns_addresses:
-                host = ''
                 gateway = 'none'
                 if idx < len(dns_hostnames) and dns_hostnames[idx] != 'none':
-                    host = dns_hostnames[idx]
+                    obj['dns{0}host'.format(idx + 1)] = dns_hostnames[idx]
                 if idx < len(dns_gateways) and dns_gateways[idx] != 'none':
                     gateway = dns_gateways[idx]
 
-                obj['dns{0}host'.format(idx + 1)] = host
                 gw_key = 'dns{0}gw'.format(idx + 1)
                 if gw_key not in obj or gateway != obj[gw_key]:
                     obj[gw_key] = gateway
@@ -304,87 +360,8 @@ def _dns_params_to_obj(self, params, obj):
 
     def _params_to_obj(self):
         """ return a dict from module params """
-        params = self.params
-
-        obj = self.pfsense.element_to_dict(self.root_elt)
-        self.before = deepcopy(obj)
-        self.before_elt = deepcopy(self.root_elt)
-        self.obj = obj
-        webgui = obj['webgui']
-
-        def _set_param(target, param, strip=False):
-            if params.get(param) is not None:
-                if param == 'dnslocalhost':
-                    if str(params.get(param)).lower() in ['', 'false']:
-                        if self.pfsense.is_at_least_2_5_0():
-                            target[param] = ''
-                        else:
-                            del target[param]
-                    elif str(params.get(param)).lower() in ['remote', 'true']:
-                        if self.pfsense.is_at_least_2_5_0():
-                            target[param] = 'remote'
-                        else:
-                            target[param] = ''
-                    elif params.get(param).lower() == 'local':
-                        target[param] = 'local'
-
-                else:
-                    if strip:
-                        target[param] = ' '.join(params[param].split())
-                    elif isinstance(params[param], str):
-                        target[param] = params[param]
-                    else:
-                        target[param] = str(params[param])
-
-        def _set_param_bool(target, param):
-            if params.get(param) is not None:
-                if param == 'webguifixedmenu':
-                    if params[param] and (param not in target or target[param] != 'fixed'):
-                        target[param] = 'fixed'
-                    elif not params[param] and param in target:
-                        del target[param]
-                else:
-                    if params[param] and param not in target:
-                        target[param] = ''
-                    elif not params[param] and param in target:
-                        del target[param]
-
-        _set_param(obj, 'hostname')
-        _set_param(obj, 'domain')
-        _set_param(obj, 'timezone')
-        _set_param(obj, 'timeservers', True)
-
-        _set_param(obj, 'language')
-
-        _set_param_bool(webgui, 'interfacessort')
-        _set_param_bool(webgui, 'webguileftcolumnhyper')
-        _set_param_bool(webgui, 'disablealiaspopupdetail')
-        _set_param_bool(webgui, 'dashboardavailablewidgetspanel')
-        _set_param_bool(webgui, 'systemlogsfilterpanel')
-        _set_param_bool(webgui, 'systemlogsmanagelogpanel')
-        _set_param_bool(webgui, 'statusmonitoringsettingspanel')
-        _set_param(webgui, 'session_timeout')
-        _set_param(webgui, 'authmode')
-        if self.pfsense.is_at_least_2_5_0():
-            _set_param_bool(webgui, 'shellauth')
-
-        if params.get('webguicss') is not None:
-            webgui['webguicss'] = params['webguicss'] + '.css'
-
-        _set_param_bool(webgui, 'roworderdragging')
-
-        _set_param(webgui, 'logincss')
-        _set_param_bool(webgui, 'loginshowhost')
-        _set_param_bool(webgui, 'webguifixedmenu')
-        _set_param(webgui, 'webguihostnamemenu')
-        _set_param(webgui, 'dashboardcolumns')
-        _set_param_bool(webgui, 'requirestatefilter')
-
-        _set_param_bool(obj, 'dnsallowoverride')
-        _set_param(obj, 'dnslocalhost')
-
-        self._dns_params_to_obj(params, obj)
-
+        obj = super(PFSenseSetupModule, self)._params_to_obj()
+        self._dns_params_to_obj(self.params, obj)
         return obj
 
     def _validate_hostname(self, hostname, name, strict=False):
@@ -401,6 +378,7 @@ def _validate_hostname(self, hostname, name, strict=False):
 
     def _validate_params(self):
         """ do some extra checks on input parameters """
+        super(PFSenseSetupModule, self)._validate_params()
         params = self.params
 
         if params.get('dashboardcolumns') is not None and (params['dashboardcolumns'] < 1 or params['dashboardcolumns'] > 6):
@@ -415,12 +393,6 @@ def _validate_params(self):
         if params.get('hostname') is not None:
             self._validate_hostname(params['hostname'], 'hostname', True)
 
-        if params.get('dnslocalhost') is not None:
-            if not self.pfsense.is_at_least_2_5_0():
-                value = params.get('dnslocalhost')
-                if not (isinstance(value, bool) or str(value).lower() == '' or str(value).lower() == 'remote'):
-                    self.module.fail_json(msg="unsupported value '{0}' for parameter '{1}'".format(params.get('dnslocalhost'), 'dnslocalhost'))
-
         if params.get('logincss') is not None:
             error = False
             try:
@@ -433,9 +405,6 @@ def _validate_params(self):
         if params.get('timezone') is not None:
             self._validate_timezone(params['timezone'])
 
-        if params.get('webguicss') is not None:
-            self._validate_webguicss(params['webguicss'])
-
         if params.get('timeservers') is not None:
             for timeserver in params['timeservers'].split(' '):
                 self._validate_hostname(timeserver, 'timeserver')
@@ -447,14 +416,13 @@ def _validate_params(self):
                 if authserver_elt is None:
                     self.module.fail_json(msg="Given authserver '{0}' could not be found.".format(value))
 
-                if self.pfsense.is_at_least_2_5_0():
-                    if params.get('shellauth') is not None and params.get('shellauth') is True:
-                        if authserver_elt.find('type').text == 'ldap':
-                            # check if ldap_pam_groupdn is set
-                            if authserver_elt.find('ldap_pam_groupdn') is None or \
-                               authserver_elt.find('ldap_pam_groupdn').text is None or \
-                               authserver_elt.find('ldap_pam_groupdn').text == '':
-                                self.module.fail_json(msg="ldap_pam_groupdn not set for authserver '{0}'.".format(value))
+                if params.get('shellauth') is not None and params.get('shellauth') is True:
+                    if authserver_elt.find('type').text == 'ldap':
+                        # check if ldap_pam_groupdn is set
+                        if authserver_elt.find('ldap_pam_groupdn') is None or \
+                           authserver_elt.find('ldap_pam_groupdn').text is None or \
+                           authserver_elt.find('ldap_pam_groupdn').text == '':
+                            self.module.fail_json(msg="ldap_pam_groupdn not set for authserver '{0}'.".format(value))
 
         # DNS
         ip_types = []
@@ -502,53 +470,18 @@ def _validate_timezone(self, timezone):
         if not isfile(path + timezone) or timezone[:1] < 'A' or timezone[:1] > 'Z':
             self.module.fail_json(msg='The submitted timezone is invalid')
 
-    def _validate_webguicss(self, webguicss):
-        """ check css style """
-        path = '/usr/local/www/css/'
-        themes = [f for f in listdir(path) if isfile(join(path, f)) and f.endswith('.css') and f.find('login') == -1 and f.find('logo') == -1]
-        themes = map(lambda x: x.replace('.css', ''), themes)
-        if webguicss not in themes:
-            self.module.fail_json(msg='The submitted Theme could not be found. Pick a different theme.')
-
     ##############################
     # XML processing
     #
-    def _remove_deleted_params(self):
-        """ Remove from target_elt a few deleted params """
-        changed = False
-        params = ['dnsallowoverride']
-        if not self.pfsense.is_at_least_2_5_0():
-            params += ['dnslocalhost']
-        params += self.params_to_delete
-        for param in params:
-            if self.pfsense.remove_deleted_param_from_elt(self.target_elt, param, self.obj):
-                changed = True
-
-        webgui_elt = self.target_elt.find('webgui')
-        webgui = self.obj['webgui']
-
-        params = ['interfacessort', 'webguileftcolumnhyper', 'disablealiaspopupdetail', 'dashboardavailablewidgetspanel', 'systemlogsfilterpanel']
-        params += ['systemlogsmanagelogpanel', 'statusmonitoringsettingspanel', 'roworderdragging', 'loginshowhost', 'webguifixedmenu']
-        params += ['requirestatefilter']
-        for param in params:
-            if self.pfsense.remove_deleted_param_from_elt(webgui_elt, param, webgui):
-                changed = True
-
-        return changed
+    def _get_params_to_remove(self):
+        """ returns the list of params to remove if they are not set """
+        to_remove = super(PFSenseSetupModule, self)._get_params_to_remove()
+        to_remove.extend(self.params_to_delete)
+        return to_remove
 
     ##############################
     # run
     #
-    def run(self, params):
-        """ process input params to add/update/delete """
-        self.params = params
-        self.target_elt = self.root_elt
-        self._validate_params()
-
-        self.obj = self._params_to_obj()
-
-        self._add()
-
     def _update(self):
         """ make the target pfsense reload """
         for cmd in self.route_cmds:
@@ -570,8 +503,8 @@ def _update(self):
 $retval |= system_ntp_configure();'''
 
         if self.params.get('dnsallowoverride') is not None:
-            if (self.params['dnsallowoverride'] and 'dnsallowoverride' not in self.before or
-                    not self.params['dnsallowoverride'] and 'dnsallowoverride' in self.before):
+            if (self.params['dnsallowoverride'] and 'dnsallowoverride' not in self.diff['before'] or
+                    not self.params['dnsallowoverride'] and 'dnsallowoverride' in self.diff['before']):
                 cmd += '$retval |= send_event("service reload dns");\n'
 
         if self.params.get('shellauth') is not None:
@@ -591,29 +524,26 @@ def _get_obj_name():
 
     def _log_fields(self, before=None):
         """ generate pseudo-CLI command fields parameters to create an obj """
-        bwebgui = self.before['webgui']
+        bwebgui = self.diff['before']['webgui']
         webgui = self.obj['webgui']
 
-        obj_before = self._prepare_dns_log(self.before)
+        obj_before = self._prepare_dns_log(self.diff['before'])
         obj_after = self._prepare_dns_log(self.obj)
 
         values = ''
-        values += self.format_updated_cli_field(self.obj, self.before, 'hostname', add_comma=(values), log_none=False)
-        values += self.format_updated_cli_field(self.obj, self.before, 'domain', add_comma=(values), log_none=False)
+        values += self.format_updated_cli_field(self.obj, self.diff['before'], 'hostname', add_comma=(values), log_none=False)
+        values += self.format_updated_cli_field(self.obj, self.diff['before'], 'domain', add_comma=(values), log_none=False)
 
         values += self.format_updated_cli_field(obj_after, obj_before, 'dns_addresses', add_comma=(values), log_none=True)
         values += self.format_updated_cli_field(obj_after, obj_before, 'dns_hostnames', add_comma=(values), log_none=True)
         values += self.format_updated_cli_field(obj_after, obj_before, 'dns_gateways', add_comma=(values), log_none=True)
 
-        values += self.format_updated_cli_field(self.obj, self.before, 'timezone', add_comma=(values), log_none=False)
-        values += self.format_updated_cli_field(self.obj, self.before, 'timeservers', add_comma=(values), log_none=False)
-        values += self.format_updated_cli_field(self.obj, self.before, 'language', add_comma=(values), log_none=False)
+        values += self.format_updated_cli_field(self.obj, self.diff['before'], 'timezone', add_comma=(values), log_none=False)
+        values += self.format_updated_cli_field(self.obj, self.diff['before'], 'timeservers', add_comma=(values), log_none=False)
+        values += self.format_updated_cli_field(self.obj, self.diff['before'], 'language', add_comma=(values), log_none=False)
 
-        values += self.format_updated_cli_field(self.obj, self.before, 'dnsallowoverride', fvalue=self.fvalue_bool, add_comma=(values), log_none=False)
-        if self.pfsense.is_at_least_2_5_0:
-            values += self.format_updated_cli_field(self.obj, self.before, 'dnslocalhost', add_comma=(values), log_none=False)
-        else:
-            values += self.format_updated_cli_field(self.obj, self.before, 'dnslocalhost', fvalue=self.fvalue_bool, add_comma=(values), log_none=False)
+        values += self.format_updated_cli_field(self.obj, self.diff['before'], 'dnsallowoverride', fvalue=self.fvalue_bool, add_comma=(values), log_none=False)
+        values += self.format_updated_cli_field(self.obj, self.diff['before'], 'dnslocalhost', add_comma=(values), log_none=False)
 
         values += self.format_updated_cli_field(obj_after, obj_before, 'webguicss', add_comma=(values), log_none=False)
         values += self.format_updated_cli_field(webgui, bwebgui, 'webguifixedmenu', fvalue=self.fvalue_bool, add_comma=(values), log_none=False)
diff --git a/tests/unit/plugins/module_utils/fixtures/pfsense_setup_config.xml b/tests/unit/plugins/module_utils/fixtures/pfsense_setup_config.xml
index ceee3baf..c58bad86 100644
--- a/tests/unit/plugins/module_utils/fixtures/pfsense_setup_config.xml
+++ b/tests/unit/plugins/module_utils/fixtures/pfsense_setup_config.xml
@@ -5,7 +5,7 @@
 		<optimization>normal</optimization>
 		<hostname>pfSense</hostname>
 		<domain>acme.com</domain>
-		<dnsallowoverride>on</dnsallowoverride>
+		<dnsallowoverride></dnsallowoverride>
 		<group>
 			<name>all</name>
 			<description>All Users</description>
diff --git a/tests/unit/plugins/modules/test_pfsense_setup.py b/tests/unit/plugins/modules/test_pfsense_setup.py
index cbdad844..263e9efa 100644
--- a/tests/unit/plugins/modules/test_pfsense_setup.py
+++ b/tests/unit/plugins/modules/test_pfsense_setup.py
@@ -37,8 +37,10 @@ def setUp(self):
 
         super(TestPFSenseSetupModule, self).setUp()
 
-        self.mock_validate_webguicss = patch('ansible_collections.pfsensible.core.plugins.modules.pfsense_setup.PFSenseSetupModule._validate_webguicss')
-        self.validate_webguicss = self.mock_validate_webguicss.start()
+        # Remove validate command for webguicss which references files on the pfSense instance
+        self.mock_validate_webguicss = patch.dict('ansible_collections.pfsensible.core.plugins.modules.pfsense_setup.SETUP_ARG_ROUTE',
+                                                  dict(webguicss=dict(parse=pfsense_setup.p2o_webguicss)))
+        self.mock_validate_webguicss.start()
 
         self.mock_run_command = patch('ansible.module_utils.basic.AnsibleModule.run_command')
         self.run_command = self.mock_run_command.start()
@@ -48,7 +50,7 @@ def tearDown(self):
         """ mocking down """
         super(TestPFSenseSetupModule, self).tearDown()
 
-        self.validate_webguicss.stop()
+        self.mock_validate_webguicss.stop()
         self.run_command.stop()
 
     ##############

From 5734adb6c47000d7e6c777d66d9a6ff8da7ac228 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 22 Jan 2024 22:31:36 -0700
Subject: [PATCH 59/62] [generate_module] First stabe at config module

---
 misc/generate_module      | 238 ++++++++++++++++++++------------------
 misc/pfsense_module.py.j2 |  28 ++++-
 2 files changed, 153 insertions(+), 113 deletions(-)

diff --git a/misc/generate_module b/misc/generate_module
index 0de8d042..4a0e56eb 100755
--- a/misc/generate_module
+++ b/misc/generate_module
@@ -31,6 +31,11 @@ gitconfig = git.GitConfigParser()
 author_name = gitconfig.get_value('user', 'name')
 author_email = gitconfig.get_value('user', 'email')
 package = 'core'
+module_base = 'PFSenseModuleBase'
+module_key = None
+module_node = None
+name_param = None
+params_xml_only = []
 is_package = False
 args_imports = []
 
@@ -43,6 +48,7 @@ parser.add_argument('--author-name', default=author_name, help='The full name of
 parser.add_argument('--author-email', default=author_email, help='The email address of the module author')
 parser.add_argument('--author-handle', default='', help='The github handle of the module author')
 parser.add_argument('--module-name', help='The name of the module to generate - defaults to being based on the url')
+parser.add_argument('--is-config', action='store_true', help='This is a configuration module', )
 parser.add_argument('--name-param', help='The name of the primary module parameter - defaults to the key, but often "name" is used instead of "descr"')
 parser.add_argument('--type-param', default='type', help='The name of the parameter for selecting different types of elements', )
 parser.add_argument('--type-suffix', const=True, default=False, nargs='?', help='Suffix the module name with the item type', )
@@ -55,6 +61,10 @@ parser.add_argument('--keep-params', action=argparse.BooleanOptionalAction, help
 
 args = parser.parse_args()
 
+# TODO - require a --module-root arg or search for it
+if args.is_config:
+    module_root = 'system'
+
 # Temporary directory for files
 tmpdir = tempfile.TemporaryDirectory(prefix='pfgenmod-')
 
@@ -131,67 +141,68 @@ scp.close()
 # Parse the config.xml file
 root = ET.parse(f'{tmpdir.name}/config.xml').getroot()
 
-# Search for any element with our target text, make sure we found only one
-xpath = f'.//*[.="{args.item_min}"]'
-key_elts = root.findall(xpath)
-if len(key_elts) > 1:
-    sys.exit(f'Found {len(key_elts)} items with path "{xpath}"')
-elif len(key_elts) == 0:
-    sys.exit(f'Cannot find minimally configured item with path "{xpath}"')
-else:
-    key_elt = key_elts[0]
+params_full = dict()
+if not args.is_config:
+    # Search for any element with our target text, make sure we found only one
+    xpath = f'.//*[.="{args.item_min}"]'
+    key_elts = root.findall(xpath)
+    if len(key_elts) > 1:
+        sys.exit(f'Found {len(key_elts)} items with path "{xpath}"')
+    elif len(key_elts) == 0:
+        sys.exit(f'Cannot find minimally configured item with path "{xpath}"')
+    else:
+        key_elt = key_elts[0]
 
-# This element should be the key for the items
-module_key = key_elt.tag
-if args.name_param:
-   name_param = args.name_param
-else:
-   name_param = module_key
+    # This element should be the key for the items
+    module_key = key_elt.tag
+    if args.name_param:
+       name_param = args.name_param
+    else:
+       name_param = module_key
 
-# The full node configuration element will be the parent
-node_elt = key_elt.find('..')
-module_node = node_elt.tag
+    # The full node configuration element will be the parent
+    node_elt = key_elt.find('..')
+    module_node = node_elt.tag
 
-# The "root" for this type of element is above that
-root_elt = node_elt.find('..')
-module_root = root_elt.tag
+    # The "root" for this type of element is above that
+    root_elt = node_elt.find('..')
+    module_root = root_elt.tag
 
-# Debug
-print('item_min:\t' + ET.tostring(node_elt).decode())
+    # Debug
+    print('item_min:\t' + ET.tostring(node_elt).decode())
 
-# Let's use our node and key as a check
-full_elt = root.find(f'.//{module_node}[{module_key}="{args.item_full}"]')
-if full_elt is None:
-    sys.exit(f'Cannot find fully configured item with path ".//{module_node}[{module_key}="{args.item_full}"]"')
+    # Let's use our node and key as a check
+    full_elt = root.find(f'.//{module_node}[{module_key}="{args.item_full}"]')
+    if full_elt is None:
+        sys.exit(f'Cannot find fully configured item with path ".//{module_node}[{module_key}="{args.item_full}"]"')
 
-# Debug
-print('item_full:\t' + ET.tostring(full_elt).decode())
+    # Debug
+    print('item_full:\t' + ET.tostring(full_elt).decode())
 
-# Collect the items for comparison with web elements and example values
-params_full = dict()
-for elt in full_elt:
-    if elt.tag == '':
-        continue
-    param = dict()
-    addr_elt = elt.find('address')
-    if addr_elt is not None:
-        param['example'] = addr_elt.text
-        param['address'] = True
-    elif elt.text is not None:
-        if elt.tag in params_full:
-            # Copy example and possibly other values from previous copy
-            param = params_full[elt.tag]
-            # If we have already need one of these, then it is a list
-            param['type'] = 'list'
-            # TODO - can we determine the type?
-            param['elements'] = 'str'
-            param['example2'] = elt.text.strip()
-        else:
-            param['type'] = 'str'
-            param['example'] = elt.text.strip()
-    # else:
-        # Likely a bool?
-    params_full[elt.tag] = param
+    # Collect the items for comparison with web elements and example values
+    for elt in full_elt:
+        if elt.tag == '':
+            continue
+        param = dict()
+        addr_elt = elt.find('address')
+        if addr_elt is not None:
+            param['example'] = addr_elt.text
+            param['address'] = True
+        elif elt.text is not None:
+            if elt.tag in params_full:
+                # Copy example and possibly other values from previous copy
+                param = params_full[elt.tag]
+                # If we have already need one of these, then it is a list
+                param['type'] = 'list'
+                # TODO - can we determine the type?
+                param['elements'] = 'str'
+                param['example2'] = elt.text.strip()
+            else:
+                param['type'] = 'str'
+                param['example'] = elt.text.strip()
+        # else:
+            # Likely a bool?
+        params_full[elt.tag] = param
 
 print('')
 
@@ -318,9 +329,10 @@ for input in html.forms[0].inputs:
 
     params[input.name] = param
 
-# Key is handled separately from other parameters so remove it
-# TODO - keep the description, etc?
-params.pop(module_key, None)
+if not args.is_config:
+    # Key is handled separately from other parameters so remove it
+    # TODO - keep the description, etc?
+    params.pop(module_key, None)
 
 # Debug
 print(f'Web paramters: {params.keys()}')
@@ -345,62 +357,67 @@ elif args.type_param in params:
 else:
     module_type = None
 
-# Consistency
-params_web_only = list(set(params.keys()) - set(params_full.keys()))
-print('Web parameters not in xml: ' + str(params_web_only))
-
-# Cleanup extra web parameters
-for param in params_web_only:
-    # See if the items are numbered, likely maps to an unnumbered XML tag
-    newp = re.sub(r'0$', '', param)
-    if newp != param:
-        if newp in params_full:
-            print(f'Renaming {param} to {newp}')
-            params[newp] = params.pop(param)
-            continue
-
-    # See if the items are prefixed by a type, likely maps to un-prefixed XML tag
-    newp = re.sub(f'^{module_type}_', '', param)
-    if newp != param:
-        if newp in params_full and newp not in params:
-            print(f'Renaming {param} to {newp}')
-            params[newp] = params.pop(param)
-            continue
-
-    # Common renamings
-    for f, t in [('dst', 'destination'), ('src', 'source')]:
-        if param == f and t in params_full:
-            print(f'Renaming {f} to {t}')
-            params[t] = params.pop(f)
-            break
-    else:
-        # Otherwise, drop - probably just used to construct the final elements
-        if param in params and not args.keep_params:
-            print(f'Removing {param}')
-            del params[param]
-
-print('')
-params_xml_only = list(set(params_full.keys()) - set(params.keys()) - {module_key, 'refid'})
-print(f'XML parameters not in web: {params_xml_only}\n')
-if len(params_xml_only) > 0:
-    print(f'You may need to use {module_node.upper()}_MAP_PARAMS')
-
-for param in params_xml_only:
-    params[param] = params_full[param]
-
-# Create some sample descriptions
-for name, param in params.items():
-    # TODO - wrap long descriptions
-    if 'description' not in param or param['description'] == '':
-        param['description'] = f'The {name} of the {module_node}.'
-    if 'example' not in param or param['example'] == '':
-        if name in params_full and 'example' in params_full[name]:
-            param['example'] = params_full[name]['example']
-    if 'default' in param:
-        param['description'] += f' Defaults to {param["default"]}.'
+if not args.is_config:
+    # Consistency
+    params_web_only = list(set(params.keys()) - set(params_full.keys()))
+    print('Web parameters not in xml: ' + str(params_web_only))
+
+    # Cleanup extra web parameters
+    for param in params_web_only:
+        # See if the items are numbered, likely maps to an unnumbered XML tag
+        newp = re.sub(r'0$', '', param)
+        if newp != param:
+            if newp in params_full:
+                print(f'Renaming {param} to {newp}')
+                params[newp] = params.pop(param)
+                continue
+
+        # See if the items are prefixed by a type, likely maps to un-prefixed XML tag
+        newp = re.sub(f'^{module_type}_', '', param)
+        if newp != param:
+            if newp in params_full and newp not in params:
+                print(f'Renaming {param} to {newp}')
+                params[newp] = params.pop(param)
+                continue
+
+        # Common renamings
+        for f, t in [('dst', 'destination'), ('src', 'source')]:
+            if param == f and t in params_full:
+                print(f'Renaming {f} to {t}')
+                params[t] = params.pop(f)
+                break
+        else:
+            # Otherwise, drop - probably just used to construct the final elements
+            if param in params and not args.keep_params:
+                print(f'Removing {param}')
+                del params[param]
+
+    print('')
+    params_xml_only = list(set(params_full.keys()) - set(params.keys()) - {module_key, 'refid'})
+    print(f'XML parameters not in web: {params_xml_only}\n')
+    if len(params_xml_only) > 0:
+        print(f'You may need to use {module_node.upper()}_MAP_PARAMS')
+
+    for param in params_xml_only:
+        params[param] = params_full[param]
+
+    # Create some sample descriptions
+    for name, param in params.items():
+        # TODO - wrap long descriptions
+        if 'description' not in param or param['description'] == '':
+            param['description'] = f'The {name} of the {module_node}.'
+        if 'example' not in param or param['example'] == '':
+            if name in params_full and 'example' in params_full[name]:
+                param['example'] = params_full[name]['example']
+        if 'default' in param:
+            param['description'] += f' Defaults to {param["default"]}.'
+
+if args.is_config:
+    module_base = 'PFSenseModuleConfigBase'
 
 # Template variables
 context = dict(
+    module_base=module_base,
     module_name=module_name,
     module_root=module_root,
     module_node=module_node,
@@ -409,6 +426,7 @@ context = dict(
     params_xml_only=params_xml_only,
     name_param=name_param,
     args_imports=args_imports,
+    is_config=args.is_config,
     is_package=is_package,
     package=package,
     author_name=args.author_name,
diff --git a/misc/pfsense_module.py.j2 b/misc/pfsense_module.py.j2
index 88a07d68..e8d3028c 100644
--- a/misc/pfsense_module.py.j2
+++ b/misc/pfsense_module.py.j2
@@ -19,6 +19,7 @@ description:
   - Manage pfSense {{ module_name }}s.{{ ' This requires the pfSense ' ~ package ~ ' package to be installed.' if is_package else '' }}
 
 options:
+{% if not is_config %}
   {{ name_param }}:
     description: The {{ name_param }} of the {{ module_name }}.
     required: true
@@ -28,6 +29,7 @@ options:
     default: present
     choices: ['present', 'absent']
     type: str
+{% endif %}
 {% for name, param in params.items() %}
   {{ name }}:
     description: {{ "'" if ':' in param['description'] else '' }}{{ param['description'] | default('') }}{{ "'" if ':' in param['description'] else '' }}
@@ -44,9 +46,11 @@ author: {{ author_name }} (@{{ author_handle }})
 '''
 
 EXAMPLES = r'''
-- name: Add myitem {{ module_name }}
+- name: {{ 'Configure' if is_config else 'Add myitem' }} {{ module_name }}
   pfsensible.{{ package }}.pfsense_{{ module_name }}:
+{% if not is_config %}
     {{ name_param }}: myitem
+{% endif %}
 {% for name, param in params.items() %}
 {% if param['example'] is defined %}
 {% if param['type'] == 'list' %}
@@ -61,23 +65,33 @@ EXAMPLES = r'''
 
 {% endif %}
 {% endfor %}
+{% if not is_config %}
     state: present
 
 - name: Remove myitem {{ module_name }}
   pfsensible.{{ package }}.pfsense_{{ module_name }}:
     {{ name_param }}: myitem
     state: absent
+{% endif %}
 '''
 RETURN = r'''
 commands:
     description: the set of commands that would be pushed to the remote device (if pfSense had a CLI).
     returned: always
     type: list
+{% if is_config %}
+    sample: ["update {{ module_name }} set ..."]
+{% else %}
     sample: ["create {{ module_name }} 'myitem'", "update {{ module_name }} 'myitem' set ...", "delete {{ module_name }} 'myitem'"]
+{% endif %}
 '''
 
 from ansible.module_utils.basic import AnsibleModule
+{% if is_config %}
+from ansible_collections.pfsensible.core.plugins.module_utils.module_config_base import PFSenseModuleConfigBase
+{% else %}
 from ansible_collections.pfsensible.core.plugins.module_utils.module_base import PFSenseModuleBase
+{% endif %}
 {% if args_imports %}
 from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import {{ args_imports | sort | join(', ') }}
 {% endif %}
@@ -85,6 +99,7 @@ from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import {
 # TODO -Change to name of module, extend for needed parameters
 # TODO -Keep either this or the next compact version of {{ module_name | upper() }}_ARGUMENT_SPEC
 {{ module_name | upper() }}_ARGUMENT_SPEC = {
+{% if not is_config %}
     # Only {{ name_param }} should be required here - othewise you cannot remove an item with just '{{ name_param }}'
     # Required arguments for creation should be noted in {{ module_name | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
     '{{ name_param }}': {'required': True, 'type': 'str'},
@@ -93,6 +108,7 @@ from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import {
         'default': 'present',
         'choices': ['present', 'absent']
     },
+{% endif %}
 {% for param in params %}
     '{{ param }}': {
 {% if 'choices' in params[param] %}
@@ -105,10 +121,12 @@ from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import {
 
 # Compact style
 {{ module_name | upper() }}_ARGUMENT_SPEC = dict(
+{% if not is_config %}
     # Only {{ name_param }} should be required here - othewise you cannot remove an item with just '{{ name_param }}'
     # Required arguments for creation should be noted in {{ module_name | upper() }}_REQUIRED_IF = ['state', 'present', ...] below
     {{ name_param }}=dict(required=True, type='str'),
     state=dict(type='str', default='present', choices=['present', 'absent']),
+{% endif %}
 {% for param in params %}
     {{ param }}=dict(type='{{ params[param]['type'] | default('') }}'{% if 'choices' in params[param] %}, choices={{ params[param]['choices'] }},{% endif %}),
 {% endfor %}
@@ -116,12 +134,14 @@ from ansible_collections.pfsensible.core.plugins.module_utils.arg_route import {
 
 # TODO - check for validity - what parameters are actually required when creating a new {{ module_name }}?
 {{ module_name | upper() }}_REQUIRED_IF = [
+{% if not is_config %}
 {% if module_type %}
     ['state', 'present', ['type']],
     ['type', '{{ params['type']['example'] }}', ['{{ params | dict2items | rejectattr('key', 'equalto', 'type') | selectattr('value.required', 'defined') | rejectattr('value.default', 'defined') | map(attribute='key') | join("', '") }}']],
 {% else %}
     ['state', 'present', ['{{ params | dict2items | selectattr('value.required', 'defined') | rejectattr('value.default', 'defined') | map(attribute='key') | join("', '") }}']],
 {% endif %}
+{% endif %}
 ]
 
 {% if params_xml_only %}
@@ -154,6 +174,7 @@ PHP_VALIDATION = r'''
 {% endif %}
 )
 
+{% if not is_config %}
 # TODO - check for validity - what are default values when creating a new {{ module_name }}
 {{ module_name | upper() }}_CREATE_DEFAULT = dict(
 {% for item in params | dict2items | selectattr('value.default', 'defined') %}
@@ -164,6 +185,7 @@ PHP_VALIDATION = r'''
 {% endfor %}
 )
 
+{% endif %}
 {% if is_package %}
 {{ module_name | upper() }}_PHP_COMMAND_SET = r'''
 require_once("{{ package }}.inc");
@@ -178,7 +200,7 @@ if (filter_configure() == 0) { clear_subsystem_dirty('{{ php_subsystem }}'); }
 
 {% endif %}
 
-class PFSense{{ module_name | capitalize() }}Module(PFSenseModuleBase):
+class PFSense{{ module_name | capitalize() }}Module({{ module_base }}):
     """ module managing pfsense {{ module_name }}s """
 
     ##############################
@@ -192,7 +214,7 @@ class PFSense{{ module_name | capitalize() }}Module(PFSenseModuleBase):
 
     def __init__(self, module, pfsense=None):
         super(PFSense{{ module_name | capitalize() }}Module, self).__init__(module, pfsense, {{ 'package=\'' ~ package ~ '\', ' if is_package else ''}}root='{{ module_root }}', node='{{ module_node }}', key='{{ module_key }}'{{ ', update_php=' ~ module_name | upper() ~ '_PHP_COMMAND_SET' if 'filter.inc' in php_requires else '' }},
-                                                arg_route={{ module_name | upper() }}_ARG_ROUTE{% if params_xml_only %}, map_param={{ module_name | upper() }}_MAP_PARAM{% endif %}, create_default={{ module_name | upper() }}_CREATE_DEFAULT)
+                                                arg_route={{ module_name | upper() }}_ARG_ROUTE{% if params_xml_only %}, map_param={{ module_name | upper() }}_MAP_PARAM{% endif %}{% if not is_config %}, create_default={{ module_name | upper() }}_CREATE_DEFAULT{% endif %})
 
 
 def main():

From 85b124945d3a4b2609421d9cc84bad05760ce2e9 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Tue, 30 Jan 2024 17:07:24 -0700
Subject: [PATCH 60/62] [tests/plays] README

---
 tests/plays/README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/plays/README.md b/tests/plays/README.md
index 44d2fe36..c2dcbb1a 100644
--- a/tests/plays/README.md
+++ b/tests/plays/README.md
@@ -6,6 +6,7 @@ The following collection dependencies are needed:
  * ansible.utils
 
 You will need a fresh pfSense install available as `pfsense-test` or adjust the `hosts` file as needed.
-You need to be able to ssh to it as `root` without a password or use `--ask-pass`.
+You need to be able to ssh to it as `root` without a password or use `--ask-pass`, which you can use
+the configure.yml play to do.
 
 Update `host_vars/pfsense-test.yml` with IP addresses of your test pfSense install.

From 266ab056ab8dd18215ebb37492a040f20a5aa4e0 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Wed, 7 Feb 2024 22:14:35 -0700
Subject: [PATCH 61/62] [module_base] style

---
 plugins/module_utils/module_base.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/module_utils/module_base.py b/plugins/module_utils/module_base.py
index 1a182835..373f42d1 100644
--- a/plugins/module_utils/module_base.py
+++ b/plugins/module_utils/module_base.py
@@ -90,7 +90,8 @@ def __init__(self, module, pfsense=None, package=None, name=None, root=None, roo
                 if package is not None:
                     self.root_elt = self.pfsense.get_element(root, root_elt=self.pfsense.root.find('installedpackages'))
                     if self.root_elt is None:
-                        self.module.fail_json(msg='Unable to find configuration for the package {package}.  Are you sure that it is installed?'.format(package=package))
+                        self.module.fail_json(
+                            msg='Unable to find configuration for the package {package}.  Are you sure that it is installed?'.format(package=package))
                 else:
                     root_elt = self.pfsense.root
                     for this in root.split('/'):

From c6a6a7807f3b773fde2ffbf619e7f4a5c9cf2e16 Mon Sep 17 00:00:00 2001
From: Orion Poplawski <orion@nwra.com>
Date: Mon, 19 Feb 2024 13:30:07 -0700
Subject: [PATCH 62/62] [pfsense] Allow turning off debug in phpshell()

---
 plugins/module_utils/pfsense.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/plugins/module_utils/pfsense.py b/plugins/module_utils/pfsense.py
index 7236e04d..07581e1b 100644
--- a/plugins/module_utils/pfsense.py
+++ b/plugins/module_utils/pfsense.py
@@ -637,11 +637,14 @@ def uniqid(prefix='', more_entropy=False):
 
         return prefix + '{0:x}{1:05x}'.format(int(time.time()), int(time.time() * 1000000) % 0x100000)
 
-    def phpshell(self, command):
+    def phpshell(self, command, debug=True):
         """ Run a command in the php developer shell """
-        command = "global $debug;\n$debug = 1;\n" + command + "\nexec\nexit"
+        phpshell = ""
+        if debug:
+            phpshell = "global $debug;\n$debug = 1;\n"
+        phpshell += command + "\nexec\nexit"
         # Dummy argument suppresses displaying help message
-        return self.module.run_command('/usr/local/sbin/pfSsh.php dummy', data=command)
+        return self.module.run_command('/usr/local/sbin/pfSsh.php dummy', data=phpshell)
 
     def php(self, command):
         """ Run a command in php and return the output """