Ini parse fixes

src/ansiblecmdb/parser.py

@@ -143,53 +143,53 @@ def _parse_line_entry(self, line, type):
         Returns:
             (None, {'purpose': 'web'})
 
-        Undocumented feature:
+        In :vars sections everything to the right of key=value is a string.
+        Comments are removed and leading and trailing spaces are stripped.
             [prod:vars]
-            json_like_vars=[{'name': 'htpasswd_auth'}]
+            json_like_vars = [{'name': 'htpasswd_auth'}]
         Returns:
-            (None, {'name': 'htpasswd_auth'})
+            (None, "[{'name': 'htpasswd_auth'}]")
         """
-
-        name = None
-        key_values = {}
+        line = self.remove_comment_from_line(line)
 
         if type == 'vars':
-            key_values = self._parse_line_vars(line)
-        else:
-            tokens = shlex.split(line.strip())
-            name = tokens.pop(0)
-            try:
-                key_values = self._parse_vars(tokens)
-            except ValueError:
-                self.log.warning("Unsupported vars syntax. Skipping line: {0}".format(line))
-                return (name, {})
-        return (name, key_values)
-
-    def _parse_line_vars(self, line):
-        """
-        Parse a line in a [XXXXX:vars] section.
-        """
-        key_values = {}
+            k, v = line.split('=', 1)
+            return None, {k.strip(): v.strip()}
 
-        # Undocumented feature allows json in vars sections like so:
-        #   [prod:vars]
-        #   json_like_vars=[{'name': 'htpasswd_auth'}]
-        # We'll try this first. If it fails, we'll fall back to normal var
-        # lines. Since it's undocumented, we just assume some things.
-        k, v = line.strip().split('=', 1)
-        if v.startswith('['):
-            try:
-                list_res = ihateyaml.safe_load(v)
-                if isinstance(list_res[0], dict):
-                    key_values = list_res[0]
-                    return key_values
-            except ValueError:
-                pass
+        tokens = shlex.split(line)
+        name = tokens.pop(0)
+        try:
+            key_values = self._parse_vars(tokens)
+        except ValueError:
+            self.log.warning("Unsupported vars syntax. Skipping line: {0}", line)
+            return name, {}
+
+        return name, key_values
+
+    # Lightly adapted from https://stackoverflow.com/questions/2319019/using-regex-to-remove-comments-from-source-files
+    def remove_comment_from_line(self, line):
+        """
+        Return the string obtained by removing any # comment from the line,
+        respecting quoted strings.
+        """
+        # I don't think this is actually completely correct as quoted
+        # backslashes will probably cause trouble -- consider what
+        # might happen with multiple \', \" and \\ in a line, but it
+        # might be close enough for government work.
+        pattern = r'''(".*?"|'.*?')|(#.*)'''
+        # first group captures quoted strings (double or single)
+        # second group captures comments
+        regex = re.compile(pattern)
+
+        def _replacer(match):
+            # if we captured a comment in group 2, remove it
+            # otherwise return the captured quoted string in group 1
+            if match.group(2) is not None:
+                return ''
+            else:
+                return match.group(1)
 
-        # Guess it's not YAML. Parse as normal host variables
-        tokens = shlex.split(line.strip())
-        key_values = self._parse_vars(tokens)
-        return key_values
+        return regex.sub(_replacer, line)
 
     def _parse_vars(self, tokens):
         """
@@ -203,14 +203,8 @@ def _parse_vars(self, tokens):
         """
         key_values = {}
         for token in tokens:
-            if token.startswith('#'):
-                # End parsing if we encounter a comment, which lasts
-                # until the end of the line.
-                break
-            else:
-                k, v = token.split('=', 1)
-                key = k.strip()
-                key_values[key] = v.strip()
+            k, v = token.split('=', 1)
+            key_values[k.strip()] = v.strip()
         return key_values
 
     def _get_distinct_hostnames(self):

test/f_hostparse/hosts_ini_key_value

@@ -0,0 +1,36 @@
+# -*-conf-*-
+
+# INI format key=value documentation from docs.ansible.com
+#
+# <quote>
+# Values passed in the INI format using the key=value syntax are
+# interpreted differently depending on where they are declared:
+#
+# When declared inline with the host, INI values are interpreted as
+# Python literal structures (strings, numbers, tuples, lists, dicts,
+# booleans, None). Host lines accept multiple key=value parameters per
+# line. Therefore they need a way to indicate that a space is part of
+# a value rather than a separator.
+#
+# When declared in a :vars section, INI values are interpreted as
+# strings. For example var=FALSE would create a string equal to
+# ‘FALSE’. Unlike host lines, :vars sections accept only a single
+# entry per line, so everything after the = must be the value for the
+# entry.</quote>
+
+[db]
+db.dev.local
+
+[db:vars]
+# This is a string valued variable that Ansible will automagically
+# convert to a list when used in some contexts.  It is *not* YAML and
+# the items must be quoted as it is interpreted as a Python literal.
+# The comment should be ignored.
+simple_list = ["item1", "item2"]  # list string
+
+# This is a string valued var that Ansible will automagically convert
+# to a dictionary when used in some contexts.  It is *not* YAML so the
+# items must be quoted as Ansible will ultimately try to interpret it
+# as a Python literal.  Extra spaces in the string should be
+# preserved, but Ansible strips leading and trailing spaces.
+simple_dict = {"k1": "v1",    'k2': 'v2'}  # dict string

test/test.py

@@ -69,6 +69,18 @@ def testExpandHostDef(self):
         self.assertIn('web02.dev.local', ansible.hosts)
         self.assertIn('fe03.dev02.local', ansible.hosts)
 
+    def testIniKeyValueParse(self):
+        """
+        Verify that key=value is parsed correctly in inventory ini files.
+        """
+        fact_dirs = ['f_hostparse/out']
+        inventories = ['f_hostparse/hosts_ini_key_value']
+        ansible = ansiblecmdb.Ansible(fact_dirs, inventories)
+        host_vars = ansible.hosts['db.dev.local']['hostvars']
+        self.assertEqual(host_vars['simple_list'], '["item1", "item2"]')
+        self.assertEqual(host_vars['simple_dict'],
+                         '''{"k1": "v1",    'k2': 'v2'}''')
+
 
 class InventoryTestCase(unittest.TestCase):
     def testHostsDir(self):