From 9c5822b26790b2fba71160b80152bbc381329eab Mon Sep 17 00:00:00 2001 From: Doug Quale Date: Sat, 11 Apr 2020 15:44:48 -0500 Subject: [PATCH 1/4] new test and data to verify inventory ini key=value parsing --- test/f_hostparse/hosts_ini_key_value | 36 ++++++++++++++++++++++++++++ test/test.py | 15 +++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 test/f_hostparse/hosts_ini_key_value diff --git a/test/f_hostparse/hosts_ini_key_value b/test/f_hostparse/hosts_ini_key_value new file mode 100644 index 0000000..321f6c3 --- /dev/null +++ b/test/f_hostparse/hosts_ini_key_value @@ -0,0 +1,36 @@ +# -*-conf-*- + +# INI format key=value documentation from docs.ansible.com +# +# +# Values passed in the INI format using the key=value syntax are +# interpreted differently depending on where they are declared: +# +# When declared inline with the host, INI values are interpreted as +# Python literal structures (strings, numbers, tuples, lists, dicts, +# booleans, None). Host lines accept multiple key=value parameters per +# line. Therefore they need a way to indicate that a space is part of +# a value rather than a separator. +# +# When declared in a :vars section, INI values are interpreted as +# strings. For example var=FALSE would create a string equal to +# ‘FALSE’. Unlike host lines, :vars sections accept only a single +# entry per line, so everything after the = must be the value for the +# entry. + +[db] +db.dev.local + +[db:vars] +# This is a string valued variable that Ansible will automagically +# convert to a list when used in some contexts. It is *not* YAML and +# the items must be quoted as it is interpreted as a Python literal. +# The comment should be ignored. +simple_list = ["item1", "item2"] # list string + +# This is a string valued var that Ansible will automagically convert +# to a dictionary when used in some contexts. It is *not* YAML so the +# items must be quoted as Ansible will ultimately try to interpret it +# as a Python literal. Extra spaces in the string should be +# preserved, but Ansible strips leading and trailing spaces. +simple_dict = {"k1": "v1", 'k2': 'v2'} # dict string diff --git a/test/test.py b/test/test.py index 667c7a1..bd91805 100644 --- a/test/test.py +++ b/test/test.py @@ -1,6 +1,6 @@ +import logging import sys import unittest -import imp import os sys.path.insert(0, os.path.realpath('../lib')) @@ -69,6 +69,18 @@ def testExpandHostDef(self): self.assertIn('web02.dev.local', ansible.hosts) self.assertIn('fe03.dev02.local', ansible.hosts) + def testIniKeyValueParse(self): + """ + Verify that key=value is parsed correctly in inventory ini files. + """ + fact_dirs = ['f_hostparse/out'] + inventories = ['f_hostparse/hosts_ini_key_value'] + ansible = ansiblecmdb.Ansible(fact_dirs, inventories) + host_vars = ansible.hosts['db.dev.local']['hostvars'] + self.assertEqual(host_vars['simple_list'], '["item1", "item2"]') + self.assertEqual(host_vars['simple_dict'], + '''{"k1": "v1", 'k2': 'v2'}''') + class InventoryTestCase(unittest.TestCase): def testHostsDir(self): @@ -133,6 +145,7 @@ def testFactCache(self): if __name__ == '__main__': + logging.basicConfig(stream=sys.stderr, level=logging.WARNING) unittest.main(exit=True) try: From 869fc1dc5c5e888076eb629321e85628961afbe5 Mon Sep 17 00:00:00 2001 From: Doug Quale Date: Sat, 11 Apr 2020 16:43:53 -0500 Subject: [PATCH 2/4] _parse_line_entry: simplify and correct key=value parsing in ini files --- src/ansiblecmdb/parser.py | 80 +++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/src/ansiblecmdb/parser.py b/src/ansiblecmdb/parser.py index 3b3a885..716d19f 100644 --- a/src/ansiblecmdb/parser.py +++ b/src/ansiblecmdb/parser.py @@ -143,53 +143,53 @@ def _parse_line_entry(self, line, type): Returns: (None, {'purpose': 'web'}) - Undocumented feature: + In :vars sections everything to the right of key=value is a string. + Comments are removed and leading and trailing spaces are stripped. [prod:vars] - json_like_vars=[{'name': 'htpasswd_auth'}] + json_like_vars = [{'name': 'htpasswd_auth'}] Returns: - (None, {'name': 'htpasswd_auth'}) + (None, "[{'name': 'htpasswd_auth'}]") """ - - name = None - key_values = {} + line = self.remove_comment_from_line(line) if type == 'vars': - key_values = self._parse_line_vars(line) - else: - tokens = shlex.split(line.strip()) - name = tokens.pop(0) - try: - key_values = self._parse_vars(tokens) - except ValueError: - self.log.warning("Unsupported vars syntax. Skipping line: {0}".format(line)) - return (name, {}) - return (name, key_values) - - def _parse_line_vars(self, line): - """ - Parse a line in a [XXXXX:vars] section. - """ - key_values = {} + k, v = line.split('=', 1) + return None, {k.strip(): v.strip()} - # Undocumented feature allows json in vars sections like so: - # [prod:vars] - # json_like_vars=[{'name': 'htpasswd_auth'}] - # We'll try this first. If it fails, we'll fall back to normal var - # lines. Since it's undocumented, we just assume some things. - k, v = line.strip().split('=', 1) - if v.startswith('['): - try: - list_res = ihateyaml.safe_load(v) - if isinstance(list_res[0], dict): - key_values = list_res[0] - return key_values - except ValueError: - pass + tokens = shlex.split(line) + name = tokens.pop(0) + try: + key_values = self._parse_vars(tokens) + except ValueError: + self.log.warning("Unsupported vars syntax. Skipping line: {0}", line) + return name, {} + + return name, key_values + + # Lightly adapted from https://stackoverflow.com/questions/2319019/using-regex-to-remove-comments-from-source-files + def remove_comment_from_line(self, line): + """ + Return the string obtained by removing any # comment from the line, + respecting quoted strings. + """ + # I don't think this is actually completely correct as quoted + # backslashes will probably cause trouble -- consider what + # might happen with multiple \', \" and \\ in a line, but it + # might be close enough for government work. + pattern = r'''(".*?"|'.*?')|(#.*)''' + # first group captures quoted strings (double or single) + # second group captures comments + regex = re.compile(pattern) + + def _replacer(match): + # if we captured a comment in group 2, remove it + # otherwise return the captured quoted string in group 1 + if match.group(2) is not None: + return '' + else: + return match.group(1) - # Guess it's not YAML. Parse as normal host variables - tokens = shlex.split(line.strip()) - key_values = self._parse_vars(tokens) - return key_values + return regex.sub(_replacer, line) def _parse_vars(self, tokens): """ From 2e7b7ea6869a7e7facd4d2e95d11542536ca59ff Mon Sep 17 00:00:00 2001 From: quale1 Date: Mon, 20 Apr 2020 22:34:26 -0500 Subject: [PATCH 3/4] _parse_vars: comments have already been stripped --- src/ansiblecmdb/parser.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/ansiblecmdb/parser.py b/src/ansiblecmdb/parser.py index 716d19f..f5ea5b8 100644 --- a/src/ansiblecmdb/parser.py +++ b/src/ansiblecmdb/parser.py @@ -203,14 +203,8 @@ def _parse_vars(self, tokens): """ key_values = {} for token in tokens: - if token.startswith('#'): - # End parsing if we encounter a comment, which lasts - # until the end of the line. - break - else: - k, v = token.split('=', 1) - key = k.strip() - key_values[key] = v.strip() + k, v = token.split('=', 1) + key_values[k.strip()] = v.strip() return key_values def _get_distinct_hostnames(self): From ad72bdf1d5fd09c74f48e6caf7019bd545bfd33b Mon Sep 17 00:00:00 2001 From: quale1 Date: Mon, 20 Apr 2020 22:41:02 -0500 Subject: [PATCH 4/4] revert nonessential changes to test.py --- test/test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test.py b/test/test.py index bd91805..8e22ca9 100644 --- a/test/test.py +++ b/test/test.py @@ -1,6 +1,6 @@ -import logging import sys import unittest +import imp import os sys.path.insert(0, os.path.realpath('../lib')) @@ -145,7 +145,6 @@ def testFactCache(self): if __name__ == '__main__': - logging.basicConfig(stream=sys.stderr, level=logging.WARNING) unittest.main(exit=True) try: