Merge pull request ansible#4758 from alanfairless/group-host-var-dirs

Support organizing group and host variables across multiple files in a directory

lib/ansible/constants.py

@@ -90,6 +90,10 @@ def shell_expand_path(path):
 else:
     DIST_MODULE_PATH = '/usr/share/ansible/'
 
+# check all of these extensions when looking for yaml files for things like
+# group variables
+YAML_FILENAME_EXTENSIONS = [ "", ".yml", ".yaml" ]
+
 # sections in config file
 DEFAULTS='defaults'
 

lib/ansible/inventory/vars_plugins/group_vars.py

@@ -16,11 +16,119 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import glob
+import stat
+import errno
+
 from ansible import errors
 from ansible import utils
 import ansible.constants as C
 
+def _load_vars(basepath, results):
+    """
+    Load variables from any potential yaml filename combinations of basepath,
+    returning result.
+    """
+
+    paths_to_check = [ "".join([basepath, ext]) 
+                       for ext in C.YAML_FILENAME_EXTENSIONS ]
+
+    found_paths = []
+
+    for path in paths_to_check:
+        found, results = _load_vars_from_path(path, results)
+        if found:
+            found_paths.append(path)
+
+
+    # disallow the potentially confusing situation that there are multiple
+    # variable files for the same name. For example if both group_vars/all.yml
+    # and group_vars/all.yaml
+    if len(found_paths) > 1:
+        raise errors.AnsibleError("Multiple variable files found. "
+            "There should only be one. %s" % ( found_paths, ))
+
+    return results
+
+def _load_vars_from_path(path, results):
+    """
+    Robustly access the file at path and load variables, carefully reporting
+    errors in a friendly/informative way.
+
+    Return the tuple (found, new_results, )
+    """
+
+    try:
+        # in the case of a symbolic link, we want the stat of the link itself,
+        # not its target
+        pathstat = os.lstat(path)
+    except os.error, err:
+        # most common case is that nothing exists at that path.
+        if err.errno == errno.ENOENT:
+            return False, results
+        # otherwise this is a condition we should report to the user
+        raise errors.AnsibleError(
+            "%s is not accessible: %s." 
+            " Please check its permissions." % ( path, err.strerror))
+
+    # symbolic link
+    if stat.S_ISLNK(pathstat.st_mode):
+        try:
+            target = os.readlink(path)
+        except os.error, err2:
+            raise errors.AnsibleError("The symbolic link at %s "
+                "is not readable: %s.  Please check its permissions."
+                % (path, err2.strerror, ))
+        # follow symbolic link chains by recursing, so we repeat the same
+        # permissions checks above and provide useful errors.
+        return _load_vars_from_path(target, results)
+
+    # directory
+    if stat.S_ISDIR(pathstat.st_mode):
+
+        # support organizing variables across multiple files in a directory
+        return True, _load_vars_from_folder(path, results)
+
+    # regular file
+    elif stat.S_ISREG(pathstat.st_mode):
+        data = utils.parse_yaml_from_file(path)
+        if type(data) != dict:
+            raise errors.AnsibleError(
+                "%s must be stored as a dictionary/hash" % path)
+
+        # combine vars overrides by default but can be configured to do a
+        # hash merge in settings
+        results = utils.combine_vars(results, data)
+        return True, results
+
+    # something else? could be a fifo, socket, device, etc.
+    else:
+        raise errors.AnsibleError("Expected a variable file or directory "
+            "but found a non-file object at path %s" % (path, ))
+
+def _load_vars_from_folder(folder_path, results):
+    """
+    Load all variables within a folder recursively.
+    """
+
+    # this function and _load_vars_from_path are mutually recursive
+
+    try:
+        names = os.listdir(folder_path)
+    except os.error, err:
+        raise errors.AnsibleError(
+            "This folder cannot be listed: %s: %s." 
+             % ( folder_path, err.strerror))
+
+    # evaluate files in a stable order rather than whatever order the
+    # filesystem lists them.
+    names.sort() 
+
+    paths = [os.path.join(folder_path, name) for name in names]
+    for path in paths:
+        _found, results = _load_vars_from_path(path, results)
+    return results
+
+
 class VarsModule(object):
 
     """
@@ -73,42 +181,13 @@ def run(self, host):
                 continue
 
             # load vars in dir/group_vars/name_of_group
-            for x in groups:
-
-                p = os.path.join(basedir, "group_vars/%s" % x)
-
-                # the file can be <groupname> or end in .yml or .yaml
-                # currently ALL will be loaded, even if more than one
-                paths = [p, '.'.join([p, 'yml']), '.'.join([p, 'yaml'])]
-
-                for path in paths:
-
-                    if os.path.exists(path) and not os.path.isdir(path):
-                        data = utils.parse_yaml_from_file(path)
-                        if type(data) != dict:
-                            raise errors.AnsibleError("%s must be stored as a dictionary/hash" % path)
-
-                        # combine vars overrides by default but can be configured to do a hash
-                        # merge in settings
-
-                        results = utils.combine_vars(results, data)
-
-            # group vars have been loaded
-            # load vars in inventory_dir/hosts_vars/name_of_host
-            # these have greater precedence than group variables
-
-            p = os.path.join(basedir, "host_vars/%s" % host.name)
-
-            # again allow the file to be named filename or end in .yml or .yaml
-            paths = [p, '.'.join([p, 'yml']), '.'.join([p, 'yaml'])]
-
-            for path in paths:
+            for group in groups:
+                base_path = os.path.join(basedir, "group_vars/%s" % group)
+                results = _load_vars(base_path, results)
 
-                if os.path.exists(path) and not os.path.isdir(path):
-                    data = utils.parse_yaml_from_file(path)
-                    if type(data) != dict:
-                        raise errors.AnsibleError("%s must be stored as a dictionary/hash" % path)
-                    results = utils.combine_vars(results, data)
+            # same for hostvars in dir/host_vars/name_of_host
+            base_path = os.path.join(basedir, "host_vars/%s" % host.name)
+            results = _load_vars(base_path, results)
 
         # all done, results is a dictionary of variables for this particular host.
         return results