Merge pull request #247 from caracal-pipeline/issue-246

fixes #246

scabha/configuratt/resolvers.py

@@ -226,7 +226,7 @@ def load_include_files(keyword):
                             flags = {}
 
                         # check for (location)filename.yaml or (location)/filename.yaml style
-                        match= re.match("^\\((.+)\\)/?(.+)$", incl)
+                        match = re.match("^\\((.+)\\)/?(.+)$", incl)
                         if match:
                             modulename, filename = match.groups()
                             if modulename.startswith("."):

stimela/commands/doc.py

@@ -9,14 +9,13 @@
 from omegaconf import OmegaConf
 
 import stimela
-from scabha import configuratt
 from stimela import logger, log_exception
 from stimela.main import cli
 from scabha.cargo import ParameterCategory
 from stimela.kitchen.recipe import Recipe
 from stimela.kitchen.cab import Cab
-from stimela.config import ConfigExceptionTypes
 from stimela.exceptions import RecipeValidationError
+from stimela.task_stats import destroy_progress_bar
 
 from .run import load_recipe_files
 
@@ -39,8 +38,6 @@ def doc(what: List[str] = [], do_list=False, implicit=False, obscure=False, all=
 
     log = logger()
     top_tree = Tree(f"stimela doc {' '.join(what)}", guide_style="dim")
-    found_something = False
-    default_recipe = None
 
     if required:
         max_category = ParameterCategory.Required
@@ -70,7 +67,7 @@ def load_recipe(name: str, section: Dict):
     files_to_load = []
     names_to_document = []
     for item in what:
-        if os.path.isfile(item) and os.path.splitext(item)[1].lower() in (".yml", ".yaml"):
+        if os.path.splitext(item)[1].lower() in (".yml", ".yaml"):
             files_to_load.append(item)
             log.info(f"will load recipe/config file '{item}'")
         else:
@@ -80,49 +77,60 @@ def load_recipe(name: str, section: Dict):
     if files_to_load:
         load_recipe_files(files_to_load)
 
+    destroy_progress_bar()
+
+    log.info(f"loaded {len(stimela.CONFIG.cabs)} cab definition(s) and {len(stimela.CONFIG.lib.recipes)} recipe(s)")
+
+    if not stimela.CONFIG.lib.recipes and not stimela.CONFIG.cabs:
+        log.error(f"Nothing to document")
+        sys.exit(2)
+
+    recipes_to_document = set()
+    cabs_to_document = set()
+
     for item in names_to_document:
         recipe_names = fnmatch.filter(stimela.CONFIG.lib.recipes.keys(), item)
         cab_names = fnmatch.filter(stimela.CONFIG.cabs.keys(), item)
         if not recipe_names and not cab_names:
-            log.error(f"'{item}' does not match any files, recipes or cab names. Try -l/--list")
+            log.error(f"'{item}' does not match any recipe or cab names. Try -l/--list")
             sys.exit(2)
+        recipes_to_document.update(recipe_names)
+        cabs_to_document.update(cab_names)
+
+    # if nothing was specified, and only one cab/only one recipe is defined, print that
+    if not names_to_document:
+        if len(stimela.CONFIG.lib.recipes) == 1 and not stimela.CONFIG.cabs:
+            recipes_to_document.update(stimela.CONFIG.lib.recipes.keys())
+        elif len(stimela.CONFIG.cabs) == 1 and not stimela.CONFIG.lib.recipes:
+            cabs_to_document.update(stimela.CONFIG.cabs.keys())
 
-        for name in recipe_names:
+    if recipes_to_document or cabs_to_document:
+        for name in recipes_to_document:
             recipe = load_recipe(name, stimela.CONFIG.lib.recipes[name])
             tree = top_tree.add(f"Recipe: [bold]{name}[/bold]")
             recipe.rich_help(tree, max_category=max_category)
 
-        for name in cab_names:
+        for name in cabs_to_document:
             cab = Cab(**stimela.CONFIG.cabs[name])
             cab.finalize(config=stimela.CONFIG)
             tree = top_tree.add(f"Cab: [bold]{name}[/bold]")
             cab.rich_help(tree, max_category=max_category)
 
-        found_something = True
-
-    if do_list or (not found_something and not default_recipe):
-
+    # list recipes and cabs -- also do this by default if nothing explicit was documented    
+    if do_list or not (recipes_to_document or cabs_to_document):
         if stimela.CONFIG.lib.recipes:
             subtree = top_tree.add("Recipes:")
             table = Table.grid("", "", padding=(0,2))
             for name, recipe in stimela.CONFIG.lib.recipes.items():
                 table.add_row(f"[bold]{name}[/bold]", recipe.info)
             subtree.add(table)
-        elif not do_list and not found_something:
-            log.error(f"nothing particular to document, please specify a recipe name or a cab name, or use -l/--list")
-            sys.exit(2)
 
-    if default_recipe and not found_something:
-        recipe = load_recipe(name, stimela.CONFIG.lib.recipes[default_recipe])
-        tree = top_tree.add(f"Recipe: [bold]{default_recipe}[/bold]")
-        recipe.rich_help(tree, max_category=max_category)
-
-    if do_list:
-        subtree = top_tree.add("Cabs:")
-        table = Table.grid("", "", padding=(0,2))
-        for name, cab in stimela.CONFIG.cabs.items():
-            table.add_row(f"[bold]{name}[/bold]", cab.info)
-        subtree.add(table)            
+        if stimela.CONFIG.cabs:
+            subtree = top_tree.add("Cabs:")
+            table = Table.grid("", "", padding=(0,2))
+            for name, cab in stimela.CONFIG.cabs.items():
+                table.add_row(f"[bold]{name}[/bold]", cab.info)
+            subtree.add(table)            
 
     rich_print(top_tree)
 

stimela/commands/run.py

@@ -5,7 +5,8 @@
 import yaml
 import sys
 import traceback
-import atexit
+import re
+import importlib
 from datetime import datetime
 from typing import List, Optional, Tuple
 from collections import OrderedDict
@@ -30,6 +31,18 @@ def load_recipe_files(filenames: List[str]):
     full_conf = OmegaConf.create()
     full_deps = configuratt.ConfigDependencies()
     for filename in filenames:
+        # check for (location)filename.yaml or (location)/filename.yaml style
+        match1 = re.fullmatch("^\\((.+)\\)/?(.+)$", filename)
+        match2 = re.fullmatch("^([\w.]+)::(.+)$", filename)
+        if match1 or match2:
+            modulename, filename = (match1 or match2).groups()
+            try:
+                mod = importlib.import_module(modulename)
+            except ImportError as exc:
+                log_exception(f"error importing {modulename}", exc)
+                sys.exit(2)
+            filename = os.path.join(os.path.dirname(mod.__file__), filename)
+        # try loading
         try:
             conf, deps = configuratt.load(filename, use_sources=[stimela.CONFIG, full_conf], no_toplevel_cache=True)
         except ConfigExceptionTypes as exc:
@@ -153,7 +166,7 @@ def convert_value(value):
             except Exception as exc:
                 log_exception(f"error parsing {pp}", exc)
                 errcode = 2
-        elif os.path.isfile(pp) and os.path.splitext(pp)[1].lower() in (".yml", ".yaml"):
+        elif os.path.splitext(pp)[1].lower() in (".yml", ".yaml"):
             files_to_load.append(pp)
             log.info(f"will load recipe/config file '{pp}'")
         else: